From 35bbc4d5c79c39895017758ddbb67ddbce8d0e9d Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Tue, 6 Aug 2024 17:19:01 -0300 Subject: [PATCH 001/156] Add custom native tracer for cargo stylus replay --- execution/gethexec/stylus_tracer.go | 138 +++++++++++++++++++ system_tests/stylus_tracer_test.go | 207 ++++++++++++++++++++++++++++ 2 files changed, 345 insertions(+) create mode 100644 execution/gethexec/stylus_tracer.go create mode 100644 system_tests/stylus_tracer_test.go diff --git a/execution/gethexec/stylus_tracer.go b/execution/gethexec/stylus_tracer.go new file mode 100644 index 00000000000..5c1fc5f3a0e --- /dev/null +++ b/execution/gethexec/stylus_tracer.go @@ -0,0 +1,138 @@ +// Copyright 2024, Offchain Labs, Inc. +// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE + +package gethexec + +import ( + "encoding/json" + "math/big" + "sync/atomic" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/tracers" + "github.com/ethereum/go-ethereum/log" +) + +func init() { + tracers.DefaultDirectory.Register("stylusTracer", newStylusTracer, false) +} + +// stylusTracer captures Stylus HostIOs and returns them in a structured format to be used in Cargo +// Stylus Replay. +type stylusTracer struct { + open *[]HostioTraceInfo + stack []*[]HostioTraceInfo + interrupt atomic.Bool + reason error +} + +// HostioTraceInfo contains the captured HostIO log returned by stylusTracer. +type HostioTraceInfo struct { + Name string `json:"name"` + Args hexutil.Bytes `json:"args"` + Outs hexutil.Bytes `json:"outs"` + StartInk uint64 `json:"startInk"` + EndInk uint64 `json:"endInk"` + Address *common.Address `json:"address,omitempty"` + Steps *[]HostioTraceInfo `json:"steps,omitempty"` +} + +// nestsHostios contains the hostios with nested calls. +var nestsHostios = map[string]bool{ + "call_contract": true, + "delegate_call_contract": true, + "static_call_contract": true, +} + +func newStylusTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { + var open []HostioTraceInfo + return &stylusTracer{ + open: &open, + }, nil +} + +func (t *stylusTracer) CaptureStylusHostio(name string, args, outs []byte, startInk, endInk uint64) { + if t.interrupt.Load() { + return + } + info := HostioTraceInfo{ + Name: name, + Args: args, + Outs: outs, + StartInk: startInk, + EndInk: endInk, + } + if nestsHostios[name] { + last := pop(t.open) + info.Address = last.Address + info.Steps = last.Steps + } + *t.open = append(*t.open, info) +} + +func (t *stylusTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { + if t.interrupt.Load() { + return + } + inner := []HostioTraceInfo{} + info := HostioTraceInfo{ + Address: &to, + Steps: &inner, + } + *t.open = append(*t.open, info) + t.stack = append(t.stack, t.open) // save where we were + t.open = &inner +} + +func (t *stylusTracer) CaptureExit(output []byte, gasUsed uint64, err error) { + if t.interrupt.Load() { + return + } + t.open = pop(&t.stack) +} + +func (t *stylusTracer) GetResult() (json.RawMessage, error) { + if t.reason != nil { + return nil, t.reason + } + msg, err := json.Marshal(t.open) + if err != nil { + return nil, err + } + return msg, nil +} + +func (t *stylusTracer) Stop(err error) { + t.reason = err + t.interrupt.Store(true) +} + +func pop[T any](stack *[]T) T { + if len(*stack) == 0 { + log.Warn("stylusTracer: trying to pop empty stack") + var zeroVal T + return zeroVal + } + i := len(*stack) - 1 + val := (*stack)[i] + *stack = (*stack)[:i] + return val +} + +// Unimplemented EVMLogger interface methods + +func (t *stylusTracer) CaptureArbitrumTransfer(env *vm.EVM, from, to *common.Address, value *big.Int, before bool, purpose string) { +} +func (t *stylusTracer) CaptureArbitrumStorageGet(key common.Hash, depth int, before bool) {} +func (t *stylusTracer) CaptureArbitrumStorageSet(key, value common.Hash, depth int, before bool) {} +func (t *stylusTracer) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { +} +func (t *stylusTracer) CaptureEnd(output []byte, gasUsed uint64, err error) {} +func (t *stylusTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { +} +func (t *stylusTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, _ *vm.ScopeContext, depth int, err error) { +} +func (t *stylusTracer) CaptureTxStart(gasLimit uint64) {} +func (t *stylusTracer) CaptureTxEnd(restGas uint64) {} diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go new file mode 100644 index 00000000000..1623b0ba030 --- /dev/null +++ b/system_tests/stylus_tracer_test.go @@ -0,0 +1,207 @@ +// Copyright 2024, Offchain Labs, Inc. +// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE + +package arbtest + +import ( + "encoding/binary" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/google/go-cmp/cmp" + "github.com/offchainlabs/nitro/execution/gethexec" + "github.com/offchainlabs/nitro/solgen/go/mocksgen" + "github.com/offchainlabs/nitro/util/testhelpers" +) + +func TestStylusTracer(t *testing.T) { + const jit = false + builder, auth, cleanup := setupProgramTest(t, jit) + ctx := builder.ctx + l2client := builder.L2.Client + l2info := builder.L2Info + rpcClient := builder.L2.Client.Client() + defer cleanup() + + traceTransaction := func(tx common.Hash, tracer string) []gethexec.HostioTraceInfo { + traceOpts := struct { + Tracer string `json:"tracer"` + }{ + Tracer: tracer, + } + var result []gethexec.HostioTraceInfo + err := rpcClient.CallContext(ctx, &result, "debug_traceTransaction", tx, traceOpts) + Require(t, err, "trace transaction") + return result + } + + // Deploy contracts + stylusMulticall := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) + evmMulticall, tx, _, err := mocksgen.DeployMultiCallTest(&auth, builder.L2.Client) + Require(t, err, "deploy evm multicall") + _, err = EnsureTxSucceeded(ctx, l2client, tx) + Require(t, err, "ensure evm multicall deployment") + + // Args for tests + key := testhelpers.RandomHash().Bytes() + value := testhelpers.RandomHash().Bytes() + loadStoreArgs := storeLoadMulticallArgs(key, value) + callArgs := argsForMulticall(vm.CALL, stylusMulticall, nil, []byte{0}) + evmCall := argsForMulticall(vm.CALL, evmMulticall, nil, []byte{0}) + + for _, testCase := range []struct { + name string + contract common.Address + args []byte + want []gethexec.HostioTraceInfo + }{ + { + name: "non-recursive hostios", + contract: stylusMulticall, + args: loadStoreArgs, + want: []gethexec.HostioTraceInfo{ + {Name: "user_entrypoint", Args: intToBe32(len(loadStoreArgs)), Outs: []byte{}}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, + {Name: "read_args", Args: []byte{}, Outs: loadStoreArgs}, + {Name: "storage_cache_bytes32", Args: append(key, value...), Outs: []byte{}}, + {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, + {Name: "storage_load_bytes32", Args: key, Outs: value}, + {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, + {Name: "write_result", Args: value, Outs: []byte{}}, + {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + }, + }, + + { + name: "call stylus contract", + contract: stylusMulticall, + args: callArgs, + want: []gethexec.HostioTraceInfo{ + {Name: "user_entrypoint", Outs: []byte{}, Args: intToBe32(len(callArgs))}, + {Name: "pay_for_memory_grow", Outs: []byte{}, Args: []byte{0x00, 0x01}}, + {Name: "read_args", Args: []byte{}, Outs: callArgs}, + { + Name: "call_contract", + Args: append(stylusMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), + Outs: common.Hex2Bytes("0000000000"), + Address: &stylusMulticall, + Steps: &[]gethexec.HostioTraceInfo{ + {Name: "user_entrypoint", Args: intToBe32(1), Outs: []byte{}}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, + {Name: "read_args", Args: []byte{}, Outs: []byte{0x00}}, + {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, + {Name: "write_result", Args: []byte{}, Outs: []byte{}}, + {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + }, + }, + {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, + {Name: "write_result", Args: []byte{}, Outs: []byte{}}, + {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + }, + }, + + { + name: "call evm contract", + contract: stylusMulticall, + args: evmCall, + want: []gethexec.HostioTraceInfo{ + {Name: "user_entrypoint", Args: intToBe32(len(callArgs)), Outs: []byte{}}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, + {Name: "read_args", Args: []byte{}, Outs: evmCall}, + { + Name: "call_contract", + Args: append(evmMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), + Outs: common.Hex2Bytes("0000000000"), + Address: &evmMulticall, + Steps: &[]gethexec.HostioTraceInfo{}, + }, + {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, + {Name: "write_result", Args: []byte{}, Outs: []byte{}}, + {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + tx := l2info.PrepareTxTo("Owner", &testCase.contract, l2info.TransferGas, nil, testCase.args) + err := l2client.SendTransaction(ctx, tx) + Require(t, err, "send transaction") + + nativeResult := traceTransaction(tx.Hash(), "stylusTracer") + clearInk(nativeResult) + if diff := cmp.Diff(testCase.want, nativeResult); diff != "" { + Fatal(t, "native tracer don't match wanted result", diff) + } + + jsResult := traceTransaction(tx.Hash(), jsStylusTracer) + clearInk(jsResult) + if diff := cmp.Diff(jsResult, nativeResult); diff != "" { + Fatal(t, "native tracer don't match js trace", diff) + } + }) + } +} + +func storeLoadMulticallArgs(key, value []byte) []byte { + // Args for storing and loading from storage + const ( + storageKind = 0x10 + storeAction = storageKind | 0x00 + loadAction = storageKind | 0x01 + ) + args := []byte{2} // number of actions + // first action + args = binary.BigEndian.AppendUint32(args, 1+64) // length + args = append(args, storeAction) + args = append(args, key...) + args = append(args, value...) + // second action + args = binary.BigEndian.AppendUint32(args, 1+32) // length + args = append(args, loadAction) + args = append(args, key...) + return args +} + +func intToBe32(v int) []byte { + return binary.BigEndian.AppendUint32(nil, uint32(v)) +} + +func clearInk(trace []gethexec.HostioTraceInfo) { + for i := range trace { + trace[i].StartInk = 0 + trace[i].EndInk = 0 + if trace[i].Steps != nil { + clearInk(*trace[i].Steps) + } + } +} + +var jsStylusTracer = ` +{ + "hostio": function(info) { + info.args = toHex(info.args); + info.outs = toHex(info.outs); + if (this.nests.includes(info.name)) { + Object.assign(info, this.open.pop()); + } + this.open.push(info); + }, + "enter": function(frame) { + let inner = []; + this.open.push({ + address: toHex(frame.getTo()), + steps: inner, + }); + this.stack.push(this.open); // save where we were + this.open = inner; + }, + "exit": function(result) { + this.open = this.stack.pop(); + }, + "result": function() { return this.open; }, + "fault": function() { return this.open; }, + stack: [], + open: [], + nests: ["call_contract", "delegate_call_contract", "static_call_contract"] +} +` From fcd7810ca801d7c1f8b58e7b33aa6dbcf994471a Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Thu, 8 Aug 2024 14:49:50 -0300 Subject: [PATCH 002/156] Fix memory aliasing lint warning --- system_tests/stylus_tracer_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index 1623b0ba030..82353294571 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -123,7 +123,8 @@ func TestStylusTracer(t *testing.T) { }, } { t.Run(testCase.name, func(t *testing.T) { - tx := l2info.PrepareTxTo("Owner", &testCase.contract, l2info.TransferGas, nil, testCase.args) + to := testCase.contract + tx := l2info.PrepareTxTo("Owner", &to, l2info.TransferGas, nil, testCase.args) err := l2client.SendTransaction(ctx, tx) Require(t, err, "send transaction") From 239bad99b17c1d9f89a78a3a17ff1e3e3b165608 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 9 Aug 2024 11:39:17 -0300 Subject: [PATCH 003/156] Add helpers to append load and store to multicall --- system_tests/program_test.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 5fa5db95c2a..ae34c6c5bbd 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -1617,6 +1617,35 @@ func multicallAppend(calls []byte, opcode vm.OpCode, address common.Address, inn return calls } +func multicallEmptyArgs() []byte { + return []byte{0} // number of actions +} + +func multicallAppendStore(args []byte, key, value common.Hash, emitLog bool) []byte { + var action byte = 0x10 + if emitLog { + action |= 0x08 + } + args[0] += 1 + args = binary.BigEndian.AppendUint32(args, 1+64) // length + args = append(args, action) + args = append(args, key.Bytes()...) + args = append(args, value.Bytes()...) + return args +} + +func multicallAppendLoad(args []byte, key common.Hash, emitLog bool) []byte { + var action byte = 0x11 + if emitLog { + action |= 0x08 + } + args[0] += 1 + args = binary.BigEndian.AppendUint32(args, 1+32) // length + args = append(args, action) + args = append(args, key.Bytes()...) + return args +} + func assertStorageAt( t *testing.T, ctx context.Context, l2client *ethclient.Client, contract common.Address, key, value common.Hash, ) { From 361eff0505408df5b8b45e94a42fc3b288da7953 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 9 Aug 2024 11:53:28 -0300 Subject: [PATCH 004/156] Use helpers to build multicall args in tracer test --- system_tests/stylus_tracer_test.go | 34 +++++++----------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index 82353294571..0c1fc4fca3e 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -44,9 +44,11 @@ func TestStylusTracer(t *testing.T) { Require(t, err, "ensure evm multicall deployment") // Args for tests - key := testhelpers.RandomHash().Bytes() - value := testhelpers.RandomHash().Bytes() - loadStoreArgs := storeLoadMulticallArgs(key, value) + key := testhelpers.RandomHash() + value := testhelpers.RandomHash() + loadStoreArgs := multicallEmptyArgs() + loadStoreArgs = multicallAppendLoad(loadStoreArgs, key, false) + loadStoreArgs = multicallAppendStore(loadStoreArgs, key, value, false) callArgs := argsForMulticall(vm.CALL, stylusMulticall, nil, []byte{0}) evmCall := argsForMulticall(vm.CALL, evmMulticall, nil, []byte{0}) @@ -64,11 +66,11 @@ func TestStylusTracer(t *testing.T) { {Name: "user_entrypoint", Args: intToBe32(len(loadStoreArgs)), Outs: []byte{}}, {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, {Name: "read_args", Args: []byte{}, Outs: loadStoreArgs}, - {Name: "storage_cache_bytes32", Args: append(key, value...), Outs: []byte{}}, + {Name: "storage_cache_bytes32", Args: append(key.Bytes(), value.Bytes()...), Outs: []byte{}}, {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, - {Name: "storage_load_bytes32", Args: key, Outs: value}, + {Name: "storage_load_bytes32", Args: key.Bytes(), Outs: value.Bytes()}, {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, - {Name: "write_result", Args: value, Outs: []byte{}}, + {Name: "write_result", Args: value.Bytes(), Outs: []byte{}}, {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, }, }, @@ -143,26 +145,6 @@ func TestStylusTracer(t *testing.T) { } } -func storeLoadMulticallArgs(key, value []byte) []byte { - // Args for storing and loading from storage - const ( - storageKind = 0x10 - storeAction = storageKind | 0x00 - loadAction = storageKind | 0x01 - ) - args := []byte{2} // number of actions - // first action - args = binary.BigEndian.AppendUint32(args, 1+64) // length - args = append(args, storeAction) - args = append(args, key...) - args = append(args, value...) - // second action - args = binary.BigEndian.AppendUint32(args, 1+32) // length - args = append(args, loadAction) - args = append(args, key...) - return args -} - func intToBe32(v int) []byte { return binary.BigEndian.AppendUint32(nil, uint32(v)) } From dc712eb51decd7b23308a69ed725df135d747b5a Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 9 Aug 2024 13:35:30 -0300 Subject: [PATCH 005/156] Fix tests --- system_tests/stylus_tracer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index 0c1fc4fca3e..93d0d0c6907 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -47,8 +47,8 @@ func TestStylusTracer(t *testing.T) { key := testhelpers.RandomHash() value := testhelpers.RandomHash() loadStoreArgs := multicallEmptyArgs() - loadStoreArgs = multicallAppendLoad(loadStoreArgs, key, false) loadStoreArgs = multicallAppendStore(loadStoreArgs, key, value, false) + loadStoreArgs = multicallAppendLoad(loadStoreArgs, key, false) callArgs := argsForMulticall(vm.CALL, stylusMulticall, nil, []byte{0}) evmCall := argsForMulticall(vm.CALL, evmMulticall, nil, []byte{0}) From 5a0b003e62e6b13a390181efaa0a2a5891b83d84 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 16 Aug 2024 14:32:32 -0300 Subject: [PATCH 006/156] Add stack struct to improve readability --- execution/gethexec/stylus_tracer.go | 62 ++++++++++++++--------------- system_tests/stylus_tracer_test.go | 7 ++-- util/stack/stack.go | 39 ++++++++++++++++++ 3 files changed, 72 insertions(+), 36 deletions(-) create mode 100644 util/stack/stack.go diff --git a/execution/gethexec/stylus_tracer.go b/execution/gethexec/stylus_tracer.go index 5c1fc5f3a0e..ac80596e8df 100644 --- a/execution/gethexec/stylus_tracer.go +++ b/execution/gethexec/stylus_tracer.go @@ -12,7 +12,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers" - "github.com/ethereum/go-ethereum/log" + "github.com/offchainlabs/nitro/util/stack" ) func init() { @@ -22,21 +22,21 @@ func init() { // stylusTracer captures Stylus HostIOs and returns them in a structured format to be used in Cargo // Stylus Replay. type stylusTracer struct { - open *[]HostioTraceInfo - stack []*[]HostioTraceInfo + open *stack.Stack[HostioTraceInfo] + stack *stack.Stack[*stack.Stack[HostioTraceInfo]] interrupt atomic.Bool reason error } // HostioTraceInfo contains the captured HostIO log returned by stylusTracer. type HostioTraceInfo struct { - Name string `json:"name"` - Args hexutil.Bytes `json:"args"` - Outs hexutil.Bytes `json:"outs"` - StartInk uint64 `json:"startInk"` - EndInk uint64 `json:"endInk"` - Address *common.Address `json:"address,omitempty"` - Steps *[]HostioTraceInfo `json:"steps,omitempty"` + Name string `json:"name"` + Args hexutil.Bytes `json:"args"` + Outs hexutil.Bytes `json:"outs"` + StartInk uint64 `json:"startInk"` + EndInk uint64 `json:"endInk"` + Address *common.Address `json:"address,omitempty"` + Steps *stack.Stack[HostioTraceInfo] `json:"steps,omitempty"` } // nestsHostios contains the hostios with nested calls. @@ -47,9 +47,9 @@ var nestsHostios = map[string]bool{ } func newStylusTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { - var open []HostioTraceInfo return &stylusTracer{ - open: &open, + open: stack.NewStack[HostioTraceInfo](), + stack: stack.NewStack[*stack.Stack[HostioTraceInfo]](), }, nil } @@ -65,32 +65,40 @@ func (t *stylusTracer) CaptureStylusHostio(name string, args, outs []byte, start EndInk: endInk, } if nestsHostios[name] { - last := pop(t.open) + last, err := t.open.Pop() + if err != nil { + t.Stop(err) + return + } info.Address = last.Address info.Steps = last.Steps } - *t.open = append(*t.open, info) + t.open.Push(info) } func (t *stylusTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { if t.interrupt.Load() { return } - inner := []HostioTraceInfo{} + inner := stack.NewStack[HostioTraceInfo]() info := HostioTraceInfo{ Address: &to, - Steps: &inner, + Steps: inner, } - *t.open = append(*t.open, info) - t.stack = append(t.stack, t.open) // save where we were - t.open = &inner + t.open.Push(info) + t.stack.Push(t.open) + t.open = inner } -func (t *stylusTracer) CaptureExit(output []byte, gasUsed uint64, err error) { +func (t *stylusTracer) CaptureExit(output []byte, gasUsed uint64, _ error) { if t.interrupt.Load() { return } - t.open = pop(&t.stack) + var err error + t.open, err = t.stack.Pop() + if err != nil { + t.Stop(err) + } } func (t *stylusTracer) GetResult() (json.RawMessage, error) { @@ -109,18 +117,6 @@ func (t *stylusTracer) Stop(err error) { t.interrupt.Store(true) } -func pop[T any](stack *[]T) T { - if len(*stack) == 0 { - log.Warn("stylusTracer: trying to pop empty stack") - var zeroVal T - return zeroVal - } - i := len(*stack) - 1 - val := (*stack)[i] - *stack = (*stack)[:i] - return val -} - // Unimplemented EVMLogger interface methods func (t *stylusTracer) CaptureArbitrumTransfer(env *vm.EVM, from, to *common.Address, value *big.Int, before bool, purpose string) { diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index 93d0d0c6907..eaa71b4c4f4 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -12,6 +12,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/offchainlabs/nitro/execution/gethexec" "github.com/offchainlabs/nitro/solgen/go/mocksgen" + "github.com/offchainlabs/nitro/util/stack" "github.com/offchainlabs/nitro/util/testhelpers" ) @@ -88,14 +89,14 @@ func TestStylusTracer(t *testing.T) { Args: append(stylusMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), Outs: common.Hex2Bytes("0000000000"), Address: &stylusMulticall, - Steps: &[]gethexec.HostioTraceInfo{ + Steps: (*stack.Stack[gethexec.HostioTraceInfo])(&[]gethexec.HostioTraceInfo{ {Name: "user_entrypoint", Args: intToBe32(1), Outs: []byte{}}, {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, {Name: "read_args", Args: []byte{}, Outs: []byte{0x00}}, {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, {Name: "write_result", Args: []byte{}, Outs: []byte{}}, {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, - }, + }), }, {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, {Name: "write_result", Args: []byte{}, Outs: []byte{}}, @@ -116,7 +117,7 @@ func TestStylusTracer(t *testing.T) { Args: append(evmMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), Outs: common.Hex2Bytes("0000000000"), Address: &evmMulticall, - Steps: &[]gethexec.HostioTraceInfo{}, + Steps: stack.NewStack[gethexec.HostioTraceInfo](), }, {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, {Name: "write_result", Args: []byte{}, Outs: []byte{}}, diff --git a/util/stack/stack.go b/util/stack/stack.go new file mode 100644 index 00000000000..1b7ac3f9d94 --- /dev/null +++ b/util/stack/stack.go @@ -0,0 +1,39 @@ +// Copyright 2024, Offchain Labs, Inc. +// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE + +package stack + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/log" +) + +type Stack[T any] []T + +func NewStack[T any]() *Stack[T] { + return &Stack[T]{} +} + +func (s *Stack[T]) Push(v T) { + if s == nil { + log.Warn("trying to push nil stack") + return + } + *s = append(*s, v) +} + +func (s *Stack[T]) Pop() (T, error) { + if s == nil { + var zeroVal T + return zeroVal, fmt.Errorf("trying to pop nil stack") + } + if len(*s) == 0 { + var zeroVal T + return zeroVal, fmt.Errorf("trying to pop empty stack") + } + i := len(*s) - 1 + val := (*s)[i] + *s = (*s)[:i] + return val, nil +} From 316db16dba7826e868f1d72f66fc42fe47a1c34e Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 16 Aug 2024 14:58:52 -0300 Subject: [PATCH 007/156] Add documentation to clearink function --- system_tests/stylus_tracer_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index eaa71b4c4f4..f69bfc42bc0 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -150,6 +150,9 @@ func intToBe32(v int) []byte { return binary.BigEndian.AppendUint32(nil, uint32(v)) } +// clearInk removes the start and end ink values from the trace so we can compare them. +// In Arbitrum, the gas used by the transaction varies depending on the L1 fees, so the trace +// returns different gas values and we can't hardcode them. func clearInk(trace []gethexec.HostioTraceInfo) { for i := range trace { trace[i].StartInk = 0 From 37475c23c0632962bf9546950134a91c41c10249 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 16 Aug 2024 15:11:52 -0300 Subject: [PATCH 008/156] Add documentation to HostIO trace info fields --- execution/gethexec/stylus_tracer.go | 31 ++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/execution/gethexec/stylus_tracer.go b/execution/gethexec/stylus_tracer.go index ac80596e8df..eb43bed080e 100644 --- a/execution/gethexec/stylus_tracer.go +++ b/execution/gethexec/stylus_tracer.go @@ -30,13 +30,30 @@ type stylusTracer struct { // HostioTraceInfo contains the captured HostIO log returned by stylusTracer. type HostioTraceInfo struct { - Name string `json:"name"` - Args hexutil.Bytes `json:"args"` - Outs hexutil.Bytes `json:"outs"` - StartInk uint64 `json:"startInk"` - EndInk uint64 `json:"endInk"` - Address *common.Address `json:"address,omitempty"` - Steps *stack.Stack[HostioTraceInfo] `json:"steps,omitempty"` + // Name of the HostIO. + Name string `json:"name"` + + // Arguments of the HostIO encoded as binary. + // For details about the encoding check the HostIO implemenation on + // arbitrator/wasm-libraries/user-host-trait. + Args hexutil.Bytes `json:"args"` + + // Outputs of the HostIO encoded as binary. + // For details about the encoding check the HostIO implemenation on + // arbitrator/wasm-libraries/user-host-trait. + Outs hexutil.Bytes `json:"outs"` + + // Amount of Ink before executing the HostIO. + StartInk uint64 `json:"startInk"` + + // Amount of Ink after executing the HostIO. + EndInk uint64 `json:"endInk"` + + // For *call HostIOs, the address of the called contract. + Address *common.Address `json:"address,omitempty"` + + // For *call HostIOs, the steps performed by the called contract. + Steps *stack.Stack[HostioTraceInfo] `json:"steps,omitempty"` } // nestsHostios contains the hostios with nested calls. From cfe65120c80f5524ba33b18490df3529fcd9383c Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 16 Aug 2024 16:31:13 -0300 Subject: [PATCH 009/156] Handle EVM calls on stylus tracer --- execution/gethexec/stylus_tracer.go | 33 +++++++++ system_tests/stylus_tracer_test.go | 111 ++++++++++++++++++++-------- 2 files changed, 113 insertions(+), 31 deletions(-) diff --git a/execution/gethexec/stylus_tracer.go b/execution/gethexec/stylus_tracer.go index eb43bed080e..16c43d71d86 100644 --- a/execution/gethexec/stylus_tracer.go +++ b/execution/gethexec/stylus_tracer.go @@ -5,7 +5,9 @@ package gethexec import ( "encoding/json" + "fmt" "math/big" + "strings" "sync/atomic" "github.com/ethereum/go-ethereum/common" @@ -87,6 +89,14 @@ func (t *stylusTracer) CaptureStylusHostio(name string, args, outs []byte, start t.Stop(err) return } + if !strings.HasPrefix(last.Name, "evm_") || last.Name[4:] != info.Name { + t.Stop(fmt.Errorf("trace inconsistency for %v: last opcode is %v", info.Name, last.Name)) + return + } + if last.Steps == nil { + t.Stop(fmt.Errorf("trace inconsistency for %v: nil steps", info.Name)) + return + } info.Address = last.Address info.Steps = last.Steps } @@ -97,8 +107,28 @@ func (t *stylusTracer) CaptureEnter(typ vm.OpCode, from common.Address, to commo if t.interrupt.Load() { return } + + // This function adds the prefix evm_ because it assumes the opcode came from the EVM. + // If the opcode comes from WASM, the CaptureStylusHostio function will remove the evm prefix. + var name string + switch typ { + case vm.CALL: + name = "evm_call_contract" + case vm.DELEGATECALL: + name = "evm_delegate_call_contract" + case vm.STATICCALL: + name = "evm_static_call_contract" + case vm.CREATE: + name = "evm_create1" + case vm.CREATE2: + name = "evm_create2" + case vm.SELFDESTRUCT: + name = "evm_self_destruct" + } + inner := stack.NewStack[HostioTraceInfo]() info := HostioTraceInfo{ + Name: name, Address: &to, Steps: inner, } @@ -122,6 +152,9 @@ func (t *stylusTracer) GetResult() (json.RawMessage, error) { if t.reason != nil { return nil, t.reason } + if t.open == nil { + return nil, fmt.Errorf("trace is nil") + } msg, err := json.Marshal(t.open) if err != nil { return nil, err diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index f69bfc42bc0..f7c83c73f67 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -64,15 +64,15 @@ func TestStylusTracer(t *testing.T) { contract: stylusMulticall, args: loadStoreArgs, want: []gethexec.HostioTraceInfo{ - {Name: "user_entrypoint", Args: intToBe32(len(loadStoreArgs)), Outs: []byte{}}, - {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, - {Name: "read_args", Args: []byte{}, Outs: loadStoreArgs}, - {Name: "storage_cache_bytes32", Args: append(key.Bytes(), value.Bytes()...), Outs: []byte{}}, - {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, + {Name: "user_entrypoint", Args: intToBe32(len(loadStoreArgs))}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}}, + {Name: "read_args", Outs: loadStoreArgs}, + {Name: "storage_cache_bytes32", Args: append(key.Bytes(), value.Bytes()...)}, + {Name: "storage_flush_cache", Args: []byte{0x00}}, {Name: "storage_load_bytes32", Args: key.Bytes(), Outs: value.Bytes()}, - {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, - {Name: "write_result", Args: value.Bytes(), Outs: []byte{}}, - {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + {Name: "storage_flush_cache", Args: []byte{0x00}}, + {Name: "write_result", Args: value.Bytes()}, + {Name: "user_returned", Outs: intToBe32(0)}, }, }, @@ -81,26 +81,26 @@ func TestStylusTracer(t *testing.T) { contract: stylusMulticall, args: callArgs, want: []gethexec.HostioTraceInfo{ - {Name: "user_entrypoint", Outs: []byte{}, Args: intToBe32(len(callArgs))}, - {Name: "pay_for_memory_grow", Outs: []byte{}, Args: []byte{0x00, 0x01}}, - {Name: "read_args", Args: []byte{}, Outs: callArgs}, + {Name: "user_entrypoint", Args: intToBe32(len(callArgs))}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}}, + {Name: "read_args", Outs: callArgs}, { Name: "call_contract", Args: append(stylusMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), Outs: common.Hex2Bytes("0000000000"), Address: &stylusMulticall, Steps: (*stack.Stack[gethexec.HostioTraceInfo])(&[]gethexec.HostioTraceInfo{ - {Name: "user_entrypoint", Args: intToBe32(1), Outs: []byte{}}, - {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, - {Name: "read_args", Args: []byte{}, Outs: []byte{0x00}}, - {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, - {Name: "write_result", Args: []byte{}, Outs: []byte{}}, - {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + {Name: "user_entrypoint", Args: intToBe32(1)}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}}, + {Name: "read_args", Outs: []byte{0x00}}, + {Name: "storage_flush_cache", Args: []byte{0x00}}, + {Name: "write_result"}, + {Name: "user_returned", Outs: intToBe32(0)}, }), }, - {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, - {Name: "write_result", Args: []byte{}, Outs: []byte{}}, - {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + {Name: "storage_flush_cache", Args: []byte{0x00}}, + {Name: "write_result"}, + {Name: "user_returned", Outs: intToBe32(0)}, }, }, @@ -109,9 +109,9 @@ func TestStylusTracer(t *testing.T) { contract: stylusMulticall, args: evmCall, want: []gethexec.HostioTraceInfo{ - {Name: "user_entrypoint", Args: intToBe32(len(callArgs)), Outs: []byte{}}, - {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, - {Name: "read_args", Args: []byte{}, Outs: evmCall}, + {Name: "user_entrypoint", Args: intToBe32(len(callArgs))}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}}, + {Name: "read_args", Outs: evmCall}, { Name: "call_contract", Args: append(evmMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), @@ -119,9 +119,29 @@ func TestStylusTracer(t *testing.T) { Address: &evmMulticall, Steps: stack.NewStack[gethexec.HostioTraceInfo](), }, - {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, - {Name: "write_result", Args: []byte{}, Outs: []byte{}}, - {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + {Name: "storage_flush_cache", Args: []byte{0x00}}, + {Name: "write_result"}, + {Name: "user_returned", Outs: intToBe32(0)}, + }, + }, + + { + name: "evm contract calling wasm", + contract: evmMulticall, + args: callArgs, + want: []gethexec.HostioTraceInfo{ + { + Name: "evm_call_contract", + Address: &stylusMulticall, + Steps: (*stack.Stack[gethexec.HostioTraceInfo])(&[]gethexec.HostioTraceInfo{ + {Name: "user_entrypoint", Args: intToBe32(1)}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}}, + {Name: "read_args", Outs: []byte{0x00}}, + {Name: "storage_flush_cache", Args: []byte{0x00}}, + {Name: "write_result"}, + {Name: "user_returned", Outs: intToBe32(0)}, + }), + }, }, }, } { @@ -132,13 +152,13 @@ func TestStylusTracer(t *testing.T) { Require(t, err, "send transaction") nativeResult := traceTransaction(tx.Hash(), "stylusTracer") - clearInk(nativeResult) + normalizeHostioTrace(nativeResult) if diff := cmp.Diff(testCase.want, nativeResult); diff != "" { Fatal(t, "native tracer don't match wanted result", diff) } jsResult := traceTransaction(tx.Hash(), jsStylusTracer) - clearInk(jsResult) + normalizeHostioTrace(jsResult) if diff := cmp.Diff(jsResult, nativeResult); diff != "" { Fatal(t, "native tracer don't match js trace", diff) } @@ -150,15 +170,21 @@ func intToBe32(v int) []byte { return binary.BigEndian.AppendUint32(nil, uint32(v)) } -// clearInk removes the start and end ink values from the trace so we can compare them. +// normalize removes the start and end ink values from the trace so we can compare them. // In Arbitrum, the gas used by the transaction varies depending on the L1 fees, so the trace // returns different gas values and we can't hardcode them. -func clearInk(trace []gethexec.HostioTraceInfo) { +func normalizeHostioTrace(trace []gethexec.HostioTraceInfo) { for i := range trace { trace[i].StartInk = 0 trace[i].EndInk = 0 + if len(trace[i].Args) == 0 { + trace[i].Args = nil + } + if len(trace[i].Outs) == 0 { + trace[i].Outs = nil + } if trace[i].Steps != nil { - clearInk(*trace[i].Steps) + normalizeHostioTrace(*trace[i].Steps) } } } @@ -170,14 +196,37 @@ var jsStylusTracer = ` info.outs = toHex(info.outs); if (this.nests.includes(info.name)) { Object.assign(info, this.open.pop()); + info.name = info.name.substring(4) // remove evm_ } this.open.push(info); }, "enter": function(frame) { let inner = []; + let name = ""; + switch (frame.getType()) { + case "CALL": + name = "evm_call_contract"; + break; + case "DELEGATECALL": + name = "evm_delegate_call_contract"; + break; + case "STATICCALL": + name = "evm_static_call_contract"; + break; + case "CREATE": + name = "evm_create1"; + break; + case "CREATE2": + name = "evm_create2"; + break; + case "SELFDESTRUCT": + name = "evm_self_destruct"; + break; + } this.open.push({ address: toHex(frame.getTo()), steps: inner, + name: name, }); this.stack.push(this.open); // save where we were this.open = inner; From d14e10d101f33c1aaff40a1d0ada291589c1ac83 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 19 Aug 2024 14:30:58 +0200 Subject: [PATCH 010/156] change Ke/s to entries/s in conversion summary log message --- cmd/dbconv/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/dbconv/main.go b/cmd/dbconv/main.go index c0b5c8f8e41..2d61c965523 100644 --- a/cmd/dbconv/main.go +++ b/cmd/dbconv/main.go @@ -85,7 +85,7 @@ func main() { os.Exit(1) } stats := conv.Stats() - log.Info("Conversion finished.", "entries", stats.Entries(), "MB", stats.Bytes()/1024/1024, "avg Ke/s", stats.AverageEntriesPerSecond()/1000, "avg MB/s", stats.AverageBytesPerSecond()/1024/1024, "elapsed", stats.Elapsed()) + log.Info("Conversion finished.", "entries", stats.Entries(), "MB", stats.Bytes()/1024/1024, "avg entries/s", fmt.Sprintf("%.3e", stats.AverageEntriesPerSecond()/1000), "avg MB/s", stats.AverageBytesPerSecond()/1024/1024, "elapsed", stats.Elapsed()) } if config.Compact { From 5c482ffea39b1232d682670ed00b136ad151ff97 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 19 Aug 2024 14:33:06 +0200 Subject: [PATCH 011/156] specify unit of IdealBatchSize in the help message --- cmd/dbconv/dbconv/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/dbconv/dbconv/config.go b/cmd/dbconv/dbconv/config.go index 74623bc2649..917f34261df 100644 --- a/cmd/dbconv/dbconv/config.go +++ b/cmd/dbconv/dbconv/config.go @@ -71,7 +71,7 @@ var DefaultDBConvConfig = DBConvConfig{ func DBConvConfigAddOptions(f *flag.FlagSet) { DBConfigAddOptions("src", f, &DefaultDBConvConfig.Src) DBConfigAddOptions("dst", f, &DefaultDBConvConfig.Dst) - f.Int("ideal-batch-size", DefaultDBConvConfig.IdealBatchSize, "ideal write batch size") + f.Int("ideal-batch-size", DefaultDBConvConfig.IdealBatchSize, "ideal write batch size in bytes") f.Bool("convert", DefaultDBConvConfig.Convert, "enables conversion step") f.Bool("compact", DefaultDBConvConfig.Compact, "enables compaction step") f.String("verify", DefaultDBConvConfig.Verify, "enables verification step (\"\" = disabled, \"keys\" = only keys, \"full\" = keys and values)") From b5a224b26cbab8830f72c7ec88220cc181dd9554 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 19 Aug 2024 14:34:09 +0200 Subject: [PATCH 012/156] don't enable trace logging in TestConversion --- cmd/dbconv/dbconv/dbconv_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/dbconv/dbconv/dbconv_test.go b/cmd/dbconv/dbconv/dbconv_test.go index f31dd68618a..31aa0c39176 100644 --- a/cmd/dbconv/dbconv/dbconv_test.go +++ b/cmd/dbconv/dbconv/dbconv_test.go @@ -4,12 +4,10 @@ import ( "context" "testing" - "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/util/testhelpers" ) func TestConversion(t *testing.T) { - _ = testhelpers.InitTestLog(t, log.LvlTrace) oldDBConfig := DBConfigDefaultSrc oldDBConfig.Data = t.TempDir() From 11d72a3bc9362a1ef9cca9af067350e2bea1a9f4 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 19 Aug 2024 14:49:48 +0200 Subject: [PATCH 013/156] convert_databases.bash: remove unneeded tr command use for srcdir --- scripts/convert-databases.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/convert-databases.bash b/scripts/convert-databases.bash index bd898c2c987..3020b389b46 100755 --- a/scripts/convert-databases.bash +++ b/scripts/convert-databases.bash @@ -182,7 +182,7 @@ fi convert_result= convert () { - srcdir=$(echo $src/$1 | tr -s /) + srcdir="$src"/$1 dstdir=$(echo $dst/$1 | tr -s /) if ! [ -e $dstdir ]; then echo "== Converting $1 db" From cfb4c8736bd7885c12baf01830d7b7f3bf835a4c Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 21 Aug 2024 11:48:50 -0300 Subject: [PATCH 014/156] Ignores gosec G115 --- .golangci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.golangci.yml b/.golangci.yml index 05946701379..e788eca6c0a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -44,6 +44,7 @@ linters-settings: gosec: excludes: - G404 # checks that random numbers are securely generated + - G115 # Potential integer overflow when converting between integer types govet: enable-all: true From dc976f8b113dbae3bdb51133215147e862eecc2a Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 21 Aug 2024 12:16:24 -0500 Subject: [PATCH 015/156] Re-enable G115 lint --- .golangci.yml | 1 - arbcompress/compress_common.go | 4 +++- arbnode/batch_poster.go | 20 +++++++++++++------- arbnode/dataposter/data_poster.go | 7 +++++++ arbnode/dataposter/dbstorage/storage.go | 2 +- arbnode/dataposter/slice/slicestorage.go | 4 ++-- arbnode/dataposter/storage/time.go | 2 ++ arbnode/dataposter/storage_test.go | 1 + arbnode/inbox_reader.go | 4 ++-- arbnode/inbox_tracker.go | 3 +++ arbnode/node.go | 2 ++ arbnode/transaction_streamer.go | 1 + arbos/addressSet/addressSet_test.go | 1 + arbos/addressTable/addressTable.go | 1 + arbos/arbosState/initialization_test.go | 1 + arbos/l1pricing/l1pricing.go | 4 ++-- arbos/l1pricing_test.go | 6 +++--- arbos/l2pricing/l2pricing_test.go | 3 +++ arbos/l2pricing/model.go | 12 ++++++++---- arbos/storage/storage.go | 8 +++----- arbstate/inbox.go | 6 +++--- arbutil/block_message_relation.go | 1 + arbutil/correspondingl1blocknumber.go | 1 + blocks_reexecutor/blocks_reexecutor.go | 3 ++- broadcaster/backlog/backlog.go | 9 +++++++-- broadcaster/backlog/backlog_test.go | 4 ++-- broadcaster/broadcaster.go | 1 + cmd/nitro/nitro.go | 3 ++- cmd/staterecovery/staterecovery.go | 1 + das/aggregator.go | 1 + das/dasRpcClient.go | 1 + das/dasRpcServer.go | 2 ++ das/dastree/dastree.go | 15 +++++++++------ das/db_storage_service.go | 4 +++- das/local_file_storage_service.go | 6 ++++++ das/local_file_storage_service_test.go | 1 + das/s3_storage_service.go | 4 +++- das/sign_after_store_das_writer.go | 1 + das/simple_das_reader_aggregator.go | 12 ++++++------ das/util.go | 2 ++ execution/gethexec/api.go | 1 + execution/gethexec/sequencer.go | 5 +++++ execution/gethexec/tx_pre_checker.go | 1 + execution/nodeInterface/NodeInterface.go | 7 +++++++ precompiles/ArbAddressTable.go | 8 ++++---- precompiles/ArbRetryableTx.go | 7 ++++--- precompiles/ArbSys.go | 4 ++-- relay/relay_stress_test.go | 2 +- staker/block_challenge_backend.go | 2 +- staker/block_validator.go | 7 +++++++ staker/challenge-cache/cache.go | 4 ++-- staker/challenge_manager.go | 2 +- staker/challenge_test.go | 2 +- staker/l1_validator.go | 2 ++ staker/rollup_watcher.go | 2 +- staker/staker.go | 4 ++++ system_tests/block_validator_test.go | 1 + system_tests/forwarder_test.go | 1 + system_tests/initialization_test.go | 1 + system_tests/outbox_test.go | 1 + system_tests/program_recursive_test.go | 1 + system_tests/program_test.go | 5 +++++ system_tests/recreatestate_rpc_test.go | 7 +++++++ system_tests/seq_nonce_test.go | 1 + system_tests/seqinbox_test.go | 6 +++++- system_tests/snap_sync_test.go | 2 ++ system_tests/twonodeslong_test.go | 1 + system_tests/unsupported_txtypes_test.go | 4 ++-- util/arbmath/bips.go | 4 ++-- util/arbmath/math.go | 14 ++++++++++++++ util/arbmath/math_test.go | 3 +++ util/arbmath/uint24.go | 8 +++++--- util/headerreader/header_reader.go | 1 + util/merkletree/merkleTree.go | 4 ++-- util/rpcclient/rpcclient.go | 16 +++++++++------- util/sharedmetrics/sharedmetrics.go | 2 ++ util/testhelpers/testhelpers.go | 1 + validator/client/validation_client.go | 1 + validator/server_arb/execution_run_test.go | 2 +- validator/server_arb/machine_cache.go | 1 + validator/server_arb/validator_spawner.go | 3 +++ validator/server_jit/jit_machine.go | 15 ++++++++++++--- wavmio/stub.go | 6 +++--- wsbroadcastserver/clientconnection.go | 1 + 84 files changed, 247 insertions(+), 91 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index e788eca6c0a..05946701379 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -44,7 +44,6 @@ linters-settings: gosec: excludes: - G404 # checks that random numbers are securely generated - - G115 # Potential integer overflow when converting between integer types govet: enable-all: true diff --git a/arbcompress/compress_common.go b/arbcompress/compress_common.go index a61dd9a171c..997232e7cc8 100644 --- a/arbcompress/compress_common.go +++ b/arbcompress/compress_common.go @@ -17,6 +17,8 @@ func compressedBufferSizeFor(length int) int { return length + (length>>10)*8 + 64 // actual limit is: length + (length >> 14) * 4 + 6 } -func CompressLevel(input []byte, level int) ([]byte, error) { +func CompressLevel(input []byte, level uint64) ([]byte, error) { + // level is trusted and shouldn't be anything crazy + // #nosec G115 return Compress(input, uint32(level), EmptyDictionary) } diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 71239efdbbc..d0764cc783c 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -121,7 +121,7 @@ type BatchPoster struct { nextRevertCheckBlock int64 // the last parent block scanned for reverting batches postedFirstBatch bool // indicates if batch poster has posted the first batch - accessList func(SequencerInboxAccs, AfterDelayedMessagesRead int) types.AccessList + accessList func(SequencerInboxAccs, AfterDelayedMessagesRead uint64) types.AccessList } type l1BlockBound int @@ -374,7 +374,7 @@ func NewBatchPoster(ctx context.Context, opts *BatchPosterOpts) (*BatchPoster, e } // Dataposter sender may be external signer address, so we should initialize // access list after initializing dataposter. - b.accessList = func(SequencerInboxAccs, AfterDelayedMessagesRead int) types.AccessList { + b.accessList = func(SequencerInboxAccs, AfterDelayedMessagesRead uint64) types.AccessList { if !b.config().UseAccessLists || opts.L1Reader.IsParentChainArbitrum() { // Access lists cost gas instead of saving gas when posting to L2s, // because data is expensive in comparison to computation. @@ -433,8 +433,8 @@ type AccessListOpts struct { BridgeAddr common.Address DataPosterAddr common.Address GasRefunderAddr common.Address - SequencerInboxAccs int - AfterDelayedMessagesRead int + SequencerInboxAccs uint64 + AfterDelayedMessagesRead uint64 } // AccessList returns access list (contracts, storage slots) for batchposter. @@ -476,12 +476,12 @@ func AccessList(opts *AccessListOpts) types.AccessList { }, } - for _, v := range []struct{ slotIdx, val int }{ + for _, v := range []struct{ slotIdx, val uint64 }{ {7, opts.SequencerInboxAccs - 1}, // - sequencerInboxAccs[sequencerInboxAccs.length - 1]; (keccak256(7, sequencerInboxAccs.length - 1)) {7, opts.SequencerInboxAccs}, // - sequencerInboxAccs.push(...); (keccak256(7, sequencerInboxAccs.length)) {6, opts.AfterDelayedMessagesRead - 1}, // - delayedInboxAccs[afterDelayedMessagesRead - 1]; (keccak256(6, afterDelayedMessagesRead - 1)) } { - sb := arbutil.SumBytes(arbutil.PaddedKeccak256([]byte{byte(v.slotIdx)}), big.NewInt(int64(v.val)).Bytes()) + sb := arbutil.SumBytes(arbutil.PaddedKeccak256([]byte{byte(v.slotIdx)}), new(big.Int).SetUint64(v.val).Bytes()) l[1].StorageKeys = append(l[1].StorageKeys, common.Hash(sb)) } @@ -603,9 +603,12 @@ func (b *BatchPoster) pollForL1PriceData(ctx context.Context) { l1GasPrice = blobFeePerByte.Uint64() / 16 } } + // #nosec G115 blobGasUsedGauge.Update(int64(*h.BlobGasUsed)) } + // #nosec G115 blockGasUsedGauge.Update(int64(h.GasUsed)) + // #nosec G115 blockGasLimitGauge.Update(int64(h.GasLimit)) suggestedTipCap, err := b.l1Reader.Client().SuggestGasTipCap(ctx) if err != nil { @@ -613,6 +616,7 @@ func (b *BatchPoster) pollForL1PriceData(ctx context.Context) { } else { suggestedTipCapGauge.Update(suggestedTipCap.Int64()) } + // #nosec G115 l1GasPriceGauge.Update(int64(l1GasPrice)) case <-ctx.Done(): return @@ -1176,6 +1180,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) if err != nil { return false, err } + // #nosec G115 firstMsgTime := time.Unix(int64(firstMsg.Message.Header.Timestamp), 0) lastPotentialMsg, err := b.streamer.GetMessage(msgCount - 1) @@ -1403,7 +1408,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) if len(kzgBlobs)*params.BlobTxBlobGasPerBlob > params.MaxBlobGasPerBlock { return false, fmt.Errorf("produced %v blobs for batch but a block can only hold %v (compressed batch was %v bytes long)", len(kzgBlobs), params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob, len(sequencerMsg)) } - accessList := b.accessList(int(batchPosition.NextSeqNum), int(b.building.segments.delayedMsg)) + accessList := b.accessList(batchPosition.NextSeqNum, b.building.segments.delayedMsg) // On restart, we may be trying to estimate gas for a batch whose successor has // already made it into pending state, if not latest state. // In that case, we might get a revert with `DelayedBackwards()`. @@ -1505,6 +1510,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) messagesPerBatch = 1 } backlog := uint64(unpostedMessages) / messagesPerBatch + // #nosec G115 batchPosterEstimatedBatchBacklogGauge.Update(int64(backlog)) if backlog > 10 { logLevel := log.Warn diff --git a/arbnode/dataposter/data_poster.go b/arbnode/dataposter/data_poster.go index 5630a52947f..6a483929b26 100644 --- a/arbnode/dataposter/data_poster.go +++ b/arbnode/dataposter/data_poster.go @@ -359,6 +359,7 @@ func (p *DataPoster) canPostWithNonce(ctx context.Context, nextNonce uint64, thi if err != nil { return fmt.Errorf("getting nonce of a dataposter sender: %w", err) } + // #nosec G115 latestUnconfirmedNonceGauge.Update(int64(unconfirmedNonce)) if nextNonce >= cfg.MaxMempoolTransactions+unconfirmedNonce { return fmt.Errorf("%w: transaction nonce: %d, unconfirmed nonce: %d, max mempool size: %d", ErrExceedsMaxMempoolSize, nextNonce, unconfirmedNonce, cfg.MaxMempoolTransactions) @@ -371,6 +372,7 @@ func (p *DataPoster) canPostWithNonce(ctx context.Context, nextNonce uint64, thi if err != nil { return fmt.Errorf("getting nonce of a dataposter sender: %w", err) } + // #nosec G115 latestUnconfirmedNonceGauge.Update(int64(unconfirmedNonce)) if unconfirmedNonce > nextNonce { return fmt.Errorf("latest on-chain nonce %v is greater than to next nonce %v", unconfirmedNonce, nextNonce) @@ -525,6 +527,7 @@ func (p *DataPoster) feeAndTipCaps(ctx context.Context, nonce uint64, gasLimit u if err != nil { return nil, nil, nil, fmt.Errorf("failed to get latest nonce %v blocks ago (block %v): %w", config.NonceRbfSoftConfs, softConfBlock, err) } + // #nosec G115 latestSoftConfirmedNonceGauge.Update(int64(softConfNonce)) suggestedTip, err := p.client.SuggestGasTipCap(ctx) @@ -1052,6 +1055,7 @@ func (p *DataPoster) updateNonce(ctx context.Context) error { } return nil } + // #nosec G115 latestFinalizedNonceGauge.Update(int64(nonce)) log.Info("Data poster transactions confirmed", "previousNonce", p.nonce, "newNonce", nonce, "previousL1Block", p.lastBlock, "newL1Block", header.Number) if len(p.errorCount) > 0 { @@ -1132,6 +1136,7 @@ func (p *DataPoster) Start(ctxIn context.Context) { log.Warn("Failed to get latest nonce", "err", err) return minWait } + // #nosec G115 latestUnconfirmedNonceGauge.Update(int64(unconfirmedNonce)) // We use unconfirmedNonce here to replace-by-fee transactions that aren't in a block, // excluding those that are in an unconfirmed block. If a reorg occurs, we'll continue @@ -1154,7 +1159,9 @@ func (p *DataPoster) Start(ctxIn context.Context) { confirmedNonce := unconfirmedNonce - 1 confirmedMeta, err := p.queue.Get(ctx, confirmedNonce) if err == nil && confirmedMeta != nil { + // #nosec G115 totalQueueWeightGauge.Update(int64(arbmath.SaturatingUSub(latestCumulativeWeight, confirmedMeta.CumulativeWeight()))) + // #nosec G115 totalQueueLengthGauge.Update(int64(arbmath.SaturatingUSub(latestNonce, confirmedNonce))) } else { log.Error("Failed to fetch latest confirmed tx from queue", "confirmedNonce", confirmedNonce, "err", err, "confirmedMeta", confirmedMeta) diff --git a/arbnode/dataposter/dbstorage/storage.go b/arbnode/dataposter/dbstorage/storage.go index 97055193a64..37ebfa50991 100644 --- a/arbnode/dataposter/dbstorage/storage.go +++ b/arbnode/dataposter/dbstorage/storage.go @@ -42,7 +42,7 @@ func (s *Storage) FetchContents(_ context.Context, startingIndex uint64, maxResu var res []*storage.QueuedTransaction it := s.db.NewIterator([]byte(""), idxToKey(startingIndex)) defer it.Release() - for i := 0; i < int(maxResults); i++ { + for i := uint64(0); i < maxResults; i++ { if !it.Next() { break } diff --git a/arbnode/dataposter/slice/slicestorage.go b/arbnode/dataposter/slice/slicestorage.go index 69de7564a32..8685ed6f54b 100644 --- a/arbnode/dataposter/slice/slicestorage.go +++ b/arbnode/dataposter/slice/slicestorage.go @@ -89,8 +89,8 @@ func (s *Storage) Put(_ context.Context, index uint64, prev, new *storage.Queued } s.queue = append(s.queue, newEnc) } else if index >= s.firstNonce { - queueIdx := int(index - s.firstNonce) - if queueIdx > len(s.queue) { + queueIdx := index - s.firstNonce + if queueIdx > uint64(len(s.queue)) { return fmt.Errorf("attempted to set out-of-bounds index %v in queue starting at %v of length %v", index, s.firstNonce, len(s.queue)) } prevEnc, err := s.encDec().Encode(prev) diff --git a/arbnode/dataposter/storage/time.go b/arbnode/dataposter/storage/time.go index aa15f291702..82f8a3dbf55 100644 --- a/arbnode/dataposter/storage/time.go +++ b/arbnode/dataposter/storage/time.go @@ -34,11 +34,13 @@ func (b *RlpTime) DecodeRLP(s *rlp.Stream) error { if err != nil { return err } + // #nosec G115 *b = RlpTime(time.Unix(int64(enc.Seconds), int64(enc.Nanos))) return nil } func (b RlpTime) EncodeRLP(w io.Writer) error { + // #nosec G115 return rlp.Encode(w, rlpTimeEncoding{ Seconds: uint64(time.Time(b).Unix()), Nanos: uint64(time.Time(b).Nanosecond()), diff --git a/arbnode/dataposter/storage_test.go b/arbnode/dataposter/storage_test.go index e2aa321e0df..8934d92b456 100644 --- a/arbnode/dataposter/storage_test.go +++ b/arbnode/dataposter/storage_test.go @@ -362,6 +362,7 @@ func TestLength(t *testing.T) { if err != nil { t.Fatalf("Length() unexpected error: %v", err) } + // #nosec G115 if want := arbmath.MaxInt(0, 20-int(tc.pruneFrom)); got != want { t.Errorf("Length() = %d want %d", got, want) } diff --git a/arbnode/inbox_reader.go b/arbnode/inbox_reader.go index 77a0b6e7a29..fd050b5f67b 100644 --- a/arbnode/inbox_reader.go +++ b/arbnode/inbox_reader.go @@ -437,8 +437,8 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { } delayedMessages, err := r.delayedBridge.LookupMessagesInRange(ctx, from, to, func(batchNum uint64) ([]byte, error) { if len(sequencerBatches) > 0 && batchNum >= sequencerBatches[0].SequenceNumber { - idx := int(batchNum - sequencerBatches[0].SequenceNumber) - if idx < len(sequencerBatches) { + idx := batchNum - sequencerBatches[0].SequenceNumber + if idx < uint64(len(sequencerBatches)) { return sequencerBatches[idx].Serialize(ctx, r.l1Reader.Client()) } log.Warn("missing mentioned batch in L1 message lookup", "batch", batchNum) diff --git a/arbnode/inbox_tracker.go b/arbnode/inbox_tracker.go index 23b81bde627..fe4149c80eb 100644 --- a/arbnode/inbox_tracker.go +++ b/arbnode/inbox_tracker.go @@ -804,6 +804,7 @@ func (t *InboxTracker) AddSequencerBatches(ctx context.Context, client arbutil.L if len(messages) > 0 { latestTimestamp = messages[len(messages)-1].Message.Header.Timestamp } + // #nosec G115 log.Info( "InboxTracker", "sequencerBatchCount", pos, @@ -811,7 +812,9 @@ func (t *InboxTracker) AddSequencerBatches(ctx context.Context, client arbutil.L "l1Block", latestL1Block, "l1Timestamp", time.Unix(int64(latestTimestamp), 0), ) + // #nosec G115 inboxLatestBatchGauge.Update(int64(pos)) + // #nosec G115 inboxLatestBatchMessageGauge.Update(int64(newMessageCount)) if t.validator != nil { diff --git a/arbnode/node.go b/arbnode/node.go index c66598618f9..93b58e800f9 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -515,6 +515,7 @@ func createNodeImpl( if err != nil { return nil, err } + // #nosec G115 sequencerInbox, err := NewSequencerInbox(l1client, deployInfo.SequencerInbox, int64(deployInfo.DeployedAt)) if err != nil { return nil, err @@ -639,6 +640,7 @@ func createNodeImpl( tmpAddress := common.HexToAddress(config.Staker.ContractWalletAddress) existingWalletAddress = &tmpAddress } + // #nosec G115 wallet, err = validatorwallet.NewContract(dp, existingWalletAddress, deployInfo.ValidatorWalletCreator, deployInfo.Rollup, l1Reader, txOptsValidator, int64(deployInfo.DeployedAt), func(common.Address) {}, getExtraGas) if err != nil { return nil, err diff --git a/arbnode/transaction_streamer.go b/arbnode/transaction_streamer.go index 90e7feddc65..a5bab8342f9 100644 --- a/arbnode/transaction_streamer.go +++ b/arbnode/transaction_streamer.go @@ -840,6 +840,7 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil // Active broadcast reorg and L1 messages at or before start of broadcast messages // Or no active broadcast reorg and broadcast messages start before or immediately after last L1 message if messagesAfterPos >= broadcastStartPos { + // #nosec G115 broadcastSliceIndex := int(messagesAfterPos - broadcastStartPos) messagesOldLen := len(messages) if broadcastSliceIndex < len(s.broadcasterQueuedMessages) { diff --git a/arbos/addressSet/addressSet_test.go b/arbos/addressSet/addressSet_test.go index 7d06c74f0bb..d32e07a546d 100644 --- a/arbos/addressSet/addressSet_test.go +++ b/arbos/addressSet/addressSet_test.go @@ -316,6 +316,7 @@ func checkIfRectifyMappingWorks(t *testing.T, aset *AddressSet, owners []common. Fail(t, "RectifyMapping did not fix the mismatch") } + // #nosec G115 if clearList && int(size(t, aset)) != index+1 { Fail(t, "RectifyMapping did not fix the mismatch") } diff --git a/arbos/addressTable/addressTable.go b/arbos/addressTable/addressTable.go index 3fbb7b3782e..566c71b6895 100644 --- a/arbos/addressTable/addressTable.go +++ b/arbos/addressTable/addressTable.go @@ -118,6 +118,7 @@ func (atab *AddressTable) Decompress(buf []byte) (common.Address, uint64, error) if !exists { return common.Address{}, 0, errors.New("invalid index in compressed address") } + // #nosec G115 numBytesRead := uint64(rd.Size() - int64(rd.Len())) return addr, numBytesRead, nil } diff --git a/arbos/arbosState/initialization_test.go b/arbos/arbosState/initialization_test.go index 34802392fe7..b0fe1d0dac4 100644 --- a/arbos/arbosState/initialization_test.go +++ b/arbos/arbosState/initialization_test.go @@ -109,6 +109,7 @@ func pseudorandomAccountInitInfoForTesting(prand *testhelpers.PseudoRandomDataSo } func pseudorandomHashHashMapForTesting(prand *testhelpers.PseudoRandomDataSource, maxItems uint64) map[common.Hash]common.Hash { + // #nosec G115 size := int(prand.GetUint64() % maxItems) ret := make(map[common.Hash]common.Hash) for i := 0; i < size; i++ { diff --git a/arbos/l1pricing/l1pricing.go b/arbos/l1pricing/l1pricing.go index 9e00eeb581a..392bf36d377 100644 --- a/arbos/l1pricing/l1pricing.go +++ b/arbos/l1pricing/l1pricing.go @@ -509,7 +509,7 @@ func (ps *L1PricingState) getPosterUnitsWithoutCache(tx *types.Transaction, post return 0 } - l1Bytes, err := byteCountAfterBrotliLevel(txBytes, int(brotliCompressionLevel)) + l1Bytes, err := byteCountAfterBrotliLevel(txBytes, brotliCompressionLevel) if err != nil { panic(fmt.Sprintf("failed to compress tx: %v", err)) } @@ -594,7 +594,7 @@ func (ps *L1PricingState) PosterDataCost(message *core.Message, poster common.Ad return am.BigMulByUint(pricePerUnit, units), units } -func byteCountAfterBrotliLevel(input []byte, level int) (uint64, error) { +func byteCountAfterBrotliLevel(input []byte, level uint64) (uint64, error) { compressed, err := arbcompress.CompressLevel(input, level) if err != nil { return 0, err diff --git a/arbos/l1pricing_test.go b/arbos/l1pricing_test.go index 6e2b1b7eec3..1cda4b3d82b 100644 --- a/arbos/l1pricing_test.go +++ b/arbos/l1pricing_test.go @@ -100,7 +100,7 @@ func expectedResultsForL1Test(input *l1PricingTest) *l1TestExpectedResults { availableFunds = availableFundsCap } } - fundsWantedForRewards := big.NewInt(int64(input.unitReward * input.unitsPerSecond)) + fundsWantedForRewards := new(big.Int).SetUint64(input.unitReward * input.unitsPerSecond) unitsAllocated := arbmath.UintToBig(input.unitsPerSecond) if arbmath.BigLessThan(availableFunds, fundsWantedForRewards) { ret.rewardRecipientBalance = availableFunds @@ -111,7 +111,7 @@ func expectedResultsForL1Test(input *l1PricingTest) *l1TestExpectedResults { uncappedAvailableFunds = arbmath.BigSub(uncappedAvailableFunds, ret.rewardRecipientBalance) ret.unitsRemaining = (3 * input.unitsPerSecond) - unitsAllocated.Uint64() - maxCollectable := big.NewInt(int64(input.fundsSpent)) + maxCollectable := new(big.Int).SetUint64(input.fundsSpent) if arbmath.BigLessThan(availableFunds, maxCollectable) { maxCollectable = availableFunds } @@ -170,7 +170,7 @@ func _testL1PricingFundsDue(t *testing.T, testParams *l1PricingTest, expectedRes Require(t, err) // create some fake collection - balanceAdded := big.NewInt(int64(testParams.fundsCollectedPerSecond * 3)) + balanceAdded := new(big.Int).SetUint64(testParams.fundsCollectedPerSecond * 3) unitsAdded := testParams.unitsPerSecond * 3 evm.StateDB.AddBalance(l1pricing.L1PricerFundsPoolAddress, uint256.MustFromBig(balanceAdded)) err = l1p.SetL1FeesAvailable(balanceAdded) diff --git a/arbos/l2pricing/l2pricing_test.go b/arbos/l2pricing/l2pricing_test.go index 57759d7f829..aa1e785f70a 100644 --- a/arbos/l2pricing/l2pricing_test.go +++ b/arbos/l2pricing/l2pricing_test.go @@ -40,6 +40,7 @@ func TestPricingModelExp(t *testing.T) { // show that running at the speed limit with a full pool is a steady-state colors.PrintBlue("full pool & speed limit") for seconds := 0; seconds < 4; seconds++ { + // #nosec G115 fakeBlockUpdate(t, pricing, int64(seconds)*int64(limit), uint64(seconds)) if getPrice(t, pricing) != minPrice { Fail(t, "price changed when it shouldn't have") @@ -50,6 +51,7 @@ func TestPricingModelExp(t *testing.T) { // note that for large enough spans of time the price will rise a miniscule amount due to the pool's avg colors.PrintBlue("pool target & speed limit") for seconds := 0; seconds < 4; seconds++ { + // #nosec G115 fakeBlockUpdate(t, pricing, int64(seconds)*int64(limit), uint64(seconds)) if getPrice(t, pricing) != minPrice { Fail(t, "price changed when it shouldn't have") @@ -59,6 +61,7 @@ func TestPricingModelExp(t *testing.T) { // show that running over the speed limit escalates the price before the pool drains colors.PrintBlue("exceeding the speed limit") for { + // #nosec G115 fakeBlockUpdate(t, pricing, 8*int64(limit), 1) newPrice := getPrice(t, pricing) if newPrice < price { diff --git a/arbos/l2pricing/model.go b/arbos/l2pricing/model.go index 131af2c2cf6..476effa8aa8 100644 --- a/arbos/l2pricing/model.go +++ b/arbos/l2pricing/model.go @@ -30,22 +30,26 @@ func (ps *L2PricingState) AddToGasPool(gas int64) error { return err } // pay off some of the backlog with the added gas, stopping at 0 - backlog = arbmath.SaturatingUCast[uint64](arbmath.SaturatingSub(int64(backlog), gas)) + if gas > 0 { + backlog = arbmath.SaturatingUSub(backlog, uint64(gas)) + } else { + backlog = arbmath.SaturatingUAdd(backlog, uint64(-gas)) + } return ps.SetGasBacklog(backlog) } // UpdatePricingModel updates the pricing model with info from the last block func (ps *L2PricingState) UpdatePricingModel(l2BaseFee *big.Int, timePassed uint64, debug bool) { speedLimit, _ := ps.SpeedLimitPerSecond() - _ = ps.AddToGasPool(int64(timePassed * speedLimit)) + _ = ps.AddToGasPool(arbmath.SaturatingCast[int64](arbmath.SaturatingUMul(timePassed, speedLimit))) inertia, _ := ps.PricingInertia() tolerance, _ := ps.BacklogTolerance() backlog, _ := ps.GasBacklog() minBaseFee, _ := ps.MinBaseFeeWei() baseFee := minBaseFee if backlog > tolerance*speedLimit { - excess := int64(backlog - tolerance*speedLimit) - exponentBips := arbmath.NaturalToBips(excess) / arbmath.Bips(inertia*speedLimit) + excess := arbmath.SaturatingCast[int64](backlog - tolerance*speedLimit) + exponentBips := arbmath.NaturalToBips(excess) / arbmath.SaturatingCast[arbmath.Bips](inertia*speedLimit) baseFee = arbmath.BigMulByBips(minBaseFee, arbmath.ApproxExpBasisPoints(exponentBips, 4)) } _ = ps.SetBaseFeeWei(baseFee) diff --git a/arbos/storage/storage.go b/arbos/storage/storage.go index 6e6c976644e..352726778d0 100644 --- a/arbos/storage/storage.go +++ b/arbos/storage/storage.go @@ -156,11 +156,6 @@ func (s *Storage) GetUint64ByUint64(key uint64) (uint64, error) { return s.GetUint64(util.UintToHash(key)) } -func (s *Storage) GetUint32(key common.Hash) (uint32, error) { - value, err := s.Get(key) - return uint32(value.Big().Uint64()), err -} - func (s *Storage) Set(key common.Hash, value common.Hash) error { if s.burner.ReadOnly() { log.Error("Read-only burner attempted to mutate state", "key", key, "value", value) @@ -420,6 +415,7 @@ func (sbu *StorageBackedInt64) Get() (int64, error) { if !raw.Big().IsUint64() { panic("invalid value found in StorageBackedInt64 storage") } + // #nosec G115 return int64(raw.Big().Uint64()), err // see implementation note above } @@ -477,6 +473,7 @@ func (sbu *StorageBackedUint16) Get() (uint16, error) { if !big.IsUint64() || big.Uint64() > math.MaxUint16 { panic("expected uint16 compatible value in storage") } + // #nosec G115 return uint16(big.Uint64()), err } @@ -517,6 +514,7 @@ func (sbu *StorageBackedUint32) Get() (uint32, error) { if !big.IsUint64() || big.Uint64() > math.MaxUint32 { panic("expected uint32 compatible value in storage") } + // #nosec G115 return uint32(big.Uint64()), err } diff --git a/arbstate/inbox.go b/arbstate/inbox.go index 753ca19cd6c..b58a7420b75 100644 --- a/arbstate/inbox.go +++ b/arbstate/inbox.go @@ -246,7 +246,7 @@ func (r *inboxMultiplexer) IsCachedSegementLast() bool { if r.delayedMessagesRead < seqMsg.afterDelayedMessages { return false } - for segmentNum := int(r.cachedSegmentNum) + 1; segmentNum < len(seqMsg.segments); segmentNum++ { + for segmentNum := r.cachedSegmentNum + 1; segmentNum < uint64(len(seqMsg.segments)); segmentNum++ { segment := seqMsg.segments[segmentNum] if len(segment) == 0 { continue @@ -276,7 +276,7 @@ func (r *inboxMultiplexer) getNextMsg() (*arbostypes.MessageWithMetadata, error) if segmentNum >= uint64(len(seqMsg.segments)) { break } - segment = seqMsg.segments[int(segmentNum)] + segment = seqMsg.segments[segmentNum] if len(segment) == 0 { segmentNum++ continue @@ -322,7 +322,7 @@ func (r *inboxMultiplexer) getNextMsg() (*arbostypes.MessageWithMetadata, error) log.Warn("reading virtual delayed message segment", "delayedMessagesRead", r.delayedMessagesRead, "afterDelayedMessages", seqMsg.afterDelayedMessages) segment = []byte{BatchSegmentKindDelayedMessages} } else { - segment = seqMsg.segments[int(segmentNum)] + segment = seqMsg.segments[segmentNum] } if len(segment) == 0 { log.Error("empty sequencer message segment", "sequence", r.cachedSegmentNum, "segmentNum", segmentNum) diff --git a/arbutil/block_message_relation.go b/arbutil/block_message_relation.go index a69f9079ee1..e164cf2619a 100644 --- a/arbutil/block_message_relation.go +++ b/arbutil/block_message_relation.go @@ -15,5 +15,6 @@ func SignedBlockNumberToMessageCount(blockNumber int64, genesisBlockNumber uint6 } func MessageCountToBlockNumber(messageCount MessageIndex, genesisBlockNumber uint64) int64 { + // #nosec G115 return int64(uint64(messageCount)+genesisBlockNumber) - 1 } diff --git a/arbutil/correspondingl1blocknumber.go b/arbutil/correspondingl1blocknumber.go index 05323ed1832..d654e471e25 100644 --- a/arbutil/correspondingl1blocknumber.go +++ b/arbutil/correspondingl1blocknumber.go @@ -20,6 +20,7 @@ func ParentHeaderToL1BlockNumber(header *types.Header) uint64 { } func CorrespondingL1BlockNumber(ctx context.Context, client L1Interface, parentBlockNumber uint64) (uint64, error) { + // #nosec G115 header, err := client.HeaderByNumber(ctx, big.NewInt(int64(parentBlockNumber))) if err != nil { return 0, fmt.Errorf("error getting L1 block number %d header : %w", parentBlockNumber, err) diff --git a/blocks_reexecutor/blocks_reexecutor.go b/blocks_reexecutor/blocks_reexecutor.go index 1e4a06fe904..f7cc0d8c720 100644 --- a/blocks_reexecutor/blocks_reexecutor.go +++ b/blocks_reexecutor/blocks_reexecutor.go @@ -102,7 +102,8 @@ func New(c *Config, blockchain *core.BlockChain, fatalErrChan chan error) *Block if rng > end-start { rng = end - start } - start += uint64(rand.Intn(int(end - start - rng + 1))) + // #nosec G115 + start += uint64(rand.Int63n(int64(end - start - rng + 1))) end = start + rng } // Inclusive of block reexecution [start, end] diff --git a/broadcaster/backlog/backlog.go b/broadcaster/backlog/backlog.go index f6501105c2e..b7b935fb7a2 100644 --- a/broadcaster/backlog/backlog.go +++ b/broadcaster/backlog/backlog.go @@ -97,6 +97,7 @@ func (b *backlog) Append(bm *m.BroadcastMessage) error { if err != nil { log.Warn("error calculating backlogSizeInBytes", "err", err) } else { + // #nosec G115 backlogSizeInBytesGauge.Update(int64(size)) } } @@ -108,6 +109,7 @@ func (b *backlog) Append(bm *m.BroadcastMessage) error { segment = newBacklogSegment() b.head.Store(segment) b.tail.Store(segment) + // #nosec G115 confirmedSequenceNumberGauge.Update(int64(msg.SequenceNumber)) } @@ -143,9 +145,11 @@ func (b *backlog) Append(bm *m.BroadcastMessage) error { } lookupByIndex.Store(uint64(msg.SequenceNumber), segment) b.messageCount.Add(1) + // #nosec G115 backlogSizeInBytesGauge.Inc(int64(msg.Size())) } + // #nosec G115 backlogSizeGauge.Update(int64(b.Count())) return nil } @@ -174,7 +178,7 @@ func (b *backlog) Get(start, end uint64) (*m.BroadcastMessage, error) { } bm := &m.BroadcastMessage{Version: 1} - required := int(end-start) + 1 + required := end - start + 1 for { segMsgs, err := segment.Get(arbmath.MaxInt(start, segment.Start()), arbmath.MinInt(end, segment.End())) if err != nil { @@ -183,7 +187,7 @@ func (b *backlog) Get(start, end uint64) (*m.BroadcastMessage, error) { bm.Messages = append(bm.Messages, segMsgs...) segment = segment.Next() - if len(bm.Messages) == required { + if uint64(len(bm.Messages)) == required { break } else if segment == nil { return nil, errOutOfBounds @@ -213,6 +217,7 @@ func (b *backlog) delete(confirmed uint64) { return } + // #nosec G115 confirmedSequenceNumberGauge.Update(int64(confirmed)) // find the segment containing the confirmed message diff --git a/broadcaster/backlog/backlog_test.go b/broadcaster/backlog/backlog_test.go index ee712de9ede..d74389f692b 100644 --- a/broadcaster/backlog/backlog_test.go +++ b/broadcaster/backlog/backlog_test.go @@ -33,8 +33,8 @@ func validateBacklog(t *testing.T, b *backlog, count, start, end uint64, lookupK } } - expLen := len(lookupKeys) - actualLen := int(b.Count()) + expLen := uint64(len(lookupKeys)) + actualLen := b.Count() if expLen != actualLen { t.Errorf("expected length of lookupByIndex map (%d) does not equal actual length (%d)", expLen, actualLen) } diff --git a/broadcaster/broadcaster.go b/broadcaster/broadcaster.go index ba95f2d8af7..397698635af 100644 --- a/broadcaster/broadcaster.go +++ b/broadcaster/broadcaster.go @@ -145,6 +145,7 @@ func (b *Broadcaster) ListenerAddr() net.Addr { } func (b *Broadcaster) GetCachedMessageCount() int { + // #nosec G115 return int(b.backlog.Count()) } diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index a052c146d1e..39f204980d3 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -371,6 +371,7 @@ func mainImpl() int { if err != nil { log.Crit("error getting rollup addresses config", "err", err) } + // #nosec G115 addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true) if err != nil { log.Crit("error creating validator wallet contract", "error", err, "address", l1TransactionOptsValidator.From.Hex()) @@ -582,7 +583,7 @@ func mainImpl() int { l1TransactionOptsBatchPoster, dataSigner, fatalErrChan, - big.NewInt(int64(nodeConfig.ParentChain.ID)), + new(big.Int).SetUint64(nodeConfig.ParentChain.ID), blobReader, ) if err != nil { diff --git a/cmd/staterecovery/staterecovery.go b/cmd/staterecovery/staterecovery.go index bb014774142..5486ba3726c 100644 --- a/cmd/staterecovery/staterecovery.go +++ b/cmd/staterecovery/staterecovery.go @@ -60,6 +60,7 @@ func RecreateMissingStates(chainDb ethdb.Database, bc *core.BlockChain, cacheCon break } if time.Since(logged) > 1*time.Minute { + // #nosec G115 log.Info("Recreating missing states", "block", current, "target", target, "remaining", int64(target)-int64(current), "elapsed", time.Since(start), "recreated", recreated) logged = time.Now() } diff --git a/das/aggregator.go b/das/aggregator.go index d944f8d48ab..e8972447ad6 100644 --- a/das/aggregator.go +++ b/das/aggregator.go @@ -166,6 +166,7 @@ type storeResponse struct { // If Store gets not enough successful responses by the time its context is canceled // (eg via TimeoutWrapper) then it also returns an error. func (a *Aggregator) Store(ctx context.Context, message []byte, timeout uint64) (*daprovider.DataAvailabilityCertificate, error) { + // #nosec G115 log.Trace("das.Aggregator.Store", "message", pretty.FirstFewBytes(message), "timeout", time.Unix(int64(timeout), 0)) allBackendsSucceeded := false diff --git a/das/dasRpcClient.go b/das/dasRpcClient.go index ca2ee8e7d47..635696bdab2 100644 --- a/das/dasRpcClient.go +++ b/das/dasRpcClient.go @@ -138,6 +138,7 @@ func (c *DASRPCClient) sendChunk(ctx context.Context, batchId, i uint64, chunk [ } func (c *DASRPCClient) legacyStore(ctx context.Context, message []byte, timeout uint64) (*daprovider.DataAvailabilityCertificate, error) { + // #nosec G115 log.Trace("das.DASRPCClient.Store(...)", "message", pretty.FirstFewBytes(message), "timeout", time.Unix(int64(timeout), 0), "this", *c) reqSig, err := applyDasSigner(c.signer, message, timeout) diff --git a/das/dasRpcServer.go b/das/dasRpcServer.go index 9e6228ca5dc..d14766cc7e9 100644 --- a/das/dasRpcServer.go +++ b/das/dasRpcServer.go @@ -108,6 +108,7 @@ type StoreResult struct { } func (s *DASRPCServer) Store(ctx context.Context, message hexutil.Bytes, timeout hexutil.Uint64, sig hexutil.Bytes) (*StoreResult, error) { + // #nosec G115 log.Trace("dasRpc.DASRPCServer.Store", "message", pretty.FirstFewBytes(message), "message length", len(message), "timeout", time.Unix(int64(timeout), 0), "sig", pretty.FirstFewBytes(sig), "this", s) rpcStoreRequestGauge.Inc(1) start := time.Now() @@ -277,6 +278,7 @@ func (s *DASRPCServer) StartChunkedStore(ctx context.Context, timestamp, nChunks } // Prevent replay of old messages + // #nosec G115 if time.Since(time.Unix(int64(timestamp), 0)).Abs() > time.Minute { return nil, errors.New("too much time has elapsed since request was signed") } diff --git a/das/dastree/dastree.go b/das/dastree/dastree.go index d873f0568de..2bcbccaae3b 100644 --- a/das/dastree/dastree.go +++ b/das/dastree/dastree.go @@ -61,12 +61,13 @@ func RecordHash(record func(bytes32, []byte, arbutil.PreimageType), preimage ... return arbmath.FlipBit(keccord(prepend(LeafByte, keccord([]byte{}).Bytes())), 0) } - length := uint32(len(unrolled)) + length := len(unrolled) leaves := []node{} - for bin := uint32(0); bin < length; bin += BinSize { + for bin := 0; bin < length; bin += BinSize { end := arbmath.MinInt(bin+BinSize, length) hash := keccord(prepend(LeafByte, keccord(unrolled[bin:end]).Bytes())) - leaves = append(leaves, node{hash, end - bin}) + // #nosec G115 + leaves = append(leaves, node{hash, uint32(end - bin)}) } layer := leaves @@ -186,7 +187,9 @@ func Content(root bytes32, oracle func(bytes32) ([]byte, error)) ([]byte, error) leaves = append(leaves, leaf) case NodeByte: count := binary.BigEndian.Uint32(data[64:]) - power := uint32(arbmath.NextOrCurrentPowerOf2(uint64(count))) + power := arbmath.NextOrCurrentPowerOf2(uint64(count)) + // #nosec G115 + halfPower := uint32(power / 2) if place.size != count { return nil, fmt.Errorf("invalid size data: %v vs %v for %v", count, place.size, data) @@ -194,11 +197,11 @@ func Content(root bytes32, oracle func(bytes32) ([]byte, error)) ([]byte, error) prior := node{ hash: common.BytesToHash(data[:32]), - size: power / 2, + size: halfPower, } after := node{ hash: common.BytesToHash(data[32:64]), - size: count - power/2, + size: count - halfPower, } // we want to expand leftward so we reverse their order diff --git a/das/db_storage_service.go b/das/db_storage_service.go index e3b6183c37b..1d9e5348d46 100644 --- a/das/db_storage_service.go +++ b/das/db_storage_service.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "math" "os" "path/filepath" "time" @@ -172,7 +173,8 @@ func (dbs *DBStorageService) Put(ctx context.Context, data []byte, timeout uint6 return dbs.db.Update(func(txn *badger.Txn) error { e := badger.NewEntry(dastree.HashBytes(data), data) - if dbs.discardAfterTimeout { + if dbs.discardAfterTimeout && timeout <= math.MaxInt64 { + // #nosec G115 e = e.WithTTL(time.Until(time.Unix(int64(timeout), 0))) } return txn.SetEntry(e) diff --git a/das/local_file_storage_service.go b/das/local_file_storage_service.go index 65ca6fe15cf..ce86786718d 100644 --- a/das/local_file_storage_service.go +++ b/das/local_file_storage_service.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "math" "os" "path" "path/filepath" @@ -133,6 +134,10 @@ func (s *LocalFileStorageService) GetByHash(ctx context.Context, key common.Hash func (s *LocalFileStorageService) Put(ctx context.Context, data []byte, expiry uint64) error { logPut("das.LocalFileStorageService.Store", data, expiry, s) + if expiry > math.MaxInt64 { + return fmt.Errorf("request expiry time (%v) exceeds max int64", expiry) + } + // #nosec G115 expiryTime := time.Unix(int64(expiry), 0) currentTimePlusRetention := time.Now().Add(s.config.MaxRetention) if expiryTime.After(currentTimePlusRetention) { @@ -182,6 +187,7 @@ func (s *LocalFileStorageService) Put(ctx context.Context, data []byte, expiry u // new flat layout files, set their modification time accordingly. if s.enableLegacyLayout { tv := syscall.Timeval{ + // #nosec G115 Sec: int64(expiry - uint64(s.legacyLayout.retention.Seconds())), Usec: 0, } diff --git a/das/local_file_storage_service_test.go b/das/local_file_storage_service_test.go index cc27e293e3c..01b999f3563 100644 --- a/das/local_file_storage_service_test.go +++ b/das/local_file_storage_service_test.go @@ -99,6 +99,7 @@ func TestMigrationNoExpiry(t *testing.T) { getByHashAndCheck(t, s, "a", "b", "c", "d") // Can still iterate by timestamp even if expiry disabled + // #nosec G115 countTimestampEntries(t, &s.layout, time.Unix(int64(now+11), 0), 4) } diff --git a/das/s3_storage_service.go b/das/s3_storage_service.go index a1de200c52c..f9baa96547b 100644 --- a/das/s3_storage_service.go +++ b/das/s3_storage_service.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "io" + "math" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -110,7 +111,8 @@ func (s3s *S3StorageService) Put(ctx context.Context, value []byte, timeout uint Bucket: aws.String(s3s.bucket), Key: aws.String(s3s.objectPrefix + EncodeStorageServiceKey(dastree.Hash(value))), Body: bytes.NewReader(value)} - if !s3s.discardAfterTimeout { + if !s3s.discardAfterTimeout && timeout <= math.MaxInt64 { + // #nosec G115 expires := time.Unix(int64(timeout), 0) putObjectInput.Expires = &expires } diff --git a/das/sign_after_store_das_writer.go b/das/sign_after_store_das_writer.go index 0e31d30ae97..40b03847d86 100644 --- a/das/sign_after_store_das_writer.go +++ b/das/sign_after_store_das_writer.go @@ -105,6 +105,7 @@ func NewSignAfterStoreDASWriter(ctx context.Context, config DataAvailabilityConf } func (d *SignAfterStoreDASWriter) Store(ctx context.Context, message []byte, timeout uint64) (c *daprovider.DataAvailabilityCertificate, err error) { + // #nosec G115 log.Trace("das.SignAfterStoreDASWriter.Store", "message", pretty.FirstFewBytes(message), "timeout", time.Unix(int64(timeout), 0), "this", d) c = &daprovider.DataAvailabilityCertificate{ Timeout: timeout, diff --git a/das/simple_das_reader_aggregator.go b/das/simple_das_reader_aggregator.go index dc6147a7e48..f45c56afe0e 100644 --- a/das/simple_das_reader_aggregator.go +++ b/das/simple_das_reader_aggregator.go @@ -50,8 +50,8 @@ var DefaultRestfulClientAggregatorConfig = RestfulClientAggregatorConfig{ } type SimpleExploreExploitStrategyConfig struct { - ExploreIterations int `koanf:"explore-iterations"` - ExploitIterations int `koanf:"exploit-iterations"` + ExploreIterations uint32 `koanf:"explore-iterations"` + ExploitIterations uint32 `koanf:"exploit-iterations"` } var DefaultSimpleExploreExploitStrategyConfig = SimpleExploreExploitStrategyConfig{ @@ -73,8 +73,8 @@ func RestfulClientAggregatorConfigAddOptions(prefix string, f *flag.FlagSet) { } func SimpleExploreExploitStrategyConfigAddOptions(prefix string, f *flag.FlagSet) { - f.Int(prefix+".explore-iterations", DefaultSimpleExploreExploitStrategyConfig.ExploreIterations, "number of consecutive GetByHash calls to the aggregator where each call will cause it to randomly select from REST endpoints until one returns successfully, before switching to exploit mode") - f.Int(prefix+".exploit-iterations", DefaultSimpleExploreExploitStrategyConfig.ExploitIterations, "number of consecutive GetByHash calls to the aggregator where each call will cause it to select from REST endpoints in order of best latency and success rate, before switching to explore mode") + f.Uint32(prefix+".explore-iterations", DefaultSimpleExploreExploitStrategyConfig.ExploreIterations, "number of consecutive GetByHash calls to the aggregator where each call will cause it to randomly select from REST endpoints until one returns successfully, before switching to exploit mode") + f.Uint32(prefix+".exploit-iterations", DefaultSimpleExploreExploitStrategyConfig.ExploitIterations, "number of consecutive GetByHash calls to the aggregator where each call will cause it to select from REST endpoints in order of best latency and success rate, before switching to explore mode") } func NewRestfulClientAggregator(ctx context.Context, config *RestfulClientAggregatorConfig) (*SimpleDASReaderAggregator, error) { @@ -120,8 +120,8 @@ func NewRestfulClientAggregator(ctx context.Context, config *RestfulClientAggreg switch strings.ToLower(config.Strategy) { case "simple-explore-exploit": a.strategy = &simpleExploreExploitStrategy{ - exploreIterations: uint32(config.SimpleExploreExploitStrategy.ExploreIterations), - exploitIterations: uint32(config.SimpleExploreExploitStrategy.ExploitIterations), + exploreIterations: config.SimpleExploreExploitStrategy.ExploreIterations, + exploitIterations: config.SimpleExploreExploitStrategy.ExploitIterations, } case "testing-sequential": a.strategy = &testingSequentialStrategy{} diff --git a/das/util.go b/das/util.go index de266c433f3..114e075e797 100644 --- a/das/util.go +++ b/das/util.go @@ -13,11 +13,13 @@ import ( func logPut(store string, data []byte, timeout uint64, reader daprovider.DASReader, more ...interface{}) { if len(more) == 0 { + // #nosec G115 log.Trace( store, "message", pretty.FirstFewBytes(data), "timeout", time.Unix(int64(timeout), 0), "this", reader, ) } else { + // #nosec G115 log.Trace( store, "message", pretty.FirstFewBytes(data), "timeout", time.Unix(int64(timeout), 0), "this", reader, more, diff --git a/execution/gethexec/api.go b/execution/gethexec/api.go index c19072ae771..2bff8026c27 100644 --- a/execution/gethexec/api.go +++ b/execution/gethexec/api.go @@ -78,6 +78,7 @@ func (api *ArbDebugAPI) evenlySpaceBlocks(start, end rpc.BlockNumber) (uint64, u end, _ = api.blockchain.ClipToPostNitroGenesis(end) blocks := end.Int64() - start.Int64() + 1 + // #nosec G115 bound := int64(api.blockRangeBound) step := int64(1) if blocks > bound { diff --git a/execution/gethexec/sequencer.go b/execution/gethexec/sequencer.go index 90e30820627..819cd105000 100644 --- a/execution/gethexec/sequencer.go +++ b/execution/gethexec/sequencer.go @@ -887,6 +887,7 @@ func (s *Sequencer) createBlock(ctx context.Context) (returnValue bool) { for _, queueItem := range queueItems { s.txRetryQueue.Push(queueItem) } + // #nosec G115 log.Error( "cannot sequence: unknown L1 block or L1 timestamp too far from local clock time", "l1Block", l1Block, @@ -1037,10 +1038,14 @@ func (s *Sequencer) updateExpectedSurplus(ctx context.Context) (int64, error) { if err != nil { return 0, fmt.Errorf("error encountered getting l1 pricing surplus while updating expectedSurplus: %w", err) } + // #nosec G115 backlogL1GasCharged := int64(s.execEngine.backlogL1GasCharged()) + // #nosec G115 backlogCallDataUnits := int64(s.execEngine.backlogCallDataUnits()) + // #nosec G115 expectedSurplus := int64(surplus) + backlogL1GasCharged - backlogCallDataUnits*int64(l1GasPrice) // update metrics + // #nosec G115 l1GasPriceGauge.Update(int64(l1GasPrice)) callDataUnitsBacklogGauge.Update(backlogCallDataUnits) unusedL1GasChargeGauge.Update(backlogL1GasCharged) diff --git a/execution/gethexec/tx_pre_checker.go b/execution/gethexec/tx_pre_checker.go index 191331b48a7..e0ae330148d 100644 --- a/execution/gethexec/tx_pre_checker.go +++ b/execution/gethexec/tx_pre_checker.go @@ -161,6 +161,7 @@ func PreCheckTx(bc *core.BlockChain, chainConfig *params.ChainConfig, header *ty oldHeader := header blocksTraversed := uint(0) // find a block that's old enough + // #nosec G115 for now-int64(oldHeader.Time) < config.RequiredStateAge && (config.RequiredStateMaxBlocks <= 0 || blocksTraversed < config.RequiredStateMaxBlocks) && oldHeader.Number.Uint64() > 0 { diff --git a/execution/nodeInterface/NodeInterface.go b/execution/nodeInterface/NodeInterface.go index 9179a527184..45fcebcdfaf 100644 --- a/execution/nodeInterface/NodeInterface.go +++ b/execution/nodeInterface/NodeInterface.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "math" "math/big" "sort" @@ -234,6 +235,7 @@ func (n NodeInterface) ConstructOutboxProof(c ctx, evm mech, size, leaf uint64) } balanced := size == arbmath.NextPowerOf2(size)/2 + // #nosec G115 treeLevels := int(arbmath.Log2ceil(size)) // the # of levels in the tree proofLevels := treeLevels - 1 // the # of levels where a hash is needed (all but root) walkLevels := treeLevels // the # of levels we need to consider when building walks @@ -297,6 +299,7 @@ func (n NodeInterface) ConstructOutboxProof(c ctx, evm mech, size, leaf uint64) mid := (lo + hi) / 2 + // #nosec G115 block, err := n.backend.BlockByNumber(n.context, rpc.BlockNumber(mid)) if err != nil { searchErr = err @@ -643,6 +646,10 @@ func (n NodeInterface) LegacyLookupMessageBatchProof(c ctx, evm mech, batchNum h // L2BlockRangeForL1 fetches the L1 block number of a given l2 block number. // c ctx and evm mech arguments are not used but supplied to match the precompile function type in NodeInterface contract func (n NodeInterface) BlockL1Num(c ctx, evm mech, l2BlockNum uint64) (uint64, error) { + if l2BlockNum > math.MaxInt64 { + return 0, fmt.Errorf("requested l2 block number %d out of range for int64", l2BlockNum) + } + // #nosec G115 blockHeader, err := n.backend.HeaderByNumber(n.context, rpc.BlockNumber(l2BlockNum)) if err != nil { return 0, err diff --git a/precompiles/ArbAddressTable.go b/precompiles/ArbAddressTable.go index 05f2275fd7d..102fd55c3b6 100644 --- a/precompiles/ArbAddressTable.go +++ b/precompiles/ArbAddressTable.go @@ -33,7 +33,7 @@ func (con ArbAddressTable) Decompress(c ctx, evm mech, buf []uint8, offset huge) return addr{}, nil, errors.New("invalid offset in ArbAddressTable.Decompress") } result, nbytes, err := c.State.AddressTable().Decompress(buf[ioffset:]) - return result, big.NewInt(int64(nbytes)), err + return result, new(big.Int).SetUint64(nbytes), err } // Lookup the index of an address in the table @@ -45,7 +45,7 @@ func (con ArbAddressTable) Lookup(c ctx, evm mech, addr addr) (huge, error) { if !exists { return nil, errors.New("address does not exist in AddressTable") } - return big.NewInt(int64(result)), nil + return new(big.Int).SetUint64(result), nil } // LookupIndex for an address in the table by index @@ -66,11 +66,11 @@ func (con ArbAddressTable) LookupIndex(c ctx, evm mech, index huge) (addr, error // Register adds an account to the table, shrinking its compressed representation func (con ArbAddressTable) Register(c ctx, evm mech, addr addr) (huge, error) { slot, err := c.State.AddressTable().Register(addr) - return big.NewInt(int64(slot)), err + return new(big.Int).SetUint64(slot), err } // Size gets the number of addresses in the table func (con ArbAddressTable) Size(c ctx, evm mech) (huge, error) { size, err := c.State.AddressTable().Size() - return big.NewInt(int64(size)), err + return new(big.Int).SetUint64(size), err } diff --git a/precompiles/ArbRetryableTx.go b/precompiles/ArbRetryableTx.go index d508d757521..93e80236039 100644 --- a/precompiles/ArbRetryableTx.go +++ b/precompiles/ArbRetryableTx.go @@ -149,7 +149,7 @@ func (con ArbRetryableTx) GetTimeout(c ctx, evm mech, ticketId bytes32) (huge, e if err != nil { return nil, err } - return big.NewInt(int64(timeout)), nil + return new(big.Int).SetUint64(timeout), nil } // Keepalive adds one lifetime period to the ticket's expiry @@ -176,8 +176,9 @@ func (con ArbRetryableTx) Keepalive(c ctx, evm mech, ticketId bytes32) (huge, er return big.NewInt(0), err } - err = con.LifetimeExtended(c, evm, ticketId, big.NewInt(int64(newTimeout))) - return big.NewInt(int64(newTimeout)), err + bigNewTimeout := new(big.Int).SetUint64(newTimeout) + err = con.LifetimeExtended(c, evm, ticketId, bigNewTimeout) + return bigNewTimeout, err } // GetBeneficiary gets the beneficiary of the ticket diff --git a/precompiles/ArbSys.go b/precompiles/ArbSys.go index 13f56d3b8e5..d55067a09c2 100644 --- a/precompiles/ArbSys.go +++ b/precompiles/ArbSys.go @@ -162,7 +162,7 @@ func (con *ArbSys) SendTxToL1(c ctx, evm mech, value huge, destination addr, cal } } - leafNum := big.NewInt(int64(size - 1)) + leafNum := new(big.Int).SetUint64(size - 1) var blockTime big.Int blockTime.SetUint64(evm.Context.Time) @@ -199,7 +199,7 @@ func (con ArbSys) SendMerkleTreeState(c ctx, evm mech) (huge, bytes32, []bytes32 for i, par := range rawPartials { partials[i] = par } - return big.NewInt(int64(size)), rootHash, partials, nil + return new(big.Int).SetUint64(size), rootHash, partials, nil } // WithdrawEth send paid eth to the destination on L1 diff --git a/relay/relay_stress_test.go b/relay/relay_stress_test.go index 9a8875a4295..9d5c4150566 100644 --- a/relay/relay_stress_test.go +++ b/relay/relay_stress_test.go @@ -160,7 +160,7 @@ func largeBacklogRelayTestImpl(t *testing.T, numClients, backlogSize, l2MsgSize connected++ } } - if int32(connected) != int32(numClients) { + if connected != numClients { t.Fail() } log.Info("number of clients connected", "expected", numClients, "got", connected) diff --git a/staker/block_challenge_backend.go b/staker/block_challenge_backend.go index 42351789ba0..0dd89865bd0 100644 --- a/staker/block_challenge_backend.go +++ b/staker/block_challenge_backend.go @@ -219,6 +219,6 @@ func (b *BlockChallengeBackend) IssueExecChallenge( }, machineStatuses, globalStateHashes, - big.NewInt(int64(numsteps)), + new(big.Int).SetUint64(numsteps), ) } diff --git a/staker/block_validator.go b/staker/block_validator.go index 8f5724beac4..2239952b37f 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -315,6 +315,7 @@ func NewBlockValidator( func atomicStorePos(addr *atomic.Uint64, val arbutil.MessageIndex, metr metrics.Gauge) { addr.Store(uint64(val)) + // #nosec G115 metr.Update(int64(val)) } @@ -573,6 +574,7 @@ func (v *BlockValidator) createNextValidationEntry(ctx context.Context) (bool, e v.nextCreateBatch = batch v.nextCreateBatchBlockHash = batchBlockHash v.nextCreateBatchMsgCount = count + // #nosec G115 validatorMsgCountCurrentBatch.Update(int64(count)) v.nextCreateBatchReread = false } @@ -723,6 +725,7 @@ func (v *BlockValidator) iterativeValidationPrint(ctx context.Context) time.Dura if err != nil { printedCount = -1 } else { + // #nosec G115 printedCount = int64(batchMsgs) + int64(validated.GlobalState.PosInBatch) } log.Info("validated execution", "messageCount", printedCount, "globalstate", validated.GlobalState, "WasmRoots", validated.WasmRoots) @@ -992,8 +995,10 @@ func (v *BlockValidator) UpdateLatestStaked(count arbutil.MessageIndex, globalSt if v.recordSentA.Load() < countUint64 { v.recordSentA.Store(countUint64) } + // #nosec G115 v.validatedA.Store(countUint64) v.valLoopPos = count + // #nosec G115 validatorMsgCountValidatedGauge.Update(int64(countUint64)) err = v.writeLastValidated(globalState, nil) // we don't know which wasm roots were validated if err != nil { @@ -1058,6 +1063,7 @@ func (v *BlockValidator) Reorg(ctx context.Context, count arbutil.MessageIndex) } if v.validatedA.Load() > countUint64 { v.validatedA.Store(countUint64) + // #nosec G115 validatorMsgCountValidatedGauge.Update(int64(countUint64)) err := v.writeLastValidated(v.nextCreateStartGS, nil) // we don't know which wasm roots were validated if err != nil { @@ -1249,6 +1255,7 @@ func (v *BlockValidator) checkValidatedGSCaughtUp() (bool, error) { atomicStorePos(&v.createdA, count, validatorMsgCountCreatedGauge) atomicStorePos(&v.recordSentA, count, validatorMsgCountRecordSentGauge) atomicStorePos(&v.validatedA, count, validatorMsgCountValidatedGauge) + // #nosec G115 validatorMsgCountValidatedGauge.Update(int64(count)) v.chainCaughtUp = true return true, nil diff --git a/staker/challenge-cache/cache.go b/staker/challenge-cache/cache.go index ed4fad6450e..5dca2764e89 100644 --- a/staker/challenge-cache/cache.go +++ b/staker/challenge-cache/cache.go @@ -187,12 +187,12 @@ func (c *Cache) Prune(ctx context.Context, messageNumber uint64) error { if info.IsDir() { matches := pattern.FindStringSubmatch(info.Name()) if len(matches) > 1 { - dirNameMessageNum, err := strconv.Atoi(matches[1]) + dirNameMessageNum, err := strconv.ParseUint(matches[1], 10, 64) if err != nil { return err } // Collect the directory path if the message number is <= the specified value. - if dirNameMessageNum <= int(messageNumber) { + if dirNameMessageNum <= messageNumber { pathsToDelete = append(pathsToDelete, path) } } diff --git a/staker/challenge_manager.go b/staker/challenge_manager.go index b1421d7e419..ef431d3c797 100644 --- a/staker/challenge_manager.go +++ b/staker/challenge_manager.go @@ -294,7 +294,7 @@ func (m *ChallengeManager) bisect(ctx context.Context, backend ChallengeBackend, if newChallengeLength < bisectionDegree { bisectionDegree = newChallengeLength } - newSegments := make([][32]byte, int(bisectionDegree+1)) + newSegments := make([][32]byte, bisectionDegree+1) position := startSegmentPosition normalSegmentLength := newChallengeLength / bisectionDegree for i := range newSegments { diff --git a/staker/challenge_test.go b/staker/challenge_test.go index 4534b04a257..33f1644c637 100644 --- a/staker/challenge_test.go +++ b/staker/challenge_test.go @@ -77,7 +77,7 @@ func CreateChallenge( resultReceiverAddr, maxInboxMessage, [2][32]byte{startHashBytes, endHashBytes}, - big.NewInt(int64(endMachineSteps)), + new(big.Int).SetUint64(endMachineSteps), asserter, challenger, big.NewInt(100), diff --git a/staker/l1_validator.go b/staker/l1_validator.go index dd9673ee0bf..6ea9fd8ded0 100644 --- a/staker/l1_validator.go +++ b/staker/l1_validator.go @@ -247,6 +247,7 @@ func (v *L1Validator) generateNodeAction( startStateProposedParentChain, err, ) } + // #nosec G115 startStateProposedTime := time.Unix(int64(startStateProposedHeader.Time), 0) v.txStreamer.PauseReorgs() @@ -375,6 +376,7 @@ func (v *L1Validator) generateNodeAction( return nil, false, fmt.Errorf("error getting rollup minimum assertion period: %w", err) } + // #nosec G115 timeSinceProposed := big.NewInt(int64(l1BlockNumber) - int64(startStateProposedL1)) if timeSinceProposed.Cmp(minAssertionPeriod) < 0 { // Too soon to assert diff --git a/staker/rollup_watcher.go b/staker/rollup_watcher.go index b35bebd1c62..5ef28a49dc4 100644 --- a/staker/rollup_watcher.go +++ b/staker/rollup_watcher.go @@ -196,7 +196,7 @@ func (r *RollupWatcher) LookupNodeChildren(ctx context.Context, nodeNum uint64, if logQueryRangeSize == 0 { query.ToBlock = toBlock } else { - query.ToBlock = new(big.Int).Add(fromBlock, big.NewInt(int64(logQueryRangeSize))) + query.ToBlock = new(big.Int).Add(fromBlock, new(big.Int).SetUint64(logQueryRangeSize)) } if query.ToBlock.Cmp(toBlock) > 0 { query.ToBlock = toBlock diff --git a/staker/staker.go b/staker/staker.go index c54e74be37b..6e93d273119 100644 --- a/staker/staker.go +++ b/staker/staker.go @@ -352,6 +352,7 @@ func (s *Staker) Initialize(ctx context.Context) error { if err != nil { return err } + // #nosec G115 stakerLatestStakedNodeGauge.Update(int64(latestStaked)) if latestStaked == 0 { return nil @@ -570,6 +571,7 @@ func (s *Staker) Start(ctxIn context.Context) { if err != nil && ctx.Err() == nil { log.Error("staker: error checking latest staked", "err", err) } + // #nosec G115 stakerLatestStakedNodeGauge.Update(int64(staked)) if stakedGlobalState != nil { for _, notifier := range s.stakedNotifiers { @@ -585,6 +587,7 @@ func (s *Staker) Start(ctxIn context.Context) { log.Error("staker: error checking latest confirmed", "err", err) } } + // #nosec G115 stakerLatestConfirmedNodeGauge.Update(int64(confirmed)) if confirmedGlobalState != nil { for _, notifier := range s.confirmedNotifiers { @@ -726,6 +729,7 @@ func (s *Staker) Act(ctx context.Context) (*types.Transaction, error) { if err != nil { return nil, fmt.Errorf("error getting latest staked node of own wallet %v: %w", walletAddressOrZero, err) } + // #nosec G115 stakerLatestStakedNodeGauge.Update(int64(latestStakedNodeNum)) if rawInfo != nil { rawInfo.LatestStakedNode = latestStakedNodeNum diff --git a/system_tests/block_validator_test.go b/system_tests/block_validator_test.go index bd0a1f33362..eef6c29b7a3 100644 --- a/system_tests/block_validator_test.go +++ b/system_tests/block_validator_test.go @@ -259,6 +259,7 @@ func testBlockValidatorSimple(t *testing.T, opts Options) { Require(t, err) // up to 3 extra references: awaiting validation, recently valid, lastValidatedHeader largestRefCount := lastBlockNow.NumberU64() - lastBlock.NumberU64() + 3 + // #nosec G115 if finalRefCount < 0 || finalRefCount > int64(largestRefCount) { Fatal(t, "unexpected refcount:", finalRefCount) } diff --git a/system_tests/forwarder_test.go b/system_tests/forwarder_test.go index 9fe419593ee..f87283432e8 100644 --- a/system_tests/forwarder_test.go +++ b/system_tests/forwarder_test.go @@ -246,6 +246,7 @@ func TestRedisForwarder(t *testing.T) { for i := range seqClients { userA := user("A", i) builder.L2Info.GenerateAccount(userA) + // #nosec G115 tx := builder.L2Info.PrepareTx("Owner", userA, builder.L2Info.TransferGas, big.NewInt(1e12+int64(builder.L2Info.TransferGas)*builder.L2Info.GasPrice.Int64()), nil) err := fallbackClient.SendTransaction(ctx, tx) Require(t, err) diff --git a/system_tests/initialization_test.go b/system_tests/initialization_test.go index f0797404a94..17e020e6abb 100644 --- a/system_tests/initialization_test.go +++ b/system_tests/initialization_test.go @@ -21,6 +21,7 @@ func InitOneContract(prand *testhelpers.PseudoRandomDataSource) (*statetransfer. storageMap := make(map[common.Hash]common.Hash) code := []byte{0x60, 0x0} // PUSH1 0 sum := big.NewInt(0) + // #nosec G115 numCells := int(prand.GetUint64() % 1000) for i := 0; i < numCells; i++ { storageAddr := prand.GetHash() diff --git a/system_tests/outbox_test.go b/system_tests/outbox_test.go index 739d756a310..c68df6ea229 100644 --- a/system_tests/outbox_test.go +++ b/system_tests/outbox_test.go @@ -146,6 +146,7 @@ func TestOutboxProofs(t *testing.T) { treeSize := root.size balanced := treeSize == arbmath.NextPowerOf2(treeSize)/2 + // #nosec G115 treeLevels := int(arbmath.Log2ceil(treeSize)) // the # of levels in the tree proofLevels := treeLevels - 1 // the # of levels where a hash is needed (all but root) walkLevels := treeLevels // the # of levels we need to consider when building walks diff --git a/system_tests/program_recursive_test.go b/system_tests/program_recursive_test.go index dbf527a2931..e928f9f3aa4 100644 --- a/system_tests/program_recursive_test.go +++ b/system_tests/program_recursive_test.go @@ -154,6 +154,7 @@ func testProgramResursiveCalls(t *testing.T, tests [][]multiCallRecurse, jit boo // execute transactions blockNum := uint64(0) for { + // #nosec G115 item := int(rander.GetUint64()/4) % len(tests) blockNum = testProgramRecursiveCall(t, builder, slotVals, rander, tests[item]) tests[item] = tests[len(tests)-1] diff --git a/system_tests/program_test.go b/system_tests/program_test.go index e171f2a4442..ed640809dbf 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -582,6 +582,7 @@ func testCalls(t *testing.T, jit bool) { for i := 0; i < 2; i++ { inner := nest(level - 1) + // #nosec G115 args = append(args, arbmath.Uint32ToBytes(uint32(len(inner)))...) args = append(args, inner...) } @@ -637,6 +638,7 @@ func testCalls(t *testing.T, jit bool) { colors.PrintBlue("Calling the ArbosTest precompile (Rust => precompile)") testPrecompile := func(gas uint64) uint64 { // Call the burnArbGas() precompile from Rust + // #nosec G115 burn := pack(burnArbGas(big.NewInt(int64(gas)))) args := argsForMulticall(vm.CALL, types.ArbosTestAddress, nil, burn) tx := l2info.PrepareTxTo("Owner", &callsAddr, 1e9, nil, args) @@ -650,6 +652,7 @@ func testCalls(t *testing.T, jit bool) { large := testPrecompile(largeGas) if !arbmath.Within(large-small, largeGas-smallGas, 2) { + // #nosec G115 ratio := float64(int64(large)-int64(small)) / float64(int64(largeGas)-int64(smallGas)) Fatal(t, "inconsistent burns", large, small, largeGas, smallGas, ratio) } @@ -1527,6 +1530,7 @@ func readWasmFile(t *testing.T, file string) ([]byte, []byte) { Require(t, err) // chose a random dictionary for testing, but keep the same files consistent + // #nosec G115 randDict := arbcompress.Dictionary((len(file) + len(t.Name())) % 2) wasmSource, err := programs.Wat2Wasm(source) @@ -1597,6 +1601,7 @@ func argsForMulticall(opcode vm.OpCode, address common.Address, value *big.Int, if opcode == vm.CALL { length += 32 } + // #nosec G115 args = append(args, arbmath.Uint32ToBytes(uint32(length))...) args = append(args, kinds[opcode]) if opcode == vm.CALL { diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index 09d53669ee0..cd3904ca060 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -132,6 +132,7 @@ func TestRecreateStateForRPCNoDepthLimit(t *testing.T) { func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + // #nosec G115 depthGasLimit := int64(256 * util.NormalizeL2GasForL1GasInitial(800_000, params.GWei)) execConfig := ExecConfigDefaultTest() execConfig.RPC.MaxRecreateStateDepth = depthGasLimit @@ -407,6 +408,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig gas = 0 blocks = 0 } else { + // #nosec G115 if int(i) >= int(lastBlock)-int(cacheConfig.BlockCount) { // skipping nonexistence check - the state might have been saved on node shutdown continue @@ -471,6 +473,7 @@ func TestSkippingSavingStateAndRecreatingAfterRestart(t *testing.T) { for _, skipGas := range skipGasValues { for _, skipBlocks := range skipBlockValues[:len(skipBlockValues)-2] { cacheConfig.MaxAmountOfGasToSkipStateSaving = skipGas + // #nosec G115 cacheConfig.MaxNumberOfBlocksToSkipStateSaving = uint32(skipBlocks) testSkippingSavingStateAndRecreatingAfterRestart(t, &cacheConfig, 100) } @@ -495,6 +498,7 @@ func TestGettingStateForRPCFullNode(t *testing.T) { if header == nil { Fatal(t, "failed to get current block header") } + // #nosec G115 state, _, err := api.StateAndHeaderByNumber(ctx, rpc.BlockNumber(header.Number.Uint64())) Require(t, err) addr := builder.L2Info.GetAddress("User2") @@ -505,6 +509,7 @@ func TestGettingStateForRPCFullNode(t *testing.T) { Fatal(t, "User2 address does not exist in the state") } // Get the state again to avoid caching + // #nosec G115 state, _, err = api.StateAndHeaderByNumber(ctx, rpc.BlockNumber(header.Number.Uint64())) Require(t, err) @@ -542,6 +547,7 @@ func TestGettingStateForRPCHybridArchiveNode(t *testing.T) { if header == nil { Fatal(t, "failed to get current block header") } + // #nosec G115 state, _, err := api.StateAndHeaderByNumber(ctx, rpc.BlockNumber(header.Number.Uint64())) Require(t, err) addr := builder.L2Info.GetAddress("User2") @@ -552,6 +558,7 @@ func TestGettingStateForRPCHybridArchiveNode(t *testing.T) { Fatal(t, "User2 address does not exist in the state") } // Get the state again to avoid caching + // #nosec G115 state, _, err = api.StateAndHeaderByNumber(ctx, rpc.BlockNumber(header.Number.Uint64())) Require(t, err) diff --git a/system_tests/seq_nonce_test.go b/system_tests/seq_nonce_test.go index 72629e19787..c099563e296 100644 --- a/system_tests/seq_nonce_test.go +++ b/system_tests/seq_nonce_test.go @@ -111,6 +111,7 @@ func TestSequencerNonceTooHighQueueFull(t *testing.T) { } for wait := 9; wait >= 0; wait-- { + // #nosec G115 got := int(completed.Load()) expected := count - builder.execConfig.Sequencer.NonceFailureCacheSize if got == expected { diff --git a/system_tests/seqinbox_test.go b/system_tests/seqinbox_test.go index 4dc8f4a6647..6babe5833f0 100644 --- a/system_tests/seqinbox_test.go +++ b/system_tests/seqinbox_test.go @@ -229,6 +229,7 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { reorgTargetNumber := blockStates[reorgTo].l1BlockNumber currentHeader, err := builder.L1.Client.HeaderByNumber(ctx, nil) Require(t, err) + // #nosec G115 if currentHeader.Number.Int64()-int64(reorgTargetNumber) < 65 { Fatal(t, "Less than 65 blocks of difference between current block", currentHeader.Number, "and target", reorgTargetNumber) } @@ -346,7 +347,7 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { BridgeAddr: builder.L1Info.GetAddress("Bridge"), DataPosterAddr: seqOpts.From, GasRefunderAddr: gasRefunderAddr, - SequencerInboxAccs: len(blockStates), + SequencerInboxAccs: uint64(len(blockStates)), AfterDelayedMessagesRead: 1, }) if diff := diffAccessList(accessed, *wantAL); diff != "" { @@ -374,6 +375,7 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { t.Fatalf("BalanceAt(%v) unexpected error: %v", seqOpts.From, err) } txCost := txRes.EffectiveGasPrice.Uint64() * txRes.GasUsed + // #nosec G115 if diff := before.Int64() - after.Int64(); diff >= int64(txCost) { t.Errorf("Transaction: %v was not refunded, balance diff: %v, cost: %v", tx.Hash(), diff, txCost) } @@ -424,11 +426,13 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { } for _, state := range blockStates { + // #nosec G115 block, err := l2Backend.APIBackend().BlockByNumber(ctx, rpc.BlockNumber(state.l2BlockNumber)) Require(t, err) if block == nil { Fatal(t, "missing state block", state.l2BlockNumber) } + // #nosec G115 stateDb, _, err := l2Backend.APIBackend().StateAndHeaderByNumber(ctx, rpc.BlockNumber(state.l2BlockNumber)) Require(t, err) for acct, expectedBalance := range state.balances { diff --git a/system_tests/snap_sync_test.go b/system_tests/snap_sync_test.go index a04d9f5bf39..7462b5f5f01 100644 --- a/system_tests/snap_sync_test.go +++ b/system_tests/snap_sync_test.go @@ -92,8 +92,10 @@ func TestSnapSync(t *testing.T) { waitForBlockToCatchupToMessageCount(ctx, t, nodeC.Client, finalMessageCount) // Fetching message count - 1 instead on the latest block number as the latest block number might not be // present in the snap sync node since it does not have the sequencer feed. + // #nosec G115 header, err := builder.L2.Client.HeaderByNumber(ctx, big.NewInt(int64(finalMessageCount)-1)) Require(t, err) + // #nosec G115 headerNodeC, err := nodeC.Client.HeaderByNumber(ctx, big.NewInt(int64(finalMessageCount)-1)) Require(t, err) // Once the node is synced up, check if the block hash is the same for the last block diff --git a/system_tests/twonodeslong_test.go b/system_tests/twonodeslong_test.go index 83cd975dd80..60707b83fb9 100644 --- a/system_tests/twonodeslong_test.go +++ b/system_tests/twonodeslong_test.go @@ -63,6 +63,7 @@ func testTwoNodesLong(t *testing.T, dasModeStr string) { builder.L2Info.GenerateAccount("ErrorTxSender") builder.L2.SendWaitTestTransactions(t, []*types.Transaction{ + // #nosec G115 builder.L2Info.PrepareTx("Faucet", "ErrorTxSender", builder.L2Info.TransferGas, big.NewInt(l2pricing.InitialBaseFeeWei*int64(builder.L2Info.TransferGas)), nil), }) diff --git a/system_tests/unsupported_txtypes_test.go b/system_tests/unsupported_txtypes_test.go index 4c3c8661c86..a228cb2454c 100644 --- a/system_tests/unsupported_txtypes_test.go +++ b/system_tests/unsupported_txtypes_test.go @@ -112,8 +112,8 @@ func TestBlobAndInternalTxsAsDelayedMsgReject(t *testing.T) { blocknum, err := builder.L2.Client.BlockNumber(ctx) Require(t, err) - for i := int64(0); i <= int64(blocknum); i++ { - block, err := builder.L2.Client.BlockByNumber(ctx, big.NewInt(i)) + for i := uint64(0); i <= blocknum; i++ { + block, err := builder.L2.Client.BlockByNumber(ctx, new(big.Int).SetUint64(i)) Require(t, err) for _, tx := range block.Transactions() { if _, ok := txAcceptStatus[tx.Hash()]; ok { diff --git a/util/arbmath/bips.go b/util/arbmath/bips.go index 8b7c47d82bc..646dad3a92d 100644 --- a/util/arbmath/bips.go +++ b/util/arbmath/bips.go @@ -20,7 +20,7 @@ func PercentToBips(percentage int64) Bips { } func BigToBips(natural *big.Int) Bips { - return Bips(natural.Uint64()) + return Bips(natural.Int64()) } func BigMulByBips(value *big.Int, bips Bips) *big.Int { @@ -51,5 +51,5 @@ func (bips Bips) Uint64() uint64 { func BigDivToBips(dividend, divisor *big.Int) Bips { value := BigMulByInt(dividend, int64(OneInBips)) value.Div(value, divisor) - return Bips(BigToUintSaturating(value)) + return Bips(BigToIntSaturating(value)) } diff --git a/util/arbmath/math.go b/util/arbmath/math.go index 62af1e26e01..e5bed67f6df 100644 --- a/util/arbmath/math.go +++ b/util/arbmath/math.go @@ -117,6 +117,18 @@ func BigToUintSaturating(value *big.Int) uint64 { return value.Uint64() } +// BigToUintSaturating casts a huge to an int, saturating if out of bounds +func BigToIntSaturating(value *big.Int) int64 { + if !value.IsInt64() { + if value.Sign() < 0 { + return math.MinInt64 + } else { + return math.MaxInt64 + } + } + return value.Int64() +} + // BigToUintOrPanic casts a huge to a uint, panicking if out of bounds func BigToUintOrPanic(value *big.Int) uint64 { if value.Sign() < 0 { @@ -260,10 +272,12 @@ func BigFloatMulByUint(multiplicand *big.Float, multiplier uint64) *big.Float { } func MaxSignedValue[T Signed]() T { + // #nosec G115 return T((uint64(1) << (8*unsafe.Sizeof(T(0)) - 1)) - 1) } func MinSignedValue[T Signed]() T { + // #nosec G115 return T(uint64(1) << ((8 * unsafe.Sizeof(T(0))) - 1)) } diff --git a/util/arbmath/math_test.go b/util/arbmath/math_test.go index 1be60dc58b1..528666dc19e 100644 --- a/util/arbmath/math_test.go +++ b/util/arbmath/math_test.go @@ -35,6 +35,7 @@ func TestMath(t *testing.T) { input := rand.Uint64() / 256 approx := ApproxSquareRoot(input) correct := math.Sqrt(float64(input)) + // #nosec G115 diff := int(approx) - int(correct) if diff < -1 || diff > 1 { Fail(t, "sqrt approximation off by too much", diff, input, approx, correct) @@ -46,6 +47,7 @@ func TestMath(t *testing.T) { input := uint64(i) approx := ApproxSquareRoot(input) correct := math.Sqrt(float64(input)) + // #nosec G115 diff := int(approx) - int(correct) if diff < 0 || diff > 1 { Fail(t, "sqrt approximation off by too much", diff, input, approx, correct) @@ -57,6 +59,7 @@ func TestMath(t *testing.T) { input := uint64(1 << i) approx := ApproxSquareRoot(input) correct := math.Sqrt(float64(input)) + // #nosec G115 diff := int(approx) - int(correct) if diff != 0 { Fail(t, "incorrect", "2^", i, diff, approx, correct) diff --git a/util/arbmath/uint24.go b/util/arbmath/uint24.go index 818f871a23e..a0c5aa27b79 100644 --- a/util/arbmath/uint24.go +++ b/util/arbmath/uint24.go @@ -9,10 +9,10 @@ import ( "math/big" ) -const MaxUint24 = 1<<24 - 1 // 16777215 - type Uint24 uint32 +const MaxUint24 = 1<<24 - 1 // 16777215 + func (value Uint24) ToBig() *big.Int { return UintToBig(uint64(value)) } @@ -26,8 +26,9 @@ func (value Uint24) ToUint64() uint64 { } func IntToUint24[T uint32 | uint64](value T) (Uint24, error) { + // #nosec G115 if value > T(MaxUint24) { - return Uint24(MaxUint24), errors.New("value out of range") + return MaxUint24, errors.New("value out of range") } return Uint24(value), nil } @@ -40,6 +41,7 @@ func BigToUint24OrPanic(value *big.Int) Uint24 { if !value.IsUint64() || value.Uint64() > MaxUint24 { panic("big.Int value exceeds the max Uint24") } + // #nosec G115 return Uint24(value.Uint64()) } diff --git a/util/headerreader/header_reader.go b/util/headerreader/header_reader.go index 074d24338e8..c8041dc8710 100644 --- a/util/headerreader/header_reader.go +++ b/util/headerreader/header_reader.go @@ -340,6 +340,7 @@ func (s *HeaderReader) logIfHeaderIsOld() { if storedHeader == nil { return } + // #nosec G115 l1Timetamp := time.Unix(int64(storedHeader.Time), 0) headerTime := time.Since(l1Timetamp) if headerTime >= s.config().OldHeaderTimeout { diff --git a/util/merkletree/merkleTree.go b/util/merkletree/merkleTree.go index 1b15d51d98c..fffa9bcabc9 100644 --- a/util/merkletree/merkleTree.go +++ b/util/merkletree/merkleTree.go @@ -43,8 +43,8 @@ func NewLevelAndLeaf(level, leaf uint64) LevelAndLeaf { func (place LevelAndLeaf) ToBigInt() *big.Int { return new(big.Int).Add( - new(big.Int).Lsh(big.NewInt(int64(place.Level)), 192), - big.NewInt(int64(place.Leaf)), + new(big.Int).Lsh(new(big.Int).SetUint64(place.Level), 192), + new(big.Int).SetUint64(place.Leaf), ) } diff --git a/util/rpcclient/rpcclient.go b/util/rpcclient/rpcclient.go index be5825a28db..a35d4b6665a 100644 --- a/util/rpcclient/rpcclient.go +++ b/util/rpcclient/rpcclient.go @@ -101,7 +101,7 @@ func (c *RpcClient) Close() { } type limitedMarshal struct { - limit int + limit uint value any } @@ -113,16 +113,18 @@ func (m limitedMarshal) String() string { } else { str = string(marshalled) } - if m.limit == 0 || len(str) <= m.limit { + // #nosec G115 + limit := int(m.limit) + if m.limit <= 0 || len(str) <= limit { return str } prefix := str[:m.limit/2-1] - postfix := str[len(str)-m.limit/2+1:] + postfix := str[len(str)-limit/2+1:] return fmt.Sprintf("%v..%v", prefix, postfix) } type limitedArgumentsMarshal struct { - limit int + limit uint args []any } @@ -162,9 +164,9 @@ func (c *RpcClient) CallContext(ctx_in context.Context, result interface{}, meth return errors.New("not connected") } logId := c.logId.Add(1) - log.Trace("sending RPC request", "method", method, "logId", logId, "args", limitedArgumentsMarshal{int(c.config().ArgLogLimit), args}) + log.Trace("sending RPC request", "method", method, "logId", logId, "args", limitedArgumentsMarshal{c.config().ArgLogLimit, args}) var err error - for i := 0; i < int(c.config().Retries)+1; i++ { + for i := uint(0); i < c.config().Retries+1; i++ { retryDelay := c.config().RetryDelay if i > 0 && retryDelay > 0 { select { @@ -188,7 +190,7 @@ func (c *RpcClient) CallContext(ctx_in context.Context, result interface{}, meth cancelCtx() logger := log.Trace - limit := int(c.config().ArgLogLimit) + limit := c.config().ArgLogLimit if err != nil && !IsAlreadyKnownError(err) { logger = log.Info } diff --git a/util/sharedmetrics/sharedmetrics.go b/util/sharedmetrics/sharedmetrics.go index 377eef53522..9b4b3609bc7 100644 --- a/util/sharedmetrics/sharedmetrics.go +++ b/util/sharedmetrics/sharedmetrics.go @@ -11,8 +11,10 @@ var ( ) func UpdateSequenceNumberGauge(sequenceNumber arbutil.MessageIndex) { + // #nosec G115 latestSequenceNumberGauge.Update(int64(sequenceNumber)) } func UpdateSequenceNumberInBlockGauge(sequenceNumber arbutil.MessageIndex) { + // #nosec G115 sequenceNumberInBlockGauge.Update(int64(sequenceNumber)) } diff --git a/util/testhelpers/testhelpers.go b/util/testhelpers/testhelpers.go index b1b08708e75..d681b422bf8 100644 --- a/util/testhelpers/testhelpers.go +++ b/util/testhelpers/testhelpers.go @@ -65,6 +65,7 @@ func RandomCallValue(limit int64) *big.Int { // Computes a psuedo-random uint64 on the interval [min, max] func RandomUint32(min, max uint32) uint32 { + //#nosec G115 return uint32(RandomUint64(uint64(min), uint64(max))) } diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 80cff666754..00bd992f46f 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -102,6 +102,7 @@ func (c *ValidationClient) Start(ctx context.Context) error { } else { log.Info("connected to validation server", "name", name, "room", room) } + // #nosec G115 c.room.Store(int32(room)) c.wasmModuleRoots = moduleRoots c.name = name diff --git a/validator/server_arb/execution_run_test.go b/validator/server_arb/execution_run_test.go index bdc1eefc4da..479db58515f 100644 --- a/validator/server_arb/execution_run_test.go +++ b/validator/server_arb/execution_run_test.go @@ -194,7 +194,7 @@ func Test_machineHashesWithStep(t *testing.T) { Batch: 1, PosInBatch: mm.totalSteps - 1, })) - if len(hashes) >= int(maxIterations) { + if uint64(len(hashes)) >= maxIterations { t.Fatal("Wanted fewer hashes than the max iterations") } for i := range hashes { diff --git a/validator/server_arb/machine_cache.go b/validator/server_arb/machine_cache.go index 23fcdef6d6f..55ef61cf11d 100644 --- a/validator/server_arb/machine_cache.go +++ b/validator/server_arb/machine_cache.go @@ -239,6 +239,7 @@ func (c *MachineCache) getClosestMachine(stepCount uint64) (int, MachineInterfac if c.machineStepInterval == 0 || stepsFromStart > c.machineStepInterval*uint64(len(c.machines)-1) { index = len(c.machines) - 1 } else { + // #nosec G115 index = int(stepsFromStart / c.machineStepInterval) } return index, c.machines[index] diff --git a/validator/server_arb/validator_spawner.go b/validator/server_arb/validator_spawner.go index 844a988d282..eb530703031 100644 --- a/validator/server_arb/validator_spawner.go +++ b/validator/server_arb/validator_spawner.go @@ -179,7 +179,10 @@ func (v *ArbitratorSpawner) execute( } steps += count } + + // #nosec G115 arbitratorValidationSteps.Update(int64(mach.GetStepCount())) + if mach.IsErrored() { log.Error("machine entered errored state during attempted validation", "block", entry.Id) return validator.GoGlobalState{}, errors.New("machine entered errored state during attempted validation") diff --git a/validator/server_jit/jit_machine.go b/validator/server_jit/jit_machine.go index 23a75bba833..e7753748aba 100644 --- a/validator/server_jit/jit_machine.go +++ b/validator/server_jit/jit_machine.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "math" "net" "os" "os/exec" @@ -125,6 +126,13 @@ func (machine *JitMachine) prove( writeUint32 := func(data uint32) error { return writeExact(arbmath.Uint32ToBytes(data)) } + writeIntAsUint32 := func(data int) error { + if data < 0 || data > math.MaxUint32 { + return fmt.Errorf("attempted to write out-of-bounds int %v as uint32", data) + } + // #nosec G115 + return writeUint32(uint32(data)) + } writeUint64 := func(data uint64) error { return writeExact(arbmath.UintToBytes(data)) } @@ -192,14 +200,14 @@ func (machine *JitMachine) prove( // send known preimages preimageTypes := entry.Preimages - if err := writeUint32(uint32(len(preimageTypes))); err != nil { + if err := writeIntAsUint32(len(preimageTypes)); err != nil { return state, err } for ty, preimages := range preimageTypes { if err := writeUint8(uint8(ty)); err != nil { return state, err } - if err := writeUint32(uint32(len(preimages))); err != nil { + if err := writeIntAsUint32(len(preimages)); err != nil { return state, err } for hash, preimage := range preimages { @@ -224,7 +232,7 @@ func (machine *JitMachine) prove( } } - if err := writeUint32(uint32(len(userWasms))); err != nil { + if err := writeIntAsUint32(len(userWasms)); err != nil { return state, err } for moduleHash, program := range userWasms { @@ -301,6 +309,7 @@ func (machine *JitMachine) prove( if memoryUsed > uint64(machine.wasmMemoryUsageLimit) { log.Warn("memory used by jit wasm exceeds the wasm memory usage limit", "limit", machine.wasmMemoryUsageLimit, "memoryUsed", memoryUsed) } + // #nosec G115 jitWasmMemoryUsage.Update(int64(memoryUsed)) return state, nil default: diff --git a/wavmio/stub.go b/wavmio/stub.go index 7fd29e20626..1395fb42350 100644 --- a/wavmio/stub.go +++ b/wavmio/stub.go @@ -60,13 +60,13 @@ func parsePreimageBytes(path string) { if read != len(lenBuf) { panic(fmt.Sprintf("missing bytes reading len got %d", read)) } - fieldSize := int(binary.LittleEndian.Uint64(lenBuf)) + fieldSize := binary.LittleEndian.Uint64(lenBuf) dataBuf := make([]byte, fieldSize) read, err = file.Read(dataBuf) if err != nil { panic(err) } - if read != fieldSize { + if uint64(read) != fieldSize { panic("missing bytes reading data") } hash := crypto.Keccak256Hash(dataBuf) @@ -125,7 +125,7 @@ func ReadInboxMessage(msgNum uint64) []byte { } func ReadDelayedInboxMessage(seqNum uint64) []byte { - if seqNum < delayedMsgFirstPos || (int(seqNum-delayedMsgFirstPos) > len(delayedMsgs)) { + if seqNum < delayedMsgFirstPos || (seqNum-delayedMsgFirstPos > uint64(len(delayedMsgs))) { panic(fmt.Sprintf("trying to read bad delayed msg %d", seqNum)) } return delayedMsgs[seqNum-delayedMsgFirstPos] diff --git a/wsbroadcastserver/clientconnection.go b/wsbroadcastserver/clientconnection.go index 16a8f64daf7..00ae0f0dcfc 100644 --- a/wsbroadcastserver/clientconnection.go +++ b/wsbroadcastserver/clientconnection.go @@ -135,6 +135,7 @@ func (cc *ClientConnection) writeBacklog(ctx context.Context, segment backlog.Ba msgs := prevSegment.Messages() if isFirstSegment && prevSegment.Contains(uint64(cc.requestedSeqNum)) { + // #nosec G115 requestedIdx := int(cc.requestedSeqNum) - int(prevSegment.Start()) // This might be false if messages were added after we fetched the segment's messages if len(msgs) >= requestedIdx { From 9443c1904566ee1b7b7b6eeb949a3cd01a967b0a Mon Sep 17 00:00:00 2001 From: Tristan Wilson Date: Wed, 21 Aug 2024 20:00:17 +0200 Subject: [PATCH 016/156] Fix inverted expiry logic for DAS S3 backends Also force a resync of all mirrors since if they were using S3, they may be missing data in S3. --- das/s3_storage_service.go | 2 +- das/syncing_fallback_storage.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/das/s3_storage_service.go b/das/s3_storage_service.go index a1de200c52c..a289b0d46fc 100644 --- a/das/s3_storage_service.go +++ b/das/s3_storage_service.go @@ -110,7 +110,7 @@ func (s3s *S3StorageService) Put(ctx context.Context, value []byte, timeout uint Bucket: aws.String(s3s.bucket), Key: aws.String(s3s.objectPrefix + EncodeStorageServiceKey(dastree.Hash(value))), Body: bytes.NewReader(value)} - if !s3s.discardAfterTimeout { + if s3s.discardAfterTimeout { expires := time.Unix(int64(timeout), 0) putObjectInput.Expires = &expires } diff --git a/das/syncing_fallback_storage.go b/das/syncing_fallback_storage.go index ee0ddc6b5fc..8af46b7fc58 100644 --- a/das/syncing_fallback_storage.go +++ b/das/syncing_fallback_storage.go @@ -105,9 +105,9 @@ type l1SyncService struct { lastBatchAcc common.Hash } -// The original syncing process had a bug, so the file was renamed to cause any mirrors -// in the wild to re-sync from their configured starting block number. -const nextBlockNoFilename = "nextBlockNumberV2" +// The filename has been updated when we have discovered bugs that may have impacted +// syncing, to cause mirrors to re-sync. +const nextBlockNoFilename = "nextBlockNumberV3" func readSyncStateOrDefault(syncDir string, dflt uint64) uint64 { if syncDir == "" { From b4fa9bbd97abaa2b83d8499cc769fff629566f39 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 22 Aug 2024 09:39:41 -0500 Subject: [PATCH 017/156] Lower a couple of info logs down to debug --- arbnode/batch_poster.go | 2 +- execution/gethexec/executionengine.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 71239efdbbc..b7eee5cc476 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -1439,7 +1439,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) b.building.muxBackend.delayedInboxStart = batchPosition.DelayedMessageCount b.building.muxBackend.SetPositionWithinMessage(0) simMux := arbstate.NewInboxMultiplexer(b.building.muxBackend, batchPosition.DelayedMessageCount, dapReaders, daprovider.KeysetValidate) - log.Info("Begin checking the correctness of batch against inbox multiplexer", "startMsgSeqNum", batchPosition.MessageCount, "endMsgSeqNum", b.building.msgCount-1) + log.Debug("Begin checking the correctness of batch against inbox multiplexer", "startMsgSeqNum", batchPosition.MessageCount, "endMsgSeqNum", b.building.msgCount-1) for i := batchPosition.MessageCount; i < b.building.msgCount; i++ { msg, err := simMux.Pop(ctx) if err != nil { diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 806355b2c6d..19d77fc38f9 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -138,7 +138,7 @@ func (s *ExecutionEngine) MarkFeedStart(to arbutil.MessageIndex) { defer s.cachedL1PriceData.mutex.Unlock() if to < s.cachedL1PriceData.startOfL1PriceDataCache { - log.Info("trying to trim older cache which doesnt exist anymore") + log.Debug("trying to trim older L1 price data cache which doesnt exist anymore") } else if to >= s.cachedL1PriceData.endOfL1PriceDataCache { s.cachedL1PriceData.startOfL1PriceDataCache = 0 s.cachedL1PriceData.endOfL1PriceDataCache = 0 From 5f40b1c58653120c151d329b7a3fc91818e7d6ec Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 13:46:44 -0300 Subject: [PATCH 018/156] Creates wallet contract through Staker's DataPoster --- cmd/nitro/nitro.go | 2 +- staker/validatorwallet/contract.go | 90 ++++++++++++++++++++++-------- system_tests/fast_confirm_test.go | 8 +-- system_tests/staker_test.go | 4 +- 4 files changed, 74 insertions(+), 30 deletions(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 39f204980d3..8c26b95a927 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -372,7 +372,7 @@ func mainImpl() int { log.Crit("error getting rollup addresses config", "err", err) } // #nosec G115 - addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true) + addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, nil) if err != nil { log.Crit("error creating validator wallet contract", "error", err, "address", l1TransactionOptsValidator.From.Hex()) } diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 77b403b6692..89d4fef11d2 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -26,8 +26,11 @@ import ( "github.com/offchainlabs/nitro/util/headerreader" ) -var validatorABI abi.ABI -var walletCreatedID common.Hash +var ( + validatorABI abi.ABI + validatorWalletCreatorABI abi.ABI + walletCreatedID common.Hash +) func init() { parsedValidator, err := abi.JSON(strings.NewReader(rollupgen.ValidatorWalletABI)) @@ -40,6 +43,7 @@ func init() { if err != nil { panic(err) } + validatorWalletCreatorABI = parsedValidatorWalletCreator walletCreatedID = parsedValidatorWalletCreator.Events["WalletCreated"].ID } @@ -179,6 +183,37 @@ func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), *v.Address(), data, gas, auth.Value) } +// Exposed for tests +func (v *Contract) CreateWalletContract( + ctx context.Context, + validatorWalletFactoryAddr common.Address, +) (*types.Transaction, error) { + var initialExecutorAllowedDests []common.Address + txData, err := validatorWalletCreatorABI.Pack("createWallet", initialExecutorAllowedDests) + if err != nil { + return nil, err + } + + auth, err := v.getAuth(ctx, nil) + if err != nil { + return nil, err + } + + gas, err := gasForTxData( + ctx, + v.l1Reader, + auth, + &validatorWalletFactoryAddr, + txData, + v.getExtraGas, + ) + if err != nil { + return nil, fmt.Errorf("getting gas for tx data: %w", err) + } + + return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) +} + func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) error { if v.con != nil { return nil @@ -194,7 +229,7 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err if err != nil { return err } - addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing) + addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing, v.CreateWalletContract) if err != nil { return err } @@ -295,25 +330,29 @@ func (v *Contract) ExecuteTransactions(ctx context.Context, builder *txbuilder.B return arbTx, nil } -func (v *Contract) estimateGas(ctx context.Context, value *big.Int, data []byte) (uint64, error) { - h, err := v.l1Reader.LastHeader(ctx) +func gasForTxData(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, to *common.Address, data []byte, getExtraGas func() uint64) (uint64, error) { + if auth.GasLimit != 0 { + return auth.GasLimit, nil + } + + h, err := l1Reader.LastHeader(ctx) if err != nil { return 0, fmt.Errorf("getting the last header: %w", err) } gasFeeCap := new(big.Int).Mul(h.BaseFee, big.NewInt(2)) gasFeeCap = arbmath.BigMax(gasFeeCap, arbmath.FloatToBig(params.GWei)) - gasTipCap, err := v.l1Reader.Client().SuggestGasTipCap(ctx) + gasTipCap, err := l1Reader.Client().SuggestGasTipCap(ctx) if err != nil { return 0, fmt.Errorf("getting suggested gas tip cap: %w", err) } gasFeeCap.Add(gasFeeCap, gasTipCap) - g, err := v.l1Reader.Client().EstimateGas( + g, err := l1Reader.Client().EstimateGas( ctx, ethereum.CallMsg{ - From: v.auth.From, - To: v.Address(), - Value: value, + From: auth.From, + To: to, + Value: auth.Value, Data: data, GasFeeCap: gasFeeCap, GasTipCap: gasTipCap, @@ -322,7 +361,11 @@ func (v *Contract) estimateGas(ctx context.Context, value *big.Int, data []byte) if err != nil { return 0, fmt.Errorf("estimating gas: %w", err) } - return g + v.getExtraGas(), nil + return g + getExtraGas(), nil +} + +func (v *Contract) gasForTxData(ctx context.Context, auth *bind.TransactOpts, data []byte) (uint64, error) { + return gasForTxData(ctx, v.l1Reader, auth, v.Address(), data, v.getExtraGas) } func (v *Contract) TimeoutChallenges(ctx context.Context, challenges []uint64) (*types.Transaction, error) { @@ -341,14 +384,6 @@ func (v *Contract) TimeoutChallenges(ctx context.Context, challenges []uint64) ( return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), *v.Address(), data, gas, auth.Value) } -// gasForTxData returns auth.GasLimit if it's nonzero, otherwise returns estimate. -func (v *Contract) gasForTxData(ctx context.Context, auth *bind.TransactOpts, data []byte) (uint64, error) { - if auth.GasLimit != 0 { - return auth.GasLimit, nil - } - return v.estimateGas(ctx, auth.Value, data) -} - func (v *Contract) L1Client() arbutil.L1Interface { return v.l1Reader.Client() } @@ -407,6 +442,7 @@ func GetValidatorWalletContract( transactAuth *bind.TransactOpts, l1Reader *headerreader.HeaderReader, createIfMissing bool, + createWalletFunc func(context.Context, common.Address) (*types.Transaction, error), ) (*common.Address, error) { client := l1Reader.Client() @@ -443,10 +479,18 @@ func GetValidatorWalletContract( return nil, nil } - var initialExecutorAllowedDests []common.Address - tx, err := walletCreator.CreateWallet(transactAuth, initialExecutorAllowedDests) - if err != nil { - return nil, err + var tx *types.Transaction + if createWalletFunc == nil { + var initialExecutorAllowedDests []common.Address + tx, err = walletCreator.CreateWallet(transactAuth, initialExecutorAllowedDests) + if err != nil { + return nil, err + } + } else { + tx, err = createWalletFunc(ctx, validatorWalletFactoryAddr) + if err != nil { + return nil, err + } } receipt, err := l1Reader.WaitForTxApproval(ctx, tx) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index 4a679e50772..79b77524a28 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -79,10 +79,10 @@ func TestFastConfirmation(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "Validator", balance, builder.L1Info) l1auth := builder.L1Info.GetDefaultTransactOpts("Validator", ctx) - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true) + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) Require(t, err) valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) Require(t, err) if valWalletAddr == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) @@ -278,10 +278,10 @@ func TestFastConfirmationWithSafe(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index 03c9fd36283..13530adfd56 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -132,10 +132,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) From 26bd0bd59e0ed1d0948c197937fd601809bf5342 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 15:23:46 -0300 Subject: [PATCH 019/156] Staker tests creates wallet contract through Staker's DataPoster --- system_tests/staker_test.go | 42 +++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index 13530adfd56..ea34eb8a966 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -37,6 +37,7 @@ import ( "github.com/offchainlabs/nitro/util" "github.com/offchainlabs/nitro/util/arbmath" "github.com/offchainlabs/nitro/util/colors" + "github.com/offchainlabs/nitro/util/testhelpers" "github.com/offchainlabs/nitro/validator/valnode" ) @@ -57,7 +58,8 @@ func makeBackgroundTxs(ctx context.Context, builder *NodeBuilder) error { } func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) { - t.Parallel() + logHandler := testhelpers.InitTestLog(t, log.LvlTrace) + ctx, cancelCtx := context.WithCancel(context.Background()) defer cancelCtx() srv := externalsignertest.NewServer(t) @@ -132,15 +134,6 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - if valWalletAddrA == *valWalletAddrCheck { - Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) - } - rollup, err := rollupgen.NewRollupAdminLogic(l2nodeA.DeployInfo.Rollup, builder.L1.Client) Require(t, err) @@ -149,16 +142,9 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) rollupABI, err := abi.JSON(strings.NewReader(rollupgen.RollupAdminLogicABI)) Require(t, err, "unable to parse rollup ABI") - setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address}, []bool{true, true, true}) - Require(t, err, "unable to generate setValidator calldata") - tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) - Require(t, err, "unable to set validators") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - setMinAssertPeriodCalldata, err := rollupABI.Pack("setMinimumAssertionPeriod", big.NewInt(1)) Require(t, err, "unable to generate setMinimumAssertionPeriod calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) + tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) Require(t, err, "unable to set minimum assertion period") _, err = builder.L1.EnsureTxSucceeded(tx) Require(t, err) @@ -190,6 +176,22 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) valConfigA.Strategy = "MakeNodes" } + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + Require(t, err) + valWalletAddrA := *valWalletAddrAPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + Require(t, err) + if valWalletAddrA == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) + } + + setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address}, []bool{true, true, true}) + Require(t, err, "unable to generate setValidator calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) + Require(t, err, "unable to set validators") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + _, valStack := createTestValidationNode(t, ctx, &valnode.TestValidationConfig) blockValidatorConfig := staker.TestBlockValidatorConfig @@ -464,6 +466,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) if !stakerBWasStaked { Fatal(t, "staker B was never staked") } + + if logHandler.WasLogged("data poster expected next transaction to have nonce \\d+ but was requested to post transaction with nonce \\d+") { + Fatal(t, "Staker's DataPoster inferred nonce incorrectly") + } } func TestStakersCooperative(t *testing.T) { From c3ee555bc6c3679aed7c1347fc8c771d3d02caca Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 15:24:02 -0300 Subject: [PATCH 020/156] TestGetValidatorWalletContractWithoutDataPoster --- system_tests/staker_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index ea34eb8a966..3fdf82ca744 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -475,3 +475,29 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) func TestStakersCooperative(t *testing.T) { stakerTestImpl(t, false, false) } + +func TestGetValidatorWalletContractWithoutDataPoster(t *testing.T) { + t.Parallel() + + ctx, cancelCtx := context.WithCancel(context.Background()) + defer cancelCtx() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + cleanup := builder.Build(t) + defer cleanup() + + balance := big.NewInt(params.Ether) + balance.Mul(balance, big.NewInt(100)) + builder.L1Info.GenerateAccount("ValidatorA") + builder.L1.TransferBalance(t, "Faucet", "ValidatorA", balance, builder.L1Info) + l1auth := builder.L1Info.GetDefaultTransactOpts("ValidatorA", ctx) + + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + Require(t, err) + valWalletAddrA := *valWalletAddrAPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + Require(t, err) + if valWalletAddrA == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) + } +} From 2fde94e651c991df1e4a3efe858d758bd784e4e4 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 15:52:10 -0300 Subject: [PATCH 021/156] Adds comment related to v.CreateWalletContract --- staker/validatorwallet/contract.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 89d4fef11d2..495e796cd25 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -229,6 +229,10 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err if err != nil { return err } + + // By passing v.CreateWalletContract as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. + // DataPoster keeps in its internal state information related to the transactions sent through it, which is used to infer the expected nonce in a transaction for example. + // If a transaction is sent using the Staker's DataPoster key, but not through the Staker's DataPoster object, DataPoster's internal state will be outdated, which can compromise the expected nonce inference. addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing, v.CreateWalletContract) if err != nil { return err From 04e3f12507ceff95812d0b72901d84ed643954a1 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 16:16:12 -0300 Subject: [PATCH 022/156] Fast confirmation test creates wallet contract through Staker's DataPoster --- system_tests/fast_confirm_test.go | 48 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index 79b77524a28..ae624be1e9b 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -79,15 +79,6 @@ func TestFastConfirmation(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "Validator", balance, builder.L1Info) l1auth := builder.L1Info.GetDefaultTransactOpts("Validator", ctx) - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) - Require(t, err) - valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) - Require(t, err) - if valWalletAddr == *valWalletAddrCheck { - Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) - } - rollup, err := rollupgen.NewRollupAdminLogic(l2node.DeployInfo.Rollup, builder.L1.Client) Require(t, err) @@ -96,27 +87,13 @@ func TestFastConfirmation(t *testing.T) { rollupABI, err := abi.JSON(strings.NewReader(rollupgen.RollupAdminLogicABI)) Require(t, err, "unable to parse rollup ABI") - setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddr, srv.Address}, []bool{true, true}) - Require(t, err, "unable to generate setValidator calldata") - tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setValidatorCalldata) - Require(t, err, "unable to set validators") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - setMinAssertPeriodCalldata, err := rollupABI.Pack("setMinimumAssertionPeriod", big.NewInt(1)) Require(t, err, "unable to generate setMinimumAssertionPeriod calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setMinAssertPeriodCalldata) + tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setMinAssertPeriodCalldata) Require(t, err, "unable to set minimum assertion period") _, err = builder.L1.EnsureTxSucceeded(tx) Require(t, err) - setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", valWalletAddr) - Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) - Require(t, err, "unable to set anytrust fast confirmer") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - valConfig := staker.TestL1ValidatorConfig valConfig.EnableFastConfirmation = true parentChainID, err := builder.L1.Client.ChainID(ctx) @@ -138,6 +115,29 @@ func TestFastConfirmation(t *testing.T) { Require(t, err) valConfig.Strategy = "MakeNodes" + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + Require(t, err) + valWalletAddr := *valWalletAddrPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + Require(t, err) + if valWalletAddr == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) + } + + setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddr, srv.Address}, []bool{true, true}) + Require(t, err, "unable to generate setValidator calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setValidatorCalldata) + Require(t, err, "unable to set validators") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + + setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", valWalletAddr) + Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) + Require(t, err, "unable to set anytrust fast confirmer") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + _, valStack := createTestValidationNode(t, ctx, &valnode.TestValidationConfig) blockValidatorConfig := staker.TestBlockValidatorConfig From fa3bf04edabd7d07b5986aaab73e5df4d030104b Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Thu, 22 Aug 2024 16:16:41 -0300 Subject: [PATCH 023/156] Fix linter warning --- system_tests/stylus_tracer_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index f7c83c73f67..3b95f38d218 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -167,6 +167,7 @@ func TestStylusTracer(t *testing.T) { } func intToBe32(v int) []byte { + // #nosec G115 return binary.BigEndian.AppendUint32(nil, uint32(v)) } From b933b74725e2dd34df1f472ab22fbdfcff277698 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 22 Aug 2024 22:19:16 +0200 Subject: [PATCH 024/156] pass targets to rawdb.WrapDatabaseWithWasm, change rawdb.Target to ethdb.WasmTarget --- arbnode/inbox_test.go | 4 +- arbos/programs/native.go | 11 ++--- arbos/programs/wasmstorehelper.go | 3 +- cmd/nitro/init.go | 6 +-- cmd/nitro/init_test.go | 4 ++ cmd/nitro/nitro.go | 2 +- execution/gethexec/executionengine.go | 35 ++++++++++------ execution/gethexec/node.go | 50 +++++++++++++++++++++-- go-ethereum | 2 +- staker/block_validator.go | 2 +- staker/challenge_manager.go | 3 +- staker/stateless_block_validator.go | 5 +-- system_tests/common_test.go | 47 +++++++++++---------- system_tests/forwarder_test.go | 4 +- system_tests/recreatestate_rpc_test.go | 18 ++++---- system_tests/retryable_test.go | 2 +- system_tests/validation_mock_test.go | 6 +-- validator/client/redis/producer.go | 9 ++-- validator/client/validation_client.go | 13 +++--- validator/interface.go | 4 +- validator/server_api/json.go | 8 ++-- validator/server_arb/validator_spawner.go | 5 ++- validator/server_jit/spawner.go | 5 ++- validator/validation_entry.go | 4 +- validator/valnode/validation_api.go | 4 +- 25 files changed, 161 insertions(+), 95 deletions(-) diff --git a/arbnode/inbox_test.go b/arbnode/inbox_test.go index 1c46c593b99..d579b7c2784 100644 --- a/arbnode/inbox_test.go +++ b/arbnode/inbox_test.go @@ -72,7 +72,9 @@ func NewTransactionStreamerForTest(t *testing.T, ownerAddress common.Address) (* if err != nil { Fail(t, err) } - if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, &gethexec.DefaultStylusTargetConfig); err != nil { + stylusTargetConfig := &gethexec.DefaultStylusTargetConfig + Require(t, stylusTargetConfig.Validate()) // pre-processes config (i.a. parses wasmTargets) + if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, stylusTargetConfig); err != nil { Fail(t, err) } execSeq := &execClientWrapper{execEngine, t} diff --git a/arbos/programs/native.go b/arbos/programs/native.go index fd3dec25a04..36e200af4df 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/arbos/burn" "github.com/offchainlabs/nitro/arbos/util" @@ -71,7 +72,7 @@ func activateProgramInternal( version uint16, debug bool, gasLeft *uint64, -) (*activationInfo, map[rawdb.Target][]byte, error) { +) (*activationInfo, map[ethdb.WasmTarget][]byte, error) { output := &rustBytes{} moduleHash := &bytes32{} stylusData := &C.StylusData{} @@ -111,7 +112,7 @@ func activateProgramInternal( if status_asm != 0 { return nil, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm)) } - asmMap := map[rawdb.Target][]byte{ + asmMap := map[ethdb.WasmTarget][]byte{ rawdb.TargetWavm: module, target: asm, } @@ -171,7 +172,7 @@ func getLocalAsm(statedb vm.StateDB, moduleHash common.Hash, addressForLogging c } asm, exists := asmMap[localTarget] if !exists { - var availableTargets []rawdb.Target + var availableTargets []ethdb.WasmTarget for target := range asmMap { availableTargets = append(availableTargets, target) } @@ -203,7 +204,7 @@ func callProgram( } if db, ok := db.(*state.StateDB); ok { - targets := []rawdb.Target{ + targets := []ethdb.WasmTarget{ rawdb.TargetWavm, rawdb.LocalTarget(), } @@ -291,7 +292,7 @@ func ResizeWasmLruCache(size uint32) { const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon" const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2" -func SetTarget(name rawdb.Target, description string, native bool) error { +func SetTarget(name ethdb.WasmTarget, description string, native bool) error { output := &rustBytes{} status := userStatus(C.stylus_target_set( goSlice([]byte(name)), diff --git a/arbos/programs/wasmstorehelper.go b/arbos/programs/wasmstorehelper.go index 4f82d802828..95daaf544ab 100644 --- a/arbos/programs/wasmstorehelper.go +++ b/arbos/programs/wasmstorehelper.go @@ -12,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -44,7 +45,7 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash } // If already in wasm store then return early - _, err = statedb.TryGetActivatedAsmMap([]rawdb.Target{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash) + _, err = statedb.TryGetActivatedAsmMap([]ethdb.WasmTarget{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash) if err == nil { return nil } diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index d9ae0df3b04..14d9a522030 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -475,7 +475,7 @@ func validateOrUpgradeWasmStoreSchemaVersion(db ethdb.Database) error { return nil } -func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { +func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, targetConfig *gethexec.StylusTargetConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { if !config.Init.Force { if readOnlyDb, err := stack.OpenDatabaseWithFreezerWithExtraOptions("l2chaindata", 0, 0, config.Persistent.Ancient, "l2chaindata/", true, persistentConfig.Pebble.ExtraOptions("l2chaindata")); err == nil { if chainConfig := gethexec.TryReadStoredChainConfig(readOnlyDb); chainConfig != nil { @@ -500,7 +500,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err := dbutil.UnfinishedConversionCheck(wasmDb); err != nil { return nil, nil, fmt.Errorf("wasm unfinished database conversion check error: %w", err) } - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets()) _, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb) if err != nil { return nil, nil, err @@ -617,7 +617,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err := validateOrUpgradeWasmStoreSchemaVersion(wasmDb); err != nil { return nil, nil, err } - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets()) _, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb) if err != nil { return nil, nil, err diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index b2773ed8613..af2242af783 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -377,6 +377,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -393,6 +394,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -410,6 +412,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -545,6 +548,7 @@ func TestOpenInitializeChainDbEmptyInit(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index a052c146d1e..052755d8a34 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -481,7 +481,7 @@ func mainImpl() int { } } - chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Persistent, l1Client, rollupAddrs) + chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Execution.StylusTarget, &nodeConfig.Persistent, l1Client, rollupAddrs) if l2BlockChain != nil { deferFuncs = append(deferFuncs, func() { l2BlockChain.Stop() }) } diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 806355b2c6d..4e0217f527e 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -154,20 +154,31 @@ func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusT if rustCacheSize != 0 { programs.ResizeWasmLruCache(rustCacheSize) } - var effectiveStylusTarget string - target := rawdb.LocalTarget() - switch target { - case rawdb.TargetArm64: - effectiveStylusTarget = targetConfig.Arm64 - case rawdb.TargetAmd64: - effectiveStylusTarget = targetConfig.Amd64 - case rawdb.TargetHost: - effectiveStylusTarget = targetConfig.Host + + localTarget := rawdb.LocalTarget() + targets := targetConfig.WasmTargets() + var nativeSet bool + for _, target := range targets { + var effectiveStylusTarget string + switch target { + case rawdb.TargetArm64: + effectiveStylusTarget = targetConfig.Arm64 + case rawdb.TargetAmd64: + effectiveStylusTarget = targetConfig.Amd64 + case rawdb.TargetHost: + effectiveStylusTarget = targetConfig.Host + } + isNative := target == localTarget + err := programs.SetTarget(target, effectiveStylusTarget, isNative) + if err != nil { + return fmt.Errorf("Failed to set stylus target: %w", err) + } + nativeSet = nativeSet || isNative } - err := programs.SetTarget(target, effectiveStylusTarget, true) - if err != nil { - return fmt.Errorf("Failed to set stylus target: %w", err) + if !nativeSet { + return fmt.Errorf("local target %v missing in list of archs %v", localTarget, targets) } + return nil } diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index b864332e83d..f519cce6ed7 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/arbitrum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/filters" @@ -29,21 +30,61 @@ import ( ) type StylusTargetConfig struct { - Arm64 string `koanf:"arm64"` - Amd64 string `koanf:"amd64"` - Host string `koanf:"host"` + Arm64 string `koanf:"arm64"` + Amd64 string `koanf:"amd64"` + Host string `koanf:"host"` + Archs []string `koanf:"archs"` + + wasmTargets []ethdb.WasmTarget +} + +func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { + return c.wasmTargets +} + +func (c *StylusTargetConfig) Validate() error { + localTarget := rawdb.LocalTarget() + var nativeFound bool + targets := make([]ethdb.WasmTarget, 0, len(c.Archs)) + for _, arch := range c.Archs { + target := ethdb.WasmTarget(arch) + if !rawdb.IsSupportedWasmTarget(target) { + return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) + } + var alreadyInList bool + for _, t := range targets { + if target == t { + alreadyInList = true + break + } + } + if alreadyInList { + continue + } + if target == localTarget { + nativeFound = true + } + targets = append(targets, target) + } + if !nativeFound { + return fmt.Errorf("native target not found in archs list, native target: %v, archs: %v", localTarget, c.Archs) + } + c.wasmTargets = targets + return nil } var DefaultStylusTargetConfig = StylusTargetConfig{ Arm64: programs.DefaultTargetDescriptionArm, Amd64: programs.DefaultTargetDescriptionX86, Host: "", + Archs: []string{string(rawdb.TargetWavm), string(rawdb.LocalTarget())}, } func StylusTargetConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".arm64", DefaultStylusTargetConfig.Arm64, "stylus programs compilation target for arm64 linux") f.String(prefix+".amd64", DefaultStylusTargetConfig.Amd64, "stylus programs compilation target for amd64 linux") f.String(prefix+".host", DefaultStylusTargetConfig.Host, "stylus programs compilation target for system other than 64-bit ARM or 64-bit x86") + f.StringSlice(prefix+".archs", DefaultStylusTargetConfig.Archs, fmt.Sprintf("Comma separated architectures list to compile stylus program to and cache in wasm store (available targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)) } type Config struct { @@ -82,6 +123,9 @@ func (c *Config) Validate() error { if c.forwardingTarget != "" && c.Sequencer.Enable { return errors.New("ForwardingTarget set and sequencer enabled") } + if err := c.StylusTarget.Validate(); err != nil { + return err + } return nil } diff --git a/go-ethereum b/go-ethereum index 575062fad7f..cb03c12f7f0 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 575062fad7ff4db9d7c235f49472f658be29e2fe +Subproject commit cb03c12f7f02d47344d2d089eef748e92b99c95e diff --git a/staker/block_validator.go b/staker/block_validator.go index 8f5724beac4..5cfce84c991 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -499,7 +499,7 @@ func (v *BlockValidator) sendRecord(s *validationStatus) error { //nolint:gosec func (v *BlockValidator) writeToFile(validationEntry *validationEntry, moduleRoot common.Hash) error { - input, err := validationEntry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + input, err := validationEntry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return err } diff --git a/staker/challenge_manager.go b/staker/challenge_manager.go index b1421d7e419..978a876fd0f 100644 --- a/staker/challenge_manager.go +++ b/staker/challenge_manager.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbutil" @@ -468,7 +469,7 @@ func (m *ChallengeManager) createExecutionBackend(ctx context.Context, step uint if err != nil { return fmt.Errorf("error creating validation entry for challenge %v msg %v for execution challenge: %w", m.challengeIndex, initialCount, err) } - input, err := entry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + input, err := entry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return fmt.Errorf("error getting validation entry input of challenge %v msg %v: %w", m.challengeIndex, initialCount, err) } diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index d5eeb8eb694..f54ec8b58c1 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -12,7 +12,6 @@ import ( "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -134,7 +133,7 @@ type validationEntry struct { DelayedMsg []byte } -func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.ValidationInput, error) { +func (e *validationEntry) ToInput(stylusArchs []ethdb.WasmTarget) (*validator.ValidationInput, error) { if e.Stage != Ready { return nil, errors.New("cannot create input from non-ready entry") } @@ -143,7 +142,7 @@ func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.Valida HasDelayedMsg: e.HasDelayedMsg, DelayedMsgNr: e.DelayedMsgNr, Preimages: e.Preimages, - UserWasms: make(map[rawdb.Target]map[common.Hash][]byte, len(e.UserWasms)), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash][]byte, len(e.UserWasms)), BatchInfo: e.BatchInfo, DelayedMsg: e.DelayedMsg, StartState: e.Start, diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 6e7375a921d..392f6de8059 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -197,7 +197,7 @@ var TestSequencerConfig = gethexec.SequencerConfig{ EnableProfiling: false, } -func ExecConfigDefaultNonSequencerTest() *gethexec.Config { +func ExecConfigDefaultNonSequencerTest(t *testing.T) *gethexec.Config { config := gethexec.ConfigDefault config.Caching = TestCachingConfig config.ParentChainReader = headerreader.TestConfig @@ -206,12 +206,12 @@ func ExecConfigDefaultNonSequencerTest() *gethexec.Config { config.ForwardingTarget = "null" config.TxPreChecker.Strictness = gethexec.TxPreCheckerStrictnessNone - _ = config.Validate() + Require(t, config.Validate()) return &config } -func ExecConfigDefaultTest() *gethexec.Config { +func ExecConfigDefaultTest(t *testing.T) *gethexec.Config { config := gethexec.ConfigDefault config.Caching = TestCachingConfig config.Sequencer = TestSequencerConfig @@ -219,7 +219,7 @@ func ExecConfigDefaultTest() *gethexec.Config { config.ForwardingTarget = "null" config.TxPreChecker.Strictness = gethexec.TxPreCheckerStrictnessNone - _ = config.Validate() + Require(t, config.Validate()) return &config } @@ -272,7 +272,7 @@ func (b *NodeBuilder) DefaultConfig(t *testing.T, withL1 bool) *NodeBuilder { b.l2StackConfig = testhelpers.CreateStackConfigForTest(b.dataDir) cp := valnode.TestValidationConfig b.valnodeConfig = &cp - b.execConfig = ExecConfigDefaultTest() + b.execConfig = ExecConfigDefaultTest(t) return b } @@ -310,7 +310,7 @@ func (b *NodeBuilder) CheckConfig(t *testing.T) { b.nodeConfig = arbnode.ConfigDefaultL1Test() } if b.execConfig == nil { - b.execConfig = ExecConfigDefaultTest() + b.execConfig = ExecConfigDefaultTest(t) } if b.L1Info == nil { b.L1Info = NewL1TestInfo(t) @@ -346,7 +346,7 @@ func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() { var l2arbDb ethdb.Database var l2blockchain *core.BlockChain _, b.L2.Stack, l2chainDb, l2arbDb, l2blockchain = createL2BlockChainWithStackConfig( - t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, &b.execConfig.Caching) + t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) var sequencerTxOptsPtr *bind.TransactOpts var dataSigner signature.DataSignerFunc @@ -409,7 +409,7 @@ func (b *NodeBuilder) BuildL2(t *testing.T) func() { var arbDb ethdb.Database var blockchain *core.BlockChain b.L2Info, b.L2.Stack, chainDb, arbDb, blockchain = createL2BlockChain( - t, b.L2Info, b.dataDir, b.chainConfig, &b.execConfig.Caching) + t, b.L2Info, b.dataDir, b.chainConfig, b.execConfig) Require(t, b.execConfig.Validate()) execConfig := b.execConfig @@ -460,7 +460,7 @@ func (b *NodeBuilder) RestartL2Node(t *testing.T) { } b.L2.cleanup() - l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, &b.execConfig.Caching) + l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) execConfigFetcher := func() *gethexec.Config { return b.execConfig } execNode, err := gethexec.CreateExecutionNode(b.ctx, stack, chainDb, blockchain, nil, execConfigFetcher) @@ -1047,30 +1047,32 @@ func DeployOnTestL1( } func createL2BlockChain( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, cacheConfig *gethexec.CachingConfig, + t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { - return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, cacheConfig) + return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, execConfig) } func createL2BlockChainWithStackConfig( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, cacheConfig *gethexec.CachingConfig, + t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { if l2info == nil { l2info = NewArbTestInfo(t, chainConfig.ChainID) } - var stack *node.Node - var err error if stackConfig == nil { stackConfig = testhelpers.CreateStackConfigForTest(dataDir) } - stack, err = node.New(stackConfig) + if execConfig == nil { + execConfig = ExecConfigDefaultTest(t) + } + + stack, err := node.New(stackConfig) Require(t, err) chainData, err := stack.OpenDatabaseWithExtraOptions("l2chaindata", 0, 0, "l2chaindata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("l2chaindata")) Require(t, err) wasmData, err := stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) arbDb, err := stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) @@ -1085,11 +1087,8 @@ func createL2BlockChainWithStackConfig( SerializedChainConfig: serializedChainConfig, } } - var coreCacheConfig *core.CacheConfig - if cacheConfig != nil { - coreCacheConfig = gethexec.DefaultCacheConfigFor(stack, cacheConfig) - } - blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest().TxLookupLimit, 0) + coreCacheConfig := gethexec.DefaultCacheConfigFor(stack, &execConfig.Caching) + blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) return l2info, stack, chainDb, arbDb, blockchain @@ -1149,7 +1148,7 @@ func Create2ndNodeWithConfig( nodeConfig = arbnode.ConfigDefaultL1NonSequencerTest() } if execConfig == nil { - execConfig = ExecConfigDefaultNonSequencerTest() + execConfig = ExecConfigDefaultNonSequencerTest(t) } feedErrChan := make(chan error, 10) l1rpcClient := l1stack.Attach() @@ -1165,7 +1164,7 @@ func Create2ndNodeWithConfig( Require(t, err) wasmData, err := l2stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0) + l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) l2arbDb, err := l2stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) @@ -1179,7 +1178,7 @@ func Create2ndNodeWithConfig( chainConfig := firstExec.ArbInterface.BlockChain().Config() coreCacheConfig := gethexec.DefaultCacheConfigFor(l2stack, &execConfig.Caching) - l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest().TxLookupLimit, 0) + l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) AddValNodeIfNeeded(t, ctx, nodeConfig, true, "", valnodeConfig.Wasm.RootPath) diff --git a/system_tests/forwarder_test.go b/system_tests/forwarder_test.go index 9fe419593ee..1b083d2089d 100644 --- a/system_tests/forwarder_test.go +++ b/system_tests/forwarder_test.go @@ -38,7 +38,7 @@ func TestStaticForwarder(t *testing.T) { clientA := builder.L2.Client nodeConfigB := arbnode.ConfigDefaultL1Test() - execConfigB := ExecConfigDefaultTest() + execConfigB := ExecConfigDefaultTest(t) execConfigB.Sequencer.Enable = false nodeConfigB.Sequencer = false nodeConfigB.DelayedSequencer.Enable = false @@ -109,7 +109,7 @@ func createForwardingNode(t *testing.T, builder *NodeBuilder, ipcPath string, re nodeConfig.Sequencer = false nodeConfig.DelayedSequencer.Enable = false nodeConfig.BatchPoster.Enable = false - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Sequencer.Enable = false execConfig.Forwarder.RedisUrl = redisUrl execConfig.ForwardingTarget = fallbackPath diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index 09d53669ee0..d5a6311cb45 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -95,7 +95,7 @@ func removeStatesFromDb(t *testing.T, bc *core.BlockChain, db ethdb.Database, fr func TestRecreateStateForRPCNoDepthLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -133,7 +133,7 @@ func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() depthGasLimit := int64(256 * util.NormalizeL2GasForL1GasInitial(800_000, params.GWei)) - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = depthGasLimit execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -170,7 +170,7 @@ func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { func TestRecreateStateForRPCDepthLimitExceeded(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = int64(200) execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -207,7 +207,7 @@ func TestRecreateStateForRPCMissingBlockParent(t *testing.T) { var headerCacheLimit uint64 = 512 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -255,7 +255,7 @@ func TestRecreateStateForRPCBeyondGenesis(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -293,7 +293,7 @@ func TestRecreateStateForRPCBlockNotFoundWhileRecreating(t *testing.T) { var blockCacheLimit uint64 = 256 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -341,7 +341,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = maxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -480,7 +480,7 @@ func TestSkippingSavingStateAndRecreatingAfterRestart(t *testing.T) { func TestGettingStateForRPCFullNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Caching.SnapshotCache = 0 // disable snapshots execConfig.Caching.BlockAge = 0 // use only Caching.BlockCount to keep only last N blocks in dirties cache, no matter how new they are execConfig.Sequencer.MaxBlockSpeed = 0 @@ -522,7 +522,7 @@ func TestGettingStateForRPCFullNode(t *testing.T) { func TestGettingStateForRPCHybridArchiveNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Caching.Archive = true // For now Archive node should use HashScheme execConfig.Caching.StateScheme = rawdb.HashScheme diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index 106dfc6d463..aa9fbfd72ea 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -1042,7 +1042,7 @@ func elevateL2Basefee(t *testing.T, ctx context.Context, builder *NodeBuilder) { _, err = precompilesgen.NewArbosTest(common.HexToAddress("0x69"), builder.L2.Client) Require(t, err, "failed to deploy ArbosTest") - burnAmount := ExecConfigDefaultTest().RPC.RPCGasCap + burnAmount := ExecConfigDefaultTest(t).RPC.RPCGasCap burnTarget := uint64(5 * l2pricing.InitialSpeedLimitPerSecondV6 * l2pricing.InitialBacklogTolerance) for i := uint64(0); i < (burnTarget+burnAmount)/burnAmount; i++ { burnArbGas := arbosTestAbi.Methods["burnArbGas"] diff --git a/system_tests/validation_mock_test.go b/system_tests/validation_mock_test.go index 88421e4c4b5..2739c7545e6 100644 --- a/system_tests/validation_mock_test.go +++ b/system_tests/validation_mock_test.go @@ -8,9 +8,9 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbnode" @@ -61,8 +61,8 @@ func (s *mockSpawner) WasmModuleRoots() ([]common.Hash, error) { return mockWasmModuleRoots, nil } -func (s *mockSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{"mock"} +func (s *mockSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{"mock"} } func (s *mockSpawner) Launch(entry *validator.ValidationInput, moduleRoot common.Hash) validator.ValidationRun { diff --git a/validator/client/redis/producer.go b/validator/client/redis/producer.go index f98c246d0e6..adc2f34af57 100644 --- a/validator/client/redis/producer.go +++ b/validator/client/redis/producer.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/go-redis/redis/v8" "github.com/offchainlabs/nitro/pubsub" @@ -35,7 +36,7 @@ func (c ValidationClientConfig) Enabled() bool { func (c ValidationClientConfig) Validate() error { for _, arch := range c.StylusArchs { - if !rawdb.Target(arch).IsValid() { + if !rawdb.IsSupportedWasmTarget(ethdb.WasmTarget(arch)) { return fmt.Errorf("Invalid stylus arch: %v", arch) } } @@ -162,10 +163,10 @@ func (c *ValidationClient) Name() string { return c.config.Name } -func (c *ValidationClient) StylusArchs() []rawdb.Target { - stylusArchs := make([]rawdb.Target, 0, len(c.config.StylusArchs)) +func (c *ValidationClient) StylusArchs() []ethdb.WasmTarget { + stylusArchs := make([]ethdb.WasmTarget, 0, len(c.config.StylusArchs)) for _, arch := range c.config.StylusArchs { - stylusArchs = append(stylusArchs, rawdb.Target(arch)) + stylusArchs = append(stylusArchs, ethdb.WasmTarget(arch)) } return stylusArchs } diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 80cff666754..9dd7780fbc2 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" @@ -31,7 +32,7 @@ type ValidationClient struct { stopwaiter.StopWaiter client *rpcclient.RpcClient name string - stylusArchs []rawdb.Target + stylusArchs []ethdb.WasmTarget room atomic.Int32 wasmModuleRoots []common.Hash } @@ -40,7 +41,7 @@ func NewValidationClient(config rpcclient.ClientConfigFetcher, stack *node.Node) return &ValidationClient{ client: rpcclient.NewRpcClient(config, stack), name: "not started", - stylusArchs: []rawdb.Target{"not started"}, + stylusArchs: []ethdb.WasmTarget{"not started"}, } } @@ -67,14 +68,14 @@ func (c *ValidationClient) Start(ctx context.Context) error { if len(name) == 0 { return errors.New("couldn't read name from server") } - var stylusArchs []rawdb.Target + var stylusArchs []ethdb.WasmTarget if err := c.client.CallContext(ctx, &stylusArchs, server_api.Namespace+"_stylusArchs"); err != nil { var rpcError rpc.Error ok := errors.As(err, &rpcError) if !ok || rpcError.ErrorCode() != -32601 { return fmt.Errorf("could not read stylus arch from server: %w", err) } - stylusArchs = []rawdb.Target{rawdb.Target("pre-stylus")} // invalid, will fail if trying to validate block with stylus + stylusArchs = []ethdb.WasmTarget{ethdb.WasmTarget("pre-stylus")} // invalid, will fail if trying to validate block with stylus } else { if len(stylusArchs) == 0 { return fmt.Errorf("could not read stylus archs from validation server") @@ -117,11 +118,11 @@ func (c *ValidationClient) WasmModuleRoots() ([]common.Hash, error) { return nil, errors.New("not started") } -func (c *ValidationClient) StylusArchs() []rawdb.Target { +func (c *ValidationClient) StylusArchs() []ethdb.WasmTarget { if c.Started() { return c.stylusArchs } - return []rawdb.Target{"not started"} + return []ethdb.WasmTarget{"not started"} } func (c *ValidationClient) Stop() { diff --git a/validator/interface.go b/validator/interface.go index 81b40ae5cfc..af086291372 100644 --- a/validator/interface.go +++ b/validator/interface.go @@ -4,7 +4,7 @@ import ( "context" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/containers" ) @@ -14,7 +14,7 @@ type ValidationSpawner interface { Start(context.Context) error Stop() Name() string - StylusArchs() []rawdb.Target + StylusArchs() []ethdb.WasmTarget Room() int } diff --git a/validator/server_api/json.go b/validator/server_api/json.go index dbe2bb1fee6..6fe936e17d6 100644 --- a/validator/server_api/json.go +++ b/validator/server_api/json.go @@ -11,7 +11,7 @@ import ( "os" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbcompress" "github.com/offchainlabs/nitro/arbutil" @@ -64,7 +64,7 @@ type InputJSON struct { BatchInfo []BatchInfoJson DelayedMsgB64 string StartState validator.GoGlobalState - UserWasms map[rawdb.Target]map[common.Hash]string + UserWasms map[ethdb.WasmTarget]map[common.Hash]string DebugChain bool } @@ -96,7 +96,7 @@ func ValidationInputToJson(entry *validator.ValidationInput) *InputJSON { DelayedMsgB64: base64.StdEncoding.EncodeToString(entry.DelayedMsg), StartState: entry.StartState, PreimagesB64: jsonPreimagesMap, - UserWasms: make(map[rawdb.Target]map[common.Hash]string), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash]string), DebugChain: entry.DebugChain, } for _, binfo := range entry.BatchInfo { @@ -128,7 +128,7 @@ func ValidationInputFromJson(entry *InputJSON) (*validator.ValidationInput, erro DelayedMsgNr: entry.DelayedMsgNr, StartState: entry.StartState, Preimages: preimages, - UserWasms: make(map[rawdb.Target]map[common.Hash][]byte), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash][]byte), DebugChain: entry.DebugChain, } delayed, err := base64.StdEncoding.DecodeString(entry.DelayedMsgB64) diff --git a/validator/server_arb/validator_spawner.go b/validator/server_arb/validator_spawner.go index 844a988d282..96bc5d619d0 100644 --- a/validator/server_arb/validator_spawner.go +++ b/validator/server_arb/validator_spawner.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/pflag" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/stopwaiter" @@ -89,8 +90,8 @@ func (s *ArbitratorSpawner) WasmModuleRoots() ([]common.Hash, error) { return s.locator.ModuleRoots(), nil } -func (s *ArbitratorSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{rawdb.TargetWavm} +func (s *ArbitratorSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{rawdb.TargetWavm} } func (s *ArbitratorSpawner) Name() string { diff --git a/validator/server_jit/spawner.go b/validator/server_jit/spawner.go index 92b50b17cba..d77317d2184 100644 --- a/validator/server_jit/spawner.go +++ b/validator/server_jit/spawner.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" @@ -72,8 +73,8 @@ func (v *JitSpawner) WasmModuleRoots() ([]common.Hash, error) { return v.locator.ModuleRoots(), nil } -func (v *JitSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{rawdb.LocalTarget()} +func (v *JitSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{rawdb.LocalTarget()} } func (v *JitSpawner) execute( diff --git a/validator/validation_entry.go b/validator/validation_entry.go index 2c357659ad2..d340993fa2e 100644 --- a/validator/validation_entry.go +++ b/validator/validation_entry.go @@ -2,7 +2,7 @@ package validator import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbutil" ) @@ -17,7 +17,7 @@ type ValidationInput struct { HasDelayedMsg bool DelayedMsgNr uint64 Preimages map[arbutil.PreimageType]map[common.Hash][]byte - UserWasms map[rawdb.Target]map[common.Hash][]byte + UserWasms map[ethdb.WasmTarget]map[common.Hash][]byte BatchInfo []BatchInfo DelayedMsg []byte StartState GoGlobalState diff --git a/validator/valnode/validation_api.go b/validator/valnode/validation_api.go index a79ac7fa55e..a10d931dfc2 100644 --- a/validator/valnode/validation_api.go +++ b/validator/valnode/validation_api.go @@ -12,7 +12,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" @@ -45,7 +45,7 @@ func (a *ValidationServerAPI) WasmModuleRoots() ([]common.Hash, error) { return a.spawner.WasmModuleRoots() } -func (a *ValidationServerAPI) StylusArchs() ([]rawdb.Target, error) { +func (a *ValidationServerAPI) StylusArchs() ([]ethdb.WasmTarget, error) { return a.spawner.StylusArchs(), nil } From eefd7294777775f69689c316ca97c4b9d1338172 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 00:52:30 +0200 Subject: [PATCH 025/156] compile to targets in WasmTargets list --- arbos/programs/native.go | 61 +++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 36e200af4df..98abf9443ca 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -100,25 +100,50 @@ func activateProgramInternal( } return nil, nil, err } - target := rawdb.LocalTarget() - status_asm := C.stylus_compile( - goSlice(wasm), - u16(version), - cbool(debug), - goSlice([]byte(target)), - output, - ) - asm := output.intoBytes() - if status_asm != 0 { - return nil, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm)) + targets := db.Database().WasmTargets() + type result struct { + target ethdb.WasmTarget + asm []byte + err error } - asmMap := map[ethdb.WasmTarget][]byte{ - rawdb.TargetWavm: module, - target: asm, + results := make(chan result, len(targets)) + for _, target := range targets { + if target == rawdb.TargetWavm { + results <- result{target, module, nil} + } else { + target := target + go func() { + output := &rustBytes{} + status_asm := C.stylus_compile( + goSlice(wasm), + u16(version), + cbool(debug), + goSlice([]byte(target)), + output, + ) + asm := output.intoBytes() + if status_asm != 0 { + results <- result{target, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm))} + return + } + results <- result{target, asm, nil} + }() + } + } + asmMap := make(map[ethdb.WasmTarget][]byte, len(targets)) + for range targets { + res := <-results + if res.err != nil { + err = errors.Join(res.err, err) + } else { + asmMap[res.target] = res.asm + } + } + if err != nil { + return nil, nil, fmt.Errorf("compilation failed for one or more targets: %w", err) } hash := moduleHash.toHash() - info := &activationInfo{ moduleHash: hash, initGas: uint16(stylusData.init_cost), @@ -204,11 +229,7 @@ func callProgram( } if db, ok := db.(*state.StateDB); ok { - targets := []ethdb.WasmTarget{ - rawdb.TargetWavm, - rawdb.LocalTarget(), - } - db.RecordProgram(targets, moduleHash) + db.RecordProgram(moduleHash) } evmApi := newApi(interpreter, tracingInfo, scope, memoryModel) From 2832703599b4749c32e9af9760b4d3ffdacbba9c Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 01:21:02 +0200 Subject: [PATCH 026/156] use wasm targets list in SaveActiveProgramToWasmStore --- arbos/programs/wasmstorehelper.go | 4 ++-- go-ethereum | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arbos/programs/wasmstorehelper.go b/arbos/programs/wasmstorehelper.go index 95daaf544ab..434820dd9c9 100644 --- a/arbos/programs/wasmstorehelper.go +++ b/arbos/programs/wasmstorehelper.go @@ -12,7 +12,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -44,8 +43,9 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash return err } + targets := statedb.Database().WasmTargets() // If already in wasm store then return early - _, err = statedb.TryGetActivatedAsmMap([]ethdb.WasmTarget{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash) + _, err = statedb.TryGetActivatedAsmMap(targets, moduleHash) if err == nil { return nil } diff --git a/go-ethereum b/go-ethereum index cb03c12f7f0..c7d753f2c4d 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit cb03c12f7f02d47344d2d089eef748e92b99c95e +Subproject commit c7d753f2c4d5eb41c94e2e2499219a72fbf241a0 From 10f9b159550b59d97304cd935db25b076adf6443 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 01:48:47 +0200 Subject: [PATCH 027/156] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index c7d753f2c4d..7fc4c646075 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit c7d753f2c4d5eb41c94e2e2499219a72fbf241a0 +Subproject commit 7fc4c646075eb9b6e26374886ea6fe9463b106aa From 8bfc9e680ad20ef8e5d4d292167af3c73a6149e1 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 23 Aug 2024 17:21:21 +0530 Subject: [PATCH 028/156] Allow force rebuilding of local wasm store --- cmd/conf/init.go | 14 +++++++++++--- cmd/nitro/init.go | 16 ++++++++++++---- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index d88bcdd241d..f3606916936 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -30,7 +30,7 @@ type InitConfig struct { PruneThreads int `koanf:"prune-threads"` PruneTrieCleanCache int `koanf:"prune-trie-clean-cache"` RecreateMissingStateFrom uint64 `koanf:"recreate-missing-state-from"` - RebuildLocalWasm bool `koanf:"rebuild-local-wasm"` + RebuildLocalWasm string `koanf:"rebuild-local-wasm"` ReorgToBatch int64 `koanf:"reorg-to-batch"` ReorgToMessageBatch int64 `koanf:"reorg-to-message-batch"` ReorgToBlockBatch int64 `koanf:"reorg-to-block-batch"` @@ -56,7 +56,7 @@ var InitConfigDefault = InitConfig{ PruneThreads: runtime.NumCPU(), PruneTrieCleanCache: 600, RecreateMissingStateFrom: 0, // 0 = disabled - RebuildLocalWasm: true, + RebuildLocalWasm: "auto", ReorgToBatch: -1, ReorgToMessageBatch: -1, ReorgToBlockBatch: -1, @@ -82,10 +82,14 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Int(prefix+".prune-threads", InitConfigDefault.PruneThreads, "the number of threads to use when pruning") f.Int(prefix+".prune-trie-clean-cache", InitConfigDefault.PruneTrieCleanCache, "amount of memory in megabytes to cache unchanged state trie nodes with when traversing state database during pruning") f.Uint64(prefix+".recreate-missing-state-from", InitConfigDefault.RecreateMissingStateFrom, "block number to start recreating missing states from (0 = disabled)") - f.Bool(prefix+".rebuild-local-wasm", InitConfigDefault.RebuildLocalWasm, "rebuild local wasm database on boot if needed (otherwise-will be done lazily)") f.Int64(prefix+".reorg-to-batch", InitConfigDefault.ReorgToBatch, "rolls back the blockchain to a specified batch number") f.Int64(prefix+".reorg-to-message-batch", InitConfigDefault.ReorgToMessageBatch, "rolls back the blockchain to the first batch at or before a given message index") f.Int64(prefix+".reorg-to-block-batch", InitConfigDefault.ReorgToBlockBatch, "rolls back the blockchain to the first batch at or before a given block number") + f.String(prefix+".rebuild-local-wasm", InitConfigDefault.RebuildLocalWasm, "rebuild local wasm database on boot if needed (otherwise-will be done lazily). Three modes are supported \n"+ + "\"auto\"- (enabled by default) if any previous rebuilding attempt was successful then rebuilding is disabled else continues to rebuild,\n"+ + "\"force\"- force rebuilding which would commence rebuilding despite the status of previous attempts,\n"+ + "\"false\"- do not rebuild on startup", + ) } func (c *InitConfig) Validate() error { @@ -110,6 +114,10 @@ func (c *InitConfig) Validate() error { } } } + c.RebuildLocalWasm = strings.ToLower(c.RebuildLocalWasm) + if c.RebuildLocalWasm != "auto" && c.RebuildLocalWasm != "force" && c.RebuildLocalWasm != "false" { + return fmt.Errorf("invalid value of rebuild-local-wasm, want: auto or force or false, got: %s", c.RebuildLocalWasm) + } return nil } diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index d9ae0df3b04..fc3439a56dc 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -532,13 +532,21 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err = gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, gethexec.RebuildingDone); err != nil { return nil, nil, fmt.Errorf("unable to set rebuilding status of wasm store to done: %w", err) } - } else if config.Init.RebuildLocalWasm { - position, err := gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingPositionKey) - if err != nil { - log.Info("Unable to get codehash position in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it and starting rebuilding", "err", err) + } else if config.Init.RebuildLocalWasm != "false" { + var position common.Hash + if config.Init.RebuildLocalWasm == "force" { + log.Info("Commencing force rebuilding of wasm store by setting codehash position in rebuilding to beginning") if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) } + } else { + position, err = gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingPositionKey) + if err != nil { + log.Info("Unable to get codehash position in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it and starting rebuilding", "err", err) + if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { + return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) + } + } } if position != gethexec.RebuildingDone { startBlockHash, err := gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingStartBlockHashKey) From 803384395ad64c116ec543f459888e059fa9e95a Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 23 Aug 2024 19:36:24 +0530 Subject: [PATCH 029/156] fix G115 lint errors --- system_tests/contract_tx_test.go | 1 + util/blobs/blobs.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/system_tests/contract_tx_test.go b/system_tests/contract_tx_test.go index 7d66e516b4d..c1ef840c434 100644 --- a/system_tests/contract_tx_test.go +++ b/system_tests/contract_tx_test.go @@ -51,6 +51,7 @@ func TestContractTxDeploy(t *testing.T) { 0xF3, // RETURN } var requestId common.Hash + // #nosec G115 requestId[0] = uint8(stateNonce) contractTx := &types.ArbitrumContractTx{ ChainId: params.ArbitrumDevTestChainConfig().ChainID, diff --git a/util/blobs/blobs.go b/util/blobs/blobs.go index 405c776bad7..f5914edd2e3 100644 --- a/util/blobs/blobs.go +++ b/util/blobs/blobs.go @@ -41,6 +41,7 @@ func fillBlobBits(blob []byte, data []byte) ([]byte, error) { accBits += 8 data = data[1:] } + // #nosec G115 blob[fieldElement*32] = uint8(acc & ((1 << spareBlobBits) - 1)) accBits -= spareBlobBits if accBits < 0 { @@ -88,6 +89,7 @@ func DecodeBlobs(blobs []kzg4844.Blob) ([]byte, error) { acc |= uint16(blob[fieldIndex*32]) << accBits accBits += spareBlobBits if accBits >= 8 { + // #nosec G115 rlpData = append(rlpData, uint8(acc)) acc >>= 8 accBits -= 8 From 17319ad2d1bb00d0c63a22693844652f6b739b1e Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 16:18:01 +0200 Subject: [PATCH 030/156] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 7fc4c646075..0a376604c9c 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 7fc4c646075eb9b6e26374886ea6fe9463b106aa +Subproject commit 0a376604c9ce0c588de0d943184380acd7e25db4 From cdb43d77e5ef976c6ad3e09ac63d07034c85bfcf Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 17:57:18 +0200 Subject: [PATCH 031/156] update validation client stylus archs sanity check --- validator/client/validation_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 36316a83288..3b18ad18513 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -81,7 +81,7 @@ func (c *ValidationClient) Start(ctx context.Context) error { return fmt.Errorf("could not read stylus archs from validation server") } for _, stylusArch := range stylusArchs { - if stylusArch != rawdb.TargetWavm && stylusArch != rawdb.LocalTarget() && stylusArch != "mock" { + if !rawdb.IsSupportedWasmTarget(ethdb.WasmTarget(stylusArch)) && stylusArch != "mock" { return fmt.Errorf("unsupported stylus architecture: %v", stylusArch) } } From 65b58b69fc644cf4036422107d0683126c3791fd Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 23 Aug 2024 11:18:58 -0500 Subject: [PATCH 032/156] Fix new integer cast overflow lints --- system_tests/contract_tx_test.go | 1 + util/blobs/blobs.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/system_tests/contract_tx_test.go b/system_tests/contract_tx_test.go index 7d66e516b4d..c1ef840c434 100644 --- a/system_tests/contract_tx_test.go +++ b/system_tests/contract_tx_test.go @@ -51,6 +51,7 @@ func TestContractTxDeploy(t *testing.T) { 0xF3, // RETURN } var requestId common.Hash + // #nosec G115 requestId[0] = uint8(stateNonce) contractTx := &types.ArbitrumContractTx{ ChainId: params.ArbitrumDevTestChainConfig().ChainID, diff --git a/util/blobs/blobs.go b/util/blobs/blobs.go index 405c776bad7..f5914edd2e3 100644 --- a/util/blobs/blobs.go +++ b/util/blobs/blobs.go @@ -41,6 +41,7 @@ func fillBlobBits(blob []byte, data []byte) ([]byte, error) { accBits += 8 data = data[1:] } + // #nosec G115 blob[fieldElement*32] = uint8(acc & ((1 << spareBlobBits) - 1)) accBits -= spareBlobBits if accBits < 0 { @@ -88,6 +89,7 @@ func DecodeBlobs(blobs []kzg4844.Blob) ([]byte, error) { acc |= uint16(blob[fieldIndex*32]) << accBits accBits += spareBlobBits if accBits >= 8 { + // #nosec G115 rlpData = append(rlpData, uint8(acc)) acc >>= 8 accBits -= 8 From 71ab928cb42b66247ac9fdc3acd82280255de712 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 19:27:15 +0200 Subject: [PATCH 033/156] change archs list option to extra-archs that must include wavm and is appended with local target if not in list --- execution/gethexec/node.go | 47 ++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index f519cce6ed7..bc29e9b7f23 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -30,10 +30,10 @@ import ( ) type StylusTargetConfig struct { - Arm64 string `koanf:"arm64"` - Amd64 string `koanf:"amd64"` - Host string `koanf:"host"` - Archs []string `koanf:"archs"` + Arm64 string `koanf:"arm64"` + Amd64 string `koanf:"amd64"` + Host string `koanf:"host"` + ExtraArchs []string `koanf:"extra-archs"` wasmTargets []ethdb.WasmTarget } @@ -44,47 +44,40 @@ func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { func (c *StylusTargetConfig) Validate() error { localTarget := rawdb.LocalTarget() - var nativeFound bool - targets := make([]ethdb.WasmTarget, 0, len(c.Archs)) - for _, arch := range c.Archs { + targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)) + targetsSet := make(map[ethdb.WasmTarget]struct{}, len(c.ExtraArchs)+1) + for _, arch := range c.ExtraArchs { target := ethdb.WasmTarget(arch) if !rawdb.IsSupportedWasmTarget(target) { return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) } - var alreadyInList bool - for _, t := range targets { - if target == t { - alreadyInList = true - break - } - } - if alreadyInList { - continue + if _, duplicate := targetsSet[target]; !duplicate { + targets = append(targets, target) + targetsSet[target] = struct{}{} } - if target == localTarget { - nativeFound = true - } - targets = append(targets, target) } - if !nativeFound { - return fmt.Errorf("native target not found in archs list, native target: %v, archs: %v", localTarget, c.Archs) + if _, has := targetsSet[rawdb.TargetWavm]; !has { + return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) + } + if _, has := targetsSet[localTarget]; !has { + targets = append(targets, localTarget) } c.wasmTargets = targets return nil } var DefaultStylusTargetConfig = StylusTargetConfig{ - Arm64: programs.DefaultTargetDescriptionArm, - Amd64: programs.DefaultTargetDescriptionX86, - Host: "", - Archs: []string{string(rawdb.TargetWavm), string(rawdb.LocalTarget())}, + Arm64: programs.DefaultTargetDescriptionArm, + Amd64: programs.DefaultTargetDescriptionX86, + Host: "", + ExtraArchs: []string{string(rawdb.TargetWavm)}, } func StylusTargetConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".arm64", DefaultStylusTargetConfig.Arm64, "stylus programs compilation target for arm64 linux") f.String(prefix+".amd64", DefaultStylusTargetConfig.Amd64, "stylus programs compilation target for amd64 linux") f.String(prefix+".host", DefaultStylusTargetConfig.Host, "stylus programs compilation target for system other than 64-bit ARM or 64-bit x86") - f.StringSlice(prefix+".archs", DefaultStylusTargetConfig.Archs, fmt.Sprintf("Comma separated architectures list to compile stylus program to and cache in wasm store (available targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)) + f.StringSlice(prefix+".extra-archs", DefaultStylusTargetConfig.ExtraArchs, fmt.Sprintf("Comma separated list of extra architectures to cross-compile stylus program to and cache in wasm store (additionally to local target). Currently must include at least %s. (supported targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)) } type Config struct { From 29df43a287833aa9541fa12db853bc6e3b07fd5f Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 23 Aug 2024 12:06:22 -0300 Subject: [PATCH 034/156] Remove wasm database after downloading snapshot By default, remove the wasm DB because it contains native executables. Hence, it is a security concern when downloading a snapshot from a untrusted source. --- cmd/conf/init.go | 3 +++ cmd/nitro/init.go | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index d88bcdd241d..242f43dc82e 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -22,6 +22,7 @@ type InitConfig struct { DevInitAddress string `koanf:"dev-init-address"` DevInitBlockNum uint64 `koanf:"dev-init-blocknum"` Empty bool `koanf:"empty"` + ExtractWasm bool `koanf:"extract-wasm"` AccountsPerSync uint `koanf:"accounts-per-sync"` ImportFile string `koanf:"import-file"` ThenQuit bool `koanf:"then-quit"` @@ -48,6 +49,7 @@ var InitConfigDefault = InitConfig{ DevInitAddress: "", DevInitBlockNum: 0, Empty: false, + ExtractWasm: false, ImportFile: "", AccountsPerSync: 100000, ThenQuit: false, @@ -74,6 +76,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".dev-init-address", InitConfigDefault.DevInitAddress, "Address of dev-account. Leave empty to use the dev-wallet.") f.Uint64(prefix+".dev-init-blocknum", InitConfigDefault.DevInitBlockNum, "Number of preinit blocks. Must exist in ancient database.") f.Bool(prefix+".empty", InitConfigDefault.Empty, "init with empty state") + f.Bool(prefix+".extract-wasm", InitConfigDefault.ExtractWasm, "if set, extract the wasm directory when downloading a database (this can be security concern because it contains native executables)") f.Bool(prefix+".then-quit", InitConfigDefault.ThenQuit, "quit after init is done") f.String(prefix+".import-file", InitConfigDefault.ImportFile, "path for json data to import") f.Uint(prefix+".accounts-per-sync", InitConfigDefault.AccountsPerSync, "during init - sync database every X accounts. Lower value for low-memory systems. 0 disables.") diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index d9ae0df3b04..b5cab75cf61 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -602,6 +602,17 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err != nil { return nil, nil, fmt.Errorf("couln't extract init archive '%v' err:%w", initFile, err) } + wasmDb := path.Join(stack.InstanceDir(), "wasm") + if dirExists(wasmDb) && !config.Init.ExtractWasm { + // By default, remove the wasm DB because it contains native executables. + // Hence, it is a security concern when downloading a snapshot from a + // untrusted source. + err := os.RemoveAll(wasmDb) + if err != nil { + return nil, nil, fmt.Errorf("failed to remove extracted wasm database: %w", err) + } + log.Debug("init: removed wasm database") + } } var initDataReader statetransfer.InitDataReader = nil From a84a88803db272922160d620d1d4df9bdd550064 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 23 Aug 2024 14:53:37 -0300 Subject: [PATCH 035/156] Don't recurse submodules in submodule-pin-check github action --- .github/workflows/submodule-pin-check.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index e459bad34d3..b8b97a27c5b 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -14,8 +14,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - submodules: recursive + submodules: true - name: Check all submodules are ancestors of origin/HEAD or configured branch run: ${{ github.workspace }}/.github/workflows/submodule-pin-check.sh - From f4da3d42d2f7166f4acddcb5d79e8cc7d4c0f05c Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 23 Aug 2024 15:07:58 -0300 Subject: [PATCH 036/156] Workflow "Merge Checks" creates "Submodule Pin Check" through github API --- .github/workflows/submodule-pin-check.yml | 32 +++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index b8b97a27c5b..12a9656e846 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,10 +1,13 @@ -name: Submodule Pin Check +name: Merge Checks on: pull_request: branches: [ master ] types: [synchronize, opened, reopened] +permissions: + statuses: write + jobs: submodule-pin-check: name: Submodule Pin Check @@ -17,4 +20,29 @@ jobs: submodules: true - name: Check all submodules are ancestors of origin/HEAD or configured branch - run: ${{ github.workspace }}/.github/workflows/submodule-pin-check.sh + run: | + status_state="pending" + if ${{ github.workspace }}/.github/workflows/submodule-pin-check.sh; then + status_state="success" + else + resp="$(curl -sSL --fail-with-body \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/commits/${{ github.event.pull_request.head.sha }}/statuses")" + + if ! jq -e '.[] | select(.context == "Submodule Pin Check")' > /dev/null <<< "$resp"; then + # Submodule pin check is failling and no status exists + # Keep it without a status to keep the green checkmark appearing + # Otherwise, the commit and PR's CI will appear to be indefinitely pending + # Merging will still be blocked until the required status appears + exit 0 + fi + fi + curl -sSL --fail-with-body \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/${{ github.event.pull_request.head.sha }}" \ + -d '{"context":"Submodule Pin Check","state":"'"$status_state"'"}' From 966ec11ac96d892c56d7f4dcffb6b4dadd2e7abb Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 26 Aug 2024 11:13:45 -0300 Subject: [PATCH 037/156] Rename conf init.extract-wasm to init.import-wasm --- cmd/conf/init.go | 6 +++--- cmd/nitro/init.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index d0ca592e429..dddf3c0c28c 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -22,7 +22,7 @@ type InitConfig struct { DevInitAddress string `koanf:"dev-init-address"` DevInitBlockNum uint64 `koanf:"dev-init-blocknum"` Empty bool `koanf:"empty"` - ExtractWasm bool `koanf:"extract-wasm"` + ImportWasm bool `koanf:"import-wasm"` AccountsPerSync uint `koanf:"accounts-per-sync"` ImportFile string `koanf:"import-file"` ThenQuit bool `koanf:"then-quit"` @@ -49,7 +49,7 @@ var InitConfigDefault = InitConfig{ DevInitAddress: "", DevInitBlockNum: 0, Empty: false, - ExtractWasm: false, + ImportWasm: false, ImportFile: "", AccountsPerSync: 100000, ThenQuit: false, @@ -76,7 +76,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".dev-init-address", InitConfigDefault.DevInitAddress, "Address of dev-account. Leave empty to use the dev-wallet.") f.Uint64(prefix+".dev-init-blocknum", InitConfigDefault.DevInitBlockNum, "Number of preinit blocks. Must exist in ancient database.") f.Bool(prefix+".empty", InitConfigDefault.Empty, "init with empty state") - f.Bool(prefix+".extract-wasm", InitConfigDefault.ExtractWasm, "if set, extract the wasm directory when downloading a database (this can be security concern because it contains native executables)") + f.Bool(prefix+".import-wasm", InitConfigDefault.ImportWasm, "if set, extract the wasm directory when downloading a database (this can be security concern because it contains native executables)") f.Bool(prefix+".then-quit", InitConfigDefault.ThenQuit, "quit after init is done") f.String(prefix+".import-file", InitConfigDefault.ImportFile, "path for json data to import") f.Uint(prefix+".accounts-per-sync", InitConfigDefault.AccountsPerSync, "during init - sync database every X accounts. Lower value for low-memory systems. 0 disables.") diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index aaef5798f56..55325f1b934 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -611,7 +611,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return nil, nil, fmt.Errorf("couln't extract init archive '%v' err:%w", initFile, err) } wasmDb := path.Join(stack.InstanceDir(), "wasm") - if dirExists(wasmDb) && !config.Init.ExtractWasm { + if dirExists(wasmDb) && !config.Init.ImportWasm { // By default, remove the wasm DB because it contains native executables. // Hence, it is a security concern when downloading a snapshot from a // untrusted source. From a3bf49a54e5dddba0e36f80162f7e2d9115e4e44 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 26 Aug 2024 11:14:52 -0300 Subject: [PATCH 038/156] Improve init.import-wasm description --- cmd/conf/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index dddf3c0c28c..f01d99f8b7d 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -76,7 +76,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".dev-init-address", InitConfigDefault.DevInitAddress, "Address of dev-account. Leave empty to use the dev-wallet.") f.Uint64(prefix+".dev-init-blocknum", InitConfigDefault.DevInitBlockNum, "Number of preinit blocks. Must exist in ancient database.") f.Bool(prefix+".empty", InitConfigDefault.Empty, "init with empty state") - f.Bool(prefix+".import-wasm", InitConfigDefault.ImportWasm, "if set, extract the wasm directory when downloading a database (this can be security concern because it contains native executables)") + f.Bool(prefix+".import-wasm", InitConfigDefault.ImportWasm, "if set, import the wasm directory when downloading a database (contains executable code - only use with highly trusted source)") f.Bool(prefix+".then-quit", InitConfigDefault.ThenQuit, "quit after init is done") f.String(prefix+".import-file", InitConfigDefault.ImportFile, "path for json data to import") f.Uint(prefix+".accounts-per-sync", InitConfigDefault.AccountsPerSync, "during init - sync database every X accounts. Lower value for low-memory systems. 0 disables.") From 57249c83fc7ad215a874d56c5e7200b0ae77cefb Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 26 Aug 2024 11:27:35 -0300 Subject: [PATCH 039/156] Remove WASM database even if the extraction fails --- cmd/nitro/init.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 55325f1b934..318c134eb12 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -608,18 +608,22 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo log.Info("extracting downloaded init archive", "size", fmt.Sprintf("%dMB", stat.Size()/1024/1024)) err = extract.Archive(context.Background(), reader, stack.InstanceDir(), nil) if err != nil { - return nil, nil, fmt.Errorf("couln't extract init archive '%v' err:%w", initFile, err) + err = fmt.Errorf("couln't extract init archive '%v' err:%w", initFile, err) } wasmDb := path.Join(stack.InstanceDir(), "wasm") if dirExists(wasmDb) && !config.Init.ImportWasm { // By default, remove the wasm DB because it contains native executables. // Hence, it is a security concern when downloading a snapshot from a // untrusted source. - err := os.RemoveAll(wasmDb) - if err != nil { - return nil, nil, fmt.Errorf("failed to remove extracted wasm database: %w", err) + removeErr := os.RemoveAll(wasmDb) + if removeErr != nil { + err = errors.Join(err, fmt.Errorf("failed to remove extracted wasm database: %w", removeErr)) + } else { + log.Debug("init: removed wasm database") } - log.Debug("init: removed wasm database") + } + if err != nil { + return nil, nil, err } } From 304f2c1127a4def9d292b617b3b52a437e9891b2 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 26 Aug 2024 16:53:34 +0200 Subject: [PATCH 040/156] add cross compilation to progam test --- system_tests/common_test.go | 5 ++ system_tests/program_test.go | 92 +++++++++++++++++++++++++++++++++--- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 392f6de8059..457dae09101 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -293,6 +293,11 @@ func (b *NodeBuilder) WithWasmRootDir(wasmRootDir string) *NodeBuilder { return b } +func (b *NodeBuilder) WithExtraArchs(targets []string) *NodeBuilder { + b.execConfig.StylusTarget.ExtraArchs = targets + return b +} + func (b *NodeBuilder) Build(t *testing.T) func() { b.CheckConfig(t) if b.withL1 { diff --git a/system_tests/program_test.go b/system_tests/program_test.go index ed640809dbf..8c4ca2e4386 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -45,18 +45,31 @@ import ( var oneEth = arbmath.UintToBig(1e18) +var allWasmTargets = []string{string(rawdb.TargetWavm), string(rawdb.TargetArm64), string(rawdb.TargetAmd64), string(rawdb.TargetHost)} + func TestProgramKeccak(t *testing.T) { t.Parallel() - keccakTest(t, true) + t.Run("WithDefaultWasmTargets", func(t *testing.T) { + keccakTest(t, true) + }) + + t.Run("WithAllWasmTargets", func(t *testing.T) { + keccakTest(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs(allWasmTargets) + }) + }) } -func keccakTest(t *testing.T, jit bool) { - builder, auth, cleanup := setupProgramTest(t, jit) +func keccakTest(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { + builder, auth, cleanup := setupProgramTest(t, jit, builderOpts...) ctx := builder.ctx l2client := builder.L2.Client defer cleanup() programAddress := deployWasm(t, ctx, auth, l2client, rustFile("keccak")) + wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + wasm, _ := readWasmFile(t, rustFile("keccak")) otherAddressSameCode := deployContract(t, ctx, auth, l2client, wasm) arbWasm, err := pgen.NewArbWasm(types.ArbWasmAddress, l2client) @@ -68,6 +81,7 @@ func keccakTest(t *testing.T, jit bool) { Fatal(t, "activate should have failed with ProgramUpToDate", err) } }) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) if programAddress == otherAddressSameCode { Fatal(t, "expected to deploy at two separate program addresses") @@ -141,11 +155,18 @@ func keccakTest(t *testing.T, jit bool) { func TestProgramActivateTwice(t *testing.T) { t.Parallel() - testActivateTwice(t, true) + t.Run("WithDefaultWasmTargets", func(t *testing.T) { + testActivateTwice(t, true) + }) + t.Run("WithAllWasmTargets", func(t *testing.T) { + testActivateTwice(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs(allWasmTargets) + }) + }) } -func testActivateTwice(t *testing.T, jit bool) { - builder, auth, cleanup := setupProgramTest(t, jit) +func testActivateTwice(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { + builder, auth, cleanup := setupProgramTest(t, jit, builderOpts...) ctx := builder.ctx l2info := builder.L2Info l2client := builder.L2.Client @@ -171,6 +192,10 @@ func testActivateTwice(t *testing.T, jit bool) { colors.PrintBlue("keccak program B deployed to ", keccakB) multiAddr := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) + + wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + preimage := []byte("it's time to du-du-du-du d-d-d-d-d-d-d de-duplicate") keccakArgs := []byte{0x01} // keccak the preimage once @@ -194,6 +219,7 @@ func testActivateTwice(t *testing.T, jit bool) { // Calling the contract pre-activation should fail. checkReverts() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // mechanisms for creating calldata activateProgram, _ := util.NewCallParser(pgen.ArbWasmABI, "activateProgram") @@ -216,6 +242,7 @@ func testActivateTwice(t *testing.T, jit bool) { // Ensure the revert also reverted keccak's activation checkReverts() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // Activate keccak program A, then call into B, which should succeed due to being the same codehash args = argsForMulticall(vm.CALL, types.ArbWasmAddress, oneEth, pack(activateProgram(keccakA))) @@ -223,6 +250,7 @@ func testActivateTwice(t *testing.T, jit bool) { tx = l2info.PrepareTxTo("Owner", &multiAddr, 1e9, oneEth, args) ensure(tx, l2client.SendTransaction(ctx, tx)) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 2) validateBlocks(t, 7, jit, builder) } @@ -1834,7 +1862,9 @@ func createMapFromDb(db ethdb.KeyValueStore) (map[string][]byte, error) { } func TestWasmStoreRebuilding(t *testing.T) { - builder, auth, cleanup := setupProgramTest(t, true) + builder, auth, cleanup := setupProgramTest(t, true, func(b *NodeBuilder) { + b.WithExtraArchs(allWasmTargets) + }) ctx := builder.ctx l2info := builder.L2Info l2client := builder.L2.Client @@ -1871,6 +1901,7 @@ func TestWasmStoreRebuilding(t *testing.T) { storeMap, err := createMapFromDb(wasmDb) Require(t, err) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // close nodeB cleanupB() @@ -1926,5 +1957,52 @@ func TestWasmStoreRebuilding(t *testing.T) { } } + checkWasmStoreContent(t, wasmDbAfterRebuild, builder.execConfig.StylusTarget.ExtraArchs, 1) cleanupB() } + +func readModuleHashes(t *testing.T, wasmDb ethdb.KeyValueStore) []common.Hash { + modulesSet := make(map[common.Hash]struct{}) + asmPrefix := []byte{0x00, 'w'} + it := wasmDb.NewIterator(asmPrefix, nil) + defer it.Release() + for it.Next() { + key := it.Key() + if len(key) != rawdb.WasmKeyLen { + t.Fatalf("unexpected activated module key length, len: %d, key: %v", len(key), key) + } + moduleHash := key[rawdb.WasmPrefixLen:] + if len(moduleHash) != common.HashLength { + t.Fatalf("Invalid moduleHash length in key: %v, moduleHash: %v", key, moduleHash) + } + modulesSet[common.BytesToHash(moduleHash)] = struct{}{} + } + modules := make([]common.Hash, 0, len(modulesSet)) + for module := range modulesSet { + modules = append(modules, module) + } + return modules +} + +func checkWasmStoreContent(t *testing.T, wasmDb ethdb.KeyValueStore, targets []string, numModules int) { + modules := readModuleHashes(t, wasmDb) + if len(modules) != numModules { + t.Fatalf("Unexpected number of module hashes found in wasm store, want: %d, have: %d", numModules, len(modules)) + } + for _, module := range modules { + for _, target := range targets { + wasmTarget := ethdb.WasmTarget(target) + if !rawdb.IsSupportedWasmTarget(wasmTarget) { + t.Fatalf("internal test error - unsupported target passed to checkWasmStoreContent: %v", target) + } + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("Failed to read activated asm for target: %v, module: %v", target, module) + } + }() + _ = rawdb.ReadActivatedAsm(wasmDb, wasmTarget, module) + }() + } + } +} From 4a29e0d21be963f17d3cfbcdf105799096a76a26 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 26 Aug 2024 12:02:16 -0300 Subject: [PATCH 041/156] Move stack struct to util/containers pkg --- execution/gethexec/stylus_tracer.go | 14 +++++++------- system_tests/stylus_tracer_test.go | 8 ++++---- util/{stack => containers}/stack.go | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) rename util/{stack => containers}/stack.go (97%) diff --git a/execution/gethexec/stylus_tracer.go b/execution/gethexec/stylus_tracer.go index 16c43d71d86..d1246989f0a 100644 --- a/execution/gethexec/stylus_tracer.go +++ b/execution/gethexec/stylus_tracer.go @@ -14,7 +14,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers" - "github.com/offchainlabs/nitro/util/stack" + "github.com/offchainlabs/nitro/util/containers" ) func init() { @@ -24,8 +24,8 @@ func init() { // stylusTracer captures Stylus HostIOs and returns them in a structured format to be used in Cargo // Stylus Replay. type stylusTracer struct { - open *stack.Stack[HostioTraceInfo] - stack *stack.Stack[*stack.Stack[HostioTraceInfo]] + open *containers.Stack[HostioTraceInfo] + stack *containers.Stack[*containers.Stack[HostioTraceInfo]] interrupt atomic.Bool reason error } @@ -55,7 +55,7 @@ type HostioTraceInfo struct { Address *common.Address `json:"address,omitempty"` // For *call HostIOs, the steps performed by the called contract. - Steps *stack.Stack[HostioTraceInfo] `json:"steps,omitempty"` + Steps *containers.Stack[HostioTraceInfo] `json:"steps,omitempty"` } // nestsHostios contains the hostios with nested calls. @@ -67,8 +67,8 @@ var nestsHostios = map[string]bool{ func newStylusTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { return &stylusTracer{ - open: stack.NewStack[HostioTraceInfo](), - stack: stack.NewStack[*stack.Stack[HostioTraceInfo]](), + open: containers.NewStack[HostioTraceInfo](), + stack: containers.NewStack[*containers.Stack[HostioTraceInfo]](), }, nil } @@ -126,7 +126,7 @@ func (t *stylusTracer) CaptureEnter(typ vm.OpCode, from common.Address, to commo name = "evm_self_destruct" } - inner := stack.NewStack[HostioTraceInfo]() + inner := containers.NewStack[HostioTraceInfo]() info := HostioTraceInfo{ Name: name, Address: &to, diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index 3b95f38d218..7fda39f04e8 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -12,7 +12,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/offchainlabs/nitro/execution/gethexec" "github.com/offchainlabs/nitro/solgen/go/mocksgen" - "github.com/offchainlabs/nitro/util/stack" + "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/testhelpers" ) @@ -89,7 +89,7 @@ func TestStylusTracer(t *testing.T) { Args: append(stylusMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), Outs: common.Hex2Bytes("0000000000"), Address: &stylusMulticall, - Steps: (*stack.Stack[gethexec.HostioTraceInfo])(&[]gethexec.HostioTraceInfo{ + Steps: (*containers.Stack[gethexec.HostioTraceInfo])(&[]gethexec.HostioTraceInfo{ {Name: "user_entrypoint", Args: intToBe32(1)}, {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}}, {Name: "read_args", Outs: []byte{0x00}}, @@ -117,7 +117,7 @@ func TestStylusTracer(t *testing.T) { Args: append(evmMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), Outs: common.Hex2Bytes("0000000000"), Address: &evmMulticall, - Steps: stack.NewStack[gethexec.HostioTraceInfo](), + Steps: containers.NewStack[gethexec.HostioTraceInfo](), }, {Name: "storage_flush_cache", Args: []byte{0x00}}, {Name: "write_result"}, @@ -133,7 +133,7 @@ func TestStylusTracer(t *testing.T) { { Name: "evm_call_contract", Address: &stylusMulticall, - Steps: (*stack.Stack[gethexec.HostioTraceInfo])(&[]gethexec.HostioTraceInfo{ + Steps: (*containers.Stack[gethexec.HostioTraceInfo])(&[]gethexec.HostioTraceInfo{ {Name: "user_entrypoint", Args: intToBe32(1)}, {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}}, {Name: "read_args", Outs: []byte{0x00}}, diff --git a/util/stack/stack.go b/util/containers/stack.go similarity index 97% rename from util/stack/stack.go rename to util/containers/stack.go index 1b7ac3f9d94..97fcd44a6d9 100644 --- a/util/stack/stack.go +++ b/util/containers/stack.go @@ -1,7 +1,7 @@ // Copyright 2024, Offchain Labs, Inc. // For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE -package stack +package containers import ( "fmt" From 2e5a6eadc03a562fa32e96781fda77ab430b55a0 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 26 Aug 2024 12:22:25 -0300 Subject: [PATCH 042/156] Renames job name related to checking submodule pin --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 12a9656e846..870f5208770 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -10,7 +10,7 @@ permissions: jobs: submodule-pin-check: - name: Submodule Pin Check + name: Check Submodule Pin runs-on: ubuntu-latest steps: - name: Checkout From 82b48d414ec4fd9369d1ca0fa74c62d6108a700d Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 26 Aug 2024 17:35:30 +0200 Subject: [PATCH 043/156] populate stylus target cache before rebuilding wasm store --- cmd/nitro/init.go | 2 +- execution/gethexec/executionengine.go | 15 +++++++++++---- execution/gethexec/wasmstorerebuilder.go | 7 ++++++- system_tests/program_test.go | 3 ++- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index fc3439a56dc..5573bd6ab27 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -558,7 +558,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo startBlockHash = latestBlock.Hash() } log.Info("Starting or continuing rebuilding of wasm store", "codeHash", position, "startBlockHash", startBlockHash) - if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.Execution.RPC.MaxRecreateStateDepth, l2BlockChain, position, startBlockHash); err != nil { + if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.Execution.RPC.MaxRecreateStateDepth, &config.Execution.StylusTarget, l2BlockChain, position, startBlockHash); err != nil { return nil, nil, fmt.Errorf("error rebuilding of wasm store: %w", err) } } diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 19d77fc38f9..7eb7920d8ad 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -150,10 +150,7 @@ func (s *ExecutionEngine) MarkFeedStart(to arbutil.MessageIndex) { } } -func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusTargetConfig) error { - if rustCacheSize != 0 { - programs.ResizeWasmLruCache(rustCacheSize) - } +func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { var effectiveStylusTarget string target := rawdb.LocalTarget() switch target { @@ -171,6 +168,16 @@ func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusT return nil } +func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusTargetConfig) error { + if rustCacheSize != 0 { + programs.ResizeWasmLruCache(rustCacheSize) + } + if err := populateStylusTargetCache(targetConfig); err != nil { + return err + } + return nil +} + func (s *ExecutionEngine) SetRecorder(recorder *BlockRecorder) { if s.Started() { panic("trying to set recorder after start") diff --git a/execution/gethexec/wasmstorerebuilder.go b/execution/gethexec/wasmstorerebuilder.go index dcbee45a3f9..698ba3ec8a4 100644 --- a/execution/gethexec/wasmstorerebuilder.go +++ b/execution/gethexec/wasmstorerebuilder.go @@ -59,9 +59,14 @@ func WriteToKeyValueStore[T any](store ethdb.KeyValueStore, key []byte, val T) e // It also stores a special value that is only set once when rebuilding commenced in RebuildingStartBlockHashKey as the block // time of the latest block when rebuilding was first called, this is used to avoid recomputing of assembly and module of // contracts that were created after rebuilding commenced since they would anyway already be added during sync. -func RebuildWasmStore(ctx context.Context, wasmStore ethdb.KeyValueStore, chainDb ethdb.Database, maxRecreateStateDepth int64, l2Blockchain *core.BlockChain, position, rebuildingStartBlockHash common.Hash) error { +func RebuildWasmStore(ctx context.Context, wasmStore ethdb.KeyValueStore, chainDb ethdb.Database, maxRecreateStateDepth int64, targetConfig *StylusTargetConfig, l2Blockchain *core.BlockChain, position, rebuildingStartBlockHash common.Hash) error { var err error var stateDb *state.StateDB + + if err := populateStylusTargetCache(targetConfig); err != nil { + return fmt.Errorf("error populating stylus target cache: %w", err) + } + latestHeader := l2Blockchain.CurrentBlock() // Attempt to get state at the start block when rebuilding commenced, if not available (in case of non-archival nodes) use latest state rebuildingStartHeader := l2Blockchain.GetHeaderByHash(rebuildingStartBlockHash) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index ed640809dbf..c89a13e2050 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -1896,7 +1896,8 @@ func TestWasmStoreRebuilding(t *testing.T) { // Start rebuilding and wait for it to finish log.Info("starting rebuilding of wasm store") - Require(t, gethexec.RebuildWasmStore(ctx, wasmDbAfterDelete, nodeB.ExecNode.ChainDB, nodeB.ExecNode.ConfigFetcher().RPC.MaxRecreateStateDepth, bc, common.Hash{}, bc.CurrentBlock().Hash())) + execConfig := nodeB.ExecNode.ConfigFetcher() + Require(t, gethexec.RebuildWasmStore(ctx, wasmDbAfterDelete, nodeB.ExecNode.ChainDB, execConfig.RPC.MaxRecreateStateDepth, &execConfig.StylusTarget, bc, common.Hash{}, bc.CurrentBlock().Hash())) wasmDbAfterRebuild := nodeB.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() From 4d16436406e3381612d9849d07b68fd498f02b62 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 26 Aug 2024 17:42:05 +0200 Subject: [PATCH 044/156] add error description --- execution/gethexec/executionengine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 7eb7920d8ad..991c94540ed 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -173,7 +173,7 @@ func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusT programs.ResizeWasmLruCache(rustCacheSize) } if err := populateStylusTargetCache(targetConfig); err != nil { - return err + return fmt.Errorf("error populating stylus target cache: %w", err) } return nil } From 5e178f4c149ad6c3d8a825b7bf0956ef4f2194d8 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 26 Aug 2024 12:58:23 -0300 Subject: [PATCH 045/156] Add sanity checks to stylus tracer --- execution/gethexec/stylus_tracer.go | 17 ++++++++++++++++- util/containers/stack.go | 13 ++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/execution/gethexec/stylus_tracer.go b/execution/gethexec/stylus_tracer.go index d1246989f0a..4c18bb2ebe2 100644 --- a/execution/gethexec/stylus_tracer.go +++ b/execution/gethexec/stylus_tracer.go @@ -5,6 +5,7 @@ package gethexec import ( "encoding/json" + "errors" "fmt" "math/big" "strings" @@ -14,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers" + "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/util/containers" ) @@ -152,9 +154,22 @@ func (t *stylusTracer) GetResult() (json.RawMessage, error) { if t.reason != nil { return nil, t.reason } + + var internalErr error if t.open == nil { - return nil, fmt.Errorf("trace is nil") + internalErr = errors.Join(internalErr, fmt.Errorf("tracer.open is nil")) + } + if t.stack == nil { + internalErr = errors.Join(internalErr, fmt.Errorf("tracer.stack is nil")) + } + if !t.stack.Empty() { + internalErr = errors.Join(internalErr, fmt.Errorf("tracer.stack should be empty, but has %d values", t.stack.Len())) } + if internalErr != nil { + log.Error("stylusTracer: internal error when generating a trace", "error", internalErr) + return nil, fmt.Errorf("internal error: %w", internalErr) + } + msg, err := json.Marshal(t.open) if err != nil { return nil, err diff --git a/util/containers/stack.go b/util/containers/stack.go index 97fcd44a6d9..ea7f31013b5 100644 --- a/util/containers/stack.go +++ b/util/containers/stack.go @@ -28,7 +28,7 @@ func (s *Stack[T]) Pop() (T, error) { var zeroVal T return zeroVal, fmt.Errorf("trying to pop nil stack") } - if len(*s) == 0 { + if s.Empty() { var zeroVal T return zeroVal, fmt.Errorf("trying to pop empty stack") } @@ -37,3 +37,14 @@ func (s *Stack[T]) Pop() (T, error) { *s = (*s)[:i] return val, nil } + +func (s *Stack[T]) Empty() bool { + return s == nil || len(*s) == 0 +} + +func (s *Stack[T]) Len() int { + if s == nil { + return 0 + } + return len(*s) +} From 18de0ad594cf3882f3d2c6a85a497080f3816f1b Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Mon, 26 Aug 2024 18:32:29 -0600 Subject: [PATCH 046/156] update default x86 architecture --- arbos/programs/native.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index fd3dec25a04..9316e8f230e 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -289,7 +289,7 @@ func ResizeWasmLruCache(size uint32) { } const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon" -const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2" +const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2+lzcnt+bmi" func SetTarget(name rawdb.Target, description string, native bool) error { output := &rustBytes{} From 9e9bcc43cfe6b2d0bf0c3710e416130ac9321328 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Mon, 26 Aug 2024 19:17:02 -0600 Subject: [PATCH 047/156] testcompile: test loaded compile with default target --- arbos/programs/cgo_test.go | 8 ++++++++ arbos/programs/testcompile.go | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/arbos/programs/cgo_test.go b/arbos/programs/cgo_test.go index c0e146d98d7..e16c362ef8d 100644 --- a/arbos/programs/cgo_test.go +++ b/arbos/programs/cgo_test.go @@ -40,5 +40,13 @@ func TestCompileArch(t *testing.T) { if err != nil { t.Fatal(err) } + err = resetNativeTarget() + if err != nil { + t.Fatal(err) + } + err = testCompileLoad() + if err != nil { + t.Fatal(err) + } } } diff --git a/arbos/programs/testcompile.go b/arbos/programs/testcompile.go index a16bae52c0c..1daf470620e 100644 --- a/arbos/programs/testcompile.go +++ b/arbos/programs/testcompile.go @@ -178,6 +178,28 @@ func testCompileArch(store bool) error { return nil } +func resetNativeTarget() error { + output := &rustBytes{} + + _, err := fmt.Print("resetting native target\n") + if err != nil { + return err + } + + localCompileName := []byte("local") + + status := C.stylus_target_set(goSlice(localCompileName), + goSlice([]byte{}), + output, + cbool(true)) + + if status != 0 { + return fmt.Errorf("failed setting compilation target arm: %v", string(output.intoBytes())) + } + + return nil +} + func testCompileLoad() error { filePath := "../../target/testdata/host.bin" localTarget := rawdb.LocalTarget() From ea0cbc672d82802e91be407b595463d6d9415f8c Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Tue, 27 Aug 2024 15:05:59 +0200 Subject: [PATCH 048/156] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 0a376604c9c..4ca4b4d7340 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 0a376604c9ce0c588de0d943184380acd7e25db4 +Subproject commit 4ca4b4d734021df09cb42a796ca8f263d47383f9 From af6320388b2bf9928251115d7edfafb33cbcf81b Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 27 Aug 2024 14:03:04 -0500 Subject: [PATCH 049/156] Return false if nil is passed to IsNotExistError --- util/dbutil/dbutil.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/util/dbutil/dbutil.go b/util/dbutil/dbutil.go index ca0f5aaaebe..6573c5742cd 100644 --- a/util/dbutil/dbutil.go +++ b/util/dbutil/dbutil.go @@ -6,7 +6,7 @@ package dbutil import ( "errors" "fmt" - "os" + "io/fs" "regexp" "github.com/cockroachdb/pebble" @@ -22,13 +22,15 @@ func IsErrNotFound(err error) bool { var pebbleNotExistErrorRegex = regexp.MustCompile("pebble: database .* does not exist") func isPebbleNotExistError(err error) bool { - return pebbleNotExistErrorRegex.MatchString(err.Error()) + return err != nil && pebbleNotExistErrorRegex.MatchString(err.Error()) } func isLeveldbNotExistError(err error) bool { - return os.IsNotExist(err) + return errors.Is(err, fs.ErrNotExist) } +// IsNotExistError returns true if the error is a "database not found" error. +// It must return false if err is nil. func IsNotExistError(err error) bool { return isLeveldbNotExistError(err) || isPebbleNotExistError(err) } From a32669038ec7d6009d3614973f8b779890aa13c4 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 27 Aug 2024 14:05:11 -0500 Subject: [PATCH 050/156] Add test case --- util/dbutil/dbutil_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/util/dbutil/dbutil_test.go b/util/dbutil/dbutil_test.go index b28f8a2c230..b303bb56b6e 100644 --- a/util/dbutil/dbutil_test.go +++ b/util/dbutil/dbutil_test.go @@ -28,6 +28,9 @@ func testIsNotExistError(t *testing.T, dbEngine string, isNotExist func(error) b if isNotExist(err) { t.Fatalf("Classified other error as not exist, err: %v", err) } + if isNotExist(nil) { + t.Fatal("Classified nil as not exist") + } } func TestIsNotExistError(t *testing.T) { From eb1fd9e1bd366b89431e29b3ddcc05586f90d0b4 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Tue, 27 Aug 2024 17:55:09 -0300 Subject: [PATCH 051/156] Do not extract wasm db instead of deleting it Use the extract.Renamer function to avoid extracting the wasm DB when the init.import-wasm flag is set to false. --- cmd/nitro/init.go | 57 ++++++++++--------- cmd/nitro/init_test.go | 121 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 27 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 9a1230651b0..53048d34861 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -24,7 +24,7 @@ import ( "time" "github.com/cavaliergopher/grab/v3" - extract "github.com/codeclysm/extract/v3" + "github.com/codeclysm/extract/v3" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -405,6 +405,34 @@ func databaseIsEmpty(db ethdb.Database) bool { return !it.Next() } +func extractSnapshot(archive string, location string, importWasm bool) error { + isWasmDB := regexp.MustCompile("^(./)?wasm") + reader, err := os.Open(archive) + if err != nil { + return fmt.Errorf("couln't open init '%v' archive: %w", archive, err) + } + stat, err := reader.Stat() + if err != nil { + return err + } + log.Info("extracting downloaded init archive", "size", fmt.Sprintf("%dMB", stat.Size()/1024/1024)) + var rename extract.Renamer + if !importWasm { + rename = func(path string) string { + path = strings.ToLower(path) + if isWasmDB.MatchString(path) { + return "" // do not extract wasm files + } + return path + } + } + err = extract.Archive(context.Background(), reader, location, rename) + if err != nil { + return fmt.Errorf("couln't extract init archive '%v' err: %w", archive, err) + } + return nil +} + // removes all entries with keys prefixed with prefixes and of length used in initial version of wasm store schema func purgeVersion0WasmStoreEntries(db ethdb.Database) error { prefixes, keyLength := rawdb.DeprecatedPrefixesV0() @@ -597,32 +625,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo } if initFile != "" { - reader, err := os.Open(initFile) - if err != nil { - return nil, nil, fmt.Errorf("couln't open init '%v' archive: %w", initFile, err) - } - stat, err := reader.Stat() - if err != nil { - return nil, nil, err - } - log.Info("extracting downloaded init archive", "size", fmt.Sprintf("%dMB", stat.Size()/1024/1024)) - err = extract.Archive(context.Background(), reader, stack.InstanceDir(), nil) - if err != nil { - err = fmt.Errorf("couln't extract init archive '%v' err:%w", initFile, err) - } - wasmDb := path.Join(stack.InstanceDir(), "wasm") - if dirExists(wasmDb) && !config.Init.ImportWasm { - // By default, remove the wasm DB because it contains native executables. - // Hence, it is a security concern when downloading a snapshot from a - // untrusted source. - removeErr := os.RemoveAll(wasmDb) - if removeErr != nil { - err = errors.Join(err, fmt.Errorf("failed to remove extracted wasm database: %w", removeErr)) - } else { - log.Debug("init: removed wasm database") - } - } - if err != nil { + if err := extractSnapshot(initFile, stack.InstanceDir(), config.Init.ImportWasm); err != nil { return nil, nil, err } } diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index b2773ed8613..9282839f75c 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -4,18 +4,21 @@ package main import ( + "archive/tar" "bytes" "context" "crypto/sha256" "encoding/hex" "errors" "fmt" + "io" "math/big" "net" "net/http" "os" "path" "path/filepath" + "slices" "strings" "testing" "time" @@ -24,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/node" + "github.com/google/go-cmp/cmp" "github.com/offchainlabs/nitro/arbnode" "github.com/offchainlabs/nitro/cmd/chaininfo" "github.com/offchainlabs/nitro/cmd/conf" @@ -554,3 +558,120 @@ func TestOpenInitializeChainDbEmptyInit(t *testing.T) { err = chainDb.Close() Require(t, err) } + +func TestExtractSnapshot(t *testing.T) { + testCases := []struct { + name string + archiveFiles []string + importWasm bool + wantFiles []string + }{ + { + name: "extractAll", + importWasm: true, + archiveFiles: []string{ + "arbitrumdata/000001.ldb", + "l2chaindata/000001.ldb", + "l2chaindata/ancients/000001.ldb", + "nodes/000001.ldb", + "wasm/000001.ldb", + }, + wantFiles: []string{ + "arbitrumdata/000001.ldb", + "l2chaindata/000001.ldb", + "l2chaindata/ancients/000001.ldb", + "nodes/000001.ldb", + "wasm/000001.ldb", + }, + }, + { + name: "extractAllButWasm", + importWasm: false, + archiveFiles: []string{ + "arbitrumdata/000001.ldb", + "l2chaindata/000001.ldb", + "nodes/000001.ldb", + "wasm/000001.ldb", + }, + wantFiles: []string{ + "arbitrumdata/000001.ldb", + "l2chaindata/000001.ldb", + "nodes/000001.ldb", + }, + }, + { + name: "extractAllButWasmWithPrefixDot", + importWasm: false, + archiveFiles: []string{ + "./arbitrumdata/000001.ldb", + "./l2chaindata/000001.ldb", + "./nodes/000001.ldb", + "./wasm/000001.ldb", + }, + wantFiles: []string{ + "arbitrumdata/000001.ldb", + "l2chaindata/000001.ldb", + "nodes/000001.ldb", + }, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + // Create archive with dummy files + archiveDir := t.TempDir() + archivePath := path.Join(archiveDir, "archive.tar") + { + // Create context to close the file handlers + archiveFile, err := os.Create(archivePath) + Require(t, err) + defer archiveFile.Close() + tarWriter := tar.NewWriter(archiveFile) + defer tarWriter.Close() + for _, relativePath := range testCase.archiveFiles { + filePath := path.Join(archiveDir, relativePath) + dir := filepath.Dir(filePath) + const dirPerm = 0700 + err := os.MkdirAll(dir, dirPerm) + Require(t, err) + const filePerm = 0600 + err = os.WriteFile(filePath, []byte{0xbe, 0xef}, filePerm) + Require(t, err) + file, err := os.Open(filePath) + Require(t, err) + info, err := file.Stat() + Require(t, err) + header, err := tar.FileInfoHeader(info, "") + Require(t, err) + header.Name = relativePath + err = tarWriter.WriteHeader(header) + Require(t, err) + _, err = io.Copy(tarWriter, file) + Require(t, err) + } + } + + // Extract archive and compare contents + targetDir := t.TempDir() + err := extractSnapshot(archivePath, targetDir, testCase.importWasm) + Require(t, err, "failed to extract snapshot") + gotFiles := []string{} + err = filepath.WalkDir(targetDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + gotFiles = append(gotFiles, path) + } + return nil + }) + Require(t, err) + slices.Sort(gotFiles) + for i, f := range testCase.wantFiles { + testCase.wantFiles[i] = path.Join(targetDir, f) + } + if diff := cmp.Diff(gotFiles, testCase.wantFiles); diff != "" { + t.Fatal("extracted files don't match", diff) + } + }) + } +} From 8ffca7e70304cede8048afa981b442b14677d3bb Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Tue, 27 Aug 2024 17:59:27 -0300 Subject: [PATCH 052/156] Do not change case when extracting DB --- cmd/nitro/init.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 53048d34861..435c73c997e 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -419,8 +419,7 @@ func extractSnapshot(archive string, location string, importWasm bool) error { var rename extract.Renamer if !importWasm { rename = func(path string) string { - path = strings.ToLower(path) - if isWasmDB.MatchString(path) { + if isWasmDB.MatchString(strings.ToLower(path)) { return "" // do not extract wasm files } return path From 7b4834facf7e1c5834ea630551b9908a15183321 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 13:24:35 +0200 Subject: [PATCH 053/156] pass WasmTargets to RecordProgram call --- arbos/programs/native.go | 4 ++-- go-ethereum | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 98abf9443ca..08431b12096 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -228,8 +228,8 @@ func callProgram( panic("missing asm") } - if db, ok := db.(*state.StateDB); ok { - db.RecordProgram(moduleHash) + if stateDb, ok := db.(*state.StateDB); ok { + stateDb.RecordProgram(db.Database().WasmTargets(), moduleHash) } evmApi := newApi(interpreter, tracingInfo, scope, memoryModel) diff --git a/go-ethereum b/go-ethereum index 4ca4b4d7340..81114dde8a2 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 4ca4b4d734021df09cb42a796ca8f263d47383f9 +Subproject commit 81114dde8a26bae90c188605c4a36d5919a4a265 From 99b34b301af91888b32b3395b58aad7eb0ad46b9 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 13:39:09 +0200 Subject: [PATCH 054/156] init test: call Validate on StylusTargetConfig --- cmd/nitro/init_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index af2242af783..eadc6234c0d 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -350,6 +350,12 @@ func TestEmptyDatabaseDir(t *testing.T) { } } +func defaultStylusTargetConfigForTest(t *testing.T) *gethexec.StylusTargetConfig { + targetConfig := gethexec.DefaultStylusTargetConfig + Require(t, targetConfig.Validate()) + return &targetConfig +} + func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { t.Parallel() @@ -377,7 +383,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -394,7 +400,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -412,7 +418,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -548,7 +554,7 @@ func TestOpenInitializeChainDbEmptyInit(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, From bf735f466ce13e25006ce2f87f6362692df51659 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 14:15:46 +0200 Subject: [PATCH 055/156] don't call programs.SetTarget for TargetWavm --- execution/gethexec/executionengine.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 192408794bf..56b52ed986e 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -157,6 +157,9 @@ func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { for _, target := range targets { var effectiveStylusTarget string switch target { + case rawdb.TargetWavm: + // skip wavm target + continue case rawdb.TargetArm64: effectiveStylusTarget = targetConfig.Arm64 case rawdb.TargetAmd64: From 29327a8be174e5c582caef4fbe8d4fd55839688e Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 14:16:57 +0200 Subject: [PATCH 056/156] refactor StylusTargetConfig.Validate --- execution/gethexec/node.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index bc29e9b7f23..01464123fdd 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -44,22 +44,24 @@ func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { func (c *StylusTargetConfig) Validate() error { localTarget := rawdb.LocalTarget() - targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)) - targetsSet := make(map[ethdb.WasmTarget]struct{}, len(c.ExtraArchs)+1) + targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) + targetsSet := make(map[ethdb.WasmTarget]bool, len(c.ExtraArchs)) for _, arch := range c.ExtraArchs { target := ethdb.WasmTarget(arch) + if targetsSet[target] { + // skip duplicate + continue + } if !rawdb.IsSupportedWasmTarget(target) { return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) } - if _, duplicate := targetsSet[target]; !duplicate { - targets = append(targets, target) - targetsSet[target] = struct{}{} - } + targets = append(targets, target) + targetsSet[target] = true } - if _, has := targetsSet[rawdb.TargetWavm]; !has { + if !targetsSet[rawdb.TargetWavm] { return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) } - if _, has := targetsSet[localTarget]; !has { + if !targetsSet[localTarget] { targets = append(targets, localTarget) } c.wasmTargets = targets From aba94150dbb0a1b4c6533762284c83948a299e89 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 14:21:21 +0200 Subject: [PATCH 057/156] error for unsupported stylus target in populateStylusTargetCache --- execution/gethexec/executionengine.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 56b52ed986e..8594d5867df 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -166,11 +166,13 @@ func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { effectiveStylusTarget = targetConfig.Amd64 case rawdb.TargetHost: effectiveStylusTarget = targetConfig.Host + default: + return fmt.Errorf("unsupported stylus target: %v", target) } isNative := target == localTarget err := programs.SetTarget(target, effectiveStylusTarget, isNative) if err != nil { - return fmt.Errorf("Failed to set stylus target: %w", err) + return fmt.Errorf("failed to set stylus target: %w", err) } nativeSet = nativeSet || isNative } From a69cb63efc66e86bd5f5bce998fdc66998e536ae Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 11:43:11 -0300 Subject: [PATCH 058/156] Replace regex with filepath functions --- cmd/nitro/init.go | 15 +++++++++++++-- cmd/nitro/init_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 435c73c997e..e0709bb3c94 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -405,8 +405,19 @@ func databaseIsEmpty(db ethdb.Database) bool { return !it.Next() } +func isWasmDb(path string) bool { + path = filepath.Clean(strings.ToLower(path)) + parts := strings.Split(path, string(filepath.Separator)) + if len(parts) >= 1 && parts[0] == "wasm" { + return true + } + if len(parts) >= 2 && parts[0] == "" && parts[1] == "wasm" { // Cover "/wasm" case + return true + } + return false +} + func extractSnapshot(archive string, location string, importWasm bool) error { - isWasmDB := regexp.MustCompile("^(./)?wasm") reader, err := os.Open(archive) if err != nil { return fmt.Errorf("couln't open init '%v' archive: %w", archive, err) @@ -419,7 +430,7 @@ func extractSnapshot(archive string, location string, importWasm bool) error { var rename extract.Renamer if !importWasm { rename = func(path string) string { - if isWasmDB.MatchString(strings.ToLower(path)) { + if isWasmDb(path) { return "" // do not extract wasm files } return path diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index 9282839f75c..7b9153a9cf7 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -675,3 +675,35 @@ func TestExtractSnapshot(t *testing.T) { }) } } + +func TestIsWasmDb(t *testing.T) { + testCases := []struct { + path string + want bool + }{ + {"wasm", true}, + {"wasm/", true}, + {"wasm/something", true}, + {"/wasm", true}, + {"./wasm", true}, + {"././wasm", true}, + {"/./wasm", true}, + {"WASM", true}, + {"wAsM", true}, + {"nitro/../wasm", true}, + {"/nitro/../wasm", true}, + {".//nitro/.//../wasm", true}, + {"not-wasm", false}, + {"l2chaindata/example@@", false}, + {"somedir/wasm", false}, + } + for _, testCase := range testCases { + name := fmt.Sprintf("%q", testCase.path) + t.Run(name, func(t *testing.T) { + got := isWasmDb(testCase.path) + if testCase.want != got { + t.Fatalf("want %v, but got %v", testCase.want, got) + } + }) + } +} From 2e9f4a5a1e77c0c63da526d89874f1225550fab0 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 12:07:46 -0300 Subject: [PATCH 059/156] Disallow any wasm prefix --- cmd/nitro/init.go | 4 ++-- cmd/nitro/init_test.go | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index e0709bb3c94..92398c92075 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -408,10 +408,10 @@ func databaseIsEmpty(db ethdb.Database) bool { func isWasmDb(path string) bool { path = filepath.Clean(strings.ToLower(path)) parts := strings.Split(path, string(filepath.Separator)) - if len(parts) >= 1 && parts[0] == "wasm" { + if len(parts) >= 1 && strings.HasPrefix(parts[0], "wasm") { return true } - if len(parts) >= 2 && parts[0] == "" && parts[1] == "wasm" { // Cover "/wasm" case + if len(parts) >= 2 && parts[0] == "" && strings.HasPrefix(parts[1], "wasm") { // Cover "/wasm" case return true } return false diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index 7b9153a9cf7..50d88c9b402 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -693,6 +693,9 @@ func TestIsWasmDb(t *testing.T) { {"nitro/../wasm", true}, {"/nitro/../wasm", true}, {".//nitro/.//../wasm", true}, + {"wasm-something", true}, + {"wasm\x00something", true}, + {"wasm\x00test/example", true}, {"not-wasm", false}, {"l2chaindata/example@@", false}, {"somedir/wasm", false}, From 3f9461d132899134e527bd7fdaf84cd8efbdf773 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 28 Aug 2024 22:40:17 +0530 Subject: [PATCH 060/156] Support rebuilding of wasm store on init --- cmd/nitro/init.go | 92 ++++++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 41 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 5573bd6ab27..db9f3bc0fc5 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -475,6 +475,51 @@ func validateOrUpgradeWasmStoreSchemaVersion(db ethdb.Database) error { return nil } +func rebuildLocalWasm(ctx context.Context, config *NodeConfig, l2BlockChain *core.BlockChain, chainConfig *params.ChainConfig, chainDb, wasmDb ethdb.Database, rebuildMode string) (ethdb.Database, *core.BlockChain, error) { + var err error + latestBlock := l2BlockChain.CurrentBlock() + if latestBlock == nil || latestBlock.Number.Uint64() <= chainConfig.ArbitrumChainParams.GenesisBlockNum || + types.DeserializeHeaderExtraInformation(latestBlock).ArbOSFormatVersion < params.ArbosVersion_Stylus { + // If there is only genesis block or no blocks in the blockchain, set Rebuilding of wasm store to Done + // If Stylus upgrade hasn't yet happened, skipping rebuilding of wasm store + log.Info("Setting rebuilding of wasm store to done") + if err = gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, gethexec.RebuildingDone); err != nil { + return nil, nil, fmt.Errorf("unable to set rebuilding status of wasm store to done: %w", err) + } + } else if rebuildMode != "false" { + var position common.Hash + if rebuildMode == "force" { + log.Info("Commencing force rebuilding of wasm store by setting codehash position in rebuilding to beginning") + if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { + return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) + } + } else { + position, err = gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingPositionKey) + if err != nil { + log.Info("Unable to get codehash position in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it and starting rebuilding", "err", err) + if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { + return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) + } + } + } + if position != gethexec.RebuildingDone { + startBlockHash, err := gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingStartBlockHashKey) + if err != nil { + log.Info("Unable to get start block hash in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it to latest block hash", "err", err) + if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingStartBlockHashKey, latestBlock.Hash()); err != nil { + return nil, nil, fmt.Errorf("unable to initialize start block hash in rebuilding of wasm store to latest block hash: %w", err) + } + startBlockHash = latestBlock.Hash() + } + log.Info("Starting or continuing rebuilding of wasm store", "codeHash", position, "startBlockHash", startBlockHash) + if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.Execution.RPC.MaxRecreateStateDepth, &config.Execution.StylusTarget, l2BlockChain, position, startBlockHash); err != nil { + return nil, nil, fmt.Errorf("error rebuilding of wasm store: %w", err) + } + } + } + return chainDb, l2BlockChain, nil +} + func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { if !config.Init.Force { if readOnlyDb, err := stack.OpenDatabaseWithFreezerWithExtraOptions("l2chaindata", 0, 0, config.Persistent.Ancient, "l2chaindata/", true, persistentConfig.Pebble.ExtraOptions("l2chaindata")); err == nil { @@ -523,47 +568,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return chainDb, l2BlockChain, fmt.Errorf("failed to recreate missing states: %w", err) } } - latestBlock := l2BlockChain.CurrentBlock() - if latestBlock == nil || latestBlock.Number.Uint64() <= chainConfig.ArbitrumChainParams.GenesisBlockNum || - types.DeserializeHeaderExtraInformation(latestBlock).ArbOSFormatVersion < params.ArbosVersion_Stylus { - // If there is only genesis block or no blocks in the blockchain, set Rebuilding of wasm store to Done - // If Stylus upgrade hasn't yet happened, skipping rebuilding of wasm store - log.Info("Setting rebuilding of wasm store to done") - if err = gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, gethexec.RebuildingDone); err != nil { - return nil, nil, fmt.Errorf("unable to set rebuilding status of wasm store to done: %w", err) - } - } else if config.Init.RebuildLocalWasm != "false" { - var position common.Hash - if config.Init.RebuildLocalWasm == "force" { - log.Info("Commencing force rebuilding of wasm store by setting codehash position in rebuilding to beginning") - if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { - return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) - } - } else { - position, err = gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingPositionKey) - if err != nil { - log.Info("Unable to get codehash position in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it and starting rebuilding", "err", err) - if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { - return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) - } - } - } - if position != gethexec.RebuildingDone { - startBlockHash, err := gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingStartBlockHashKey) - if err != nil { - log.Info("Unable to get start block hash in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it to latest block hash", "err", err) - if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingStartBlockHashKey, latestBlock.Hash()); err != nil { - return nil, nil, fmt.Errorf("unable to initialize start block hash in rebuilding of wasm store to latest block hash: %w", err) - } - startBlockHash = latestBlock.Hash() - } - log.Info("Starting or continuing rebuilding of wasm store", "codeHash", position, "startBlockHash", startBlockHash) - if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.Execution.RPC.MaxRecreateStateDepth, &config.Execution.StylusTarget, l2BlockChain, position, startBlockHash); err != nil { - return nil, nil, fmt.Errorf("error rebuilding of wasm store: %w", err) - } - } - } - return chainDb, l2BlockChain, nil + return rebuildLocalWasm(ctx, config, l2BlockChain, chainConfig, chainDb, wasmDb, config.Init.RebuildLocalWasm) } readOnlyDb.Close() } else if !dbutil.IsNotExistError(err) { @@ -795,6 +800,11 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return chainDb, l2BlockChain, err } + if config.Init.RebuildLocalWasm != "false" { + // In order to rebuild wasm store correctly we have to use force option + return rebuildLocalWasm(ctx, config, l2BlockChain, chainConfig, chainDb, wasmDb, "force") + } + return chainDb, l2BlockChain, nil } From 2499675af22d1c760a59dbed4b21cc6cd3821977 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 28 Aug 2024 22:46:31 +0530 Subject: [PATCH 061/156] update flag description --- cmd/conf/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index f3606916936..208969fce28 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -86,7 +86,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Int64(prefix+".reorg-to-message-batch", InitConfigDefault.ReorgToMessageBatch, "rolls back the blockchain to the first batch at or before a given message index") f.Int64(prefix+".reorg-to-block-batch", InitConfigDefault.ReorgToBlockBatch, "rolls back the blockchain to the first batch at or before a given block number") f.String(prefix+".rebuild-local-wasm", InitConfigDefault.RebuildLocalWasm, "rebuild local wasm database on boot if needed (otherwise-will be done lazily). Three modes are supported \n"+ - "\"auto\"- (enabled by default) if any previous rebuilding attempt was successful then rebuilding is disabled else continues to rebuild,\n"+ + "\"auto\"- (enabled by default) if any previous rebuilding attempt was successful then rebuilding is disabled else continues to rebuild. Is equivalent to force when starting out with --init.force or downloading db snapshot via init,\n"+ "\"force\"- force rebuilding which would commence rebuilding despite the status of previous attempts,\n"+ "\"false\"- do not rebuild on startup", ) From caa421013a96664a5f71653bde8a392165a78421 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 15:19:26 -0300 Subject: [PATCH 062/156] Revert "Disallow any wasm prefix" This reverts commit 2e9f4a5a1e77c0c63da526d89874f1225550fab0. --- cmd/nitro/init.go | 4 ++-- cmd/nitro/init_test.go | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 92398c92075..e0709bb3c94 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -408,10 +408,10 @@ func databaseIsEmpty(db ethdb.Database) bool { func isWasmDb(path string) bool { path = filepath.Clean(strings.ToLower(path)) parts := strings.Split(path, string(filepath.Separator)) - if len(parts) >= 1 && strings.HasPrefix(parts[0], "wasm") { + if len(parts) >= 1 && parts[0] == "wasm" { return true } - if len(parts) >= 2 && parts[0] == "" && strings.HasPrefix(parts[1], "wasm") { // Cover "/wasm" case + if len(parts) >= 2 && parts[0] == "" && parts[1] == "wasm" { // Cover "/wasm" case return true } return false diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index 50d88c9b402..7b9153a9cf7 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -693,9 +693,6 @@ func TestIsWasmDb(t *testing.T) { {"nitro/../wasm", true}, {"/nitro/../wasm", true}, {".//nitro/.//../wasm", true}, - {"wasm-something", true}, - {"wasm\x00something", true}, - {"wasm\x00test/example", true}, {"not-wasm", false}, {"l2chaindata/example@@", false}, {"somedir/wasm", false}, From 9644416f86ac09478f99b6c56fe3784229318abf Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 18:38:06 -0300 Subject: [PATCH 063/156] Improve out of gas warning log For the setTrieSlots and setTransientBytes32 HostIOs, when there is an out-of-gas error, Nitro emits a warning line with `Caused by:\n \x02`. This commit improves this error message by printing the debug representation of the API status code and the warning becomes: `Caused by:\n OutOfGas`. This commit also solves a potential panic when the API does not return the status code. --- arbitrator/arbutil/src/evm/req.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/arbitrator/arbutil/src/evm/req.rs b/arbitrator/arbutil/src/evm/req.rs index b1c8d999727..f76d1407f1b 100644 --- a/arbitrator/arbutil/src/evm/req.rs +++ b/arbitrator/arbutil/src/evm/req.rs @@ -140,8 +140,13 @@ impl> EvmApi for EvmApiRequestor { } let (res, _, cost) = self.request(EvmApiMethod::SetTrieSlots, data); - if res[0] != EvmApiStatus::Success.into() { - bail!("{}", String::from_utf8_or_hex(res)); + let status = res + .first() + .copied() + .map(EvmApiStatus::from) + .unwrap_or(EvmApiStatus::Failure); + if status != EvmApiStatus::Success { + bail!("{:?}", status); } Ok(cost) } @@ -156,8 +161,13 @@ impl> EvmApi for EvmApiRequestor { data.extend(key); data.extend(value); let (res, ..) = self.request(EvmApiMethod::SetTransientBytes32, data); - if res[0] != EvmApiStatus::Success.into() { - bail!("{}", String::from_utf8_or_hex(res)); + let status = res + .first() + .copied() + .map(EvmApiStatus::from) + .unwrap_or(EvmApiStatus::Failure); + if status != EvmApiStatus::Success { + bail!("{:?}", status); } Ok(()) } From 03dd3abfd7fb27f5be4bfffa6f0cb8e9deadd8ef Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 22:20:57 -0300 Subject: [PATCH 064/156] Remove unused import --- arbitrator/arbutil/src/evm/req.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/arbitrator/arbutil/src/evm/req.rs b/arbitrator/arbutil/src/evm/req.rs index f76d1407f1b..287db357f32 100644 --- a/arbitrator/arbutil/src/evm/req.rs +++ b/arbitrator/arbutil/src/evm/req.rs @@ -7,7 +7,6 @@ use crate::{ storage::{StorageCache, StorageWord}, user::UserOutcomeKind, }, - format::Utf8OrHex, pricing::EVM_API_INK, Bytes20, Bytes32, }; From f6efc1b491a39e0eb52e62dc7dc01cda829bd168 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 22:25:41 -0300 Subject: [PATCH 065/156] Add comment about case-insensitive file systems --- cmd/nitro/init.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index e0709bb3c94..b338e0a49ef 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -406,7 +406,8 @@ func databaseIsEmpty(db ethdb.Database) bool { } func isWasmDb(path string) bool { - path = filepath.Clean(strings.ToLower(path)) + path = strings.ToLower(path) // lowers the path to handle case-insensitive file systems + path = filepath.Clean(path) parts := strings.Split(path, string(filepath.Separator)) if len(parts) >= 1 && parts[0] == "wasm" { return true From 9a326e281871d3c29dd5a796972298c87e917235 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 28 Aug 2024 22:04:10 -0500 Subject: [PATCH 066/156] Disable preimage validation in JIT --- arbitrator/jit/src/wavmio.rs | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/arbitrator/jit/src/wavmio.rs b/arbitrator/jit/src/wavmio.rs index 062d18d8e92..0ca666d3b20 100644 --- a/arbitrator/jit/src/wavmio.rs +++ b/arbitrator/jit/src/wavmio.rs @@ -8,8 +8,6 @@ use crate::{ }; use arbutil::{Color, PreimageType}; use caller_env::{GuestPtr, MemAccess}; -use sha2::Sha256; -use sha3::{Digest, Keccak256}; use std::{ io, io::{BufReader, BufWriter, ErrorKind}, @@ -170,19 +168,25 @@ pub fn resolve_preimage_impl( error!("Missing requested preimage for hash {hash_hex} in {name}") }; - // Check if preimage rehashes to the provided hash. Exclude blob preimages - let calculated_hash: [u8; 32] = match preimage_type { - PreimageType::Keccak256 => Keccak256::digest(preimage).into(), - PreimageType::Sha2_256 => Sha256::digest(preimage).into(), - PreimageType::EthVersionedHash => *hash, - }; - if calculated_hash != *hash { - error!( - "Calculated hash {} of preimage {} does not match provided hash {}", - hex::encode(calculated_hash), - hex::encode(preimage), - hex::encode(*hash) - ); + #[cfg(debug_assertions)] + { + use sha2::Sha256; + use sha3::{Digest, Keccak256}; + + // Check if preimage rehashes to the provided hash. Exclude blob preimages + let calculated_hash: [u8; 32] = match preimage_type { + PreimageType::Keccak256 => Keccak256::digest(preimage).into(), + PreimageType::Sha2_256 => Sha256::digest(preimage).into(), + PreimageType::EthVersionedHash => *hash, + }; + if calculated_hash != *hash { + error!( + "Calculated hash {} of preimage {} does not match provided hash {}", + hex::encode(calculated_hash), + hex::encode(preimage), + hex::encode(*hash) + ); + } } if offset % 32 != 0 { From 972508da88fb9797c3151cbdc23349b096a098f5 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 29 Aug 2024 09:44:34 +0530 Subject: [PATCH 067/156] address PR comments --- cmd/nitro/init.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index db9f3bc0fc5..6ae3c9125d2 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -475,10 +475,10 @@ func validateOrUpgradeWasmStoreSchemaVersion(db ethdb.Database) error { return nil } -func rebuildLocalWasm(ctx context.Context, config *NodeConfig, l2BlockChain *core.BlockChain, chainConfig *params.ChainConfig, chainDb, wasmDb ethdb.Database, rebuildMode string) (ethdb.Database, *core.BlockChain, error) { +func rebuildLocalWasm(ctx context.Context, config *gethexec.Config, l2BlockChain *core.BlockChain, chainDb, wasmDb ethdb.Database, rebuildMode string) (ethdb.Database, *core.BlockChain, error) { var err error latestBlock := l2BlockChain.CurrentBlock() - if latestBlock == nil || latestBlock.Number.Uint64() <= chainConfig.ArbitrumChainParams.GenesisBlockNum || + if latestBlock == nil || latestBlock.Number.Uint64() <= l2BlockChain.Config().ArbitrumChainParams.GenesisBlockNum || types.DeserializeHeaderExtraInformation(latestBlock).ArbOSFormatVersion < params.ArbosVersion_Stylus { // If there is only genesis block or no blocks in the blockchain, set Rebuilding of wasm store to Done // If Stylus upgrade hasn't yet happened, skipping rebuilding of wasm store @@ -512,7 +512,7 @@ func rebuildLocalWasm(ctx context.Context, config *NodeConfig, l2BlockChain *cor startBlockHash = latestBlock.Hash() } log.Info("Starting or continuing rebuilding of wasm store", "codeHash", position, "startBlockHash", startBlockHash) - if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.Execution.RPC.MaxRecreateStateDepth, &config.Execution.StylusTarget, l2BlockChain, position, startBlockHash); err != nil { + if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.RPC.MaxRecreateStateDepth, &config.StylusTarget, l2BlockChain, position, startBlockHash); err != nil { return nil, nil, fmt.Errorf("error rebuilding of wasm store: %w", err) } } @@ -568,7 +568,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return chainDb, l2BlockChain, fmt.Errorf("failed to recreate missing states: %w", err) } } - return rebuildLocalWasm(ctx, config, l2BlockChain, chainConfig, chainDb, wasmDb, config.Init.RebuildLocalWasm) + return rebuildLocalWasm(ctx, &config.Execution, l2BlockChain, chainDb, wasmDb, config.Init.RebuildLocalWasm) } readOnlyDb.Close() } else if !dbutil.IsNotExistError(err) { @@ -800,12 +800,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return chainDb, l2BlockChain, err } - if config.Init.RebuildLocalWasm != "false" { - // In order to rebuild wasm store correctly we have to use force option - return rebuildLocalWasm(ctx, config, l2BlockChain, chainConfig, chainDb, wasmDb, "force") - } - - return chainDb, l2BlockChain, nil + return rebuildLocalWasm(ctx, &config.Execution, l2BlockChain, chainDb, wasmDb, config.Init.RebuildLocalWasm) } func testTxIndexUpdated(chainDb ethdb.Database, lastBlock uint64) bool { From a710d0c4a9203af55e9a43b4687647e3d1444a55 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 29 Aug 2024 09:46:29 +0530 Subject: [PATCH 068/156] undo flag description --- cmd/conf/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index 208969fce28..f3606916936 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -86,7 +86,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Int64(prefix+".reorg-to-message-batch", InitConfigDefault.ReorgToMessageBatch, "rolls back the blockchain to the first batch at or before a given message index") f.Int64(prefix+".reorg-to-block-batch", InitConfigDefault.ReorgToBlockBatch, "rolls back the blockchain to the first batch at or before a given block number") f.String(prefix+".rebuild-local-wasm", InitConfigDefault.RebuildLocalWasm, "rebuild local wasm database on boot if needed (otherwise-will be done lazily). Three modes are supported \n"+ - "\"auto\"- (enabled by default) if any previous rebuilding attempt was successful then rebuilding is disabled else continues to rebuild. Is equivalent to force when starting out with --init.force or downloading db snapshot via init,\n"+ + "\"auto\"- (enabled by default) if any previous rebuilding attempt was successful then rebuilding is disabled else continues to rebuild,\n"+ "\"force\"- force rebuilding which would commence rebuilding despite the status of previous attempts,\n"+ "\"false\"- do not rebuild on startup", ) From 138bcb7c5c3c5ab4427be249d48430afbc0a5699 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 29 Aug 2024 10:47:20 -0500 Subject: [PATCH 069/156] Add safety check that min basefee must be positive for gas estimation --- precompiles/ArbOwner.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index 066fc0a4c41..8b87445e0e5 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -69,6 +69,9 @@ func (con ArbOwner) SetL2BaseFee(c ctx, evm mech, priceInWei huge) error { // SetMinimumL2BaseFee sets the minimum base fee needed for a transaction to succeed func (con ArbOwner) SetMinimumL2BaseFee(c ctx, evm mech, priceInWei huge) error { + if c.txProcessor.MsgIsNonMutating() && priceInWei.Sign() == 0 { + return errors.New("minimum base fee must be nonzero") + } return c.State.L2PricingState().SetMinBaseFeeWei(priceInWei) } From 7a4fe007e5e3edb198130265ad167dc39d531073 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 29 Aug 2024 21:21:50 +0200 Subject: [PATCH 070/156] simplify StylusTargetConfig.Validate --- execution/gethexec/node.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 01464123fdd..21c2b4beced 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -43,26 +43,21 @@ func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { } func (c *StylusTargetConfig) Validate() error { - localTarget := rawdb.LocalTarget() - targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) targetsSet := make(map[ethdb.WasmTarget]bool, len(c.ExtraArchs)) for _, arch := range c.ExtraArchs { target := ethdb.WasmTarget(arch) - if targetsSet[target] { - // skip duplicate - continue - } if !rawdb.IsSupportedWasmTarget(target) { return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) } - targets = append(targets, target) targetsSet[target] = true } if !targetsSet[rawdb.TargetWavm] { return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) } - if !targetsSet[localTarget] { - targets = append(targets, localTarget) + targetsSet[rawdb.LocalTarget()] = true + targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) + for target := range targetsSet { + targets = append(targets, target) } c.wasmTargets = targets return nil From 01c91c72db803410631f51b8614cd69021c8e786 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 30 Aug 2024 10:11:46 -0300 Subject: [PATCH 071/156] Use pull_request_target instead of pull_request in submodule pin check --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 870f5208770..f045f71f68c 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request: + pull_request_target: branches: [ master ] types: [synchronize, opened, reopened] From ae67f4e575ac90b97119429ebf2dccc5fcc0ef17 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Fri, 30 Aug 2024 12:11:40 -0600 Subject: [PATCH 072/156] redis consimer: workers and beffers --- validator/valnode/redis/consumer.go | 69 ++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 16 deletions(-) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index fb7db1e8702..8458ec021af 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -3,6 +3,7 @@ package redis import ( "context" "fmt" + "runtime" "time" "github.com/ethereum/go-ethereum/common" @@ -22,8 +23,9 @@ type ValidationServer struct { spawner validator.ValidationSpawner // consumers stores moduleRoot to consumer mapping. - consumers map[common.Hash]*pubsub.Consumer[*validator.ValidationInput, validator.GoGlobalState] - streamTimeout time.Duration + consumers map[common.Hash]*pubsub.Consumer[*validator.ValidationInput, validator.GoGlobalState] + + config *ValidationServerConfig } func NewValidationServer(cfg *ValidationServerConfig, spawner validator.ValidationSpawner) (*ValidationServer, error) { @@ -44,9 +46,9 @@ func NewValidationServer(cfg *ValidationServerConfig, spawner validator.Validati consumers[mr] = c } return &ValidationServer{ - consumers: consumers, - spawner: spawner, - streamTimeout: cfg.StreamTimeout, + consumers: consumers, + spawner: spawner, + config: cfg, }, nil } @@ -54,6 +56,19 @@ func (s *ValidationServer) Start(ctx_in context.Context) { s.StopWaiter.Start(ctx_in, s) // Channel that all consumers use to indicate their readiness. readyStreams := make(chan struct{}, len(s.consumers)) + type workUnit struct { + req *pubsub.Message[*validator.ValidationInput] + moduleRoot common.Hash + } + workers := s.config.Workers + if workers == 0 { + workers = runtime.NumCPU() + } + bufferSize := 0 + if s.config.BufferReads { + bufferSize = workers + } + workQueue := make(chan workUnit, bufferSize) for moduleRoot, c := range s.consumers { c := c moduleRoot := moduleRoot @@ -93,17 +108,11 @@ func (s *ValidationServer) Start(ctx_in context.Context) { // There's nothing in the queue. return time.Second } - valRun := s.spawner.Launch(req.Value, moduleRoot) - res, err := valRun.Await(ctx) - if err != nil { - log.Error("Error validating", "request value", req.Value, "error", err) - return 0 - } - if err := c.SetResult(ctx, req.ID, res); err != nil { - log.Error("Error setting result for request", "id", req.ID, "result", res, "error", err) - return 0 + select { + case <-ctx.Done(): + case workQueue <- workUnit{req, moduleRoot}: } - return time.Second + return 0 }) }) } @@ -113,7 +122,7 @@ func (s *ValidationServer) Start(ctx_in context.Context) { case <-readyStreams: log.Trace("At least one stream is ready") return // Don't block Start if at least one of the stream is ready. - case <-time.After(s.streamTimeout): + case <-time.After(s.config.StreamTimeout): log.Error("Waiting for redis streams timed out") case <-ctx.Done(): log.Info("Context done while waiting redis streams to be ready, failed to start") @@ -121,6 +130,26 @@ func (s *ValidationServer) Start(ctx_in context.Context) { } } }) + for i := 0; i < workers; i++ { + s.StopWaiter.LaunchThread(func(ctx context.Context) { + for { + var work workUnit + select { + case <-ctx.Done(): + return + case work = <-workQueue: + } + valRun := s.spawner.Launch(work.req.Value, work.moduleRoot) + res, err := valRun.Await(ctx) + if err != nil { + log.Error("Error validating", "request value", work.req.Value, "error", err) + } + if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { + log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } + } + }) + } } type ValidationServerConfig struct { @@ -131,6 +160,8 @@ type ValidationServerConfig struct { // Timeout on polling for existence of each redis stream. StreamTimeout time.Duration `koanf:"stream-timeout"` StreamPrefix string `koanf:"stream-prefix"` + Workers int `koanf:"workers"` + BufferReads bool `koanf:"buffer-reads"` } var DefaultValidationServerConfig = ValidationServerConfig{ @@ -139,6 +170,8 @@ var DefaultValidationServerConfig = ValidationServerConfig{ ConsumerConfig: pubsub.DefaultConsumerConfig, ModuleRoots: []string{}, StreamTimeout: 10 * time.Minute, + Workers: 1, + BufferReads: true, } var TestValidationServerConfig = ValidationServerConfig{ @@ -147,6 +180,8 @@ var TestValidationServerConfig = ValidationServerConfig{ ConsumerConfig: pubsub.TestConsumerConfig, ModuleRoots: []string{}, StreamTimeout: time.Minute, + Workers: 1, + BufferReads: true, } func ValidationServerConfigAddOptions(prefix string, f *pflag.FlagSet) { @@ -155,6 +190,8 @@ func ValidationServerConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".redis-url", DefaultValidationServerConfig.RedisURL, "url of redis server") f.String(prefix+".stream-prefix", DefaultValidationServerConfig.StreamPrefix, "prefix for stream name") f.Duration(prefix+".stream-timeout", DefaultValidationServerConfig.StreamTimeout, "Timeout on polling for existence of redis streams") + f.Int(prefix+".workers", DefaultValidationServerConfig.Workers, "number of validation threads (0 to use number of CPUs)") + f.Bool(prefix+".buffer-reads", DefaultValidationServerConfig.BufferReads, "buffer reads (read next while working)") } func (cfg *ValidationServerConfig) Enabled() bool { From d7ee2eed4b5e07c609d95d6cba60e625d9df67ae Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Fri, 30 Aug 2024 15:41:57 -0600 Subject: [PATCH 073/156] redis consumer: use tokens for buffer, default 0 --- validator/valnode/redis/consumer.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 8458ec021af..74564665336 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -64,11 +64,15 @@ func (s *ValidationServer) Start(ctx_in context.Context) { if workers == 0 { workers = runtime.NumCPU() } - bufferSize := 0 + workQueue := make(chan workUnit, workers) + tokensCount := workers if s.config.BufferReads { - bufferSize = workers + tokensCount += workers + } + requestTokenQueue := make(chan struct{}, tokensCount) + for i := 0; i < tokensCount; i++ { + requestTokenQueue <- struct{}{} } - workQueue := make(chan workUnit, bufferSize) for moduleRoot, c := range s.consumers { c := c moduleRoot := moduleRoot @@ -99,6 +103,11 @@ func (s *ValidationServer) Start(ctx_in context.Context) { case <-ready: // Wait until the stream exists and start consuming iteratively. } s.StopWaiter.CallIteratively(func(ctx context.Context) time.Duration { + select { + case <-ctx.Done(): + return 0 + case <-requestTokenQueue: + } req, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) @@ -147,6 +156,11 @@ func (s *ValidationServer) Start(ctx_in context.Context) { if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) } + select { + case <-ctx.Done(): + return + case requestTokenQueue <- struct{}{}: + } } }) } @@ -170,7 +184,7 @@ var DefaultValidationServerConfig = ValidationServerConfig{ ConsumerConfig: pubsub.DefaultConsumerConfig, ModuleRoots: []string{}, StreamTimeout: 10 * time.Minute, - Workers: 1, + Workers: 0, BufferReads: true, } From 0d08e481f039c778ff0f9a7c60f38167eadb29f2 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 30 Aug 2024 20:46:02 -0300 Subject: [PATCH 074/156] Also uses DataPoster when only creating validator wallet contract and exiting --- arbnode/node.go | 23 ++++++++++ cmd/nitro/nitro.go | 15 ++++++- staker/validatorwallet/contract.go | 72 +++++++++++++++--------------- system_tests/fast_confirm_test.go | 54 +++++++++++----------- system_tests/staker_test.go | 25 ++++++++--- 5 files changed, 119 insertions(+), 70 deletions(-) diff --git a/arbnode/node.go b/arbnode/node.go index 93b58e800f9..e13f92f30f3 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -339,6 +339,29 @@ func checkArbDbSchemaVersion(arbDb ethdb.Database) error { return nil } +func DataposterOnlyUsedToCreateValidatorWalletContract( + ctx context.Context, + l1Reader *headerreader.HeaderReader, + transactOpts *bind.TransactOpts, + cfg *dataposter.DataPosterConfig, + parentChainID *big.Int, +) (*dataposter.DataPoster, error) { + return dataposter.NewDataPoster(ctx, + &dataposter.DataPosterOpts{ + HeaderReader: l1Reader, + Auth: transactOpts, + Config: func() *dataposter.DataPosterConfig { + cfg.UseNoOpStorage = true + return cfg + }, + MetadataRetriever: func(ctx context.Context, blockNum *big.Int) ([]byte, error) { + return nil, nil + }, + ParentChainID: parentChainID, + }, + ) +} + func StakerDataposter( ctx context.Context, db ethdb.Database, l1Reader *headerreader.HeaderReader, transactOpts *bind.TransactOpts, cfgFetcher ConfigFetcher, syncMonitor *SyncMonitor, diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 8c26b95a927..bad48f430ca 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -371,8 +371,21 @@ func mainImpl() int { if err != nil { log.Crit("error getting rollup addresses config", "err", err) } + + dataPoster, err := arbnode.DataposterOnlyUsedToCreateValidatorWalletContract( + ctx, + l1Reader, + l1TransactionOptsBatchPoster, + &nodeConfig.Node.Staker.DataPoster, + new(big.Int).SetUint64(nodeConfig.ParentChain.ID), + ) + if err != nil { + log.Crit("error creating data poster to create validator wallet contract", "err", err) + } + getExtraGas := func() uint64 { return nodeConfig.Node.Staker.ExtraGas } + // #nosec G115 - addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, nil) + addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, dataPoster, getExtraGas) if err != nil { log.Crit("error creating validator wallet contract", "error", err, "address", l1TransactionOptsValidator.From.Hex()) } diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 495e796cd25..3db35d8232c 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -155,16 +155,19 @@ func (v *Contract) From() common.Address { } // nil value == 0 value -func (v *Contract) getAuth(ctx context.Context, value *big.Int) (*bind.TransactOpts, error) { - newAuth := *v.auth - newAuth.Context = ctx - newAuth.Value = value - nonce, err := v.L1Client().NonceAt(ctx, v.auth.From, nil) +func getAuthWithUpdatedNonce(ctx context.Context, l1Reader *headerreader.HeaderReader, auth bind.TransactOpts, value *big.Int) (*bind.TransactOpts, error) { + auth.Context = ctx + auth.Value = value + nonce, err := l1Reader.Client().NonceAt(ctx, auth.From, nil) if err != nil { return nil, err } - newAuth.Nonce = new(big.Int).SetUint64(nonce) - return &newAuth, nil + auth.Nonce = new(big.Int).SetUint64(nonce) + return &auth, nil +} + +func (v *Contract) getAuth(ctx context.Context, value *big.Int) (*bind.TransactOpts, error) { + return getAuthWithUpdatedNonce(ctx, v.l1Reader, *v.auth, value) } func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction, gasRefunder common.Address) (*types.Transaction, error) { @@ -183,9 +186,12 @@ func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), *v.Address(), data, gas, auth.Value) } -// Exposed for tests -func (v *Contract) CreateWalletContract( +func createWalletContract( ctx context.Context, + l1Reader *headerreader.HeaderReader, + auth *bind.TransactOpts, + dataPoster *dataposter.DataPoster, + getExtraGas func() uint64, validatorWalletFactoryAddr common.Address, ) (*types.Transaction, error) { var initialExecutorAllowedDests []common.Address @@ -194,24 +200,19 @@ func (v *Contract) CreateWalletContract( return nil, err } - auth, err := v.getAuth(ctx, nil) - if err != nil { - return nil, err - } - gas, err := gasForTxData( ctx, - v.l1Reader, + l1Reader, auth, &validatorWalletFactoryAddr, txData, - v.getExtraGas, + getExtraGas, ) if err != nil { return nil, fmt.Errorf("getting gas for tx data: %w", err) } - return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) + return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) } func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) error { @@ -225,15 +226,10 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err return nil } if v.address.Load() == nil { - auth, err := v.getAuth(ctx, nil) - if err != nil { - return err - } - - // By passing v.CreateWalletContract as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. + // By passing v.dataPoster as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. // DataPoster keeps in its internal state information related to the transactions sent through it, which is used to infer the expected nonce in a transaction for example. // If a transaction is sent using the Staker's DataPoster key, but not through the Staker's DataPoster object, DataPoster's internal state will be outdated, which can compromise the expected nonce inference. - addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing, v.CreateWalletContract) + addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, v.auth, v.l1Reader, createIfMissing, v.dataPoster, v.getExtraGas) if err != nil { return err } @@ -439,6 +435,11 @@ func (b *Contract) DataPoster() *dataposter.DataPoster { return b.dataPoster } +// Exported for testing +func (b *Contract) GetExtraGas() func() uint64 { + return b.getExtraGas +} + func GetValidatorWalletContract( ctx context.Context, validatorWalletFactoryAddr common.Address, @@ -446,7 +447,8 @@ func GetValidatorWalletContract( transactAuth *bind.TransactOpts, l1Reader *headerreader.HeaderReader, createIfMissing bool, - createWalletFunc func(context.Context, common.Address) (*types.Transaction, error), + dataPoster *dataposter.DataPoster, + getExtraGas func() uint64, ) (*common.Address, error) { client := l1Reader.Client() @@ -483,18 +485,14 @@ func GetValidatorWalletContract( return nil, nil } - var tx *types.Transaction - if createWalletFunc == nil { - var initialExecutorAllowedDests []common.Address - tx, err = walletCreator.CreateWallet(transactAuth, initialExecutorAllowedDests) - if err != nil { - return nil, err - } - } else { - tx, err = createWalletFunc(ctx, validatorWalletFactoryAddr) - if err != nil { - return nil, err - } + transactAuth, err = getAuthWithUpdatedNonce(ctx, l1Reader, *transactAuth, nil) + if err != nil { + return nil, err + } + + tx, err := createWalletContract(ctx, l1Reader, transactAuth, dataPoster, getExtraGas, validatorWalletFactoryAddr) + if err != nil { + return nil, err } receipt, err := l1Reader.WaitForTxApproval(ctx, tx) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index ae624be1e9b..45988274186 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -115,10 +115,10 @@ func TestFastConfirmation(t *testing.T) { Require(t, err) valConfig.Strategy = "MakeNodes" - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) if valWalletAddr == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) @@ -278,15 +278,6 @@ func TestFastConfirmationWithSafe(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - if valWalletAddrA == *valWalletAddrCheck { - Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) - } - rollup, err := rollupgen.NewRollupAdminLogic(l2nodeA.DeployInfo.Rollup, builder.L1.Client) Require(t, err) @@ -295,28 +286,13 @@ func TestFastConfirmationWithSafe(t *testing.T) { rollupABI, err := abi.JSON(strings.NewReader(rollupgen.RollupAdminLogicABI)) Require(t, err, "unable to parse rollup ABI") - safeAddress := deploySafe(t, builder.L1, builder.L1.Client, deployAuth, []common.Address{valWalletAddrA, srv.Address}) - setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address, safeAddress}, []bool{true, true, true, true}) - Require(t, err, "unable to generate setValidator calldata") - tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) - Require(t, err, "unable to set validators") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - setMinAssertPeriodCalldata, err := rollupABI.Pack("setMinimumAssertionPeriod", big.NewInt(1)) Require(t, err, "unable to generate setMinimumAssertionPeriod calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) + tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) Require(t, err, "unable to set minimum assertion period") _, err = builder.L1.EnsureTxSucceeded(tx) Require(t, err) - setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", safeAddress) - Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) - Require(t, err, "unable to set anytrust fast confirmer") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - valConfigA := staker.TestL1ValidatorConfig valConfigA.EnableFastConfirmation = true @@ -339,6 +315,30 @@ func TestFastConfirmationWithSafe(t *testing.T) { Require(t, err) valConfigA.Strategy = "MakeNodes" + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + Require(t, err) + valWalletAddrA := *valWalletAddrAPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + Require(t, err) + if valWalletAddrA == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) + } + + safeAddress := deploySafe(t, builder.L1, builder.L1.Client, deployAuth, []common.Address{valWalletAddrA, srv.Address}) + setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address, safeAddress}, []bool{true, true, true, true}) + Require(t, err, "unable to generate setValidator calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) + Require(t, err, "unable to set validators") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + + setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", safeAddress) + Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) + Require(t, err, "unable to set anytrust fast confirmer") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + _, valStack := createTestValidationNode(t, ctx, &valnode.TestValidationConfig) blockValidatorConfig := staker.TestBlockValidatorConfig diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index 3fdf82ca744..e61dcedc2b4 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -176,10 +176,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) valConfigA.Strategy = "MakeNodes" } - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) @@ -476,7 +476,7 @@ func TestStakersCooperative(t *testing.T) { stakerTestImpl(t, false, false) } -func TestGetValidatorWalletContractWithoutDataPoster(t *testing.T) { +func TestGetValidatorWalletContractWithDataposterOnlyUsedToCreateValidatorWalletContract(t *testing.T) { t.Parallel() ctx, cancelCtx := context.WithCancel(context.Background()) @@ -492,10 +492,25 @@ func TestGetValidatorWalletContractWithoutDataPoster(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "ValidatorA", balance, builder.L1Info) l1auth := builder.L1Info.GetDefaultTransactOpts("ValidatorA", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + parentChainID, err := builder.L1.Client.ChainID(ctx) + Require(t, err) + + dataPoster, err := arbnode.DataposterOnlyUsedToCreateValidatorWalletContract( + ctx, + builder.L2.ConsensusNode.L1Reader, + &l1auth, + &builder.nodeConfig.Staker.DataPoster, + parentChainID, + ) + if err != nil { + log.Crit("error creating data poster to create validator wallet contract", "err", err) + } + getExtraGas := func() uint64 { return builder.nodeConfig.Staker.ExtraGas } + + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) From 6eae487866df45f46db437141e30254c0eefa4c6 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 30 Aug 2024 20:47:15 -0300 Subject: [PATCH 075/156] Creates validator wallet contract with transaction with value set to zero --- staker/validatorwallet/contract.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 3db35d8232c..8a522b44a3d 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -212,7 +212,7 @@ func createWalletContract( return nil, fmt.Errorf("getting gas for tx data: %w", err) } - return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) + return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, common.Big0) } func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) error { From caadac62a7be06a2296a0c1c0db0330e5745d800 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 30 Aug 2024 21:05:48 -0300 Subject: [PATCH 076/156] Improves error message when getting gas to create validator wallet --- staker/validatorwallet/contract.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 8a522b44a3d..1910db1d637 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -209,7 +209,7 @@ func createWalletContract( getExtraGas, ) if err != nil { - return nil, fmt.Errorf("getting gas for tx data: %w", err) + return nil, fmt.Errorf("getting gas for tx data when creating validator wallet, validatorWalletFactory=%v: %w", validatorWalletFactoryAddr, err) } return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, common.Big0) From 716bfea74964a6f2e51a2a7f119a1d9f9d314149 Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Mon, 2 Sep 2024 12:23:03 +0200 Subject: [PATCH 077/156] Add Mac OSX environment variables The unfortunate thing is that it would still be good for developers to set these in their shell environment so that if they do end up running any "go" commands directly from the CLI, they won't be bombarded with all the linker warnings. But, at least this makes the default experience of using the make files a little nicer. --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index e46bdbbe612..0a71d64f12c 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,14 @@ ifneq ($(origin GOLANG_LDFLAGS),undefined) GOLANG_PARAMS = -ldflags="-extldflags '-ldl' $(GOLANG_LDFLAGS)" endif +UNAME_S := $(shell uname -s) + +# In Mac OSX, there are a lot of warnings emitted if these environment variables aren't set. +ifeq ($(UNAME_S), Darwin) + export MACOSX_DEPLOYMENT_TARGET := $(shell sw_vers -productVersion) + export CGO_LDFLAGS := -Wl,-no_warn_duplicate_libraries +endif + precompile_names = AddressTable Aggregator BLS Debug FunctionTable GasInfo Info osTest Owner RetryableTx Statistics Sys precompiles = $(patsubst %,./solgen/generated/%.go, $(precompile_names)) From 33337b8cd074c4a2ddf38dbcbe574e9712d1123f Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 10:39:38 -0300 Subject: [PATCH 078/156] Renames getAuthWithUpdatedNonce to getAuthWithUpdatedNonceFromL1 --- staker/validatorwallet/contract.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 1910db1d637..83dedcad841 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -155,7 +155,7 @@ func (v *Contract) From() common.Address { } // nil value == 0 value -func getAuthWithUpdatedNonce(ctx context.Context, l1Reader *headerreader.HeaderReader, auth bind.TransactOpts, value *big.Int) (*bind.TransactOpts, error) { +func getAuthWithUpdatedNonceFromL1(ctx context.Context, l1Reader *headerreader.HeaderReader, auth bind.TransactOpts, value *big.Int) (*bind.TransactOpts, error) { auth.Context = ctx auth.Value = value nonce, err := l1Reader.Client().NonceAt(ctx, auth.From, nil) @@ -167,7 +167,7 @@ func getAuthWithUpdatedNonce(ctx context.Context, l1Reader *headerreader.HeaderR } func (v *Contract) getAuth(ctx context.Context, value *big.Int) (*bind.TransactOpts, error) { - return getAuthWithUpdatedNonce(ctx, v.l1Reader, *v.auth, value) + return getAuthWithUpdatedNonceFromL1(ctx, v.l1Reader, *v.auth, value) } func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction, gasRefunder common.Address) (*types.Transaction, error) { @@ -485,7 +485,7 @@ func GetValidatorWalletContract( return nil, nil } - transactAuth, err = getAuthWithUpdatedNonce(ctx, l1Reader, *transactAuth, nil) + transactAuth, err = getAuthWithUpdatedNonceFromL1(ctx, l1Reader, *transactAuth, nil) if err != nil { return nil, err } From 9150808093cbfbc2fb98c9bb44e34b3b4ad927a7 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 16:12:49 -0300 Subject: [PATCH 079/156] Don't modify config inside config fetcher --- arbnode/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbnode/node.go b/arbnode/node.go index e13f92f30f3..a9da4ea24bb 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -346,12 +346,12 @@ func DataposterOnlyUsedToCreateValidatorWalletContract( cfg *dataposter.DataPosterConfig, parentChainID *big.Int, ) (*dataposter.DataPoster, error) { + cfg.UseNoOpStorage = true return dataposter.NewDataPoster(ctx, &dataposter.DataPosterOpts{ HeaderReader: l1Reader, Auth: transactOpts, Config: func() *dataposter.DataPosterConfig { - cfg.UseNoOpStorage = true return cfg }, MetadataRetriever: func(ctx context.Context, blockNum *big.Int) ([]byte, error) { From 1bc34f20ff8f21a7c860dc059393309fd11d1d00 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 2 Sep 2024 14:23:02 -0500 Subject: [PATCH 080/156] Close opened files explicitly instead of relying on the GC --- arbnode/resourcemanager/resource_management.go | 2 ++ cmd/nitro/init.go | 1 + 2 files changed, 3 insertions(+) diff --git a/arbnode/resourcemanager/resource_management.go b/arbnode/resourcemanager/resource_management.go index aba823cc252..249b6894436 100644 --- a/arbnode/resourcemanager/resource_management.go +++ b/arbnode/resourcemanager/resource_management.go @@ -256,6 +256,7 @@ func readIntFromFile(fileName string) (int, error) { if err != nil { return 0, err } + defer file.Close() var limit int if _, err = fmt.Fscanf(file, "%d", &limit); err != nil { @@ -269,6 +270,7 @@ func readFromMemStats(fileName string, re *regexp.Regexp) (int, error) { if err != nil { return 0, err } + defer file.Close() scanner := bufio.NewScanner(file) for scanner.Scan() { diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 26cd698efc8..51d895c0697 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -423,6 +423,7 @@ func extractSnapshot(archive string, location string, importWasm bool) error { if err != nil { return fmt.Errorf("couln't open init '%v' archive: %w", archive, err) } + defer reader.Close() stat, err := reader.Stat() if err != nil { return err From 0bf0b9accf2ca8b6e6ef81d2ac6f9fa67bd6bdaa Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 18:07:15 -0300 Subject: [PATCH 081/156] Uses auth from data poster in GetValidatorWalletContract --- cmd/nitro/nitro.go | 2 +- staker/validatorwallet/contract.go | 4 ++-- system_tests/fast_confirm_test.go | 8 ++++---- system_tests/staker_test.go | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index bad48f430ca..0cf81c45e9d 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -385,7 +385,7 @@ func mainImpl() int { getExtraGas := func() uint64 { return nodeConfig.Node.Staker.ExtraGas } // #nosec G115 - addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, dataPoster, getExtraGas) + addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1Reader, true, dataPoster, getExtraGas) if err != nil { log.Crit("error creating validator wallet contract", "error", err, "address", l1TransactionOptsValidator.From.Hex()) } diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 83dedcad841..6346029c3a3 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -229,7 +229,7 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err // By passing v.dataPoster as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. // DataPoster keeps in its internal state information related to the transactions sent through it, which is used to infer the expected nonce in a transaction for example. // If a transaction is sent using the Staker's DataPoster key, but not through the Staker's DataPoster object, DataPoster's internal state will be outdated, which can compromise the expected nonce inference. - addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, v.auth, v.l1Reader, createIfMissing, v.dataPoster, v.getExtraGas) + addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, v.l1Reader, createIfMissing, v.dataPoster, v.getExtraGas) if err != nil { return err } @@ -444,13 +444,13 @@ func GetValidatorWalletContract( ctx context.Context, validatorWalletFactoryAddr common.Address, fromBlock int64, - transactAuth *bind.TransactOpts, l1Reader *headerreader.HeaderReader, createIfMissing bool, dataPoster *dataposter.DataPoster, getExtraGas func() uint64, ) (*common.Address, error) { client := l1Reader.Client() + transactAuth := dataPoster.Auth() // TODO: If we just save a mapping in the wallet creator we won't need log search walletCreator, err := rollupgen.NewValidatorWalletCreator(validatorWalletFactoryAddr, client) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index 45988274186..dae2699b9f7 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -115,10 +115,10 @@ func TestFastConfirmation(t *testing.T) { Require(t, err) valConfig.Strategy = "MakeNodes" - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) if valWalletAddr == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) @@ -315,10 +315,10 @@ func TestFastConfirmationWithSafe(t *testing.T) { Require(t, err) valConfigA.Strategy = "MakeNodes" - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index e61dcedc2b4..67ce2605290 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -176,10 +176,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) valConfigA.Strategy = "MakeNodes" } - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) @@ -507,10 +507,10 @@ func TestGetValidatorWalletContractWithDataposterOnlyUsedToCreateValidatorWallet } getExtraGas := func() uint64 { return builder.nodeConfig.Staker.ExtraGas } - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) From 857073870e455354ba13aa99c6bf06b81ba846ba Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 18:08:05 -0300 Subject: [PATCH 082/156] Uses validator auth instead of batch poster auth when creating data poster to create validator wallet contract --- cmd/nitro/nitro.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 0cf81c45e9d..d4b2a87a304 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -375,7 +375,7 @@ func mainImpl() int { dataPoster, err := arbnode.DataposterOnlyUsedToCreateValidatorWalletContract( ctx, l1Reader, - l1TransactionOptsBatchPoster, + l1TransactionOptsValidator, &nodeConfig.Node.Staker.DataPoster, new(big.Int).SetUint64(nodeConfig.ParentChain.ID), ) From 8e4e8f9dafcb40057ab59516e2dbc7604ef6e39d Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 3 Sep 2024 10:34:17 -0500 Subject: [PATCH 083/156] Use a macro to define native WASM imports --- arbitrator/stylus/src/host.rs | 88 +++++++++++---- arbitrator/stylus/src/native.rs | 186 +++++++++----------------------- 2 files changed, 118 insertions(+), 156 deletions(-) diff --git a/arbitrator/stylus/src/host.rs b/arbitrator/stylus/src/host.rs index 1afc1b4e51e..ca48da35db6 100644 --- a/arbitrator/stylus/src/host.rs +++ b/arbitrator/stylus/src/host.rs @@ -441,26 +441,76 @@ pub(crate) fn pay_for_memory_grow>( hostio!(env, pay_for_memory_grow(pages)) } -pub(crate) fn console_log_text>( - mut env: WasmEnvMut, - ptr: GuestPtr, - len: u32, -) -> MaybeEscape { - hostio!(env, console_log_text(ptr, len)) -} +pub(crate) mod console { + use super::*; -pub(crate) fn console_log, T: Into>( - mut env: WasmEnvMut, - value: T, -) -> MaybeEscape { - hostio!(env, console_log(value)) -} + pub(crate) fn log_txt>( + mut env: WasmEnvMut, + ptr: GuestPtr, + len: u32, + ) -> MaybeEscape { + hostio!(env, console_log_text(ptr, len)) + } -pub(crate) fn console_tee, T: Into + Copy>( - mut env: WasmEnvMut, - value: T, -) -> Result { - hostio!(env, console_tee(value)) + pub(crate) fn log_i32>( + mut env: WasmEnvMut, + value: u32, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_i32>( + mut env: WasmEnvMut, + value: u32, + ) -> Result { + hostio!(env, console_tee(value)) + } + + pub(crate) fn log_i64>( + mut env: WasmEnvMut, + value: u64, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_i64>( + mut env: WasmEnvMut, + value: u64, + ) -> Result { + hostio!(env, console_tee(value)) + } + + pub(crate) fn log_f32>( + mut env: WasmEnvMut, + value: f32, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_f32>( + mut env: WasmEnvMut, + value: f32, + ) -> Result { + hostio!(env, console_tee(value)) + } + + pub(crate) fn log_f64>( + mut env: WasmEnvMut, + value: f64, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_f64>( + mut env: WasmEnvMut, + value: f64, + ) -> Result { + hostio!(env, console_tee(value)) + } } -pub(crate) fn null_host>(_: WasmEnvMut) {} +pub(crate) mod debug { + use super::*; + + pub(crate) fn null_host>(_: WasmEnvMut) {} +} diff --git a/arbitrator/stylus/src/native.rs b/arbitrator/stylus/src/native.rs index cc1d191fe2b..7a82314fbca 100644 --- a/arbitrator/stylus/src/native.rs +++ b/arbitrator/stylus/src/native.rs @@ -33,7 +33,7 @@ use std::{ ops::{Deref, DerefMut}, }; use wasmer::{ - imports, AsStoreMut, Function, FunctionEnv, Instance, Memory, Module, Pages, Store, Target, + AsStoreMut, Function, FunctionEnv, Imports, Instance, Memory, Module, Pages, Store, Target, TypedFunction, Value, WasmTypeList, }; use wasmer_vm::VMExtern; @@ -151,68 +151,58 @@ impl> NativeInstance { fn from_module(module: Module, mut store: Store, env: WasmEnv) -> Result { let debug_funcs = env.compile.debug.debug_funcs; let func_env = FunctionEnv::new(&mut store, env); + let mut imports = Imports::new(); macro_rules! func { - ($func:expr) => { - Function::new_typed_with_env(&mut store, &func_env, $func) + ($rust_mod:path, $func:ident) => {{ + use $rust_mod as rust_mod; + Function::new_typed_with_env(&mut store, &func_env, rust_mod::$func) + }}; + } + macro_rules! define_imports { + ($($wasm_mod:literal => $rust_mod:path { $( $import:ident ),* $(,)? },)* $(,)?) => { + $( + $( + define_imports!(@@imports $wasm_mod, func!($rust_mod, $import), $import, "arbitrator_forward__"); + )* + )* + }; + (@@imports $wasm_mod:literal, $func:expr, $import:ident, $($p:expr),*) => { + define_imports!(@imports $wasm_mod, $func, $import, $($p),*, ""); + }; + (@imports $wasm_mod:literal, $func:expr, $import:ident, $($p:expr),*) => { + $( + imports.define($wasm_mod, concat!($p, stringify!($import)), $func); + )* }; } - let mut imports = imports! { - "vm_hooks" => { - "read_args" => func!(host::read_args), - "write_result" => func!(host::write_result), - "exit_early" => func!(host::exit_early), - "storage_load_bytes32" => func!(host::storage_load_bytes32), - "storage_cache_bytes32" => func!(host::storage_cache_bytes32), - "storage_flush_cache" => func!(host::storage_flush_cache), - "transient_load_bytes32" => func!(host::transient_load_bytes32), - "transient_store_bytes32" => func!(host::transient_store_bytes32), - "call_contract" => func!(host::call_contract), - "delegate_call_contract" => func!(host::delegate_call_contract), - "static_call_contract" => func!(host::static_call_contract), - "create1" => func!(host::create1), - "create2" => func!(host::create2), - "read_return_data" => func!(host::read_return_data), - "return_data_size" => func!(host::return_data_size), - "emit_log" => func!(host::emit_log), - "account_balance" => func!(host::account_balance), - "account_code" => func!(host::account_code), - "account_codehash" => func!(host::account_codehash), - "account_code_size" => func!(host::account_code_size), - "evm_gas_left" => func!(host::evm_gas_left), - "evm_ink_left" => func!(host::evm_ink_left), - "block_basefee" => func!(host::block_basefee), - "chainid" => func!(host::chainid), - "block_coinbase" => func!(host::block_coinbase), - "block_gas_limit" => func!(host::block_gas_limit), - "block_number" => func!(host::block_number), - "block_timestamp" => func!(host::block_timestamp), - "contract_address" => func!(host::contract_address), - "math_div" => func!(host::math_div), - "math_mod" => func!(host::math_mod), - "math_pow" => func!(host::math_pow), - "math_add_mod" => func!(host::math_add_mod), - "math_mul_mod" => func!(host::math_mul_mod), - "msg_reentrant" => func!(host::msg_reentrant), - "msg_sender" => func!(host::msg_sender), - "msg_value" => func!(host::msg_value), - "tx_gas_price" => func!(host::tx_gas_price), - "tx_ink_price" => func!(host::tx_ink_price), - "tx_origin" => func!(host::tx_origin), - "pay_for_memory_grow" => func!(host::pay_for_memory_grow), - "native_keccak256" => func!(host::native_keccak256), + define_imports!( + "vm_hooks" => host { + read_args, write_result, exit_early, + storage_load_bytes32, storage_cache_bytes32, storage_flush_cache, transient_load_bytes32, transient_store_bytes32, + call_contract, delegate_call_contract, static_call_contract, create1, create2, read_return_data, return_data_size, + emit_log, + account_balance, account_code, account_codehash, account_code_size, + evm_gas_left, evm_ink_left, + block_basefee, chainid, block_coinbase, block_gas_limit, block_number, block_timestamp, + contract_address, + math_div, math_mod, math_pow, math_add_mod, math_mul_mod, + msg_reentrant, msg_sender, msg_value, + tx_gas_price, tx_ink_price, tx_origin, + pay_for_memory_grow, + native_keccak256, }, - }; + ); if debug_funcs { - imports.define("console", "log_txt", func!(host::console_log_text)); - imports.define("console", "log_i32", func!(host::console_log::)); - imports.define("console", "log_i64", func!(host::console_log::)); - imports.define("console", "log_f32", func!(host::console_log::)); - imports.define("console", "log_f64", func!(host::console_log::)); - imports.define("console", "tee_i32", func!(host::console_tee::)); - imports.define("console", "tee_i64", func!(host::console_tee::)); - imports.define("console", "tee_f32", func!(host::console_tee::)); - imports.define("console", "tee_f64", func!(host::console_tee::)); - imports.define("debug", "null_host", func!(host::null_host)); + define_imports!( + "console" => host::console { + log_txt, + log_i32, log_i64, log_f32, log_f64, + tee_i32, tee_i64, tee_f32, tee_f64, + }, + "debug" => host::debug { + null_host, + }, + ); } let instance = Instance::new(&mut store, &module, &imports)?; let exports = &instance.exports; @@ -351,86 +341,8 @@ impl> StartlessMachine for NativeInstance { } pub fn module(wasm: &[u8], compile: CompileConfig, target: Target) -> Result> { - let mut store = compile.store(target); + let store = compile.store(target); let module = Module::new(&store, wasm)?; - macro_rules! stub { - (u8 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> u8 { panic!("incomplete import") }) - }; - (u32 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> u32 { panic!("incomplete import") }) - }; - (u64 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> u64 { panic!("incomplete import") }) - }; - (f32 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> f32 { panic!("incomplete import") }) - }; - (f64 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> f64 { panic!("incomplete import") }) - }; - ($($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> () { panic!("incomplete import") }) - }; - } - let mut imports = imports! { - "vm_hooks" => { - "read_args" => stub!(|_: u32|), - "write_result" => stub!(|_: u32, _: u32|), - "exit_early" => stub!(|_: u32|), - "storage_load_bytes32" => stub!(|_: u32, _: u32|), - "storage_cache_bytes32" => stub!(|_: u32, _: u32|), - "storage_flush_cache" => stub!(|_: u32|), - "transient_load_bytes32" => stub!(|_: u32, _: u32|), - "transient_store_bytes32" => stub!(|_: u32, _: u32|), - "call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u32, _: u64, _: u32|), - "delegate_call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u64, _: u32|), - "static_call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u64, _: u32|), - "create1" => stub!(|_: u32, _: u32, _: u32, _: u32, _: u32|), - "create2" => stub!(|_: u32, _: u32, _: u32, _: u32, _: u32, _: u32|), - "read_return_data" => stub!(u32 <- |_: u32, _: u32, _: u32|), - "return_data_size" => stub!(u32 <- ||), - "emit_log" => stub!(|_: u32, _: u32, _: u32|), - "account_balance" => stub!(|_: u32, _: u32|), - "account_code" => stub!(u32 <- |_: u32, _: u32, _: u32, _: u32|), - "account_codehash" => stub!(|_: u32, _: u32|), - "account_code_size" => stub!(u32 <- |_: u32|), - "evm_gas_left" => stub!(u64 <- ||), - "evm_ink_left" => stub!(u64 <- ||), - "block_basefee" => stub!(|_: u32|), - "chainid" => stub!(u64 <- ||), - "block_coinbase" => stub!(|_: u32|), - "block_gas_limit" => stub!(u64 <- ||), - "block_number" => stub!(u64 <- ||), - "block_timestamp" => stub!(u64 <- ||), - "contract_address" => stub!(|_: u32|), - "math_div" => stub!(|_: u32, _: u32|), - "math_mod" => stub!(|_: u32, _: u32|), - "math_pow" => stub!(|_: u32, _: u32|), - "math_add_mod" => stub!(|_: u32, _: u32, _: u32|), - "math_mul_mod" => stub!(|_: u32, _: u32, _: u32|), - "msg_reentrant" => stub!(u32 <- ||), - "msg_sender" => stub!(|_: u32|), - "msg_value" => stub!(|_: u32|), - "tx_gas_price" => stub!(|_: u32|), - "tx_ink_price" => stub!(u32 <- ||), - "tx_origin" => stub!(|_: u32|), - "pay_for_memory_grow" => stub!(|_: u16|), - "native_keccak256" => stub!(|_: u32, _: u32, _: u32|), - }, - }; - if compile.debug.debug_funcs { - imports.define("console", "log_txt", stub!(|_: u32, _: u32|)); - imports.define("console", "log_i32", stub!(|_: u32|)); - imports.define("console", "log_i64", stub!(|_: u64|)); - imports.define("console", "log_f32", stub!(|_: f32|)); - imports.define("console", "log_f64", stub!(|_: f64|)); - imports.define("console", "tee_i32", stub!(u32 <- |_: u32|)); - imports.define("console", "tee_i64", stub!(u64 <- |_: u64|)); - imports.define("console", "tee_f32", stub!(f32 <- |_: f32|)); - imports.define("console", "tee_f64", stub!(f64 <- |_: f64|)); - imports.define("debug", "null_host", stub!(||)); - } let module = module.serialize()?; Ok(module.to_vec()) From b913d6b53779489593fe4289d4a2335c86ce2b51 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 3 Sep 2024 10:51:48 -0500 Subject: [PATCH 084/156] Remove unused import --- arbitrator/stylus/src/host.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/arbitrator/stylus/src/host.rs b/arbitrator/stylus/src/host.rs index ca48da35db6..fbe5657c5f4 100644 --- a/arbitrator/stylus/src/host.rs +++ b/arbitrator/stylus/src/host.rs @@ -13,7 +13,6 @@ use arbutil::{ }; use caller_env::GuestPtr; use eyre::Result; -use prover::value::Value; use std::{ fmt::Display, mem::{self, MaybeUninit}, From d39f362da5c86964a68992b984a732ad618bf16c Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 4 Sep 2024 14:58:22 -0600 Subject: [PATCH 085/156] redis consumer fix --- validator/valnode/redis/consumer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 74564665336..6432741eba3 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -111,10 +111,12 @@ func (s *ValidationServer) Start(ctx_in context.Context) { req, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) + requestTokenQueue <- struct{}{} return 0 } if req == nil { - // There's nothing in the queue. + // There's nothing in the queue + requestTokenQueue <- struct{}{} return time.Second } select { From 8d61504e109a89b12112ddbd4be94eb2753b0802 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 4 Sep 2024 15:00:53 -0600 Subject: [PATCH 086/156] redis consumer: dont set result if error --- validator/valnode/redis/consumer.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 6432741eba3..2b025600cc0 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -154,9 +154,10 @@ func (s *ValidationServer) Start(ctx_in context.Context) { res, err := valRun.Await(ctx) if err != nil { log.Error("Error validating", "request value", work.req.Value, "error", err) - } - if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { - log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } else { + if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { + log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } } select { case <-ctx.Done(): From 8f946b3d46c4db46b03389c3502ac4ddcae323e5 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 5 Sep 2024 07:48:00 -0600 Subject: [PATCH 087/156] block_validator: introduce recording-iter-limit --- staker/block_validator.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/staker/block_validator.go b/staker/block_validator.go index e6773a08541..7a7efca8460 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -106,6 +106,7 @@ type BlockValidatorConfig struct { ValidationServerConfigs []rpcclient.ClientConfig `koanf:"validation-server-configs"` ValidationPoll time.Duration `koanf:"validation-poll" reload:"hot"` PrerecordedBlocks uint64 `koanf:"prerecorded-blocks" reload:"hot"` + RecordingIterLimit uint64 `koanf:"recording-iter-limit"` ForwardBlocks uint64 `koanf:"forward-blocks" reload:"hot"` CurrentModuleRoot string `koanf:"current-module-root"` // TODO(magic) requires reinitialization on hot reload PendingUpgradeModuleRoot string `koanf:"pending-upgrade-module-root"` // TODO(magic) requires StatelessBlockValidator recreation on hot reload @@ -174,6 +175,7 @@ func BlockValidatorConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Uint64(prefix+".forward-blocks", DefaultBlockValidatorConfig.ForwardBlocks, "prepare entries for up to that many blocks ahead of validation (small footprint)") f.Uint64(prefix+".prerecorded-blocks", DefaultBlockValidatorConfig.PrerecordedBlocks, "record that many blocks ahead of validation (larger footprint)") f.String(prefix+".current-module-root", DefaultBlockValidatorConfig.CurrentModuleRoot, "current wasm module root ('current' read from chain, 'latest' from machines/latest dir, or provide hash)") + f.Uint64(prefix+".recording-iter-limit", DefaultBlockValidatorConfig.RecordingIterLimit, "limit on block recordings sent per iteration") f.String(prefix+".pending-upgrade-module-root", DefaultBlockValidatorConfig.PendingUpgradeModuleRoot, "pending upgrade wasm module root to additionally validate (hash, 'latest' or empty)") f.Bool(prefix+".failure-is-fatal", DefaultBlockValidatorConfig.FailureIsFatal, "failing a validation is treated as a fatal error") BlockValidatorDangerousConfigAddOptions(prefix+".dangerous", f) @@ -197,6 +199,7 @@ var DefaultBlockValidatorConfig = BlockValidatorConfig{ FailureIsFatal: true, Dangerous: DefaultBlockValidatorDangerousConfig, MemoryFreeLimit: "default", + RecordingIterLimit: 20, } var TestBlockValidatorConfig = BlockValidatorConfig{ @@ -207,6 +210,7 @@ var TestBlockValidatorConfig = BlockValidatorConfig{ ValidationPoll: 100 * time.Millisecond, ForwardBlocks: 128, PrerecordedBlocks: uint64(2 * runtime.NumCPU()), + RecordingIterLimit: 20, CurrentModuleRoot: "latest", PendingUpgradeModuleRoot: "latest", FailureIsFatal: true, @@ -652,6 +656,10 @@ func (v *BlockValidator) sendNextRecordRequests(ctx context.Context) (bool, erro if recordUntil < pos { return false, nil } + recordUntilLimit := pos + arbutil.MessageIndex(v.config().RecordingIterLimit) + if recordUntil > recordUntilLimit { + recordUntil = recordUntilLimit + } log.Trace("preparing to record", "pos", pos, "until", recordUntil) // prepare could take a long time so we do it without a lock err := v.recorder.PrepareForRecord(ctx, pos, recordUntil) From 0d5293d24de3c995609c063bdc8743622c6d0762 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 6 Sep 2024 13:43:11 +0530 Subject: [PATCH 088/156] Fix ArbWASM precompile minInitGas() method from reverting --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index f7894d3a6d4..a5c2d54561c 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit f7894d3a6d4035ba60f51a7f1334f0f2d4f02dce +Subproject commit a5c2d54561c2547e05d6118d6f072882c5fc7a9d From 19ef7eaf78ceb07ad6eb5ddf3a7df39b8e920f95 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 6 Sep 2024 09:56:11 -0500 Subject: [PATCH 089/156] Revert "Fix ArbWASM precompile minInitGas() method from reverting" --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index a5c2d54561c..f7894d3a6d4 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit a5c2d54561c2547e05d6118d6f072882c5fc7a9d +Subproject commit f7894d3a6d4035ba60f51a7f1334f0f2d4f02dce From 077213c4593ade8d5afac8eb54db5dd467cd033b Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 6 Sep 2024 11:39:09 -0500 Subject: [PATCH 090/156] Bump contracts to latest develop --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index f7894d3a6d4..23fc7962829 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit f7894d3a6d4035ba60f51a7f1334f0f2d4f02dce +Subproject commit 23fc79628292aa5d604d449fed48937ae7faeb2f From 7591f1c5a9625aaced402f3e5160217c6633a665 Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Sun, 8 Sep 2024 02:11:49 +0200 Subject: [PATCH 091/156] Temporarily increase the time for bridging balances This change allows the test to pass even on a heavily-loaded mac. Related to: NIT-2768 --- system_tests/common_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 457dae09101..caf749788ce 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -618,7 +618,7 @@ func BridgeBalance( break } TransferBalance(t, "Faucet", "User", big.NewInt(1), l1info, l1client, ctx) - if i > 20 { + if i > 200 { Fatal(t, "bridging failed") } <-time.After(time.Millisecond * 100) From bf7b6ac85058feab3e42f9f85edeb5618935b02b Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Sun, 8 Sep 2024 11:13:31 -0500 Subject: [PATCH 092/156] Resolve parent chain block by hash when resolving delayed messages --- arbnode/delayed.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arbnode/delayed.go b/arbnode/delayed.go index c166aa2b906..082f0ecf9db 100644 --- a/arbnode/delayed.go +++ b/arbnode/delayed.go @@ -215,7 +215,7 @@ func (b *DelayedBridge) logsToDeliveredMessages(ctx context.Context, logs []type } messages := make([]*DelayedInboxMessage, 0, len(logs)) - var lastParentChainBlockNumber uint64 + var lastParentChainBlockHash common.Hash var lastL1BlockNumber uint64 for _, parsedLog := range parsedLogs { msgKey := common.BigToHash(parsedLog.MessageIndex) @@ -228,17 +228,17 @@ func (b *DelayedBridge) logsToDeliveredMessages(ctx context.Context, logs []type } requestId := common.BigToHash(parsedLog.MessageIndex) - parentChainBlockNumber := parsedLog.Raw.BlockNumber + parentChainBlockHash := parsedLog.Raw.BlockHash var l1BlockNumber uint64 - if lastParentChainBlockNumber == parentChainBlockNumber && lastParentChainBlockNumber > 0 { + if lastParentChainBlockHash == parentChainBlockHash && lastParentChainBlockHash != (common.Hash{}) { l1BlockNumber = lastL1BlockNumber } else { - var err error - l1BlockNumber, err = arbutil.CorrespondingL1BlockNumber(ctx, b.client, parentChainBlockNumber) + parentChainHeader, err := b.client.HeaderByHash(ctx, parentChainBlockHash) if err != nil { return nil, err } - lastParentChainBlockNumber = parentChainBlockNumber + l1BlockNumber = arbutil.ParentHeaderToL1BlockNumber(parentChainHeader) + lastParentChainBlockHash = parentChainBlockHash lastL1BlockNumber = l1BlockNumber } msg := &DelayedInboxMessage{ From 8b9f4ae319d11630af5a5ff53ff770071f26691b Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 9 Sep 2024 12:09:46 -0500 Subject: [PATCH 093/156] Improve block validator test stability --- system_tests/block_validator_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/system_tests/block_validator_test.go b/system_tests/block_validator_test.go index eef6c29b7a3..cca5572f4e7 100644 --- a/system_tests/block_validator_test.go +++ b/system_tests/block_validator_test.go @@ -63,7 +63,6 @@ func testBlockValidatorSimple(t *testing.T, opts Options) { var delayEvery int if opts.workloadLoops > 1 { - l1NodeConfigA.BatchPoster.MaxDelay = time.Millisecond * 500 delayEvery = opts.workloadLoops / 3 } From 36dc6fc827fd6475c582e8a93433e6b33124c8f8 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Mon, 9 Sep 2024 12:34:23 -0600 Subject: [PATCH 094/156] fix activateProgramInternal --- arbos/programs/native.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 377e25a31e5..e834888b470 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -108,10 +108,10 @@ func activateProgramInternal( } results := make(chan result, len(targets)) for _, target := range targets { + target := target if target == rawdb.TargetWavm { results <- result{target, module, nil} } else { - target := target go func() { output := &rustBytes{} status_asm := C.stylus_compile( From 01e1c4af5b4dcd3f05c364bccc1f29e409ef25e9 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 9 Sep 2024 16:04:34 -0500 Subject: [PATCH 095/156] Fix new G115 lints in golangci-lint v1.61.0 --- arbnode/batch_poster.go | 13 +++++++--- arbnode/dataposter/data_poster.go | 10 ++++---- arbnode/dataposter/dataposter_test.go | 4 +-- arbnode/dataposter/dbstorage/storage.go | 4 +-- arbnode/dataposter/storage_test.go | 17 +++++++++--- arbnode/delayed_sequencer.go | 2 ++ arbnode/inbox_reader.go | 1 + arbnode/transaction_streamer.go | 17 +++++++----- arbos/activate_test.go | 1 + arbos/addressSet/addressSet.go | 1 + arbos/addressTable/addressTable.go | 1 + arbos/arbosState/initialization_test.go | 1 + arbos/arbosState/initialize.go | 4 +-- arbos/l1pricing_test.go | 2 ++ arbos/merkleAccumulator/merkleAccumulator.go | 1 + arbos/programs/data_pricer.go | 4 +-- arbos/retryable_test.go | 3 +++ arbos/retryables/retryable.go | 4 ++- arbos/storage/storage.go | 9 ++++--- arbutil/block_message_relation.go | 1 + blocks_reexecutor/blocks_reexecutor.go | 1 + broadcastclient/broadcastclient_test.go | 2 ++ broadcaster/broadcaster.go | 1 + broadcaster/message/message.go | 1 + cmd/datool/datool.go | 3 +++ cmd/nitro/init.go | 1 + cmd/nitro/nitro.go | 4 +++ cmd/pruning/pruning.go | 1 + das/aggregator.go | 1 + das/dasRpcClient.go | 1 + das/dasRpcServer.go | 4 +-- das/das_test.go | 2 ++ das/db_storage_service.go | 1 + das/factory.go | 8 +----- das/fallback_storage_service.go | 1 + das/local_file_storage_service.go | 1 + das/local_file_storage_service_test.go | 13 ++++++++++ das/redis_storage_service_test.go | 1 + das/redundant_storage_test.go | 1 + das/restful_server_test.go | 1 + das/rpc_aggregator.go | 2 +- das/s3_storage_service_test.go | 1 + das/simple_das_reader_aggregator_test.go | 4 +++ das/store_signing_test.go | 1 + das/syncing_fallback_storage.go | 1 + execution/gethexec/api.go | 4 +++ execution/gethexec/blockchain.go | 1 + execution/gethexec/sequencer.go | 4 +-- execution/nodeInterface/NodeInterface.go | 3 +++ precompiles/precompile.go | 3 +++ precompiles/precompile_test.go | 2 ++ relay/relay_stress_test.go | 1 + staker/challenge-cache/cache_test.go | 15 ++++++----- staker/challenge_manager.go | 1 + system_tests/bloom_test.go | 6 +++++ system_tests/common_test.go | 2 +- system_tests/estimation_test.go | 2 +- system_tests/nodeinterface_test.go | 2 ++ system_tests/outbox_test.go | 3 +++ system_tests/recreatestate_rpc_test.go | 5 ++++ system_tests/seqfeed_test.go | 4 +-- system_tests/seqinbox_test.go | 3 +++ system_tests/stylus_trace_test.go | 1 + util/arbmath/bips.go | 27 ++++++++++++++------ util/arbmath/math.go | 7 +++-- util/arbmath/math_test.go | 1 + util/headerreader/blob_client.go | 1 + util/merkletree/merkleEventProof_test.go | 1 + validator/server_arb/machine_cache.go | 11 ++++---- validator/server_jit/jit_machine.go | 1 + wavmio/stub.go | 13 +++++----- 71 files changed, 207 insertions(+), 75 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 6b4b95f8e00..44b360e76e1 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -168,7 +168,7 @@ type BatchPosterConfig struct { L1BlockBound string `koanf:"l1-block-bound" reload:"hot"` L1BlockBoundBypass time.Duration `koanf:"l1-block-bound-bypass" reload:"hot"` UseAccessLists bool `koanf:"use-access-lists" reload:"hot"` - GasEstimateBaseFeeMultipleBips arbmath.Bips `koanf:"gas-estimate-base-fee-multiple-bips"` + GasEstimateBaseFeeMultipleBips arbmath.UBips `koanf:"gas-estimate-base-fee-multiple-bips"` Dangerous BatchPosterDangerousConfig `koanf:"dangerous"` ReorgResistanceMargin time.Duration `koanf:"reorg-resistance-margin" reload:"hot"` CheckBatchCorrectness bool `koanf:"check-batch-correctness"` @@ -253,7 +253,7 @@ var DefaultBatchPosterConfig = BatchPosterConfig{ L1BlockBoundBypass: time.Hour, UseAccessLists: true, RedisLock: redislock.DefaultCfg, - GasEstimateBaseFeeMultipleBips: arbmath.OneInBips * 3 / 2, + GasEstimateBaseFeeMultipleBips: arbmath.OneInUBips * 3 / 2, ReorgResistanceMargin: 10 * time.Minute, CheckBatchCorrectness: true, } @@ -285,7 +285,7 @@ var TestBatchPosterConfig = BatchPosterConfig{ L1BlockBound: "", L1BlockBoundBypass: time.Hour, UseAccessLists: true, - GasEstimateBaseFeeMultipleBips: arbmath.OneInBips * 3 / 2, + GasEstimateBaseFeeMultipleBips: arbmath.OneInUBips * 3 / 2, CheckBatchCorrectness: true, } @@ -1035,7 +1035,7 @@ func (b *BatchPoster) estimateGas(ctx context.Context, sequencerMessage []byte, if err != nil { return 0, err } - maxFeePerGas := arbmath.BigMulByBips(latestHeader.BaseFee, config.GasEstimateBaseFeeMultipleBips) + maxFeePerGas := arbmath.BigMulByUBips(latestHeader.BaseFee, config.GasEstimateBaseFeeMultipleBips) if useNormalEstimation { _, realBlobHashes, err := blobs.ComputeCommitmentsAndHashes(realBlobs) if err != nil { @@ -1250,7 +1250,9 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) l1BoundMinTimestamp = arbmath.SaturatingUSub(latestHeader.Time, arbmath.BigToUintSaturating(maxTimeVariationDelaySeconds)) if config.L1BlockBoundBypass > 0 { + // #nosec G115 blockNumberWithPadding := arbmath.SaturatingUAdd(latestBlockNumber, uint64(config.L1BlockBoundBypass/ethPosBlockTime)) + // #nosec G115 timestampWithPadding := arbmath.SaturatingUAdd(latestHeader.Time, uint64(config.L1BlockBoundBypass/time.Second)) l1BoundMinBlockNumberWithBypass = arbmath.SaturatingUSub(blockNumberWithPadding, arbmath.BigToUintSaturating(maxTimeVariationDelayBlocks)) l1BoundMinTimestampWithBypass = arbmath.SaturatingUSub(timestampWithPadding, arbmath.BigToUintSaturating(maxTimeVariationDelaySeconds)) @@ -1316,7 +1318,9 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) if hasL1Bound && config.ReorgResistanceMargin > 0 { firstMsgBlockNumber := firstMsg.Message.Header.BlockNumber firstMsgTimeStamp := firstMsg.Message.Header.Timestamp + // #nosec G115 batchNearL1BoundMinBlockNumber := firstMsgBlockNumber <= arbmath.SaturatingUAdd(l1BoundMinBlockNumber, uint64(config.ReorgResistanceMargin/ethPosBlockTime)) + // #nosec G115 batchNearL1BoundMinTimestamp := firstMsgTimeStamp <= arbmath.SaturatingUAdd(l1BoundMinTimestamp, uint64(config.ReorgResistanceMargin/time.Second)) if batchNearL1BoundMinTimestamp || batchNearL1BoundMinBlockNumber { log.Error( @@ -1361,6 +1365,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) batchPosterDAFailureCounter.Inc(1) return false, fmt.Errorf("%w: nonce changed from %d to %d while creating batch", storage.ErrStorageRace, nonce, gotNonce) } + // #nosec G115 sequencerMsg, err = b.dapWriter.Store(ctx, sequencerMsg, uint64(time.Now().Add(config.DASRetentionPeriod).Unix()), config.DisableDapFallbackStoreDataOnChain) if err != nil { batchPosterDAFailureCounter.Inc(1) diff --git a/arbnode/dataposter/data_poster.go b/arbnode/dataposter/data_poster.go index 6a483929b26..31b9a983dbf 100644 --- a/arbnode/dataposter/data_poster.go +++ b/arbnode/dataposter/data_poster.go @@ -638,11 +638,11 @@ func (p *DataPoster) feeAndTipCaps(ctx context.Context, nonce uint64, gasLimit u if config.MaxFeeBidMultipleBips > 0 { // Limit the fee caps to be no greater than max(MaxFeeBidMultipleBips, minRbf) - maxNonBlobFee := arbmath.BigMulByBips(currentNonBlobFee, config.MaxFeeBidMultipleBips) + maxNonBlobFee := arbmath.BigMulByUBips(currentNonBlobFee, config.MaxFeeBidMultipleBips) if lastTx != nil { maxNonBlobFee = arbmath.BigMax(maxNonBlobFee, arbmath.BigMulByBips(lastTx.GasFeeCap(), minRbfIncrease)) } - maxBlobFee := arbmath.BigMulByBips(currentBlobFee, config.MaxFeeBidMultipleBips) + maxBlobFee := arbmath.BigMulByUBips(currentBlobFee, config.MaxFeeBidMultipleBips) if lastTx != nil && lastTx.BlobGasFeeCap() != nil { maxBlobFee = arbmath.BigMax(maxBlobFee, arbmath.BigMulByBips(lastTx.BlobGasFeeCap(), minRbfIncrease)) } @@ -1241,7 +1241,7 @@ type DataPosterConfig struct { MinBlobTxTipCapGwei float64 `koanf:"min-blob-tx-tip-cap-gwei" reload:"hot"` MaxTipCapGwei float64 `koanf:"max-tip-cap-gwei" reload:"hot"` MaxBlobTxTipCapGwei float64 `koanf:"max-blob-tx-tip-cap-gwei" reload:"hot"` - MaxFeeBidMultipleBips arbmath.Bips `koanf:"max-fee-bid-multiple-bips" reload:"hot"` + MaxFeeBidMultipleBips arbmath.UBips `koanf:"max-fee-bid-multiple-bips" reload:"hot"` NonceRbfSoftConfs uint64 `koanf:"nonce-rbf-soft-confs" reload:"hot"` AllocateMempoolBalance bool `koanf:"allocate-mempool-balance" reload:"hot"` UseDBStorage bool `koanf:"use-db-storage"` @@ -1345,7 +1345,7 @@ var DefaultDataPosterConfig = DataPosterConfig{ MinBlobTxTipCapGwei: 1, // default geth minimum, and relays aren't likely to accept lower values given propagation time MaxTipCapGwei: 1.2, MaxBlobTxTipCapGwei: 1, // lower than normal because 4844 rbf is a minimum of a 2x - MaxFeeBidMultipleBips: arbmath.OneInBips * 10, + MaxFeeBidMultipleBips: arbmath.OneInUBips * 10, NonceRbfSoftConfs: 1, AllocateMempoolBalance: true, UseDBStorage: true, @@ -1380,7 +1380,7 @@ var TestDataPosterConfig = DataPosterConfig{ MinBlobTxTipCapGwei: 1, MaxTipCapGwei: 5, MaxBlobTxTipCapGwei: 1, - MaxFeeBidMultipleBips: arbmath.OneInBips * 10, + MaxFeeBidMultipleBips: arbmath.OneInUBips * 10, NonceRbfSoftConfs: 1, AllocateMempoolBalance: true, UseDBStorage: false, diff --git a/arbnode/dataposter/dataposter_test.go b/arbnode/dataposter/dataposter_test.go index 7f2f61c07e3..d2c49427bef 100644 --- a/arbnode/dataposter/dataposter_test.go +++ b/arbnode/dataposter/dataposter_test.go @@ -204,7 +204,7 @@ func TestFeeAndTipCaps_EnoughBalance_NoBacklog_NoUnconfirmed_BlobTx(t *testing.T MinBlobTxTipCapGwei: 1, MaxTipCapGwei: 5, MaxBlobTxTipCapGwei: 10, - MaxFeeBidMultipleBips: arbmath.OneInBips * 10, + MaxFeeBidMultipleBips: arbmath.OneInUBips * 10, AllocateMempoolBalance: true, UrgencyGwei: 2., @@ -335,7 +335,7 @@ func TestFeeAndTipCaps_RBF_RisingBlobFee_FallingBaseFee(t *testing.T) { MinBlobTxTipCapGwei: 1, MaxTipCapGwei: 5, MaxBlobTxTipCapGwei: 10, - MaxFeeBidMultipleBips: arbmath.OneInBips * 10, + MaxFeeBidMultipleBips: arbmath.OneInUBips * 10, AllocateMempoolBalance: true, UrgencyGwei: 2., diff --git a/arbnode/dataposter/dbstorage/storage.go b/arbnode/dataposter/dbstorage/storage.go index 37ebfa50991..6a6cd3cfa43 100644 --- a/arbnode/dataposter/dbstorage/storage.go +++ b/arbnode/dataposter/dbstorage/storage.go @@ -95,11 +95,11 @@ func (s *Storage) PruneAll(ctx context.Context) error { if err != nil { return fmt.Errorf("pruning all keys: %w", err) } - until, err := strconv.Atoi(string(idx)) + until, err := strconv.ParseUint(string(idx), 10, 64) if err != nil { return fmt.Errorf("converting last item index bytes to integer: %w", err) } - return s.Prune(ctx, uint64(until+1)) + return s.Prune(ctx, until+1) } func (s *Storage) Prune(ctx context.Context, until uint64) error { diff --git a/arbnode/dataposter/storage_test.go b/arbnode/dataposter/storage_test.go index 8934d92b456..c6316caea7e 100644 --- a/arbnode/dataposter/storage_test.go +++ b/arbnode/dataposter/storage_test.go @@ -72,24 +72,29 @@ func newRedisStorage(ctx context.Context, t *testing.T, encF storage.EncoderDeco func valueOf(t *testing.T, i int) *storage.QueuedTransaction { t.Helper() + // #nosec G115 meta, err := rlp.EncodeToBytes(storage.BatchPosterPosition{DelayedMessageCount: uint64(i)}) if err != nil { t.Fatalf("Encoding batch poster position, error: %v", err) } return &storage.QueuedTransaction{ FullTx: types.NewTransaction( + // #nosec G115 uint64(i), common.Address{}, big.NewInt(int64(i)), + // #nosec G115 uint64(i), big.NewInt(int64(i)), []byte{byte(i)}), Meta: meta, DeprecatedData: types.DynamicFeeTx{ - ChainID: big.NewInt(int64(i)), - Nonce: uint64(i), - GasTipCap: big.NewInt(int64(i)), - GasFeeCap: big.NewInt(int64(i)), + ChainID: big.NewInt(int64(i)), + // #nosec G115 + Nonce: uint64(i), + GasTipCap: big.NewInt(int64(i)), + GasFeeCap: big.NewInt(int64(i)), + // #nosec G115 Gas: uint64(i), Value: big.NewInt(int64(i)), Data: []byte{byte(i % 8)}, @@ -113,6 +118,7 @@ func values(t *testing.T, from, to int) []*storage.QueuedTransaction { func initStorage(ctx context.Context, t *testing.T, s QueueStorage) QueueStorage { t.Helper() for i := 0; i < 20; i++ { + // #nosec G115 if err := s.Put(ctx, uint64(i), nil, valueOf(t, i)); err != nil { t.Fatalf("Error putting a key/value: %v", err) } @@ -153,6 +159,7 @@ func TestPruneAll(t *testing.T) { s := newLevelDBStorage(t, func() storage.EncoderDecoderInterface { return &storage.EncoderDecoder{} }) ctx := context.Background() for i := 0; i < 20; i++ { + // #nosec G115 if err := s.Put(ctx, uint64(i), nil, valueOf(t, i)); err != nil { t.Fatalf("Error putting a key/value: %v", err) } @@ -236,6 +243,7 @@ func TestLast(t *testing.T) { ctx := context.Background() for i := 0; i < cnt; i++ { val := valueOf(t, i) + // #nosec G115 if err := s.Put(ctx, uint64(i), nil, val); err != nil { t.Fatalf("Error putting a key/value: %v", err) } @@ -255,6 +263,7 @@ func TestLast(t *testing.T) { for i := 0; i < cnt-1; i++ { prev := valueOf(t, i) newVal := valueOf(t, cnt+i) + // #nosec G115 if err := s.Put(ctx, uint64(i), prev, newVal); err != nil { t.Fatalf("Error putting a key/value: %v, prev: %v, new: %v", err, prev, newVal) } diff --git a/arbnode/delayed_sequencer.go b/arbnode/delayed_sequencer.go index 4f18531a764..b29a66dd054 100644 --- a/arbnode/delayed_sequencer.go +++ b/arbnode/delayed_sequencer.go @@ -121,6 +121,7 @@ func (d *DelayedSequencer) sequenceWithoutLockout(ctx context.Context, lastBlock if currentNum < config.FinalizeDistance { return nil } + // #nosec G115 finalized = uint64(currentNum - config.FinalizeDistance) } @@ -189,6 +190,7 @@ func (d *DelayedSequencer) sequenceWithoutLockout(ctx context.Context, lastBlock return fmt.Errorf("inbox reader at delayed message %v db accumulator %v doesn't match delayed bridge accumulator %v at L1 block %v", pos-1, lastDelayedAcc, delayedBridgeAcc, finalized) } for i, msg := range messages { + // #nosec G115 err = d.exec.SequenceDelayedMessage(msg, startPos+uint64(i)) if err != nil { return err diff --git a/arbnode/inbox_reader.go b/arbnode/inbox_reader.go index fd050b5f67b..c596cfa9b0d 100644 --- a/arbnode/inbox_reader.go +++ b/arbnode/inbox_reader.go @@ -542,6 +542,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { } else { from = arbmath.BigAddByUint(to, 1) } + // #nosec G115 haveMessages := uint64(len(delayedMessages) + len(sequencerBatches)) if haveMessages <= (config.TargetMessagesRead / 2) { blocksToFetch += (blocksToFetch + 4) / 5 diff --git a/arbnode/transaction_streamer.go b/arbnode/transaction_streamer.go index a5bab8342f9..24a0564b976 100644 --- a/arbnode/transaction_streamer.go +++ b/arbnode/transaction_streamer.go @@ -279,6 +279,7 @@ func (s *TransactionStreamer) reorg(batch ethdb.Batch, count arbutil.MessageInde return err } config := s.config() + // #nosec G115 maxResequenceMsgCount := count + arbutil.MessageIndex(config.MaxReorgResequenceDepth) if config.MaxReorgResequenceDepth >= 0 && maxResequenceMsgCount < targetMsgCount { log.Error( @@ -388,6 +389,7 @@ func (s *TransactionStreamer) reorg(batch ethdb.Batch, count arbutil.MessageInde } for i := 0; i < len(messagesResults); i++ { + // #nosec G115 pos := count + arbutil.MessageIndex(i) err = s.storeResult(pos, *messagesResults[i], batch) if err != nil { @@ -680,7 +682,7 @@ func (s *TransactionStreamer) AddMessagesAndEndBatch(pos arbutil.MessageIndex, m if err != nil { return err } - if dups == len(messages) { + if dups == uint64(len(messages)) { return endBatch(batch) } // cant keep reorg lock when catching insertionMutex. @@ -715,10 +717,10 @@ func (s *TransactionStreamer) countDuplicateMessages( pos arbutil.MessageIndex, messages []arbostypes.MessageWithMetadataAndBlockHash, batch *ethdb.Batch, -) (int, bool, *arbostypes.MessageWithMetadata, error) { - curMsg := 0 +) (uint64, bool, *arbostypes.MessageWithMetadata, error) { + var curMsg uint64 for { - if len(messages) == curMsg { + if uint64(len(messages)) == curMsg { break } key := dbKey(messagePrefix, uint64(pos)) @@ -818,7 +820,7 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil broadcastStartPos := arbutil.MessageIndex(s.broadcasterQueuedMessagesPos.Load()) if messagesAreConfirmed { - var duplicates int + var duplicates uint64 var err error duplicates, confirmedReorg, oldMsg, err = s.countDuplicateMessages(messageStartPos, messages, &batch) if err != nil { @@ -857,7 +859,7 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil var feedReorg bool if !hasNewConfirmedMessages { - var duplicates int + var duplicates uint64 var err error duplicates, feedReorg, oldMsg, err = s.countDuplicateMessages(messageStartPos, messages, nil) if err != nil { @@ -889,6 +891,7 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil // Validate delayed message counts of remaining messages for i, msg := range messages { + // #nosec G115 msgPos := messageStartPos + arbutil.MessageIndex(i) diff := msg.MessageWithMeta.DelayedMessagesRead - lastDelayedRead if diff != 0 && diff != 1 { @@ -924,6 +927,7 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil // Check if new messages were added at the end of cache, if they were, then dont remove those particular messages if len(s.broadcasterQueuedMessages) > cacheClearLen { s.broadcasterQueuedMessages = s.broadcasterQueuedMessages[cacheClearLen:] + // #nosec G115 s.broadcasterQueuedMessagesPos.Store(uint64(broadcastStartPos) + uint64(cacheClearLen)) } else { s.broadcasterQueuedMessages = s.broadcasterQueuedMessages[:0] @@ -1044,6 +1048,7 @@ func (s *TransactionStreamer) writeMessages(pos arbutil.MessageIndex, messages [ batch = s.db.NewBatch() } for i, msg := range messages { + // #nosec G115 err := s.writeMessage(pos+arbutil.MessageIndex(i), msg, batch) if err != nil { return err diff --git a/arbos/activate_test.go b/arbos/activate_test.go index 55440bb208b..a89a38639a9 100644 --- a/arbos/activate_test.go +++ b/arbos/activate_test.go @@ -20,6 +20,7 @@ func TestActivationDataFee(t *testing.T) { rand.Seed(time.Now().UTC().UnixNano()) state, _ := arbosState.NewArbosMemoryBackedArbOSState() pricer := state.Programs().DataPricer() + // #nosec G115 time := uint64(time.Now().Unix()) assert := func(cond bool) { diff --git a/arbos/addressSet/addressSet.go b/arbos/addressSet/addressSet.go index 1f09ff1440f..156f36e7e77 100644 --- a/arbos/addressSet/addressSet.go +++ b/arbos/addressSet/addressSet.go @@ -79,6 +79,7 @@ func (as *AddressSet) AllMembers(maxNumToReturn uint64) ([]common.Address, error } ret := make([]common.Address, size) for i := range ret { + // #nosec G115 sba := as.backingStorage.OpenStorageBackedAddress(uint64(i + 1)) ret[i], err = sba.Get() if err != nil { diff --git a/arbos/addressTable/addressTable.go b/arbos/addressTable/addressTable.go index 566c71b6895..6ae271060da 100644 --- a/arbos/addressTable/addressTable.go +++ b/arbos/addressTable/addressTable.go @@ -103,6 +103,7 @@ func (atab *AddressTable) Decompress(buf []byte) (common.Address, uint64, error) return common.Address{}, 0, err } if len(input) == 20 { + // #nosec G115 numBytesRead := uint64(rd.Size() - int64(rd.Len())) return common.BytesToAddress(input), numBytesRead, nil } else { diff --git a/arbos/arbosState/initialization_test.go b/arbos/arbosState/initialization_test.go index b0fe1d0dac4..5e605b8bd26 100644 --- a/arbos/arbosState/initialization_test.go +++ b/arbos/arbosState/initialization_test.go @@ -126,6 +126,7 @@ func checkAddressTable(arbState *ArbosState, addrTable []common.Address, t *test Fail(t) } for i, addr := range addrTable { + // #nosec G115 res, exists, err := atab.LookupIndex(uint64(i)) Require(t, err) if !exists { diff --git a/arbos/arbosState/initialize.go b/arbos/arbosState/initialize.go index 56fa579c153..427bdc30877 100644 --- a/arbos/arbosState/initialize.go +++ b/arbos/arbosState/initialize.go @@ -108,7 +108,7 @@ func InitializeArbosInDatabase(db ethdb.Database, cacheConfig *core.CacheConfig, if err != nil { return common.Hash{}, err } - for i := 0; addressReader.More(); i++ { + for i := uint64(0); addressReader.More(); i++ { addr, err := addressReader.GetNext() if err != nil { return common.Hash{}, err @@ -117,7 +117,7 @@ func InitializeArbosInDatabase(db ethdb.Database, cacheConfig *core.CacheConfig, if err != nil { return common.Hash{}, err } - if uint64(i) != slot { + if i != slot { return common.Hash{}, errors.New("address table slot mismatch") } } diff --git a/arbos/l1pricing_test.go b/arbos/l1pricing_test.go index 1cda4b3d82b..6f9e3ecb356 100644 --- a/arbos/l1pricing_test.go +++ b/arbos/l1pricing_test.go @@ -279,7 +279,9 @@ func _testL1PriceEquilibration(t *testing.T, initialL1BasefeeEstimate *big.Int, evm.StateDB, evm, 3, + // #nosec G115 uint64(10*(i+1)), + // #nosec G115 uint64(10*(i+1)+5), bpAddr, arbmath.BigMulByUint(equilibriumL1BasefeeEstimate, unitsToAdd), diff --git a/arbos/merkleAccumulator/merkleAccumulator.go b/arbos/merkleAccumulator/merkleAccumulator.go index 2e060c58400..e62303e5fd7 100644 --- a/arbos/merkleAccumulator/merkleAccumulator.go +++ b/arbos/merkleAccumulator/merkleAccumulator.go @@ -97,6 +97,7 @@ func (acc *MerkleAccumulator) GetPartials() ([]*common.Hash, error) { } partials := make([]*common.Hash, CalcNumPartials(size)) for i := range partials { + // #nosec G115 p, err := acc.getPartial(uint64(i)) if err != nil { return nil, err diff --git a/arbos/programs/data_pricer.go b/arbos/programs/data_pricer.go index ed7c98556db..d82aa81f045 100644 --- a/arbos/programs/data_pricer.go +++ b/arbos/programs/data_pricer.go @@ -83,8 +83,8 @@ func (p *DataPricer) UpdateModel(tempBytes uint32, time uint64) (*big.Int, error } exponent := arbmath.OneInBips * arbmath.Bips(demand) / arbmath.Bips(inertia) - multiplier := arbmath.ApproxExpBasisPoints(exponent, 12).Uint64() - costPerByte := arbmath.SaturatingUMul(uint64(minPrice), multiplier) / 10000 + multiplier := arbmath.ApproxExpBasisPoints(exponent, 12) + costPerByte := arbmath.UintSaturatingMulByBips(uint64(minPrice), multiplier) costInWei := arbmath.SaturatingUMul(costPerByte, uint64(tempBytes)) return arbmath.UintToBig(costInWei), nil } diff --git a/arbos/retryable_test.go b/arbos/retryable_test.go index ddb88348ddf..2eccaea6c2a 100644 --- a/arbos/retryable_test.go +++ b/arbos/retryable_test.go @@ -38,6 +38,7 @@ func TestRetryableLifecycle(t *testing.T) { retryableState := state.RetryableState() lifetime := uint64(retryables.RetryableLifetimeSeconds) + // #nosec G115 timestampAtCreation := uint64(rand.Int63n(1 << 16)) timeoutAtCreation := timestampAtCreation + lifetime currentTime := timeoutAtCreation @@ -57,6 +58,7 @@ func TestRetryableLifecycle(t *testing.T) { checkQueueSize := func(expected int, message string) { timeoutQueueSize, err := retryableState.TimeoutQueue.Size() Require(t, err) + // #nosec G115 if timeoutQueueSize != uint64(expected) { Fail(t, currentTime, message, timeoutQueueSize) } @@ -167,6 +169,7 @@ func TestRetryableCleanup(t *testing.T) { callvalue := big.NewInt(0) calldata := testhelpers.RandomizeSlice(make([]byte, rand.Intn(1<<12))) + // #nosec G115 timeout := uint64(rand.Int63n(1 << 16)) timestamp := 2 * timeout diff --git a/arbos/retryables/retryable.go b/arbos/retryables/retryable.go index e1cfe48bcf3..5938244782a 100644 --- a/arbos/retryables/retryable.go +++ b/arbos/retryables/retryable.go @@ -367,5 +367,7 @@ func RetryableEscrowAddress(ticketId common.Hash) common.Address { } func RetryableSubmissionFee(calldataLengthInBytes int, l1BaseFee *big.Int) *big.Int { - return arbmath.BigMulByUint(l1BaseFee, uint64(1400+6*calldataLengthInBytes)) + // This can't overflow because calldataLengthInBytes would need to be 3 exabytes + // #nosec G115 + return arbmath.BigMulByUint(l1BaseFee, 1400+6*uint64(calldataLengthInBytes)) } diff --git a/arbos/storage/storage.go b/arbos/storage/storage.go index 352726778d0..bc16491af03 100644 --- a/arbos/storage/storage.go +++ b/arbos/storage/storage.go @@ -322,11 +322,11 @@ func (s *Storage) Burner() burn.Burner { } func (s *Storage) Keccak(data ...[]byte) ([]byte, error) { - byteCount := 0 + var byteCount uint64 for _, part := range data { - byteCount += len(part) + byteCount += uint64(len(part)) } - cost := 30 + 6*arbmath.WordsForBytes(uint64(byteCount)) + cost := 30 + 6*arbmath.WordsForBytes(byteCount) if err := s.burner.Burn(cost); err != nil { return nil, err } @@ -420,6 +420,7 @@ func (sbu *StorageBackedInt64) Get() (int64, error) { } func (sbu *StorageBackedInt64) Set(value int64) error { + // #nosec G115 return sbu.StorageSlot.Set(util.UintToHash(uint64(value))) // see implementation note above } @@ -456,7 +457,7 @@ func (sbu *StorageBackedUBips) Get() (arbmath.UBips, error) { } func (sbu *StorageBackedUBips) Set(bips arbmath.UBips) error { - return sbu.backing.Set(bips.Uint64()) + return sbu.backing.Set(uint64(bips)) } type StorageBackedUint16 struct { diff --git a/arbutil/block_message_relation.go b/arbutil/block_message_relation.go index e164cf2619a..dcf4c86084e 100644 --- a/arbutil/block_message_relation.go +++ b/arbutil/block_message_relation.go @@ -11,6 +11,7 @@ func BlockNumberToMessageCount(blockNumber uint64, genesisBlockNumber uint64) Me // Block number must correspond to a message count, meaning it may not be less than -1 func SignedBlockNumberToMessageCount(blockNumber int64, genesisBlockNumber uint64) MessageIndex { + // #nosec G115 return MessageIndex(uint64(blockNumber+1) - genesisBlockNumber) } diff --git a/blocks_reexecutor/blocks_reexecutor.go b/blocks_reexecutor/blocks_reexecutor.go index f7cc0d8c720..b43999a7dbc 100644 --- a/blocks_reexecutor/blocks_reexecutor.go +++ b/blocks_reexecutor/blocks_reexecutor.go @@ -113,6 +113,7 @@ func New(c *Config, blockchain *core.BlockChain, fatalErrChan chan error) *Block } // Divide work equally among available threads when BlocksPerThread is zero if c.BlocksPerThread == 0 { + // #nosec G115 work := (end - start) / uint64(c.Room) if work > 0 { blocksPerThread = work diff --git a/broadcastclient/broadcastclient_test.go b/broadcastclient/broadcastclient_test.go index 44b48192abb..a499628cd5c 100644 --- a/broadcastclient/broadcastclient_test.go +++ b/broadcastclient/broadcastclient_test.go @@ -105,6 +105,7 @@ func testReceiveMessages(t *testing.T, clientCompression bool, serverCompression go func() { for i := 0; i < messageCount; i++ { + // #nosec G115 Require(t, b.BroadcastSingle(arbostypes.TestMessageWithMetadataAndRequestId, arbutil.MessageIndex(i), nil)) } }() @@ -156,6 +157,7 @@ func TestInvalidSignature(t *testing.T) { go func() { for i := 0; i < messageCount; i++ { + // #nosec G115 Require(t, b.BroadcastSingle(arbostypes.TestMessageWithMetadataAndRequestId, arbutil.MessageIndex(i), nil)) } }() diff --git a/broadcaster/broadcaster.go b/broadcaster/broadcaster.go index 397698635af..4fe8657bfa5 100644 --- a/broadcaster/broadcaster.go +++ b/broadcaster/broadcaster.go @@ -104,6 +104,7 @@ func (b *Broadcaster) BroadcastMessages( }() var feedMessages []*m.BroadcastFeedMessage for i, msg := range messagesWithBlockHash { + // #nosec G115 bfm, err := b.NewBroadcastFeedMessage(msg.MessageWithMeta, seq+arbutil.MessageIndex(i), msg.BlockHash) if err != nil { return err diff --git a/broadcaster/message/message.go b/broadcaster/message/message.go index aca95987540..1e26e6da5e6 100644 --- a/broadcaster/message/message.go +++ b/broadcaster/message/message.go @@ -41,6 +41,7 @@ type BroadcastFeedMessage struct { } func (m *BroadcastFeedMessage) Size() uint64 { + // #nosec G115 return uint64(len(m.Signature) + len(m.Message.Message.L2msg) + 160) } diff --git a/cmd/datool/datool.go b/cmd/datool/datool.go index ba60cbbd4d9..f791d8cbc4e 100644 --- a/cmd/datool/datool.go +++ b/cmd/datool/datool.go @@ -166,8 +166,10 @@ func startClientStore(args []string) error { if err != nil { return err } + // #nosec G115 cert, err = client.Store(ctx, message, uint64(time.Now().Add(config.DASRetentionPeriod).Unix())) } else if len(config.Message) > 0 { + // #nosec G115 cert, err = client.Store(ctx, []byte(config.Message), uint64(time.Now().Add(config.DASRetentionPeriod).Unix())) } else { return errors.New("--message or --random-message-size must be specified") @@ -363,6 +365,7 @@ func dumpKeyset(args []string) error { return err } + // #nosec G115 keysetHash, keysetBytes, err := das.KeysetHashFromServices(services, uint64(config.Keyset.AssumedHonest)) if err != nil { return err diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index a8463a7d210..fc59f2d231a 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -876,6 +876,7 @@ func testUpdateTxIndex(chainDb ethdb.Database, chainConfig *params.ChainConfig, localWg.Add(1) go func() { batch := chainDb.NewBatch() + // #nosec G115 for blockNum := uint64(thread); blockNum <= lastBlock; blockNum += uint64(threads) { blockHash := rawdb.ReadCanonicalHash(chainDb, blockNum) block := rawdb.ReadBlock(chainDb, blockHash, blockNum) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 146a0049e72..e66d99b56e7 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -892,6 +892,7 @@ func ParseNode(ctx context.Context, args []string) (*NodeConfig, *genericconf.Wa l2ChainInfoIpfsDownloadPath := k.String("chain.info-ipfs-download-path") l2ChainInfoFiles := k.Strings("chain.info-files") l2ChainInfoJson := k.String("chain.info-json") + // #nosec G115 err = applyChainParameters(ctx, k, uint64(l2ChainId), l2ChainName, l2ChainInfoFiles, l2ChainInfoJson, l2ChainInfoIpfsUrl, l2ChainInfoIpfsDownloadPath) if err != nil { return nil, nil, err @@ -1037,13 +1038,16 @@ func applyChainParameters(ctx context.Context, k *koanf.Koanf, chainId uint64, c func initReorg(initConfig conf.InitConfig, chainConfig *params.ChainConfig, inboxTracker *arbnode.InboxTracker) error { var batchCount uint64 if initConfig.ReorgToBatch >= 0 { + // #nosec G115 batchCount = uint64(initConfig.ReorgToBatch) + 1 } else { var messageIndex arbutil.MessageIndex if initConfig.ReorgToMessageBatch >= 0 { + // #nosec G115 messageIndex = arbutil.MessageIndex(initConfig.ReorgToMessageBatch) } else if initConfig.ReorgToBlockBatch > 0 { genesis := chainConfig.ArbitrumChainParams.GenesisBlockNum + // #nosec G115 blockNum := uint64(initConfig.ReorgToBlockBatch) if blockNum < genesis { return fmt.Errorf("ReorgToBlockBatch %d before genesis %d", blockNum, genesis) diff --git a/cmd/pruning/pruning.go b/cmd/pruning/pruning.go index 096bb4b1aec..6fc7741478c 100644 --- a/cmd/pruning/pruning.go +++ b/cmd/pruning/pruning.go @@ -212,6 +212,7 @@ func findImportantRoots(ctx context.Context, chainDb ethdb.Database, stack *node } if meta.ParentChainBlock <= l1BlockNum { signedBlockNum := arbutil.MessageCountToBlockNumber(meta.MessageCount, genesisNum) + // #nosec G115 blockNum := uint64(signedBlockNum) l2Hash := rawdb.ReadCanonicalHash(chainDb, blockNum) l2Header := rawdb.ReadHeader(chainDb, l2Hash, blockNum) diff --git a/das/aggregator.go b/das/aggregator.go index e8972447ad6..e7460fa371a 100644 --- a/das/aggregator.go +++ b/das/aggregator.go @@ -130,6 +130,7 @@ func NewAggregatorWithSeqInboxCaller( seqInboxCaller *bridgegen.SequencerInboxCaller, ) (*Aggregator, error) { + // #nosec G115 keysetHash, keysetBytes, err := KeysetHashFromServices(services, uint64(config.RPCAggregator.AssumedHonest)) if err != nil { return nil, err diff --git a/das/dasRpcClient.go b/das/dasRpcClient.go index 635696bdab2..7d48ed796d8 100644 --- a/das/dasRpcClient.go +++ b/das/dasRpcClient.go @@ -58,6 +58,7 @@ func NewDASRPCClient(target string, signer signature.DataSignerFunc, maxStoreChu } func (c *DASRPCClient) Store(ctx context.Context, message []byte, timeout uint64) (*daprovider.DataAvailabilityCertificate, error) { + // #nosec G115 timestamp := uint64(time.Now().Unix()) nChunks := uint64(len(message)) / c.chunkSize lastChunkSize := uint64(len(message)) % c.chunkSize diff --git a/das/dasRpcServer.go b/das/dasRpcServer.go index d14766cc7e9..bb1be0384ed 100644 --- a/das/dasRpcServer.go +++ b/das/dasRpcServer.go @@ -153,7 +153,7 @@ type SendChunkResult struct { type batch struct { chunks [][]byte expectedChunks uint64 - seenChunks atomic.Int64 + seenChunks atomic.Uint64 expectedChunkSize, expectedSize uint64 timeout uint64 startTime time.Time @@ -248,7 +248,7 @@ func (b *batchBuilder) close(id uint64) ([]byte, uint64, time.Time, error) { return nil, 0, time.Time{}, fmt.Errorf("unknown batch(%d)", id) } - if batch.expectedChunks != uint64(batch.seenChunks.Load()) { + if batch.expectedChunks != batch.seenChunks.Load() { return nil, 0, time.Time{}, fmt.Errorf("incomplete batch(%d): got %d/%d chunks", id, batch.seenChunks.Load(), batch.expectedChunks) } diff --git a/das/das_test.go b/das/das_test.go index 179734c8b11..4971d454e54 100644 --- a/das/das_test.go +++ b/das/das_test.go @@ -55,6 +55,7 @@ func testDASStoreRetrieveMultipleInstances(t *testing.T, storageType string) { Require(t, err, "no das") var daReader DataAvailabilityServiceReader = storageService + // #nosec G115 timeout := uint64(time.Now().Add(time.Hour * 24).Unix()) messageSaved := []byte("hello world") cert, err := daWriter.Store(firstCtx, messageSaved, timeout) @@ -146,6 +147,7 @@ func testDASMissingMessage(t *testing.T, storageType string) { var daReader DataAvailabilityServiceReader = storageService messageSaved := []byte("hello world") + // #nosec G115 timeout := uint64(time.Now().Add(time.Hour * 24).Unix()) cert, err := daWriter.Store(ctx, messageSaved, timeout) Require(t, err, "Error storing message") diff --git a/das/db_storage_service.go b/das/db_storage_service.go index 1d9e5348d46..74bf12b927e 100644 --- a/das/db_storage_service.go +++ b/das/db_storage_service.go @@ -267,6 +267,7 @@ func (dbs *DBStorageService) String() string { func (dbs *DBStorageService) HealthCheck(ctx context.Context) error { testData := []byte("Test-Data") + // #nosec G115 err := dbs.Put(ctx, testData, uint64(time.Now().Add(time.Minute).Unix())) if err != nil { return err diff --git a/das/factory.go b/das/factory.go index 5742a394794..7f696912b3c 100644 --- a/das/factory.go +++ b/das/factory.go @@ -7,7 +7,6 @@ import ( "context" "errors" "fmt" - "math" "github.com/ethereum/go-ethereum/common" @@ -187,12 +186,7 @@ func CreateDAComponentsForDaserver( dasLifecycleManager.Register(restAgg) syncConf := &config.RestAggregator.SyncToStorage - var retentionPeriodSeconds uint64 - if uint64(syncConf.RetentionPeriod) == math.MaxUint64 { - retentionPeriodSeconds = math.MaxUint64 - } else { - retentionPeriodSeconds = uint64(syncConf.RetentionPeriod.Seconds()) - } + retentionPeriodSeconds := uint64(syncConf.RetentionPeriod.Seconds()) if syncConf.Eager { if l1Reader == nil || seqInboxAddress == nil { diff --git a/das/fallback_storage_service.go b/das/fallback_storage_service.go index 49f961da60d..0a451678d08 100644 --- a/das/fallback_storage_service.go +++ b/das/fallback_storage_service.go @@ -85,6 +85,7 @@ func (f *FallbackStorageService) GetByHash(ctx context.Context, key common.Hash) } if dastree.ValidHash(key, data) { putErr := f.StorageService.Put( + // #nosec G115 ctx, data, arbmath.SaturatingUAdd(uint64(time.Now().Unix()), f.backupRetentionSeconds), ) if putErr != nil && !f.ignoreRetentionWriteErrors { diff --git a/das/local_file_storage_service.go b/das/local_file_storage_service.go index ce86786718d..5e64c34b100 100644 --- a/das/local_file_storage_service.go +++ b/das/local_file_storage_service.go @@ -377,6 +377,7 @@ func migrate(fl *flatLayout, tl *trieLayout) error { return err } + // #nosec G115 expiryPath := tl.expiryPath(batch.key, uint64(batch.expiry.Unix())) if err = createHardLink(newPath, expiryPath); err != nil { return err diff --git a/das/local_file_storage_service_test.go b/das/local_file_storage_service_test.go index 01b999f3563..8a36664670e 100644 --- a/das/local_file_storage_service_test.go +++ b/das/local_file_storage_service_test.go @@ -78,6 +78,7 @@ func TestMigrationNoExpiry(t *testing.T) { Require(t, err) s.enableLegacyLayout = true + // #nosec G115 now := uint64(time.Now().Unix()) err = s.Put(ctx, []byte("a"), now+1) @@ -121,14 +122,19 @@ func TestMigrationExpiry(t *testing.T) { now := time.Now() // Use increments of expiry divisor in order to span multiple by-expiry-timestamp dirs + // #nosec G115 err = s.Put(ctx, []byte("a"), uint64(now.Add(-2*time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("b"), uint64(now.Add(-1*time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("c"), uint64(now.Add(time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("d"), uint64(now.Add(time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("e"), uint64(now.Add(2*time.Second*expiryDivisor).Unix())) Require(t, err) @@ -171,19 +177,26 @@ func TestExpiryDuplicates(t *testing.T) { now := time.Now() // Use increments of expiry divisor in order to span multiple by-expiry-timestamp dirs + // #nosec G115 err = s.Put(ctx, []byte("a"), uint64(now.Add(-2*time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("a"), uint64(now.Add(-1*time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("a"), uint64(now.Add(time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("d"), uint64(now.Add(time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("e"), uint64(now.Add(2*time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("f"), uint64(now.Add(3*time.Second*expiryDivisor).Unix())) Require(t, err) // Put the same entry and expiry again, should have no effect + // #nosec G115 err = s.Put(ctx, []byte("f"), uint64(now.Add(3*time.Second*expiryDivisor).Unix())) Require(t, err) diff --git a/das/redis_storage_service_test.go b/das/redis_storage_service_test.go index 55f3ecd82c9..77d3e8cd0f5 100644 --- a/das/redis_storage_service_test.go +++ b/das/redis_storage_service_test.go @@ -16,6 +16,7 @@ import ( func TestRedisStorageService(t *testing.T) { ctx := context.Background() + // #nosec G115 timeout := uint64(time.Now().Add(time.Hour).Unix()) baseStorageService := NewMemoryBackedStorageService(ctx) server, err := miniredis.Run() diff --git a/das/redundant_storage_test.go b/das/redundant_storage_test.go index b56f62ee248..11d3b58264e 100644 --- a/das/redundant_storage_test.go +++ b/das/redundant_storage_test.go @@ -17,6 +17,7 @@ const NumServices = 3 func TestRedundantStorageService(t *testing.T) { ctx := context.Background() + // #nosec G115 timeout := uint64(time.Now().Add(time.Hour).Unix()) services := []StorageService{} for i := 0; i < NumServices; i++ { diff --git a/das/restful_server_test.go b/das/restful_server_test.go index 1d3675749a6..e6982f9db5a 100644 --- a/das/restful_server_test.go +++ b/das/restful_server_test.go @@ -48,6 +48,7 @@ func TestRestfulClientServer(t *testing.T) { server, port, err := NewRestfulDasServerOnRandomPort(LocalServerAddressForTest, storage) Require(t, err) + // #nosec G115 err = storage.Put(ctx, data, uint64(time.Now().Add(time.Hour).Unix())) Require(t, err) diff --git a/das/rpc_aggregator.go b/das/rpc_aggregator.go index 24a470be5bf..1b3e2b8f447 100644 --- a/das/rpc_aggregator.go +++ b/das/rpc_aggregator.go @@ -119,7 +119,7 @@ func ParseServices(config AggregatorConfig, signer signature.DataSignerFunc) ([] return nil, err } - d, err := NewServiceDetails(service, *pubKey, 1< uint64(countsNum) && sectionNum > 1 { break } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 457dae09101..7a78cee309e 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -671,7 +671,7 @@ func l2MessageBatchDataFromTxes(txes types.Transactions) ([]byte, error) { if err != nil { return nil, err } - binary.BigEndian.PutUint64(sizeBuf, uint64(len(txBytes)+1)) + binary.BigEndian.PutUint64(sizeBuf, uint64(len(txBytes))+1) l2Message = append(l2Message, sizeBuf...) l2Message = append(l2Message, arbos.L2MessageKind_SignedTx) l2Message = append(l2Message, txBytes...) diff --git a/system_tests/estimation_test.go b/system_tests/estimation_test.go index 284c709fad3..62857023429 100644 --- a/system_tests/estimation_test.go +++ b/system_tests/estimation_test.go @@ -214,7 +214,7 @@ func TestComponentEstimate(t *testing.T) { userBalance := big.NewInt(1e16) maxPriorityFeePerGas := big.NewInt(0) - maxFeePerGas := arbmath.BigMulByUfrac(l2BaseFee, 3, 2) + maxFeePerGas := arbmath.BigMulByUFrac(l2BaseFee, 3, 2) builder.L2Info.GenerateAccount("User") builder.L2.TransferBalance(t, "Owner", "User", userBalance, builder.L2Info) diff --git a/system_tests/nodeinterface_test.go b/system_tests/nodeinterface_test.go index 17bfb18892e..927dc1b6304 100644 --- a/system_tests/nodeinterface_test.go +++ b/system_tests/nodeinterface_test.go @@ -163,6 +163,7 @@ func TestGetL1Confirmations(t *testing.T) { numTransactions := 200 + // #nosec G115 if l1Confs >= uint64(numTransactions) { t.Fatalf("L1Confirmations for latest block %v is already %v (over %v)", genesisBlock.Number(), l1Confs, numTransactions) } @@ -175,6 +176,7 @@ func TestGetL1Confirmations(t *testing.T) { Require(t, err) // Allow a gap of 10 for asynchronicity, just in case + // #nosec G115 if l1Confs+10 < uint64(numTransactions) { t.Fatalf("L1Confirmations for latest block %v is only %v (did not hit expected %v)", genesisBlock.Number(), l1Confs, numTransactions) } diff --git a/system_tests/outbox_test.go b/system_tests/outbox_test.go index c68df6ea229..25c52396f93 100644 --- a/system_tests/outbox_test.go +++ b/system_tests/outbox_test.go @@ -175,6 +175,7 @@ func TestOutboxProofs(t *testing.T) { sibling := place ^ which position := merkletree.LevelAndLeaf{ + // #nosec G115 Level: uint64(level), Leaf: sibling, } @@ -201,6 +202,7 @@ func TestOutboxProofs(t *testing.T) { leaf := total - 1 // preceding it. We subtract 1 since we count from 0 partial := merkletree.LevelAndLeaf{ + // #nosec G115 Level: uint64(level), Leaf: leaf, } @@ -289,6 +291,7 @@ func TestOutboxProofs(t *testing.T) { step.Leaf += 1 << step.Level // we start on the min partial's zero-hash sibling known[step] = zero + // #nosec G115 for step.Level < uint64(treeLevels) { curr, ok := known[step] diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index 7b09552e28a..4833d355361 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -362,12 +362,14 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig Require(t, err) l2info.GenerateAccount("User2") + // #nosec G115 for i := genesis; i < uint64(txCount)+genesis; i++ { tx := l2info.PrepareTx("Owner", "User2", l2info.TransferGas, common.Big1, nil) err := client.SendTransaction(ctx, tx) Require(t, err) receipt, err := EnsureTxSucceeded(ctx, client, tx) Require(t, err) + // #nosec G115 if have, want := receipt.BlockNumber.Uint64(), uint64(i)+1; have != want { Fatal(t, "internal test error - tx got included in unexpected block number, have:", have, "want:", want) } @@ -378,6 +380,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig Fatal(t, "missing current block") } lastBlock := currentHeader.Number.Uint64() + // #nosec G115 if want := genesis + uint64(txCount); lastBlock < want { Fatal(t, "internal test error - not enough blocks produced during preparation, want:", want, "have:", lastBlock) } @@ -391,6 +394,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig bc = builder.L2.ExecNode.Backend.ArbInterface().BlockChain() gas := skipGas blocks := skipBlocks + // #nosec G115 for i := genesis; i <= genesis+uint64(txCount); i++ { block := bc.GetBlockByNumber(i) if block == nil { @@ -423,6 +427,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig } } } + // #nosec G115 for i := genesis + 1; i <= genesis+uint64(txCount); i += i % 10 { _, err = client.BalanceAt(ctx, GetTestAddressForAccountName(t, "User2"), new(big.Int).SetUint64(i)) if err != nil { diff --git a/system_tests/seqfeed_test.go b/system_tests/seqfeed_test.go index 5e70fdf0982..21f07552250 100644 --- a/system_tests/seqfeed_test.go +++ b/system_tests/seqfeed_test.go @@ -164,12 +164,12 @@ func compareAllMsgResultsFromConsensusAndExecution( } var lastResult *execution.MessageResult - for msgCount := 1; arbutil.MessageIndex(msgCount) <= consensusMsgCount; msgCount++ { + for msgCount := arbutil.MessageIndex(1); msgCount <= consensusMsgCount; msgCount++ { pos := msgCount - 1 resultExec, err := testClient.ExecNode.ResultAtPos(arbutil.MessageIndex(pos)) Require(t, err) - resultConsensus, err := testClient.ConsensusNode.TxStreamer.ResultAtCount(arbutil.MessageIndex(msgCount)) + resultConsensus, err := testClient.ConsensusNode.TxStreamer.ResultAtCount(msgCount) Require(t, err) if !reflect.DeepEqual(resultExec, resultConsensus) { diff --git a/system_tests/seqinbox_test.go b/system_tests/seqinbox_test.go index 6babe5833f0..a9f66b0e2f9 100644 --- a/system_tests/seqinbox_test.go +++ b/system_tests/seqinbox_test.go @@ -265,6 +265,7 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { for j := 0; j < numMessages; j++ { sourceNum := rand.Int() % len(state.accounts) source := state.accounts[sourceNum] + // #nosec G115 amount := new(big.Int).SetUint64(uint64(rand.Int()) % state.balances[source].Uint64()) reserveAmount := new(big.Int).SetUint64(l2pricing.InitialBaseFeeWei * 100000000) if state.balances[source].Cmp(new(big.Int).Add(amount, reserveAmount)) < 0 { @@ -314,6 +315,7 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { for j := 0; ; j++ { haveNonce, err := builder.L1.Client.PendingNonceAt(ctx, seqOpts.From) Require(t, err) + // #nosec G115 if haveNonce == uint64(seqNonce) { break } @@ -380,6 +382,7 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { t.Errorf("Transaction: %v was not refunded, balance diff: %v, cost: %v", tx.Hash(), diff, txCost) } + // #nosec G115 state.l2BlockNumber += uint64(numMessages) state.l1BlockNumber = txRes.BlockNumber.Uint64() blockStates = append(blockStates, state) diff --git a/system_tests/stylus_trace_test.go b/system_tests/stylus_trace_test.go index cb303874d61..5c4463d9f77 100644 --- a/system_tests/stylus_trace_test.go +++ b/system_tests/stylus_trace_test.go @@ -76,6 +76,7 @@ func sendAndTraceTransaction( } func intToBytes(v int) []byte { + // #nosec G115 return binary.BigEndian.AppendUint64(nil, uint64(v)) } diff --git a/util/arbmath/bips.go b/util/arbmath/bips.go index 646dad3a92d..39b014f3ac8 100644 --- a/util/arbmath/bips.go +++ b/util/arbmath/bips.go @@ -27,24 +27,35 @@ func BigMulByBips(value *big.Int, bips Bips) *big.Int { return BigMulByFrac(value, int64(bips), int64(OneInBips)) } +func BigMulByUBips(value *big.Int, bips UBips) *big.Int { + return BigMulByUFrac(value, uint64(bips), uint64(OneInUBips)) +} + func IntMulByBips(value int64, bips Bips) int64 { return value * int64(bips) / int64(OneInBips) } +// UintMulByBips multiplies a uint value by a bips value +// bips must be positive and not cause an overflow func UintMulByBips(value uint64, bips Bips) uint64 { + // #nosec G115 return value * uint64(bips) / uint64(OneInBips) } -func SaturatingCastToBips(value uint64) Bips { - return Bips(SaturatingCast[int64](value)) -} - -func (bips UBips) Uint64() uint64 { - return uint64(bips) +// UintSaturatingMulByBips multiplies a uint value by a bips value, +// saturating at the maximum bips value (not the maximum uint64 result), +// then rounding down and returning a uint64. +// Returns 0 if bips is less than or equal to zero +func UintSaturatingMulByBips(value uint64, bips Bips) uint64 { + if bips <= 0 { + return 0 + } + // #nosec G115 + return SaturatingUMul(value, uint64(bips)) / uint64(OneInBips) } -func (bips Bips) Uint64() uint64 { - return uint64(bips) +func SaturatingCastToBips(value uint64) Bips { + return Bips(SaturatingCast[int64](value)) } // BigDivToBips returns dividend/divisor as bips, saturating if out of bounds diff --git a/util/arbmath/math.go b/util/arbmath/math.go index e5bed67f6df..07a9941b659 100644 --- a/util/arbmath/math.go +++ b/util/arbmath/math.go @@ -29,6 +29,7 @@ func NextOrCurrentPowerOf2(value uint64) uint64 { // Log2ceil the log2 of the int, rounded up func Log2ceil(value uint64) uint64 { + // #nosec G115 return uint64(64 - bits.LeadingZeros64(value)) } @@ -228,8 +229,8 @@ func BigMulByFrac(value *big.Int, numerator, denominator int64) *big.Int { return value } -// BigMulByUfrac multiply a huge by a rational whose components are non-negative -func BigMulByUfrac(value *big.Int, numerator, denominator uint64) *big.Int { +// BigMulByUFrac multiply a huge by a rational whose components are non-negative +func BigMulByUFrac(value *big.Int, numerator, denominator uint64) *big.Int { value = new(big.Int).Set(value) value.Mul(value, new(big.Int).SetUint64(numerator)) value.Div(value, new(big.Int).SetUint64(denominator)) @@ -407,6 +408,8 @@ func ApproxExpBasisPoints(value Bips, accuracy uint64) Bips { if negative { input = -value } + // This cast is safe because input is always positive + // #nosec G115 x := uint64(input) bips := uint64(OneInBips) diff --git a/util/arbmath/math_test.go b/util/arbmath/math_test.go index 528666dc19e..3660f3657e1 100644 --- a/util/arbmath/math_test.go +++ b/util/arbmath/math_test.go @@ -44,6 +44,7 @@ func TestMath(t *testing.T) { // try the first million sqrts for i := 0; i < 1000000; i++ { + // #nosec G115 input := uint64(i) approx := ApproxSquareRoot(input) correct := math.Sqrt(float64(input)) diff --git a/util/headerreader/blob_client.go b/util/headerreader/blob_client.go index 2b47a940c31..160323cf603 100644 --- a/util/headerreader/blob_client.go +++ b/util/headerreader/blob_client.go @@ -191,6 +191,7 @@ func (b *BlobClient) blobSidecars(ctx context.Context, slot uint64, versionedHas rawData, err := beaconRequest[json.RawMessage](b, ctx, fmt.Sprintf("/eth/v1/beacon/blob_sidecars/%d", slot)) if err != nil || len(rawData) == 0 { // blobs are pruned after 4096 epochs (1 epoch = 32 slots), we determine if the requested slot were to be pruned by a non-archive endpoint + // #nosec G115 roughAgeOfSlot := uint64(time.Now().Unix()) - (b.genesisTime + slot*b.secondsPerSlot) if roughAgeOfSlot > b.secondsPerSlot*32*4096 { return nil, fmt.Errorf("beacon client in blobSidecars got error or empty response fetching older blobs in slot: %d, an archive endpoint is required, please refer to https://docs.arbitrum.io/run-arbitrum-node/l1-ethereum-beacon-chain-rpc-providers, err: %w", slot, err) diff --git a/util/merkletree/merkleEventProof_test.go b/util/merkletree/merkleEventProof_test.go index b64cc88c2a9..6af84791904 100644 --- a/util/merkletree/merkleEventProof_test.go +++ b/util/merkletree/merkleEventProof_test.go @@ -22,6 +22,7 @@ func initializedMerkleAccumulatorForTesting() *merkleAccumulator.MerkleAccumulat func TestProofForNext(t *testing.T) { leaves := make([]common.Hash, 13) for i := range leaves { + // #nosec G115 leaves[i] = pseudorandomForTesting(uint64(i)) } diff --git a/validator/server_arb/machine_cache.go b/validator/server_arb/machine_cache.go index 55ef61cf11d..35f34062364 100644 --- a/validator/server_arb/machine_cache.go +++ b/validator/server_arb/machine_cache.go @@ -31,7 +31,7 @@ type MachineCache struct { } type MachineCacheConfig struct { - CachedChallengeMachines int `koanf:"cached-challenge-machines"` + CachedChallengeMachines uint64 `koanf:"cached-challenge-machines"` InitialSteps uint64 `koanf:"initial-steps"` } @@ -42,7 +42,7 @@ var DefaultMachineCacheConfig = MachineCacheConfig{ func MachineCacheConfigConfigAddOptions(prefix string, f *flag.FlagSet) { f.Uint64(prefix+".initial-steps", DefaultMachineCacheConfig.InitialSteps, "initial steps between machines") - f.Int(prefix+".cached-challenge-machines", DefaultMachineCacheConfig.CachedChallengeMachines, "how many machines to store in cache while working on a challenge (should be even)") + f.Uint64(prefix+".cached-challenge-machines", DefaultMachineCacheConfig.CachedChallengeMachines, "how many machines to store in cache while working on a challenge (should be even)") } // `initialMachine` won't be mutated by this function. @@ -140,7 +140,7 @@ func (c *MachineCache) unlockBuild(err error) { } func (c *MachineCache) setRangeLocked(ctx context.Context, start uint64, end uint64) error { - newInterval := (end - start) / uint64(c.config.CachedChallengeMachines) + newInterval := (end - start) / c.config.CachedChallengeMachines if newInterval == 0 { newInterval = 2 } @@ -150,7 +150,7 @@ func (c *MachineCache) setRangeLocked(ctx context.Context, start uint64, end uin if end >= c.finalMachineStep { end = c.finalMachineStep - newInterval/2 } - newInterval = (end - start) / uint64(c.config.CachedChallengeMachines) + newInterval = (end - start) / c.config.CachedChallengeMachines if newInterval == 0 { newInterval = 1 } @@ -212,7 +212,7 @@ func (c *MachineCache) populateCache(ctx context.Context) error { if nextMachine.GetStepCount()+c.machineStepInterval >= c.finalMachineStep { break } - if len(c.machines) >= c.config.CachedChallengeMachines { + if uint64(len(c.machines)) >= c.config.CachedChallengeMachines { break } nextMachine = nextMachine.CloneMachineInterface() @@ -236,6 +236,7 @@ func (c *MachineCache) getClosestMachine(stepCount uint64) (int, MachineInterfac } stepsFromStart := stepCount - c.firstMachineStep var index int + // #nosec G115 if c.machineStepInterval == 0 || stepsFromStart > c.machineStepInterval*uint64(len(c.machines)-1) { index = len(c.machines) - 1 } else { diff --git a/validator/server_jit/jit_machine.go b/validator/server_jit/jit_machine.go index e7753748aba..06c451bda18 100644 --- a/validator/server_jit/jit_machine.go +++ b/validator/server_jit/jit_machine.go @@ -306,6 +306,7 @@ func (machine *JitMachine) prove( if err != nil { return state, fmt.Errorf("failed to read memory usage from Jit machine: %w", err) } + // #nosec G115 if memoryUsed > uint64(machine.wasmMemoryUsageLimit) { log.Warn("memory used by jit wasm exceeds the wasm memory usage limit", "limit", machine.wasmMemoryUsageLimit, "memoryUsed", memoryUsed) } diff --git a/wavmio/stub.go b/wavmio/stub.go index 1395fb42350..0c82506ff39 100644 --- a/wavmio/stub.go +++ b/wavmio/stub.go @@ -66,6 +66,7 @@ func parsePreimageBytes(path string) { if err != nil { panic(err) } + // #nosec G115 if uint64(read) != fieldSize { panic("missing bytes reading data") } @@ -77,18 +78,18 @@ func parsePreimageBytes(path string) { func StubInit() { preimages = make(map[common.Hash][]byte) var delayedMsgPath arrayFlags - seqMsgPosFlag := flag.Int("inbox-position", 0, "position for sequencer inbox message") - posWithinMsgFlag := flag.Int("position-within-message", 0, "position inside sequencer inbox message") - delayedPositionFlag := flag.Int("delayed-inbox-position", 0, "position for first delayed inbox message") + seqMsgPosFlag := flag.Uint64("inbox-position", 0, "position for sequencer inbox message") + posWithinMsgFlag := flag.Uint64("position-within-message", 0, "position inside sequencer inbox message") + delayedPositionFlag := flag.Uint64("delayed-inbox-position", 0, "position for first delayed inbox message") lastBlockFlag := flag.String("last-block-hash", "0000000000000000000000000000000000000000000000000000000000000000", "lastBlockHash") flag.Var(&delayedMsgPath, "delayed-inbox", "delayed inbox messages (multiple values)") inboxPath := flag.String("inbox", "", "file to load sequencer message") preimagesPath := flag.String("preimages", "", "file to load preimages from") flag.Parse() - seqMsgPos = uint64(*seqMsgPosFlag) - posWithinMsg = uint64(*posWithinMsgFlag) - delayedMsgFirstPos = uint64(*delayedPositionFlag) + seqMsgPos = *seqMsgPosFlag + posWithinMsg = *posWithinMsgFlag + delayedMsgFirstPos = *delayedPositionFlag lastBlockHash = common.HexToHash(*lastBlockFlag) for _, path := range delayedMsgPath { msg, err := os.ReadFile(path) From 6cc3a5dfe8ecf248c950264684612cdce958caf7 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 10:12:38 -0600 Subject: [PATCH 096/156] block_validator: read batches once per batch --- staker/block_validator.go | 60 +++++++--- staker/stateless_block_validator.go | 171 ++++++++++++++++------------ 2 files changed, 145 insertions(+), 86 deletions(-) diff --git a/staker/block_validator.go b/staker/block_validator.go index 7a7efca8460..18411c6163c 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -56,12 +56,12 @@ type BlockValidator struct { chainCaughtUp bool // can only be accessed from creation thread or if holding reorg-write - nextCreateBatch []byte - nextCreateBatchBlockHash common.Hash - nextCreateBatchMsgCount arbutil.MessageIndex - nextCreateBatchReread bool - nextCreateStartGS validator.GoGlobalState - nextCreatePrevDelayed uint64 + nextCreateBatch *FullBatchInfo + nextCreateBatchReread bool + prevBatchCache map[uint64]*FullBatchInfo + + nextCreateStartGS validator.GoGlobalState + nextCreatePrevDelayed uint64 // can only be accessed from from validation thread or if holding reorg-write lastValidGS validator.GoGlobalState @@ -108,6 +108,7 @@ type BlockValidatorConfig struct { PrerecordedBlocks uint64 `koanf:"prerecorded-blocks" reload:"hot"` RecordingIterLimit uint64 `koanf:"recording-iter-limit"` ForwardBlocks uint64 `koanf:"forward-blocks" reload:"hot"` + BatchCacheLimit uint32 `koanf:"batch-cache-limit"` CurrentModuleRoot string `koanf:"current-module-root"` // TODO(magic) requires reinitialization on hot reload PendingUpgradeModuleRoot string `koanf:"pending-upgrade-module-root"` // TODO(magic) requires StatelessBlockValidator recreation on hot reload FailureIsFatal bool `koanf:"failure-is-fatal" reload:"hot"` @@ -174,6 +175,7 @@ func BlockValidatorConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Duration(prefix+".validation-poll", DefaultBlockValidatorConfig.ValidationPoll, "poll time to check validations") f.Uint64(prefix+".forward-blocks", DefaultBlockValidatorConfig.ForwardBlocks, "prepare entries for up to that many blocks ahead of validation (small footprint)") f.Uint64(prefix+".prerecorded-blocks", DefaultBlockValidatorConfig.PrerecordedBlocks, "record that many blocks ahead of validation (larger footprint)") + f.Uint32(prefix+".batch-cache-limit", DefaultBlockValidatorConfig.BatchCacheLimit, "limit number of old batches to keep in block-validator") f.String(prefix+".current-module-root", DefaultBlockValidatorConfig.CurrentModuleRoot, "current wasm module root ('current' read from chain, 'latest' from machines/latest dir, or provide hash)") f.Uint64(prefix+".recording-iter-limit", DefaultBlockValidatorConfig.RecordingIterLimit, "limit on block recordings sent per iteration") f.String(prefix+".pending-upgrade-module-root", DefaultBlockValidatorConfig.PendingUpgradeModuleRoot, "pending upgrade wasm module root to additionally validate (hash, 'latest' or empty)") @@ -194,6 +196,7 @@ var DefaultBlockValidatorConfig = BlockValidatorConfig{ ValidationPoll: time.Second, ForwardBlocks: 1024, PrerecordedBlocks: uint64(2 * runtime.NumCPU()), + BatchCacheLimit: 20, CurrentModuleRoot: "current", PendingUpgradeModuleRoot: "latest", FailureIsFatal: true, @@ -209,6 +212,7 @@ var TestBlockValidatorConfig = BlockValidatorConfig{ RedisValidationClientConfig: redis.TestValidationClientConfig, ValidationPoll: 100 * time.Millisecond, ForwardBlocks: 128, + BatchCacheLimit: 20, PrerecordedBlocks: uint64(2 * runtime.NumCPU()), RecordingIterLimit: 20, CurrentModuleRoot: "latest", @@ -271,6 +275,7 @@ func NewBlockValidator( progressValidationsChan: make(chan struct{}, 1), config: config, fatalErr: fatalErr, + prevBatchCache: make(map[uint64]*FullBatchInfo), } if !config().Dangerous.ResetBlockValidation { validated, err := ret.ReadLastValidatedInfo() @@ -571,33 +576,58 @@ func (v *BlockValidator) createNextValidationEntry(ctx context.Context) (bool, e } if v.nextCreateStartGS.PosInBatch == 0 || v.nextCreateBatchReread { // new batch - found, batch, batchBlockHash, count, err := v.readBatch(ctx, v.nextCreateStartGS.Batch) + found, fullBatchInfo, err := v.readBatch(ctx, v.nextCreateStartGS.Batch) if !found { return false, err } - v.nextCreateBatch = batch - v.nextCreateBatchBlockHash = batchBlockHash - v.nextCreateBatchMsgCount = count + if v.nextCreateBatch != nil { + v.prevBatchCache[v.nextCreateBatch.Number] = v.nextCreateBatch + } + v.nextCreateBatch = fullBatchInfo // #nosec G115 - validatorMsgCountCurrentBatch.Update(int64(count)) + validatorMsgCountCurrentBatch.Update(int64(fullBatchInfo.MsgCount)) + batchCacheLimit := v.config().BatchCacheLimit + if len(v.prevBatchCache) > int(batchCacheLimit) { + for num := range v.prevBatchCache { + if num < v.nextCreateStartGS.Batch-uint64(batchCacheLimit) { + delete(v.prevBatchCache, num) + } + } + } v.nextCreateBatchReread = false } endGS := validator.GoGlobalState{ BlockHash: endRes.BlockHash, SendRoot: endRes.SendRoot, } - if pos+1 < v.nextCreateBatchMsgCount { + if pos+1 < v.nextCreateBatch.MsgCount { endGS.Batch = v.nextCreateStartGS.Batch endGS.PosInBatch = v.nextCreateStartGS.PosInBatch + 1 - } else if pos+1 == v.nextCreateBatchMsgCount { + } else if pos+1 == v.nextCreateBatch.MsgCount { endGS.Batch = v.nextCreateStartGS.Batch + 1 endGS.PosInBatch = 0 } else { - return false, fmt.Errorf("illegal batch msg count %d pos %d batch %d", v.nextCreateBatchMsgCount, pos, endGS.Batch) + return false, fmt.Errorf("illegal batch msg count %d pos %d batch %d", v.nextCreateBatch.MsgCount, pos, endGS.Batch) } chainConfig := v.streamer.ChainConfig() + batchReader := func(batchNum uint64) (*FullBatchInfo, error) { + if batchNum == v.nextCreateBatch.Number { + return v.nextCreateBatch, nil + } + if entry, found := v.prevBatchCache[batchNum]; found { + return entry, nil + } + found, entry, err := v.readBatch(ctx, batchNum) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("batch %d not found", batchNum) + } + return entry, nil + } entry, err := newValidationEntry( - pos, v.nextCreateStartGS, endGS, msg, v.nextCreateBatch, v.nextCreateBatchBlockHash, v.nextCreatePrevDelayed, chainConfig, + pos, v.nextCreateStartGS, endGS, msg, batchReader, v.nextCreatePrevDelayed, chainConfig, ) if err != nil { return false, err diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index f54ec8b58c1..857bc255ada 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -114,6 +114,14 @@ const ( Ready ) +type FullBatchInfo struct { + Number uint64 + BlockHash common.Hash + PostedData []byte + MsgCount arbutil.MessageIndex + Preimages map[arbutil.PreimageType]map[common.Hash][]byte +} + type validationEntry struct { Stage ValidationEntryStage // Valid since ReadyforRecord: @@ -171,16 +179,27 @@ func newValidationEntry( start validator.GoGlobalState, end validator.GoGlobalState, msg *arbostypes.MessageWithMetadata, - batch []byte, - batchBlockHash common.Hash, + fullBatchFetcher func(uint64) (*FullBatchInfo, error), prevDelayed uint64, chainConfig *params.ChainConfig, ) (*validationEntry, error) { - batchInfo := validator.BatchInfo{ - Number: start.Batch, - BlockHash: batchBlockHash, - Data: batch, + preimages := make(map[arbutil.PreimageType]map[common.Hash][]byte) + valBatches := make([]validator.BatchInfo, 0) + + valBatchFetcher := func(batchNum uint64) ([]byte, error) { + fullBatchInfo, err := fullBatchFetcher(batchNum) + if err != nil { + return nil, err + } + valBatches = append(valBatches, validator.BatchInfo{ + Number: batchNum, + BlockHash: fullBatchInfo.BlockHash, + Data: fullBatchInfo.PostedData, + }) + clonePreimagesInto(preimages, fullBatchInfo.Preimages) + return fullBatchInfo.PostedData, nil } + hasDelayed := false var delayedNum uint64 if msg.DelayedMessagesRead == prevDelayed+1 { @@ -189,6 +208,15 @@ func newValidationEntry( } else if msg.DelayedMessagesRead != prevDelayed { return nil, fmt.Errorf("illegal validation entry delayedMessage %d, previous %d", msg.DelayedMessagesRead, prevDelayed) } + + if _, err := valBatchFetcher(start.Batch); err != nil { + return nil, err + } + msg.Message.BatchGasCost = nil + if err := msg.Message.FillInBatchGasCost(valBatchFetcher); err != nil { + return nil, err + } + return &validationEntry{ Stage: ReadyForRecord, Pos: pos, @@ -197,8 +225,9 @@ func newValidationEntry( HasDelayedMsg: hasDelayed, DelayedMsgNr: delayedNum, msg: msg, - BatchInfo: []validator.BatchInfo{batchInfo}, + BatchInfo: valBatches, ChainConfig: chainConfig, + Preimages: preimages, }, nil } @@ -246,30 +275,73 @@ func NewStatelessBlockValidator( }, nil } -func (v *StatelessBlockValidator) readBatch(ctx context.Context, batchNum uint64) (bool, []byte, common.Hash, arbutil.MessageIndex, error) { +func (v *StatelessBlockValidator) readBatch(ctx context.Context, batchNum uint64) (bool, *FullBatchInfo, error) { batchCount, err := v.inboxTracker.GetBatchCount() if err != nil { - return false, nil, common.Hash{}, 0, err + return false, nil, err } if batchCount <= batchNum { - return false, nil, common.Hash{}, 0, nil + return false, nil, nil } batchMsgCount, err := v.inboxTracker.GetBatchMessageCount(batchNum) if err != nil { - return false, nil, common.Hash{}, 0, err + return false, nil, err } - batch, batchBlockHash, err := v.inboxReader.GetSequencerMessageBytes(ctx, batchNum) + postedData, batchBlockHash, err := v.inboxReader.GetSequencerMessageBytes(ctx, batchNum) if err != nil { - return false, nil, common.Hash{}, 0, err + return false, nil, err + } + preimages := make(map[arbutil.PreimageType]map[common.Hash][]byte) + if len(postedData) > 40 { + foundDA := false + for _, dapReader := range v.dapReaders { + if dapReader != nil && dapReader.IsValidHeaderByte(postedData[40]) { + preimageRecorder := daprovider.RecordPreimagesTo(preimages) + _, err := dapReader.RecoverPayloadFromBatch(ctx, batchNum, batchBlockHash, postedData, preimageRecorder, true) + if err != nil { + // Matches the way keyset validation was done inside DAS readers i.e logging the error + // But other daproviders might just want to return the error + if errors.Is(err, daprovider.ErrSeqMsgValidation) && daprovider.IsDASMessageHeaderByte(postedData[40]) { + log.Error(err.Error()) + } else { + return false, nil, err + } + } + foundDA = true + break + } + } + if !foundDA { + if daprovider.IsDASMessageHeaderByte(postedData[40]) { + log.Error("No DAS Reader configured, but sequencer message found with DAS header") + } + } + } + fullInfo := FullBatchInfo{ + Number: batchNum, + BlockHash: batchBlockHash, + PostedData: postedData, + MsgCount: batchMsgCount, + Preimages: preimages, + } + return true, &fullInfo, nil +} + +func clonePreimagesInto(dest, source map[arbutil.PreimageType]map[common.Hash][]byte) { + for piType, piMap := range source { + if dest[piType] == nil { + dest[piType] = make(map[common.Hash][]byte, len(source[piType])) + } + for hash, preimage := range piMap { + dest[piType][hash] = preimage + } } - return true, batch, batchBlockHash, batchMsgCount, nil } func (v *StatelessBlockValidator) ValidationEntryRecord(ctx context.Context, e *validationEntry) error { if e.Stage != ReadyForRecord { return fmt.Errorf("validation entry should be ReadyForRecord, is: %v", e.Stage) } - e.Preimages = make(map[arbutil.PreimageType]map[common.Hash][]byte) if e.Pos != 0 { recording, err := v.recorder.RecordBlockCreation(ctx, e.Pos, e.msg) if err != nil { @@ -278,30 +350,10 @@ func (v *StatelessBlockValidator) ValidationEntryRecord(ctx context.Context, e * if recording.BlockHash != e.End.BlockHash { return fmt.Errorf("recording failed: pos %d, hash expected %v, got %v", e.Pos, e.End.BlockHash, recording.BlockHash) } - // record any additional batch fetching - batchFetcher := func(batchNum uint64) ([]byte, error) { - found, data, hash, _, err := v.readBatch(ctx, batchNum) - if err != nil { - return nil, err - } - if !found { - return nil, errors.New("batch not found") - } - e.BatchInfo = append(e.BatchInfo, validator.BatchInfo{ - Number: batchNum, - BlockHash: hash, - Data: data, - }) - return data, nil - } - e.msg.Message.BatchGasCost = nil - err = e.msg.Message.FillInBatchGasCost(batchFetcher) - if err != nil { - return err - } - if recording.Preimages != nil { - e.Preimages[arbutil.Keccak256PreimageType] = recording.Preimages + recordingPreimages := make(map[arbutil.PreimageType]map[common.Hash][]byte) + recordingPreimages[arbutil.Keccak256PreimageType] = recording.Preimages + clonePreimagesInto(e.Preimages, recordingPreimages) } e.UserWasms = recording.UserWasms } @@ -316,35 +368,6 @@ func (v *StatelessBlockValidator) ValidationEntryRecord(ctx context.Context, e * } e.DelayedMsg = delayedMsg } - for _, batch := range e.BatchInfo { - if len(batch.Data) <= 40 { - continue - } - foundDA := false - for _, dapReader := range v.dapReaders { - if dapReader != nil && dapReader.IsValidHeaderByte(batch.Data[40]) { - preimageRecorder := daprovider.RecordPreimagesTo(e.Preimages) - _, err := dapReader.RecoverPayloadFromBatch(ctx, batch.Number, batch.BlockHash, batch.Data, preimageRecorder, true) - if err != nil { - // Matches the way keyset validation was done inside DAS readers i.e logging the error - // But other daproviders might just want to return the error - if errors.Is(err, daprovider.ErrSeqMsgValidation) && daprovider.IsDASMessageHeaderByte(batch.Data[40]) { - log.Error(err.Error()) - } else { - return err - } - } - foundDA = true - break - } - } - if !foundDA { - if daprovider.IsDASMessageHeaderByte(batch.Data[40]) { - log.Error("No DAS Reader configured, but sequencer message found with DAS header") - } - } - } - e.msg = nil // no longer needed e.Stage = Ready return nil @@ -404,11 +427,17 @@ func (v *StatelessBlockValidator) CreateReadyValidationEntry(ctx context.Context } start := buildGlobalState(*prevResult, startPos) end := buildGlobalState(*result, endPos) - seqMsg, batchBlockHash, err := v.inboxReader.GetSequencerMessageBytes(ctx, startPos.BatchNumber) - if err != nil { - return nil, err + fullBatchReader := func(batchNum uint64) (*FullBatchInfo, error) { + found, fullBlockInfo, err := v.readBatch(ctx, batchNum) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("batch %d not found", startPos.BatchNumber) + } + return fullBlockInfo, nil } - entry, err := newValidationEntry(pos, start, end, msg, seqMsg, batchBlockHash, prevDelayed, v.streamer.ChainConfig()) + entry, err := newValidationEntry(pos, start, end, msg, fullBatchReader, prevDelayed, v.streamer.ChainConfig()) if err != nil { return nil, err } From d752445bfac0da673fcfe7ec7f3dbe209223ac71 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 10:16:50 -0600 Subject: [PATCH 097/156] remove blockhash from validator.BatchInfo --- staker/stateless_block_validator.go | 5 ++--- validator/validation_entry.go | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index 857bc255ada..3d92b9df59b 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -192,9 +192,8 @@ func newValidationEntry( return nil, err } valBatches = append(valBatches, validator.BatchInfo{ - Number: batchNum, - BlockHash: fullBatchInfo.BlockHash, - Data: fullBatchInfo.PostedData, + Number: batchNum, + Data: fullBatchInfo.PostedData, }) clonePreimagesInto(preimages, fullBatchInfo.Preimages) return fullBatchInfo.PostedData, nil diff --git a/validator/validation_entry.go b/validator/validation_entry.go index d340993fa2e..4ec6919d3b9 100644 --- a/validator/validation_entry.go +++ b/validator/validation_entry.go @@ -7,9 +7,8 @@ import ( ) type BatchInfo struct { - Number uint64 - BlockHash common.Hash - Data []byte + Number uint64 + Data []byte } type ValidationInput struct { From 25b4afe0363f65a9df15ee27a20917151dddf3f8 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 10:31:01 -0600 Subject: [PATCH 098/156] update default forward-blocks value --- staker/block_validator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/staker/block_validator.go b/staker/block_validator.go index 18411c6163c..847b6f63499 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -173,7 +173,7 @@ func BlockValidatorConfigAddOptions(prefix string, f *pflag.FlagSet) { redis.ValidationClientConfigAddOptions(prefix+".redis-validation-client-config", f) f.String(prefix+".validation-server-configs-list", DefaultBlockValidatorConfig.ValidationServerConfigsList, "array of execution rpc configs given as a json string. time duration should be supplied in number indicating nanoseconds") f.Duration(prefix+".validation-poll", DefaultBlockValidatorConfig.ValidationPoll, "poll time to check validations") - f.Uint64(prefix+".forward-blocks", DefaultBlockValidatorConfig.ForwardBlocks, "prepare entries for up to that many blocks ahead of validation (small footprint)") + f.Uint64(prefix+".forward-blocks", DefaultBlockValidatorConfig.ForwardBlocks, "prepare entries for up to that many blocks ahead of validation (stores batch-copy per block)") f.Uint64(prefix+".prerecorded-blocks", DefaultBlockValidatorConfig.PrerecordedBlocks, "record that many blocks ahead of validation (larger footprint)") f.Uint32(prefix+".batch-cache-limit", DefaultBlockValidatorConfig.BatchCacheLimit, "limit number of old batches to keep in block-validator") f.String(prefix+".current-module-root", DefaultBlockValidatorConfig.CurrentModuleRoot, "current wasm module root ('current' read from chain, 'latest' from machines/latest dir, or provide hash)") @@ -194,7 +194,7 @@ var DefaultBlockValidatorConfig = BlockValidatorConfig{ ValidationServer: rpcclient.DefaultClientConfig, RedisValidationClientConfig: redis.DefaultValidationClientConfig, ValidationPoll: time.Second, - ForwardBlocks: 1024, + ForwardBlocks: 128, PrerecordedBlocks: uint64(2 * runtime.NumCPU()), BatchCacheLimit: 20, CurrentModuleRoot: "current", From c98d652ce97e99063596096f5b0bd993c3cf22d4 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 10:38:08 -0600 Subject: [PATCH 099/156] block_validator: prevBatchCache in reorgs --- staker/block_validator.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/staker/block_validator.go b/staker/block_validator.go index 847b6f63499..49dc59c4a2c 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -1027,6 +1027,9 @@ func (v *BlockValidator) UpdateLatestStaked(count arbutil.MessageIndex, globalSt v.nextCreateStartGS = globalState v.nextCreatePrevDelayed = msg.DelayedMessagesRead v.nextCreateBatchReread = true + if v.nextCreateBatch != nil { + v.prevBatchCache[v.nextCreateBatch.Number] = v.nextCreateBatch + } v.createdA.Store(countUint64) } // under the reorg mutex we don't need atomic access @@ -1053,6 +1056,7 @@ func (v *BlockValidator) ReorgToBatchCount(count uint64) { defer v.reorgMutex.Unlock() if v.nextCreateStartGS.Batch >= count { v.nextCreateBatchReread = true + v.prevBatchCache = make(map[uint64]*FullBatchInfo) } } @@ -1093,6 +1097,7 @@ func (v *BlockValidator) Reorg(ctx context.Context, count arbutil.MessageIndex) v.nextCreateStartGS = buildGlobalState(*res, endPosition) v.nextCreatePrevDelayed = msg.DelayedMessagesRead v.nextCreateBatchReread = true + v.prevBatchCache = make(map[uint64]*FullBatchInfo) countUint64 := uint64(count) v.createdA.Store(countUint64) // under the reorg mutex we don't need atomic access From b71fd7b5befb34901b2fdfbe1ead03dd1b5eeb26 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 11:03:59 -0600 Subject: [PATCH 100/156] block_recorder: separate config, add place to max blocks --- execution/gethexec/block_recorder.go | 32 +++++++++++++++++++++++++--- execution/gethexec/node.go | 30 +++++++++++++------------- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/execution/gethexec/block_recorder.go b/execution/gethexec/block_recorder.go index 8879c907023..a31b6b37361 100644 --- a/execution/gethexec/block_recorder.go +++ b/execution/gethexec/block_recorder.go @@ -16,6 +16,7 @@ import ( "github.com/offchainlabs/nitro/arbos/arbostypes" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/execution" + flag "github.com/spf13/pflag" ) // BlockRecorder uses a separate statedatabase from the blockchain. @@ -25,6 +26,8 @@ import ( // Most recent/advanced header we ever computed (lastHdr) // Hopefully - some recent valid block. For that we always keep one candidate block until it becomes validated. type BlockRecorder struct { + config *BlockRecorderConfig + recordingDatabase *arbitrum.RecordingDatabase execEngine *ExecutionEngine @@ -39,10 +42,33 @@ type BlockRecorder struct { preparedLock sync.Mutex } -func NewBlockRecorder(config *arbitrum.RecordingDatabaseConfig, execEngine *ExecutionEngine, ethDb ethdb.Database) *BlockRecorder { +type BlockRecorderConfig struct { + TrieDirtyCache int `koanf:"trie-dirty-cache"` + TrieCleanCache int `koanf:"trie-clean-cache"` + MaxPrepared int `koanf:"max-prepared"` +} + +var DefaultBlockRecorderConfig = BlockRecorderConfig{ + TrieDirtyCache: 1024, + TrieCleanCache: 16, + MaxPrepared: 1000, +} + +func BlockRecorderConfigAddOptions(prefix string, f *flag.FlagSet) { + f.Int(prefix+".trie-dirty-cache", DefaultBlockRecorderConfig.TrieDirtyCache, "like trie-dirty-cache for the separate, recording database (used for validation)") + f.Int(prefix+".trie-clean-cache", DefaultBlockRecorderConfig.TrieCleanCache, "like trie-clean-cache for the separate, recording database (used for validation)") + f.Int(prefix+".max-prepared", DefaultBlockRecorderConfig.MaxPrepared, "max references to store in the recording database") +} + +func NewBlockRecorder(config *BlockRecorderConfig, execEngine *ExecutionEngine, ethDb ethdb.Database) *BlockRecorder { + dbConfig := arbitrum.RecordingDatabaseConfig{ + TrieDirtyCache: config.TrieDirtyCache, + TrieCleanCache: config.TrieCleanCache, + } recorder := &BlockRecorder{ + config: config, execEngine: execEngine, - recordingDatabase: arbitrum.NewRecordingDatabase(config, ethDb, execEngine.bc), + recordingDatabase: arbitrum.NewRecordingDatabase(&dbConfig, ethDb, execEngine.bc), } execEngine.SetRecorder(recorder) return recorder @@ -303,7 +329,7 @@ func (r *BlockRecorder) PrepareForRecord(ctx context.Context, start, end arbutil r.updateLastHdr(header) hdrNum++ } - r.preparedAddTrim(references, 1000) + r.preparedAddTrim(references, r.config.MaxPrepared) return nil } diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 21c2b4beced..5847f392b1d 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -78,19 +78,19 @@ func StylusTargetConfigAddOptions(prefix string, f *flag.FlagSet) { } type Config struct { - ParentChainReader headerreader.Config `koanf:"parent-chain-reader" reload:"hot"` - Sequencer SequencerConfig `koanf:"sequencer" reload:"hot"` - RecordingDatabase arbitrum.RecordingDatabaseConfig `koanf:"recording-database"` - TxPreChecker TxPreCheckerConfig `koanf:"tx-pre-checker" reload:"hot"` - Forwarder ForwarderConfig `koanf:"forwarder"` - ForwardingTarget string `koanf:"forwarding-target"` - SecondaryForwardingTarget []string `koanf:"secondary-forwarding-target"` - Caching CachingConfig `koanf:"caching"` - RPC arbitrum.Config `koanf:"rpc"` - TxLookupLimit uint64 `koanf:"tx-lookup-limit"` - EnablePrefetchBlock bool `koanf:"enable-prefetch-block"` - SyncMonitor SyncMonitorConfig `koanf:"sync-monitor"` - StylusTarget StylusTargetConfig `koanf:"stylus-target"` + ParentChainReader headerreader.Config `koanf:"parent-chain-reader" reload:"hot"` + Sequencer SequencerConfig `koanf:"sequencer" reload:"hot"` + RecordingDatabase BlockRecorderConfig `koanf:"recording-database"` + TxPreChecker TxPreCheckerConfig `koanf:"tx-pre-checker" reload:"hot"` + Forwarder ForwarderConfig `koanf:"forwarder"` + ForwardingTarget string `koanf:"forwarding-target"` + SecondaryForwardingTarget []string `koanf:"secondary-forwarding-target"` + Caching CachingConfig `koanf:"caching"` + RPC arbitrum.Config `koanf:"rpc"` + TxLookupLimit uint64 `koanf:"tx-lookup-limit"` + EnablePrefetchBlock bool `koanf:"enable-prefetch-block"` + SyncMonitor SyncMonitorConfig `koanf:"sync-monitor"` + StylusTarget StylusTargetConfig `koanf:"stylus-target"` forwardingTarget string } @@ -123,7 +123,7 @@ func ConfigAddOptions(prefix string, f *flag.FlagSet) { arbitrum.ConfigAddOptions(prefix+".rpc", f) SequencerConfigAddOptions(prefix+".sequencer", f) headerreader.AddOptions(prefix+".parent-chain-reader", f) - arbitrum.RecordingDatabaseConfigAddOptions(prefix+".recording-database", f) + BlockRecorderConfigAddOptions(prefix+".recording-database", f) f.String(prefix+".forwarding-target", ConfigDefault.ForwardingTarget, "transaction forwarding target URL, or \"null\" to disable forwarding (iff not sequencer)") f.StringSlice(prefix+".secondary-forwarding-target", ConfigDefault.SecondaryForwardingTarget, "secondary transaction forwarding target URL") AddOptionsForNodeForwarderConfig(prefix+".forwarder", f) @@ -139,7 +139,7 @@ var ConfigDefault = Config{ RPC: arbitrum.DefaultConfig, Sequencer: DefaultSequencerConfig, ParentChainReader: headerreader.DefaultConfig, - RecordingDatabase: arbitrum.DefaultRecordingDatabaseConfig, + RecordingDatabase: DefaultBlockRecorderConfig, ForwardingTarget: "", SecondaryForwardingTarget: []string{}, TxPreChecker: DefaultTxPreCheckerConfig, From 747a0f37c0f72a1135ecfb3eb4d1eab6820b0ca1 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 13:31:09 -0600 Subject: [PATCH 101/156] block_validator: delete batch from cache after used by report --- staker/block_validator.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/staker/block_validator.go b/staker/block_validator.go index 49dc59c4a2c..4a5bd740a86 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -614,7 +614,9 @@ func (v *BlockValidator) createNextValidationEntry(ctx context.Context) (bool, e if batchNum == v.nextCreateBatch.Number { return v.nextCreateBatch, nil } + // only batch-posting-reports will get here, and there's only one per batch if entry, found := v.prevBatchCache[batchNum]; found { + delete(v.prevBatchCache, batchNum) return entry, nil } found, entry, err := v.readBatch(ctx, batchNum) From a32334f70e0aa89ed1010bafae4718035f86d677 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 17:55:39 -0600 Subject: [PATCH 102/156] move forwarder to prover build step --- Makefile | 31 +- arbitrator/prover/Cargo.toml | 3 + .../forward/src/main.rs => prover/build.rs} | 76 +- arbitrator/prover/src/binary.rs | 12 +- arbitrator/prover/src/lib.rs | 5 +- arbitrator/prover/src/machine.rs | 44 +- arbitrator/prover/src/main.rs | 3 + arbitrator/prover/src/test.rs | 2 +- arbitrator/prover/test-cases/forward-test.wat | 32 - .../prover/test-cases/forward/forward.wat | 8 - .../prover/test-cases/forward/target.wat | 27 - arbitrator/stylus/src/test/mod.rs | 1 + arbitrator/tools/module_roots/src/main.rs | 1 + arbitrator/wasm-libraries/Cargo.lock | 668 +++++++++++++++++- arbitrator/wasm-libraries/Cargo.toml | 1 - arbitrator/wasm-libraries/forward/.gitignore | 1 - arbitrator/wasm-libraries/forward/Cargo.toml | 8 - arbitrator/wasm-testsuite/src/main.rs | 2 + validator/server_arb/machine.go | 2 +- 19 files changed, 735 insertions(+), 192 deletions(-) rename arbitrator/{wasm-libraries/forward/src/main.rs => prover/build.rs} (78%) delete mode 100644 arbitrator/prover/test-cases/forward-test.wat delete mode 100644 arbitrator/prover/test-cases/forward/forward.wat delete mode 100644 arbitrator/prover/test-cases/forward/target.wat delete mode 100644 arbitrator/wasm-libraries/forward/.gitignore delete mode 100644 arbitrator/wasm-libraries/forward/Cargo.toml diff --git a/Makefile b/Makefile index 0a71d64f12c..9ecba1ce2d1 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ replay_wasm=$(output_latest)/replay.wasm arb_brotli_files = $(wildcard arbitrator/brotli/src/*.* arbitrator/brotli/src/*/*.* arbitrator/brotli/*.toml arbitrator/brotli/*.rs) .make/cbrotli-lib .make/cbrotli-wasm arbitrator_generated_header=$(output_root)/include/arbitrator.h -arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, forward wasi_stub host_io soft-float arbcompress user_host program_exec) +arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, wasi_stub host_io soft-float arbcompress user_host program_exec) arbitrator_stylus_lib=$(output_root)/lib/libstylus.a prover_bin=$(output_root)/bin/prover arbitrator_jit=$(output_root)/bin/jit @@ -75,16 +75,12 @@ arbitrator_test_wasms=$(patsubst %.wat,%.wasm, $(arbitrator_tests_wat)) $(patsub arbitrator_tests_link_info = $(shell cat $(arbitrator_cases)/link.txt | xargs) arbitrator_tests_link_deps = $(patsubst %,$(arbitrator_cases)/%.wasm, $(arbitrator_tests_link_info)) -arbitrator_tests_forward_wats = $(wildcard $(arbitrator_cases)/forward/*.wat) -arbitrator_tests_forward_deps = $(arbitrator_tests_forward_wats:wat=wasm) - WASI_SYSROOT?=/opt/wasi-sdk/wasi-sysroot arbitrator_wasm_lib_flags=$(patsubst %, -l %, $(arbitrator_wasm_libs)) rust_arbutil_files = $(wildcard arbitrator/arbutil/src/*.* arbitrator/arbutil/src/*/*.* arbitrator/arbutil/*.toml arbitrator/caller-env/src/*.* arbitrator/caller-env/src/*/*.* arbitrator/caller-env/*.toml) .make/cbrotli-lib -prover_direct_includes = $(patsubst %,$(output_latest)/%.wasm, forward forward_stub) prover_dir = arbitrator/prover/ rust_prover_files = $(wildcard $(prover_dir)/src/*.* $(prover_dir)/src/*/*.* $(prover_dir)/*.toml $(prover_dir)/*.rs) $(rust_arbutil_files) $(prover_direct_includes) $(arb_brotli_files) @@ -92,12 +88,9 @@ wasm_lib = arbitrator/wasm-libraries wasm_lib_cargo = $(wasm_lib)/.cargo/config.toml wasm_lib_deps = $(wildcard $(wasm_lib)/$(1)/*.toml $(wasm_lib)/$(1)/src/*.rs $(wasm_lib)/$(1)/*.rs) $(wasm_lib_cargo) $(rust_arbutil_files) $(arb_brotli_files) .make/machines wasm_lib_go_abi = $(call wasm_lib_deps,go-abi) -wasm_lib_forward = $(call wasm_lib_deps,forward) wasm_lib_user_host_trait = $(call wasm_lib_deps,user-host-trait) wasm_lib_user_host = $(call wasm_lib_deps,user-host) $(wasm_lib_user_host_trait) -forward_dir = $(wasm_lib)/forward - stylus_files = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(wasm_lib_user_host_trait) $(rust_prover_files) jit_dir = arbitrator/jit @@ -281,7 +274,6 @@ clean: rm -f arbitrator/wasm-libraries/soft-float/*.o rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.o rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.a - rm -f arbitrator/wasm-libraries/forward/*.wat rm -rf arbitrator/stylus/tests/*/target/ arbitrator/stylus/tests/*/*.wasm @rm -rf contracts/build contracts/cache solgen/go/ @rm -f .make/* @@ -399,7 +391,7 @@ $(output_latest)/host_io.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,host-io) $( cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package host-io install arbitrator/wasm-libraries/$(wasm32_wasi)/host_io.wasm $@ -$(output_latest)/user_host.wasm: $(DEP_PREDICATE) $(wasm_lib_user_host) $(rust_prover_files) $(output_latest)/forward_stub.wasm .make/machines +$(output_latest)/user_host.wasm: $(DEP_PREDICATE) $(wasm_lib_user_host) $(rust_prover_files) .make/machines cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package user-host install arbitrator/wasm-libraries/$(wasm32_wasi)/user_host.wasm $@ @@ -415,17 +407,9 @@ $(output_latest)/arbcompress.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,brotli) cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package arbcompress install arbitrator/wasm-libraries/$(wasm32_wasi)/arbcompress.wasm $@ -$(output_latest)/forward.wasm: $(DEP_PREDICATE) $(wasm_lib_forward) .make/machines - cargo run --manifest-path $(forward_dir)/Cargo.toml -- --path $(forward_dir)/forward.wat - wat2wasm $(wasm_lib)/forward/forward.wat -o $@ - -$(output_latest)/forward_stub.wasm: $(DEP_PREDICATE) $(wasm_lib_forward) .make/machines - cargo run --manifest-path $(forward_dir)/Cargo.toml -- --path $(forward_dir)/forward_stub.wat --stub - wat2wasm $(wasm_lib)/forward/forward_stub.wat -o $@ - $(output_latest)/machine.wavm.br: $(DEP_PREDICATE) $(prover_bin) $(arbitrator_wasm_libs) $(replay_wasm) - $(prover_bin) $(replay_wasm) --generate-binaries $(output_latest) \ - $(patsubst %,-l $(output_latest)/%.wasm, forward soft-float wasi_stub host_io user_host arbcompress program_exec) + $(prover_bin) $(replay_wasm) --generate-binaries $(output_latest) --with-forwarder \ + $(patsubst %,-l $(output_latest)/%.wasm, soft-float wasi_stub host_io user_host arbcompress program_exec) $(arbitrator_cases)/%.wasm: $(arbitrator_cases)/%.wat wat2wasm $< -o $@ @@ -492,10 +476,10 @@ target/testdata/preimages.bin: python3 scripts/create-test-preimages.py $@ contracts/test/prover/proofs/rust-%.json: $(arbitrator_cases)/rust/$(wasm32_wasi)/%.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin - $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin + $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin --with-forwarder contracts/test/prover/proofs/go.json: $(arbitrator_cases)/go/testcase.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin $(arbitrator_tests_link_deps) $(arbitrator_cases)/user.wasm - $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm + $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm --with-forwarder # avoid testing user.wasm in onestepproofs. It can only run as stylus program. contracts/test/prover/proofs/user.json: @@ -508,9 +492,6 @@ contracts/test/prover/proofs/read-inboxmsg-10.json: contracts/test/prover/proofs/global-state.json: echo "[]" > $@ -contracts/test/prover/proofs/forward-test.json: $(arbitrator_cases)/forward-test.wasm $(arbitrator_tests_forward_deps) $(prover_bin) - $(prover_bin) $< -o $@ --allow-hostapi $(patsubst %,-l %, $(arbitrator_tests_forward_deps)) - contracts/test/prover/proofs/link.json: $(arbitrator_cases)/link.wasm $(arbitrator_tests_link_deps) $(prover_bin) $(prover_bin) $< -o $@ --allow-hostapi --stylus-modules $(arbitrator_tests_link_deps) --require-success diff --git a/arbitrator/prover/Cargo.toml b/arbitrator/prover/Cargo.toml index 54756477657..a4a69fcb057 100644 --- a/arbitrator/prover/Cargo.toml +++ b/arbitrator/prover/Cargo.toml @@ -47,6 +47,9 @@ enum-iterator = "2.0.1" criterion = { version = "0.5.0", features = ["html_reports"] } rand = "0.8.4" +[build-dependencies] +wasmer = { path = "../tools/wasmer/lib/api" } + [[bench]] name = "merkle_bench" harness = false diff --git a/arbitrator/wasm-libraries/forward/src/main.rs b/arbitrator/prover/build.rs similarity index 78% rename from arbitrator/wasm-libraries/forward/src/main.rs rename to arbitrator/prover/build.rs index 05a949e8aae..8191771c79e 100644 --- a/arbitrator/wasm-libraries/forward/src/main.rs +++ b/arbitrator/prover/build.rs @@ -1,12 +1,8 @@ -// Copyright 2022-2024, Offchain Labs, Inc. -// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE - -use eyre::Result; -use std::{fs::File, io::Write, path::PathBuf}; -use structopt::StructOpt; +use core::str; +use std::{env, fmt::Write, fs, path::Path}; /// order matters! -const HOSTIOS: [[&str; 3]; 42] = [ +pub const HOSTIOS: [[&str; 3]; 42] = [ ["read_args", "i32", ""], ["write_result", "i32 i32", ""], ["exit_early", "i32", ""], @@ -51,44 +47,16 @@ const HOSTIOS: [[&str; 3]; 42] = [ ["pay_for_memory_grow", "i32", ""], ]; -#[derive(StructOpt)] -#[structopt(name = "arbitrator-prover")] -struct Opts { - #[structopt(long)] - path: PathBuf, - #[structopt(long)] - stub: bool, -} - -fn main() -> Result<()> { - let opts = Opts::from_args(); - let file = &mut File::options() - .create(true) - .write(true) - .truncate(true) - .open(opts.path)?; - - match opts.stub { - true => forward_stub(file), - false => forward(file), - } -} - -fn forward(file: &mut File) -> Result<()> { +pub fn gen_forwarder(out_path: &Path) { + let mut wat = String::new(); macro_rules! wln { ($($text:tt)*) => { - writeln!(file, $($text)*)?; + writeln!(wat, $($text)*).unwrap(); }; } let s = " "; - wln!( - ";; Copyright 2022-2023, Offchain Labs, Inc.\n\ - ;; For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE\n\ - ;; This file is auto-generated.\n\ - \n\ - (module" - ); + wln!("(module"); macro_rules! group { ($list:expr, $kind:expr) => { @@ -102,9 +70,7 @@ fn forward(file: &mut File) -> Result<()> { for [name, ins, outs] in HOSTIOS { let params = group!(ins, "param"); let result = group!(outs, "result"); - wln!( - r#"{s}(import "user_host" "arbitrator_forward__{name}" (func ${name}{params}{result}))"# - ); + wln!(r#"{s}(import "user_host" "{name}" (func $_{name}{params}{result}))"#); } wln!(); @@ -139,20 +105,25 @@ fn forward(file: &mut File) -> Result<()> { } wln!( - "{s}{s}call ${name}\n\ + "{s}{s}call $_{name}\n\ {s}{s}call $check\n\ {s})" ); } wln!(")"); - Ok(()) + + let wasm = wasmer::wat2wasm(wat.as_bytes()).unwrap(); + + fs::write(out_path, wasm.as_ref()).unwrap(); } -fn forward_stub(file: &mut File) -> Result<()> { +pub fn gen_forwarder_stub(out_path: &Path) { + let mut wat = String::new(); + macro_rules! wln { ($($text:tt)*) => { - writeln!(file, $($text)*)?; + writeln!(wat, $($text)*).unwrap(); }; } let s = " "; @@ -202,5 +173,16 @@ fn forward_stub(file: &mut File) -> Result<()> { } wln!(")"); - Ok(()) + + let wasm = wasmer::wat2wasm(wat.as_bytes()).unwrap(); + + fs::write(out_path, wasm.as_ref()).unwrap(); +} + +fn main() { + let out_dir = env::var("OUT_DIR").unwrap(); + let forwarder_path = Path::new(&out_dir).join("forwarder.wasm"); + let forwarder_stub_path = Path::new(&out_dir).join("forwarder_stub.wasm"); + gen_forwarder(&forwarder_path); + gen_forwarder_stub(&forwarder_stub_path); } diff --git a/arbitrator/prover/src/binary.rs b/arbitrator/prover/src/binary.rs index aa5537476c0..b1d819deb26 100644 --- a/arbitrator/prover/src/binary.rs +++ b/arbitrator/prover/src/binary.rs @@ -471,10 +471,14 @@ pub fn parse<'a>(input: &'a [u8], path: &'_ Path) -> Result> { let export = export.rsplit("__").take(1); exports.extend(export); } - for import in &binary.imports { - let name = import.name; - if exports.contains(name) { - bail!("binary exports an import with the same name {}", name.red()); + // forwarder is allowed to re-export the same name + // TODO: is there a real problem if import_name == export_name but module names don't? + if path != Path::new("forwarder") { + for import in &binary.imports { + let name = import.name; + if exports.contains(name) { + bail!("binary exports an import with the same name {}", name.red()); + } } } diff --git a/arbitrator/prover/src/lib.rs b/arbitrator/prover/src/lib.rs index 0f537478eb7..20472691c18 100644 --- a/arbitrator/prover/src/lib.rs +++ b/arbitrator/prover/src/lib.rs @@ -70,9 +70,10 @@ pub unsafe extern "C" fn arbitrator_load_machine( library_paths: *const *const c_char, library_paths_size: isize, debug_chain: usize, + with_forwarder: bool, ) -> *mut Machine { let debug_chain = debug_chain != 0; - match arbitrator_load_machine_impl(binary_path, library_paths, library_paths_size, debug_chain) + match arbitrator_load_machine_impl(binary_path, library_paths, library_paths_size, debug_chain, with_forwarder) { Ok(mach) => mach, Err(err) => { @@ -87,6 +88,7 @@ unsafe fn arbitrator_load_machine_impl( library_paths: *const *const c_char, library_paths_size: isize, debug_chain: bool, + with_forwarder: bool, ) -> Result<*mut Machine> { let binary_path = cstr_to_string(binary_path); let binary_path = Path::new(&binary_path); @@ -107,6 +109,7 @@ unsafe fn arbitrator_load_machine_impl( Default::default(), Default::default(), get_empty_preimage_resolver(), + with_forwarder, )?; Ok(Box::into_raw(Box::new(mach))) } diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index 358876bd252..da3a4e3a79d 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -334,7 +334,7 @@ lazy_static! { static ref USER_IMPORTS: HashMap = { let mut imports = HashMap::default(); - let forward = include_bytes!("../../../target/machines/latest/forward_stub.wasm"); + let forward = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder_stub.wasm")); let forward = binary::parse(forward, Path::new("forward")).unwrap(); for (name, &(export, kind)) in &forward.exports { @@ -352,13 +352,12 @@ lazy_static! { } impl Module { - const FORWARDING_PREFIX: &'static str = "arbitrator_forward__"; - fn from_binary( bin: &WasmBinary, available_imports: &HashMap, floating_point_impls: &FloatingPointImpls, allow_hostapi: bool, + is_forwarder: bool, debug_funcs: bool, stylus_data: Option, ) -> Result { @@ -371,16 +370,11 @@ impl Module { for import in &bin.imports { let module = import.module; let have_ty = &bin.types[import.offset as usize]; - let (forward, import_name) = match import.name.strip_prefix(Module::FORWARDING_PREFIX) { - Some(name) => (true, name), - None => (false, import.name), - }; - let mut qualified_name = format!("{module}__{import_name}"); - qualified_name = qualified_name.replace(&['/', '.', '-'] as &[char], "_"); + let qualified_name = format!("{module}__{}", import.name); let func = if let Some(import) = available_imports.get(&qualified_name) { - let call = match forward { + let call = match is_forwarder { true => Opcode::CrossModuleForward, false => Opcode::CrossModuleCall, }; @@ -393,18 +387,18 @@ impl Module { Instruction::simple(Opcode::Return), ]; Function::new_from_wavm(wavm, import.ty.clone(), vec![]) - } else if let Ok((hostio, debug)) = host::get_impl(import.module, import_name) { + } else if let Ok((hostio, debug)) = host::get_impl(import.module, import.name) { ensure!( (debug && debug_funcs) || (!debug && allow_hostapi), "Host func {} in {} not enabled debug_funcs={debug_funcs} hostapi={allow_hostapi} debug={debug}", - import_name.red(), + import.name.red(), import.module.red(), ); hostio } else { bail!( "No such import {} in {} for {}", - import_name.red(), + import.name.red(), import.module.red(), bin_name.red() ) @@ -412,12 +406,12 @@ impl Module { ensure!( &func.ty == have_ty, "Import {} for {} has different function signature than export.\nexpected {} in {}\nbut have {}", - import_name.red(), bin_name.red(), func.ty.red(), module.red(), have_ty.red(), + import.name.red(), bin_name.red(), func.ty.red(), module.red(), have_ty.red(), ); func_type_idxs.push(import.offset); code.push(func); - host_call_hooks.push(Some((import.module.into(), import_name.into()))); + host_call_hooks.push(Some((import.module.into(), import.name.into()))); } func_type_idxs.extend(bin.functions.iter()); @@ -616,6 +610,7 @@ impl Module { &USER_IMPORTS, &HashMap::default(), false, + false, debug_funcs, stylus_data, ) @@ -1227,6 +1222,7 @@ impl Machine { global_state: GlobalState, inbox_contents: HashMap<(InboxIdentifier, u64), Vec>, preimage_resolver: PreimageResolver, + with_forwarder: bool, ) -> Result { let bin_source = file_bytes(binary_path)?; let bin = parse(&bin_source, binary_path) @@ -1252,6 +1248,7 @@ impl Machine { inbox_contents, preimage_resolver, None, + with_forwarder, ) } @@ -1281,6 +1278,7 @@ impl Machine { HashMap::default(), Arc::new(|_, _, _| panic!("tried to read preimage")), Some(stylus_data), + false, )?; let footprint: u32 = stylus_data.footprint.into(); @@ -1328,6 +1326,7 @@ impl Machine { inbox_contents: HashMap<(InboxIdentifier, u64), Vec>, preimage_resolver: PreimageResolver, stylus_data: Option, + with_forwarder: bool, ) -> Result { use ArbValueType::*; @@ -1337,6 +1336,17 @@ impl Machine { let mut floating_point_impls = HashMap::default(); let main_module_index = u32::try_from(modules.len() + libraries.len())?; + let forwarder_wasm = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); + let mut libs_vec = vec![]; + + let libraries = if with_forwarder { + libs_vec.push(parse(forwarder_wasm.as_ref(), Path::new("forwarder"))?); + libs_vec.extend_from_slice(libraries); + &libs_vec + } else { + libraries + }; + // make the main module's exports available to libraries for (name, &(export, kind)) in &bin.exports { if kind == ExportKind::Func { @@ -1367,12 +1377,13 @@ impl Machine { } } - for lib in libraries { + for (index, lib) in libraries.iter().enumerate() { let module = Module::from_binary( lib, &available_imports, &floating_point_impls, true, + with_forwarder && (index == 0), debug_funcs, None, )?; @@ -1408,6 +1419,7 @@ impl Machine { &available_imports, &floating_point_impls, allow_hostapi_from_main, + false, debug_funcs, stylus_data, )?); diff --git a/arbitrator/prover/src/main.rs b/arbitrator/prover/src/main.rs index dba32e0e727..442b35992dc 100644 --- a/arbitrator/prover/src/main.rs +++ b/arbitrator/prover/src/main.rs @@ -86,6 +86,8 @@ struct Opts { skip_until_host_io: bool, #[structopt(long)] max_steps: Option, + #[structopt(short, long)] + with_forwarder: bool, } fn file_with_stub_header(path: &Path, headerlength: usize) -> Result> { @@ -211,6 +213,7 @@ fn main() -> Result<()> { global_state, inbox_contents, preimage_resolver, + opts.with_forwarder, )?; for path in &opts.stylus_modules { diff --git a/arbitrator/prover/src/test.rs b/arbitrator/prover/src/test.rs index 97170441ffa..52bf394dd5b 100644 --- a/arbitrator/prover/src/test.rs +++ b/arbitrator/prover/src/test.rs @@ -57,7 +57,7 @@ pub fn reject_ambiguous_imports() { #[test] pub fn test_compress() -> Result<()> { - let data = include_bytes!("../../../target/machines/latest/forward_stub.wasm"); + let data = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder_stub.wasm")); let mut last = vec![]; for dict in [Dictionary::Empty, Dictionary::StylusProgram] { diff --git a/arbitrator/prover/test-cases/forward-test.wat b/arbitrator/prover/test-cases/forward-test.wat deleted file mode 100644 index b9beff0d825..00000000000 --- a/arbitrator/prover/test-cases/forward-test.wat +++ /dev/null @@ -1,32 +0,0 @@ - -(module - (import "forward" "add" (func $add (param i32 i32) (result i32))) - (import "forward" "sub" (func $sub (param i32 i32) (result i32))) - (import "forward" "mul" (func $mul (param i32 i32) (result i32))) - (func $start - ;; this address will update each time a forwarded call is made - i32.const 0xa4b - i32.const 805 - i32.store - - i32.const 11 - i32.const 5 - call $sub - - i32.const 3 - i32.const -2 - call $mul - - call $add - (if - (then (unreachable))) - - ;; check that the address has changed - i32.const 0xa4b - i32.load - i32.const 808 - i32.ne - (if - (then (unreachable)))) - (start $start) - (memory 1)) diff --git a/arbitrator/prover/test-cases/forward/forward.wat b/arbitrator/prover/test-cases/forward/forward.wat deleted file mode 100644 index ff55953e626..00000000000 --- a/arbitrator/prover/test-cases/forward/forward.wat +++ /dev/null @@ -1,8 +0,0 @@ - -(module - (import "target" "arbitrator_forward__add" (func $add (param i32 i32) (result i32))) - (import "target" "arbitrator_forward__sub" (func $sub (param i32 i32) (result i32))) - (import "target" "arbitrator_forward__mul" (func $mul (param i32 i32) (result i32))) - (export "forward__add" (func $add)) - (export "forward__sub" (func $sub)) - (export "forward__mul" (func $mul))) diff --git a/arbitrator/prover/test-cases/forward/target.wat b/arbitrator/prover/test-cases/forward/target.wat deleted file mode 100644 index 0779eb753d5..00000000000 --- a/arbitrator/prover/test-cases/forward/target.wat +++ /dev/null @@ -1,27 +0,0 @@ - -(module - (import "env" "wavm_caller_load8" (func $load (param i32) (result i32))) - (import "env" "wavm_caller_store8" (func $store (param i32 i32))) - (func (export "target__add") (param i32 i32) (result i32) - call $write_caller - local.get 0 - local.get 1 - i32.add) - (func (export "target__sub") (param i32 i32) (result i32) - call $write_caller - local.get 0 - local.get 1 - i32.sub) - (func (export "target__mul") (param i32 i32) (result i32) - call $write_caller - local.get 0 - local.get 1 - i32.mul) - (func $write_caller (export "write_caller") - ;; increment the value at address 0xa4b in the caller - i32.const 0xa4b - i32.const 0xa4b - call $load - i32.const 1 - i32.add - call $store)) diff --git a/arbitrator/stylus/src/test/mod.rs b/arbitrator/stylus/src/test/mod.rs index 00c9c62ae4b..42c94465643 100644 --- a/arbitrator/stylus/src/test/mod.rs +++ b/arbitrator/stylus/src/test/mod.rs @@ -164,6 +164,7 @@ fn new_test_machine(path: &str, compile: &CompileConfig) -> Result { HashMap::default(), Arc::new(|_, _, _| panic!("tried to read preimage")), Some(stylus_data), + false, )?; mach.set_ink(u64::MAX); mach.set_stack(u32::MAX); diff --git a/arbitrator/tools/module_roots/src/main.rs b/arbitrator/tools/module_roots/src/main.rs index 32e47648476..fc22bed5fa7 100644 --- a/arbitrator/tools/module_roots/src/main.rs +++ b/arbitrator/tools/module_roots/src/main.rs @@ -39,6 +39,7 @@ fn main() -> Result<()> { GlobalState::default(), HashMap::default(), Arc::new(|_, _, _| panic!("tried to read preimage")), + false, )?; let mut stylus = vec![]; diff --git a/arbitrator/wasm-libraries/Cargo.lock b/arbitrator/wasm-libraries/Cargo.lock index 7620ff538b8..b234424f690 100644 --- a/arbitrator/wasm-libraries/Cargo.lock +++ b/arbitrator/wasm-libraries/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +dependencies = [ + "gimli 0.29.0", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.7.8" @@ -91,6 +106,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "backtrace" +version = "0.3.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "bincode" version = "1.3.3" @@ -187,6 +217,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.7.0" @@ -203,6 +239,15 @@ dependencies = [ "rand_pcg", ] +[[package]] +name = "cc" +version = "1.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" +dependencies = [ + "shlex", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -236,6 +281,19 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "corosensei" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80128832c58ea9cbd041d2a759ec449224487b2c1e400453d99d244eead87a8e" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "libc", + "scopeguard", + "windows-sys 0.33.0", +] + [[package]] name = "cpufeatures" version = "0.2.12" @@ -245,6 +303,123 @@ dependencies = [ "libc", ] +[[package]] +name = "cranelift-bforest" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2ab4512dfd3a6f4be184403a195f76e81a8a9f9e6c898e19d2dc3ce20e0115" +dependencies = [ + "cranelift-entity", +] + +[[package]] +name = "cranelift-codegen" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b022ed2a5913a38839dfbafe6cf135342661293b08049843362df4301261dc" +dependencies = [ + "arrayvec", + "bumpalo", + "cranelift-bforest", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-egraph", + "cranelift-entity", + "cranelift-isle", + "gimli 0.26.2", + "log", + "regalloc2", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "639307b45434ad112a98f8300c0f0ab085cbefcd767efcdef9ef19d4c0756e74" +dependencies = [ + "cranelift-codegen-shared", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278e52e29c53fcf32431ef08406c295699a70306d05a0715c5b1bf50e33a9ab7" + +[[package]] +name = "cranelift-egraph" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624b54323b06e675293939311943ba82d323bb340468ce1889be5da7932c8d73" +dependencies = [ + "cranelift-entity", + "fxhash", + "hashbrown 0.12.3", + "indexmap 1.9.3", + "log", + "smallvec", +] + +[[package]] +name = "cranelift-entity" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a59bcbca89c3f1b70b93ab3cbba5e5e0cbf3e63dadb23c7525cb142e21a9d4c" + +[[package]] +name = "cranelift-frontend" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d70abacb8cfef3dc8ff7e8836e9c1d70f7967dfdac824a4cd5e30223415aca6" +dependencies = [ + "cranelift-codegen", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-isle" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "393bc73c451830ff8dbb3a07f61843d6cb41a084f9996319917c0b291ed785bb" + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "crunchy" version = "0.2.2" @@ -330,6 +505,19 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if 1.0.0", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "derivative" version = "2.2.0" @@ -456,26 +644,33 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "forward" -version = "0.1.0" -dependencies = [ - "eyre", - "structopt", -] - [[package]] name = "funty" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -497,6 +692,23 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" +dependencies = [ + "fallible-iterator", + "indexmap 1.9.3", + "stable_deref_trait", +] + +[[package]] +name = "gimli" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" + [[package]] name = "hashbrown" version = "0.12.3" @@ -595,6 +807,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "js-sys" +version = "0.3.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "keccak" version = "0.1.5" @@ -632,6 +853,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + [[package]] name = "lru" version = "0.12.4" @@ -641,12 +868,57 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + +[[package]] +name = "mach2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" +dependencies = [ + "libc", +] + [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + +[[package]] +name = "memmap2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d28bba84adfe6646737845bc5ebbfa2c08424eb1c37e94a1fd2a82adb56a872" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "memory_units" version = "0.4.0" @@ -659,6 +931,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + [[package]] name = "more-asserts" version = "0.2.2" @@ -791,6 +1072,15 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "object" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.19.0" @@ -832,6 +1122,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -912,6 +1208,7 @@ dependencies = [ "smallvec", "static_assertions", "structopt", + "wasmer", "wasmer-types", "wasmparser", "wat", @@ -976,6 +1273,26 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.3" @@ -985,6 +1302,30 @@ dependencies = [ "bitflags 2.6.0", ] +[[package]] +name = "regalloc2" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300d4fbfb40c1c66a78ba3ddd41c1110247cf52f97b87d0f2fc9209bd49b030c" +dependencies = [ + "fxhash", + "log", + "slice-group-by", + "smallvec", +] + +[[package]] +name = "region" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6b6ebd13bc009aef9cd476c1310d49ac354d36e240cf1bd753290f3dc7199a7" +dependencies = [ + "bitflags 1.3.2", + "libc", + "mach2", + "windows-sys 0.52.0", +] + [[package]] name = "rend" version = "0.4.2" @@ -1075,6 +1416,12 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "self_cell" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d369a96f978623eb3dc28807c4852d6cc617fed53da5d3c400feff1ef34a714a" + [[package]] name = "semver" version = "1.0.23" @@ -1090,6 +1437,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b4c031cd0d9014307d82b8abf653c0290fbdaeb4c02d00c63cf52f728628bf" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_derive" version = "1.0.204" @@ -1181,6 +1539,22 @@ dependencies = [ "keccak", ] +[[package]] +name = "shared-buffer" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6c99835bad52957e7aa241d3975ed17c1e5f8c92026377d117a606f36b84b16" +dependencies = [ + "bytes", + "memmap2 0.6.2", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "simdutf8" version = "0.1.4" @@ -1193,6 +1567,12 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "slice-group-by" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" + [[package]] name = "smallvec" version = "1.13.2" @@ -1202,6 +1582,12 @@ dependencies = [ "serde", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -1348,6 +1734,37 @@ dependencies = [ "winnow", ] +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + [[package]] name = "typenum" version = "1.17.0" @@ -1445,13 +1862,147 @@ dependencies = [ "wee_alloc", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" + [[package]] name = "wasm-encoder" -version = "0.215.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb56df3e06b8e6b77e37d2969a50ba51281029a9aeb3855e76b7f49b6418847" +checksum = "1ba64e81215916eaeb48fee292f29401d69235d62d8b8fd92a7b2844ec5ae5f7" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasmer" +version = "4.2.8" dependencies = [ + "bytes", + "cfg-if 1.0.0", + "derivative", + "indexmap 1.9.3", + "js-sys", + "more-asserts", + "rustc-demangle", + "serde", + "serde-wasm-bindgen", + "shared-buffer", + "target-lexicon", + "thiserror", + "tracing", + "wasm-bindgen", + "wasmer-compiler", + "wasmer-compiler-cranelift", + "wasmer-derive", + "wasmer-types", + "wasmer-vm", + "wat", + "winapi", +] + +[[package]] +name = "wasmer-compiler" +version = "4.2.8" +dependencies = [ + "backtrace", + "bytes", + "cfg-if 1.0.0", + "enum-iterator 0.7.0", + "enumset", + "lazy_static", "leb128", + "memmap2 0.5.10", + "more-asserts", + "region", + "rkyv", + "self_cell", + "shared-buffer", + "smallvec", + "thiserror", + "wasmer-types", + "wasmer-vm", + "wasmparser", + "winapi", +] + +[[package]] +name = "wasmer-compiler-cranelift" +version = "4.2.8" +dependencies = [ + "cranelift-codegen", + "cranelift-entity", + "cranelift-frontend", + "gimli 0.26.2", + "more-asserts", + "rayon", + "smallvec", + "target-lexicon", + "tracing", + "wasmer-compiler", + "wasmer-types", +] + +[[package]] +name = "wasmer-derive" +version = "4.2.8" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -1468,6 +2019,32 @@ dependencies = [ "thiserror", ] +[[package]] +name = "wasmer-vm" +version = "4.2.8" +dependencies = [ + "backtrace", + "cc", + "cfg-if 1.0.0", + "corosensei", + "crossbeam-queue", + "dashmap", + "derivative", + "enum-iterator 0.7.0", + "fnv", + "indexmap 1.9.3", + "lazy_static", + "libc", + "mach", + "memoffset", + "more-asserts", + "region", + "scopeguard", + "thiserror", + "wasmer-types", + "winapi", +] + [[package]] name = "wasmparser" version = "0.121.2" @@ -1481,11 +2058,10 @@ dependencies = [ [[package]] name = "wast" -version = "215.0.0" +version = "64.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ff1d00d893593249e60720be04a7c1f42f1c4dc3806a2869f4e66ab61eb54cb" +checksum = "a259b226fd6910225aa7baeba82f9d9933b6d00f2ce1b49b80fa4214328237cc" dependencies = [ - "bumpalo", "leb128", "memchr", "unicode-width", @@ -1494,9 +2070,9 @@ dependencies = [ [[package]] name = "wat" -version = "1.215.0" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670bf4d9c8cf76ae242d70ded47c546525b6dafaa6871f9bcb065344bf2b4e3d" +checksum = "53253d920ab413fca1c7dc2161d601c79b4fdf631d0ba51dd4343bf9b556c3f6" dependencies = [ "wast", ] @@ -1535,6 +2111,28 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43dbb096663629518eb1dfa72d80243ca5a6aca764cae62a2df70af760a9be75" +dependencies = [ + "windows_aarch64_msvc 0.33.0", + "windows_i686_gnu 0.33.0", + "windows_i686_msvc 0.33.0", + "windows_x86_64_gnu 0.33.0", + "windows_x86_64_msvc 0.33.0", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -1542,13 +2140,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -1557,12 +2155,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd761fd3eb9ab8cc1ed81e56e567f02dd82c4c837e48ac3b2181b9ffc5060807" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab0cf703a96bab2dc0c02c0fa748491294bf9b7feb27e1f4f96340f208ada0e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -1575,12 +2185,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfdbe89cc9ad7ce618ba34abc34bbb6c36d99e96cae2245b7943cd75ee773d0" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4dd9b0c0e9ece7bb22e84d70d01b71c6d6248b81a3c60d11869451b4cb24784" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -1593,6 +2215,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1e4aa646495048ec7f3ffddc411e1d829c026a2ec62b39da15c1055e406eaa" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/arbitrator/wasm-libraries/Cargo.toml b/arbitrator/wasm-libraries/Cargo.toml index 837df8f4da5..2fd585ee7df 100644 --- a/arbitrator/wasm-libraries/Cargo.toml +++ b/arbitrator/wasm-libraries/Cargo.toml @@ -7,6 +7,5 @@ members = [ "user-host-trait", "user-test", "program-exec", - "forward", ] resolver = "2" diff --git a/arbitrator/wasm-libraries/forward/.gitignore b/arbitrator/wasm-libraries/forward/.gitignore deleted file mode 100644 index 40da2042b75..00000000000 --- a/arbitrator/wasm-libraries/forward/.gitignore +++ /dev/null @@ -1 +0,0 @@ -**.wat diff --git a/arbitrator/wasm-libraries/forward/Cargo.toml b/arbitrator/wasm-libraries/forward/Cargo.toml deleted file mode 100644 index 73ed9d8827b..00000000000 --- a/arbitrator/wasm-libraries/forward/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "forward" -version = "0.1.0" -edition = "2021" - -[dependencies] -eyre = "0.6.5" -structopt = "0.3.26" diff --git a/arbitrator/wasm-testsuite/src/main.rs b/arbitrator/wasm-testsuite/src/main.rs index 2144dcf992d..cb861e3a357 100644 --- a/arbitrator/wasm-testsuite/src/main.rs +++ b/arbitrator/wasm-testsuite/src/main.rs @@ -343,6 +343,7 @@ fn main() -> eyre::Result<()> { GlobalState::default(), HashMap::default(), machine::get_empty_preimage_resolver(), + false, ); if let Err(error) = &mech { @@ -429,6 +430,7 @@ fn main() -> eyre::Result<()> { GlobalState::default(), HashMap::default(), machine::get_empty_preimage_resolver(), + false, ) .expect_err(&format!("failed to reject invalid module {}", filename)); } diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index adca9695e22..e77e071d708 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -92,7 +92,7 @@ func LoadSimpleMachine(wasm string, libraries []string, debugChain bool) (*Arbit cWasm := C.CString(wasm) cLibraries := CreateCStringList(libraries) debug := usize(arbmath.BoolToUint32(debugChain)) - mach := C.arbitrator_load_machine(cWasm, cLibraries, C.long(len(libraries)), debug) + mach := C.arbitrator_load_machine(cWasm, cLibraries, C.long(len(libraries)), debug, false) C.free(unsafe.Pointer(cWasm)) FreeCStringList(cLibraries, len(libraries)) if mach == nil { From ed18d13032542989318a664c3c85d1c33756eca7 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 5 Sep 2024 19:35:58 -0600 Subject: [PATCH 103/156] remove forwarder_stub --- arbitrator/arbutil/src/hostios.rs | 64 +++++++++++++++ arbitrator/arbutil/src/lib.rs | 1 + arbitrator/prover/Cargo.toml | 1 + arbitrator/prover/build.rs | 131 ++++-------------------------- arbitrator/prover/src/machine.rs | 118 ++++++++++++--------------- arbitrator/prover/src/test.rs | 2 +- arbitrator/prover/src/value.rs | 12 ++- 7 files changed, 145 insertions(+), 184 deletions(-) create mode 100644 arbitrator/arbutil/src/hostios.rs diff --git a/arbitrator/arbutil/src/hostios.rs b/arbitrator/arbutil/src/hostios.rs new file mode 100644 index 00000000000..05bcb94f73b --- /dev/null +++ b/arbitrator/arbutil/src/hostios.rs @@ -0,0 +1,64 @@ +use std::fmt::Display; + +pub enum ParamType { + I32, + I64, +} + +impl Display for ParamType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use ParamType::*; + match self { + I32 => write!(f, "i32"), + I64 => write!(f, "i64"), + } + } +} + +use ParamType::*; + +/// order matters! +pub const HOSTIOS: [(&str, &[ParamType], &[ParamType]); 42] = [ + ("read_args", &[I32], &[]), + ("write_result", &[I32, I32], &[]), + ("exit_early", &[I32], &[]), + ("storage_load_bytes32", &[I32, I32], &[]), + ("storage_cache_bytes32", &[I32, I32], &[]), + ("storage_flush_cache", &[I32], &[]), + ("transient_load_bytes32", &[I32, I32], &[]), + ("transient_store_bytes32", &[I32, I32], &[]), + ("call_contract", &[I32, I32, I32, I32, I64, I32], &[I32]), + ("delegate_call_contract", &[I32, I32, I32, I64, I32], &[I32]), + ("static_call_contract", &[I32, I32, I32, I64, I32], &[I32]), + ("create1", &[I32, I32, I32, I32, I32], &[]), + ("create2", &[I32, I32, I32, I32, I32, I32], &[]), + ("read_return_data", &[I32, I32, I32], &[I32]), + ("return_data_size", &[], &[I32]), + ("emit_log", &[I32, I32, I32], &[]), + ("account_balance", &[I32, I32], &[]), + ("account_code", &[I32, I32, I32, I32], &[I32]), + ("account_code_size", &[I32], &[I32]), + ("account_codehash", &[I32, I32], &[]), + ("evm_gas_left", &[], &[I64]), + ("evm_ink_left", &[], &[I64]), + ("block_basefee", &[I32], &[]), + ("chainid", &[], &[I64]), + ("block_coinbase", &[I32], &[]), + ("block_gas_limit", &[], &[I64]), + ("block_number", &[], &[I64]), + ("block_timestamp", &[], &[I64]), + ("contract_address", &[I32], &[]), + ("math_div", &[I32, I32], &[]), + ("math_mod", &[I32, I32], &[]), + ("math_pow", &[I32, I32], &[]), + ("math_add_mod", &[I32, I32, I32], &[]), + ("math_mul_mod", &[I32, I32, I32], &[]), + ("msg_reentrant", &[], &[I32]), + ("msg_sender", &[I32], &[]), + ("msg_value", &[I32], &[]), + ("native_keccak256", &[I32, I32, I32], &[]), + ("tx_gas_price", &[I32], &[]), + ("tx_ink_price", &[], &[I32]), + ("tx_origin", &[I32], &[]), + ("pay_for_memory_grow", &[I32], &[]), +]; diff --git a/arbitrator/arbutil/src/lib.rs b/arbitrator/arbutil/src/lib.rs index 9c48a9fefc2..8d8c1d0fca9 100644 --- a/arbitrator/arbutil/src/lib.rs +++ b/arbitrator/arbutil/src/lib.rs @@ -6,6 +6,7 @@ pub mod color; pub mod crypto; pub mod evm; pub mod format; +pub mod hostios; pub mod math; pub mod operator; pub mod pricing; diff --git a/arbitrator/prover/Cargo.toml b/arbitrator/prover/Cargo.toml index a4a69fcb057..a634dba19eb 100644 --- a/arbitrator/prover/Cargo.toml +++ b/arbitrator/prover/Cargo.toml @@ -49,6 +49,7 @@ rand = "0.8.4" [build-dependencies] wasmer = { path = "../tools/wasmer/lib/api" } +arbutil = { path = "../arbutil/" } [[bench]] name = "merkle_bench" diff --git a/arbitrator/prover/build.rs b/arbitrator/prover/build.rs index 8191771c79e..db0560b2c54 100644 --- a/arbitrator/prover/build.rs +++ b/arbitrator/prover/build.rs @@ -1,52 +1,6 @@ -use core::str; +use arbutil::hostios::HOSTIOS; use std::{env, fmt::Write, fs, path::Path}; -/// order matters! -pub const HOSTIOS: [[&str; 3]; 42] = [ - ["read_args", "i32", ""], - ["write_result", "i32 i32", ""], - ["exit_early", "i32", ""], - ["storage_load_bytes32", "i32 i32", ""], - ["storage_cache_bytes32", "i32 i32", ""], - ["storage_flush_cache", "i32", ""], - ["transient_load_bytes32", "i32 i32", ""], - ["transient_store_bytes32", "i32 i32", ""], - ["call_contract", "i32 i32 i32 i32 i64 i32", "i32"], - ["delegate_call_contract", "i32 i32 i32 i64 i32", "i32"], - ["static_call_contract", "i32 i32 i32 i64 i32", "i32"], - ["create1", "i32 i32 i32 i32 i32", ""], - ["create2", "i32 i32 i32 i32 i32 i32", ""], - ["read_return_data", "i32 i32 i32", "i32"], - ["return_data_size", "", "i32"], - ["emit_log", "i32 i32 i32", ""], - ["account_balance", "i32 i32", ""], - ["account_code", "i32 i32 i32 i32", "i32"], - ["account_code_size", "i32", "i32"], - ["account_codehash", "i32 i32", ""], - ["evm_gas_left", "", "i64"], - ["evm_ink_left", "", "i64"], - ["block_basefee", "i32", ""], - ["chainid", "", "i64"], - ["block_coinbase", "i32", ""], - ["block_gas_limit", "", "i64"], - ["block_number", "", "i64"], - ["block_timestamp", "", "i64"], - ["contract_address", "i32", ""], - ["math_div", "i32 i32", ""], - ["math_mod", "i32 i32", ""], - ["math_pow", "i32 i32", ""], - ["math_add_mod", "i32 i32 i32", ""], - ["math_mul_mod", "i32 i32 i32", ""], - ["msg_reentrant", "", "i32"], - ["msg_sender", "i32", ""], - ["msg_value", "i32", ""], - ["native_keccak256", "i32 i32 i32", ""], - ["tx_gas_price", "i32", ""], - ["tx_ink_price", "", "i32"], - ["tx_origin", "i32", ""], - ["pay_for_memory_grow", "i32", ""], -]; - pub fn gen_forwarder(out_path: &Path) { let mut wat = String::new(); macro_rules! wln { @@ -61,13 +15,23 @@ pub fn gen_forwarder(out_path: &Path) { macro_rules! group { ($list:expr, $kind:expr) => { (!$list.is_empty()) - .then(|| format!(" ({} {})", $kind, $list)) + .then(|| { + format!( + " ({} {})", + $kind, + $list + .iter() + .map(|x| x.to_string()) + .collect::>() + .join(" ") + ) + }) .unwrap_or_default() }; } wln!("{s};; symbols to re-export"); - for [name, ins, outs] in HOSTIOS { + for (name, ins, outs) in HOSTIOS { let params = group!(ins, "param"); let result = group!(outs, "result"); wln!(r#"{s}(import "user_host" "{name}" (func $_{name}{params}{result}))"#); @@ -94,13 +58,12 @@ pub fn gen_forwarder(out_path: &Path) { ); wln!("{s};; user linkage"); - for [name, ins, outs] in HOSTIOS { + for (name, ins, outs) in HOSTIOS { let params = group!(ins, "param"); let result = group!(outs, "result"); wln!("{s}(func (export \"vm_hooks__{name}\"){params}{result}"); - let gets = (1 + ins.len()) / 4; - for i in 0..gets { + for i in 0..ins.len() { wln!("{s}{s}local.get {i}"); } @@ -112,67 +75,7 @@ pub fn gen_forwarder(out_path: &Path) { } wln!(")"); - - let wasm = wasmer::wat2wasm(wat.as_bytes()).unwrap(); - - fs::write(out_path, wasm.as_ref()).unwrap(); -} - -pub fn gen_forwarder_stub(out_path: &Path) { - let mut wat = String::new(); - - macro_rules! wln { - ($($text:tt)*) => { - writeln!(wat, $($text)*).unwrap(); - }; - } - let s = " "; - - wln!( - ";; Copyright 2022-2023, Offchain Labs, Inc.\n\ - ;; For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE\n\ - ;; This file is auto-generated.\n\ - \n\ - (module" - ); - - macro_rules! group { - ($list:expr, $kind:expr) => { - (!$list.is_empty()) - .then(|| format!(" ({} {})", $kind, $list)) - .unwrap_or_default() - }; - } - - wln!("{s};; stubs for the symbols we re-export"); - for [name, ins, outs] in HOSTIOS { - let params = group!(ins, "param"); - let result = group!(outs, "result"); - wln!("{s}(func ${name}{params}{result} unreachable)"); - } - wln!(); - - wln!("{s};; reserved offsets for future user_host imports"); - for i in HOSTIOS.len()..512 { - wln!("{s}(func $reserved_{i} unreachable)"); - } - wln!(); - - wln!( - "{s};; allows user_host to request a trap\n\ - {s}(global $trap (mut i32) (i32.const 0))\n\ - {s}(func $check unreachable)\n\ - {s}(func (export \"forward__set_trap\") unreachable)" - ); - - wln!("{s};; user linkage"); - for [name, ins, outs] in HOSTIOS { - let params = group!(ins, "param"); - let result = group!(outs, "result"); - wln!("{s}(func (export \"vm_hooks__{name}\"){params}{result} unreachable)"); - } - - wln!(")"); + eprintln!("{}", &wat); let wasm = wasmer::wat2wasm(wat.as_bytes()).unwrap(); @@ -182,7 +85,5 @@ pub fn gen_forwarder_stub(out_path: &Path) { fn main() { let out_dir = env::var("OUT_DIR").unwrap(); let forwarder_path = Path::new(&out_dir).join("forwarder.wasm"); - let forwarder_stub_path = Path::new(&out_dir).join("forwarder_stub.wasm"); gen_forwarder(&forwarder_path); - gen_forwarder_stub(&forwarder_stub_path); } diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index da3a4e3a79d..c045fc72c9b 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -19,7 +19,7 @@ use crate::{ IBinOpType, IRelOpType, IUnOpType, Instruction, Opcode, }, }; -use arbutil::{crypto, math, Bytes32, Color, DebugColor, PreimageType}; +use arbutil::{crypto, hostios::HOSTIOS, math, Bytes32, Color, DebugColor, PreimageType}; use brotli::Dictionary; #[cfg(feature = "native")] use c_kzg::BYTES_PER_BLOB; @@ -330,22 +330,17 @@ pub struct Module { pub(crate) extra_hash: Arc, } +#[cfg(feature = "native")] +static FORWARDER_WASM: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); + lazy_static! { static ref USER_IMPORTS: HashMap = { let mut imports = HashMap::default(); - let forward = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder_stub.wasm")); - let forward = binary::parse(forward, Path::new("forward")).unwrap(); - - for (name, &(export, kind)) in &forward.exports { - if kind == ExportKind::Func { - let ty = match forward.get_function(FunctionIndex::from_u32(export)) { - Ok(ty) => ty, - Err(error) => panic!("failed to read export {name}: {error:?}"), - }; - let import = AvailableImport::new(ty, 1, export); - imports.insert(name.to_owned(), import); - } + // 0-513 are internal + for (index, (name, ins, outs)) in HOSTIOS.iter().enumerate() { + let import = AvailableImport::new(FunctionType::new(ins.iter().map(|x|x.into()).collect::>(), outs.iter().map(|x|x.into()).collect::>()), 1, (index + 514).try_into().unwrap()); + imports.insert(format!("vm_hooks__{name}"), import); } imports }; @@ -1334,36 +1329,19 @@ impl Machine { let mut modules = vec![Module::default()]; let mut available_imports = HashMap::default(); let mut floating_point_impls = HashMap::default(); - let main_module_index = u32::try_from(modules.len() + libraries.len())?; - let forwarder_wasm = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); - let mut libs_vec = vec![]; + let forwarder = if with_forwarder { + #[cfg(not(feature = "native"))] + bail!("forwarder not supported without native"); - let libraries = if with_forwarder { - libs_vec.push(parse(forwarder_wasm.as_ref(), Path::new("forwarder"))?); - libs_vec.extend_from_slice(libraries); - &libs_vec + #[cfg(feature = "native")] + Some(parse(FORWARDER_WASM, Path::new("forwarder"))?) } else { - libraries + None }; - // make the main module's exports available to libraries - for (name, &(export, kind)) in &bin.exports { - if kind == ExportKind::Func { - let index: usize = export.try_into()?; - if let Some(index) = index.checked_sub(bin.imports.len()) { - let ty: usize = bin.functions[index].try_into()?; - let ty = bin.types[ty].clone(); - available_imports.insert( - format!("env__wavm_guest_call__{name}"), - AvailableImport::new(ty, main_module_index, export), - ); - } - } - } - // collect all the library exports in advance so they can use each other's - for (index, lib) in libraries.iter().enumerate() { + for (index, lib) in forwarder.iter().chain(libraries.iter()).enumerate() { let module = 1 + index as u32; // off by one due to the entry point for (name, &(export, kind)) in &lib.exports { if kind == ExportKind::Func { @@ -1371,49 +1349,55 @@ impl Machine { Ok(ty) => ty, Err(error) => bail!("failed to read export {name}: {error}"), }; - let import = AvailableImport::new(ty, module, export); + let import = AvailableImport::new(ty.clone(), module, export); available_imports.insert(name.to_owned(), import); + if let Ok(op) = name.parse::() { + let mut sig = op.signature(); + // wavm codegen takes care of effecting this type change at callsites + for ty in sig.inputs.iter_mut().chain(sig.outputs.iter_mut()) { + if *ty == F32 { + *ty = I32; + } else if *ty == F64 { + *ty = I64; + } + } + ensure!( + ty == sig, + "Wrong type for floating point impl {} expecting {} but got {}", + name.red(), + sig.red(), + ty.red() + ); + floating_point_impls.insert(op, (module, export)); + } } } } - for (index, lib) in libraries.iter().enumerate() { - let module = Module::from_binary( + if let Some(lib) = forwarder { + modules.push(Module::from_binary( + &lib, + &available_imports, + &floating_point_impls, + true, + true, + debug_funcs, + None, + )?); + } + + for lib in libraries.iter() { + modules.push(Module::from_binary( lib, &available_imports, &floating_point_impls, true, - with_forwarder && (index == 0), + false, debug_funcs, None, - )?; - for (name, &func) in &*module.func_exports { - let ty = module.func_types[func as usize].clone(); - if let Ok(op) = name.parse::() { - let mut sig = op.signature(); - // wavm codegen takes care of effecting this type change at callsites - for ty in sig.inputs.iter_mut().chain(sig.outputs.iter_mut()) { - if *ty == F32 { - *ty = I32; - } else if *ty == F64 { - *ty = I64; - } - } - ensure!( - ty == sig, - "Wrong type for floating point impl {} expecting {} but got {}", - name.red(), - sig.red(), - ty.red() - ); - floating_point_impls.insert(op, (modules.len() as u32, func)); - } - } - modules.push(module); + )?); } - // Shouldn't be necessary, but to be safe, don't allow the main binary to import its own guest calls - available_imports.retain(|_, i| i.module as usize != modules.len()); modules.push(Module::from_binary( &bin, &available_imports, diff --git a/arbitrator/prover/src/test.rs b/arbitrator/prover/src/test.rs index 52bf394dd5b..2f32aff0125 100644 --- a/arbitrator/prover/src/test.rs +++ b/arbitrator/prover/src/test.rs @@ -57,7 +57,7 @@ pub fn reject_ambiguous_imports() { #[test] pub fn test_compress() -> Result<()> { - let data = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder_stub.wasm")); + let data = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); let mut last = vec![]; for dict in [Dictionary::Empty, Dictionary::StylusProgram] { diff --git a/arbitrator/prover/src/value.rs b/arbitrator/prover/src/value.rs index 6afffdf7a05..f2f54404f04 100644 --- a/arbitrator/prover/src/value.rs +++ b/arbitrator/prover/src/value.rs @@ -2,7 +2,7 @@ // For license information, see https://github.com/nitro/blob/master/LICENSE use crate::binary::FloatType; -use arbutil::{Bytes32, Color}; +use arbutil::{hostios::ParamType, Bytes32, Color}; use digest::Digest; use eyre::{bail, ErrReport, Result}; use serde::{Deserialize, Serialize}; @@ -77,6 +77,16 @@ impl From for ValType { } } +impl From<&ParamType> for ArbValueType { + fn from(ty: &ParamType) -> Self { + use ParamType as V; + match ty { + V::I32 => Self::I32, + V::I64 => Self::I64, + } + } +} + #[cfg(feature = "native")] pub fn parser_type(ty: &wasmer::Type) -> wasmer::wasmparser::ValType { match ty { From 8b719d0d0e7a948849f6735e597eb7d04b699594 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 18:05:27 -0600 Subject: [PATCH 104/156] Merge remote-tracking branch 'origin/master' into forwarder-refactor --- arbnode/inbox_test.go | 4 +- arbos/programs/native.go | 70 +++++++++++------ arbos/programs/wasmstorehelper.go | 3 +- cmd/nitro/init.go | 6 +- cmd/nitro/init_test.go | 10 +++ cmd/nitro/nitro.go | 2 +- execution/gethexec/executionengine.go | 40 ++++++---- execution/gethexec/node.go | 46 ++++++++++-- go-ethereum | 2 +- staker/block_validator.go | 10 ++- staker/challenge_manager.go | 3 +- staker/stateless_block_validator.go | 5 +- system_tests/common_test.go | 52 +++++++------ system_tests/forwarder_test.go | 4 +- system_tests/program_test.go | 92 +++++++++++++++++++++-- system_tests/recreatestate_rpc_test.go | 18 ++--- system_tests/retryable_test.go | 2 +- system_tests/validation_mock_test.go | 6 +- validator/client/redis/producer.go | 9 ++- validator/client/validation_client.go | 15 ++-- validator/interface.go | 4 +- validator/server_api/json.go | 8 +- validator/server_arb/validator_spawner.go | 5 +- validator/server_jit/spawner.go | 5 +- validator/validation_entry.go | 4 +- validator/valnode/redis/consumer.go | 11 ++- validator/valnode/validation_api.go | 4 +- 27 files changed, 310 insertions(+), 130 deletions(-) diff --git a/arbnode/inbox_test.go b/arbnode/inbox_test.go index 1c46c593b99..d579b7c2784 100644 --- a/arbnode/inbox_test.go +++ b/arbnode/inbox_test.go @@ -72,7 +72,9 @@ func NewTransactionStreamerForTest(t *testing.T, ownerAddress common.Address) (* if err != nil { Fail(t, err) } - if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, &gethexec.DefaultStylusTargetConfig); err != nil { + stylusTargetConfig := &gethexec.DefaultStylusTargetConfig + Require(t, stylusTargetConfig.Validate()) // pre-processes config (i.a. parses wasmTargets) + if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, stylusTargetConfig); err != nil { Fail(t, err) } execSeq := &execClientWrapper{execEngine, t} diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 9316e8f230e..377e25a31e5 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/arbos/burn" "github.com/offchainlabs/nitro/arbos/util" @@ -71,7 +72,7 @@ func activateProgramInternal( version uint16, debug bool, gasLeft *uint64, -) (*activationInfo, map[rawdb.Target][]byte, error) { +) (*activationInfo, map[ethdb.WasmTarget][]byte, error) { output := &rustBytes{} moduleHash := &bytes32{} stylusData := &C.StylusData{} @@ -99,25 +100,50 @@ func activateProgramInternal( } return nil, nil, err } - target := rawdb.LocalTarget() - status_asm := C.stylus_compile( - goSlice(wasm), - u16(version), - cbool(debug), - goSlice([]byte(target)), - output, - ) - asm := output.intoBytes() - if status_asm != 0 { - return nil, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm)) + targets := db.Database().WasmTargets() + type result struct { + target ethdb.WasmTarget + asm []byte + err error } - asmMap := map[rawdb.Target][]byte{ - rawdb.TargetWavm: module, - target: asm, + results := make(chan result, len(targets)) + for _, target := range targets { + if target == rawdb.TargetWavm { + results <- result{target, module, nil} + } else { + target := target + go func() { + output := &rustBytes{} + status_asm := C.stylus_compile( + goSlice(wasm), + u16(version), + cbool(debug), + goSlice([]byte(target)), + output, + ) + asm := output.intoBytes() + if status_asm != 0 { + results <- result{target, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm))} + return + } + results <- result{target, asm, nil} + }() + } + } + asmMap := make(map[ethdb.WasmTarget][]byte, len(targets)) + for range targets { + res := <-results + if res.err != nil { + err = errors.Join(res.err, err) + } else { + asmMap[res.target] = res.asm + } + } + if err != nil { + return nil, nil, fmt.Errorf("compilation failed for one or more targets: %w", err) } hash := moduleHash.toHash() - info := &activationInfo{ moduleHash: hash, initGas: uint16(stylusData.init_cost), @@ -171,7 +197,7 @@ func getLocalAsm(statedb vm.StateDB, moduleHash common.Hash, addressForLogging c } asm, exists := asmMap[localTarget] if !exists { - var availableTargets []rawdb.Target + var availableTargets []ethdb.WasmTarget for target := range asmMap { availableTargets = append(availableTargets, target) } @@ -202,12 +228,8 @@ func callProgram( panic("missing asm") } - if db, ok := db.(*state.StateDB); ok { - targets := []rawdb.Target{ - rawdb.TargetWavm, - rawdb.LocalTarget(), - } - db.RecordProgram(targets, moduleHash) + if stateDb, ok := db.(*state.StateDB); ok { + stateDb.RecordProgram(db.Database().WasmTargets(), moduleHash) } evmApi := newApi(interpreter, tracingInfo, scope, memoryModel) @@ -291,7 +313,7 @@ func ResizeWasmLruCache(size uint32) { const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon" const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2+lzcnt+bmi" -func SetTarget(name rawdb.Target, description string, native bool) error { +func SetTarget(name ethdb.WasmTarget, description string, native bool) error { output := &rustBytes{} status := userStatus(C.stylus_target_set( goSlice([]byte(name)), diff --git a/arbos/programs/wasmstorehelper.go b/arbos/programs/wasmstorehelper.go index 4f82d802828..434820dd9c9 100644 --- a/arbos/programs/wasmstorehelper.go +++ b/arbos/programs/wasmstorehelper.go @@ -43,8 +43,9 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash return err } + targets := statedb.Database().WasmTargets() // If already in wasm store then return early - _, err = statedb.TryGetActivatedAsmMap([]rawdb.Target{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash) + _, err = statedb.TryGetActivatedAsmMap(targets, moduleHash) if err == nil { return nil } diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 51d895c0697..a8463a7d210 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -560,7 +560,7 @@ func rebuildLocalWasm(ctx context.Context, config *gethexec.Config, l2BlockChain return chainDb, l2BlockChain, nil } -func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { +func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, targetConfig *gethexec.StylusTargetConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { if !config.Init.Force { if readOnlyDb, err := stack.OpenDatabaseWithFreezerWithExtraOptions("l2chaindata", 0, 0, config.Persistent.Ancient, "l2chaindata/", true, persistentConfig.Pebble.ExtraOptions("l2chaindata")); err == nil { if chainConfig := gethexec.TryReadStoredChainConfig(readOnlyDb); chainConfig != nil { @@ -585,7 +585,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err := dbutil.UnfinishedConversionCheck(wasmDb); err != nil { return nil, nil, fmt.Errorf("wasm unfinished database conversion check error: %w", err) } - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets()) _, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb) if err != nil { return nil, nil, err @@ -660,7 +660,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err := validateOrUpgradeWasmStoreSchemaVersion(wasmDb); err != nil { return nil, nil, err } - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets()) _, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb) if err != nil { return nil, nil, err diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index 7b9153a9cf7..48d969f053a 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -354,6 +354,12 @@ func TestEmptyDatabaseDir(t *testing.T) { } } +func defaultStylusTargetConfigForTest(t *testing.T) *gethexec.StylusTargetConfig { + targetConfig := gethexec.DefaultStylusTargetConfig + Require(t, targetConfig.Validate()) + return &targetConfig +} + func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { t.Parallel() @@ -381,6 +387,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -397,6 +404,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -414,6 +422,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -549,6 +558,7 @@ func TestOpenInitializeChainDbEmptyInit(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index d4b2a87a304..146a0049e72 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -495,7 +495,7 @@ func mainImpl() int { } } - chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Persistent, l1Client, rollupAddrs) + chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Execution.StylusTarget, &nodeConfig.Persistent, l1Client, rollupAddrs) if l2BlockChain != nil { deferFuncs = append(deferFuncs, func() { l2BlockChain.Stop() }) } diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 991c94540ed..8594d5867df 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -151,19 +151,33 @@ func (s *ExecutionEngine) MarkFeedStart(to arbutil.MessageIndex) { } func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { - var effectiveStylusTarget string - target := rawdb.LocalTarget() - switch target { - case rawdb.TargetArm64: - effectiveStylusTarget = targetConfig.Arm64 - case rawdb.TargetAmd64: - effectiveStylusTarget = targetConfig.Amd64 - case rawdb.TargetHost: - effectiveStylusTarget = targetConfig.Host - } - err := programs.SetTarget(target, effectiveStylusTarget, true) - if err != nil { - return fmt.Errorf("Failed to set stylus target: %w", err) + localTarget := rawdb.LocalTarget() + targets := targetConfig.WasmTargets() + var nativeSet bool + for _, target := range targets { + var effectiveStylusTarget string + switch target { + case rawdb.TargetWavm: + // skip wavm target + continue + case rawdb.TargetArm64: + effectiveStylusTarget = targetConfig.Arm64 + case rawdb.TargetAmd64: + effectiveStylusTarget = targetConfig.Amd64 + case rawdb.TargetHost: + effectiveStylusTarget = targetConfig.Host + default: + return fmt.Errorf("unsupported stylus target: %v", target) + } + isNative := target == localTarget + err := programs.SetTarget(target, effectiveStylusTarget, isNative) + if err != nil { + return fmt.Errorf("failed to set stylus target: %w", err) + } + nativeSet = nativeSet || isNative + } + if !nativeSet { + return fmt.Errorf("local target %v missing in list of archs %v", localTarget, targets) } return nil } diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index b864332e83d..21c2b4beced 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/arbitrum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/filters" @@ -29,21 +30,51 @@ import ( ) type StylusTargetConfig struct { - Arm64 string `koanf:"arm64"` - Amd64 string `koanf:"amd64"` - Host string `koanf:"host"` + Arm64 string `koanf:"arm64"` + Amd64 string `koanf:"amd64"` + Host string `koanf:"host"` + ExtraArchs []string `koanf:"extra-archs"` + + wasmTargets []ethdb.WasmTarget +} + +func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { + return c.wasmTargets +} + +func (c *StylusTargetConfig) Validate() error { + targetsSet := make(map[ethdb.WasmTarget]bool, len(c.ExtraArchs)) + for _, arch := range c.ExtraArchs { + target := ethdb.WasmTarget(arch) + if !rawdb.IsSupportedWasmTarget(target) { + return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) + } + targetsSet[target] = true + } + if !targetsSet[rawdb.TargetWavm] { + return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) + } + targetsSet[rawdb.LocalTarget()] = true + targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) + for target := range targetsSet { + targets = append(targets, target) + } + c.wasmTargets = targets + return nil } var DefaultStylusTargetConfig = StylusTargetConfig{ - Arm64: programs.DefaultTargetDescriptionArm, - Amd64: programs.DefaultTargetDescriptionX86, - Host: "", + Arm64: programs.DefaultTargetDescriptionArm, + Amd64: programs.DefaultTargetDescriptionX86, + Host: "", + ExtraArchs: []string{string(rawdb.TargetWavm)}, } func StylusTargetConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".arm64", DefaultStylusTargetConfig.Arm64, "stylus programs compilation target for arm64 linux") f.String(prefix+".amd64", DefaultStylusTargetConfig.Amd64, "stylus programs compilation target for amd64 linux") f.String(prefix+".host", DefaultStylusTargetConfig.Host, "stylus programs compilation target for system other than 64-bit ARM or 64-bit x86") + f.StringSlice(prefix+".extra-archs", DefaultStylusTargetConfig.ExtraArchs, fmt.Sprintf("Comma separated list of extra architectures to cross-compile stylus program to and cache in wasm store (additionally to local target). Currently must include at least %s. (supported targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)) } type Config struct { @@ -82,6 +113,9 @@ func (c *Config) Validate() error { if c.forwardingTarget != "" && c.Sequencer.Enable { return errors.New("ForwardingTarget set and sequencer enabled") } + if err := c.StylusTarget.Validate(); err != nil { + return err + } return nil } diff --git a/go-ethereum b/go-ethereum index 575062fad7f..81114dde8a2 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 575062fad7ff4db9d7c235f49472f658be29e2fe +Subproject commit 81114dde8a26bae90c188605c4a36d5919a4a265 diff --git a/staker/block_validator.go b/staker/block_validator.go index 2239952b37f..7a7efca8460 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -106,6 +106,7 @@ type BlockValidatorConfig struct { ValidationServerConfigs []rpcclient.ClientConfig `koanf:"validation-server-configs"` ValidationPoll time.Duration `koanf:"validation-poll" reload:"hot"` PrerecordedBlocks uint64 `koanf:"prerecorded-blocks" reload:"hot"` + RecordingIterLimit uint64 `koanf:"recording-iter-limit"` ForwardBlocks uint64 `koanf:"forward-blocks" reload:"hot"` CurrentModuleRoot string `koanf:"current-module-root"` // TODO(magic) requires reinitialization on hot reload PendingUpgradeModuleRoot string `koanf:"pending-upgrade-module-root"` // TODO(magic) requires StatelessBlockValidator recreation on hot reload @@ -174,6 +175,7 @@ func BlockValidatorConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Uint64(prefix+".forward-blocks", DefaultBlockValidatorConfig.ForwardBlocks, "prepare entries for up to that many blocks ahead of validation (small footprint)") f.Uint64(prefix+".prerecorded-blocks", DefaultBlockValidatorConfig.PrerecordedBlocks, "record that many blocks ahead of validation (larger footprint)") f.String(prefix+".current-module-root", DefaultBlockValidatorConfig.CurrentModuleRoot, "current wasm module root ('current' read from chain, 'latest' from machines/latest dir, or provide hash)") + f.Uint64(prefix+".recording-iter-limit", DefaultBlockValidatorConfig.RecordingIterLimit, "limit on block recordings sent per iteration") f.String(prefix+".pending-upgrade-module-root", DefaultBlockValidatorConfig.PendingUpgradeModuleRoot, "pending upgrade wasm module root to additionally validate (hash, 'latest' or empty)") f.Bool(prefix+".failure-is-fatal", DefaultBlockValidatorConfig.FailureIsFatal, "failing a validation is treated as a fatal error") BlockValidatorDangerousConfigAddOptions(prefix+".dangerous", f) @@ -197,6 +199,7 @@ var DefaultBlockValidatorConfig = BlockValidatorConfig{ FailureIsFatal: true, Dangerous: DefaultBlockValidatorDangerousConfig, MemoryFreeLimit: "default", + RecordingIterLimit: 20, } var TestBlockValidatorConfig = BlockValidatorConfig{ @@ -207,6 +210,7 @@ var TestBlockValidatorConfig = BlockValidatorConfig{ ValidationPoll: 100 * time.Millisecond, ForwardBlocks: 128, PrerecordedBlocks: uint64(2 * runtime.NumCPU()), + RecordingIterLimit: 20, CurrentModuleRoot: "latest", PendingUpgradeModuleRoot: "latest", FailureIsFatal: true, @@ -500,7 +504,7 @@ func (v *BlockValidator) sendRecord(s *validationStatus) error { //nolint:gosec func (v *BlockValidator) writeToFile(validationEntry *validationEntry, moduleRoot common.Hash) error { - input, err := validationEntry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + input, err := validationEntry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return err } @@ -652,6 +656,10 @@ func (v *BlockValidator) sendNextRecordRequests(ctx context.Context) (bool, erro if recordUntil < pos { return false, nil } + recordUntilLimit := pos + arbutil.MessageIndex(v.config().RecordingIterLimit) + if recordUntil > recordUntilLimit { + recordUntil = recordUntilLimit + } log.Trace("preparing to record", "pos", pos, "until", recordUntil) // prepare could take a long time so we do it without a lock err := v.recorder.PrepareForRecord(ctx, pos, recordUntil) diff --git a/staker/challenge_manager.go b/staker/challenge_manager.go index ef431d3c797..c488c8a96eb 100644 --- a/staker/challenge_manager.go +++ b/staker/challenge_manager.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbutil" @@ -468,7 +469,7 @@ func (m *ChallengeManager) createExecutionBackend(ctx context.Context, step uint if err != nil { return fmt.Errorf("error creating validation entry for challenge %v msg %v for execution challenge: %w", m.challengeIndex, initialCount, err) } - input, err := entry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + input, err := entry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return fmt.Errorf("error getting validation entry input of challenge %v msg %v: %w", m.challengeIndex, initialCount, err) } diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index d5eeb8eb694..f54ec8b58c1 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -12,7 +12,6 @@ import ( "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -134,7 +133,7 @@ type validationEntry struct { DelayedMsg []byte } -func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.ValidationInput, error) { +func (e *validationEntry) ToInput(stylusArchs []ethdb.WasmTarget) (*validator.ValidationInput, error) { if e.Stage != Ready { return nil, errors.New("cannot create input from non-ready entry") } @@ -143,7 +142,7 @@ func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.Valida HasDelayedMsg: e.HasDelayedMsg, DelayedMsgNr: e.DelayedMsgNr, Preimages: e.Preimages, - UserWasms: make(map[rawdb.Target]map[common.Hash][]byte, len(e.UserWasms)), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash][]byte, len(e.UserWasms)), BatchInfo: e.BatchInfo, DelayedMsg: e.DelayedMsg, StartState: e.Start, diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 6e7375a921d..457dae09101 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -197,7 +197,7 @@ var TestSequencerConfig = gethexec.SequencerConfig{ EnableProfiling: false, } -func ExecConfigDefaultNonSequencerTest() *gethexec.Config { +func ExecConfigDefaultNonSequencerTest(t *testing.T) *gethexec.Config { config := gethexec.ConfigDefault config.Caching = TestCachingConfig config.ParentChainReader = headerreader.TestConfig @@ -206,12 +206,12 @@ func ExecConfigDefaultNonSequencerTest() *gethexec.Config { config.ForwardingTarget = "null" config.TxPreChecker.Strictness = gethexec.TxPreCheckerStrictnessNone - _ = config.Validate() + Require(t, config.Validate()) return &config } -func ExecConfigDefaultTest() *gethexec.Config { +func ExecConfigDefaultTest(t *testing.T) *gethexec.Config { config := gethexec.ConfigDefault config.Caching = TestCachingConfig config.Sequencer = TestSequencerConfig @@ -219,7 +219,7 @@ func ExecConfigDefaultTest() *gethexec.Config { config.ForwardingTarget = "null" config.TxPreChecker.Strictness = gethexec.TxPreCheckerStrictnessNone - _ = config.Validate() + Require(t, config.Validate()) return &config } @@ -272,7 +272,7 @@ func (b *NodeBuilder) DefaultConfig(t *testing.T, withL1 bool) *NodeBuilder { b.l2StackConfig = testhelpers.CreateStackConfigForTest(b.dataDir) cp := valnode.TestValidationConfig b.valnodeConfig = &cp - b.execConfig = ExecConfigDefaultTest() + b.execConfig = ExecConfigDefaultTest(t) return b } @@ -293,6 +293,11 @@ func (b *NodeBuilder) WithWasmRootDir(wasmRootDir string) *NodeBuilder { return b } +func (b *NodeBuilder) WithExtraArchs(targets []string) *NodeBuilder { + b.execConfig.StylusTarget.ExtraArchs = targets + return b +} + func (b *NodeBuilder) Build(t *testing.T) func() { b.CheckConfig(t) if b.withL1 { @@ -310,7 +315,7 @@ func (b *NodeBuilder) CheckConfig(t *testing.T) { b.nodeConfig = arbnode.ConfigDefaultL1Test() } if b.execConfig == nil { - b.execConfig = ExecConfigDefaultTest() + b.execConfig = ExecConfigDefaultTest(t) } if b.L1Info == nil { b.L1Info = NewL1TestInfo(t) @@ -346,7 +351,7 @@ func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() { var l2arbDb ethdb.Database var l2blockchain *core.BlockChain _, b.L2.Stack, l2chainDb, l2arbDb, l2blockchain = createL2BlockChainWithStackConfig( - t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, &b.execConfig.Caching) + t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) var sequencerTxOptsPtr *bind.TransactOpts var dataSigner signature.DataSignerFunc @@ -409,7 +414,7 @@ func (b *NodeBuilder) BuildL2(t *testing.T) func() { var arbDb ethdb.Database var blockchain *core.BlockChain b.L2Info, b.L2.Stack, chainDb, arbDb, blockchain = createL2BlockChain( - t, b.L2Info, b.dataDir, b.chainConfig, &b.execConfig.Caching) + t, b.L2Info, b.dataDir, b.chainConfig, b.execConfig) Require(t, b.execConfig.Validate()) execConfig := b.execConfig @@ -460,7 +465,7 @@ func (b *NodeBuilder) RestartL2Node(t *testing.T) { } b.L2.cleanup() - l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, &b.execConfig.Caching) + l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) execConfigFetcher := func() *gethexec.Config { return b.execConfig } execNode, err := gethexec.CreateExecutionNode(b.ctx, stack, chainDb, blockchain, nil, execConfigFetcher) @@ -1047,30 +1052,32 @@ func DeployOnTestL1( } func createL2BlockChain( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, cacheConfig *gethexec.CachingConfig, + t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { - return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, cacheConfig) + return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, execConfig) } func createL2BlockChainWithStackConfig( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, cacheConfig *gethexec.CachingConfig, + t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { if l2info == nil { l2info = NewArbTestInfo(t, chainConfig.ChainID) } - var stack *node.Node - var err error if stackConfig == nil { stackConfig = testhelpers.CreateStackConfigForTest(dataDir) } - stack, err = node.New(stackConfig) + if execConfig == nil { + execConfig = ExecConfigDefaultTest(t) + } + + stack, err := node.New(stackConfig) Require(t, err) chainData, err := stack.OpenDatabaseWithExtraOptions("l2chaindata", 0, 0, "l2chaindata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("l2chaindata")) Require(t, err) wasmData, err := stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) arbDb, err := stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) @@ -1085,11 +1092,8 @@ func createL2BlockChainWithStackConfig( SerializedChainConfig: serializedChainConfig, } } - var coreCacheConfig *core.CacheConfig - if cacheConfig != nil { - coreCacheConfig = gethexec.DefaultCacheConfigFor(stack, cacheConfig) - } - blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest().TxLookupLimit, 0) + coreCacheConfig := gethexec.DefaultCacheConfigFor(stack, &execConfig.Caching) + blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) return l2info, stack, chainDb, arbDb, blockchain @@ -1149,7 +1153,7 @@ func Create2ndNodeWithConfig( nodeConfig = arbnode.ConfigDefaultL1NonSequencerTest() } if execConfig == nil { - execConfig = ExecConfigDefaultNonSequencerTest() + execConfig = ExecConfigDefaultNonSequencerTest(t) } feedErrChan := make(chan error, 10) l1rpcClient := l1stack.Attach() @@ -1165,7 +1169,7 @@ func Create2ndNodeWithConfig( Require(t, err) wasmData, err := l2stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0) + l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) l2arbDb, err := l2stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) @@ -1179,7 +1183,7 @@ func Create2ndNodeWithConfig( chainConfig := firstExec.ArbInterface.BlockChain().Config() coreCacheConfig := gethexec.DefaultCacheConfigFor(l2stack, &execConfig.Caching) - l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest().TxLookupLimit, 0) + l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) AddValNodeIfNeeded(t, ctx, nodeConfig, true, "", valnodeConfig.Wasm.RootPath) diff --git a/system_tests/forwarder_test.go b/system_tests/forwarder_test.go index f87283432e8..57381ca84e0 100644 --- a/system_tests/forwarder_test.go +++ b/system_tests/forwarder_test.go @@ -38,7 +38,7 @@ func TestStaticForwarder(t *testing.T) { clientA := builder.L2.Client nodeConfigB := arbnode.ConfigDefaultL1Test() - execConfigB := ExecConfigDefaultTest() + execConfigB := ExecConfigDefaultTest(t) execConfigB.Sequencer.Enable = false nodeConfigB.Sequencer = false nodeConfigB.DelayedSequencer.Enable = false @@ -109,7 +109,7 @@ func createForwardingNode(t *testing.T, builder *NodeBuilder, ipcPath string, re nodeConfig.Sequencer = false nodeConfig.DelayedSequencer.Enable = false nodeConfig.BatchPoster.Enable = false - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Sequencer.Enable = false execConfig.Forwarder.RedisUrl = redisUrl execConfig.ForwardingTarget = fallbackPath diff --git a/system_tests/program_test.go b/system_tests/program_test.go index c89a13e2050..83c066fdb5e 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -45,18 +45,31 @@ import ( var oneEth = arbmath.UintToBig(1e18) +var allWasmTargets = []string{string(rawdb.TargetWavm), string(rawdb.TargetArm64), string(rawdb.TargetAmd64), string(rawdb.TargetHost)} + func TestProgramKeccak(t *testing.T) { t.Parallel() - keccakTest(t, true) + t.Run("WithDefaultWasmTargets", func(t *testing.T) { + keccakTest(t, true) + }) + + t.Run("WithAllWasmTargets", func(t *testing.T) { + keccakTest(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs(allWasmTargets) + }) + }) } -func keccakTest(t *testing.T, jit bool) { - builder, auth, cleanup := setupProgramTest(t, jit) +func keccakTest(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { + builder, auth, cleanup := setupProgramTest(t, jit, builderOpts...) ctx := builder.ctx l2client := builder.L2.Client defer cleanup() programAddress := deployWasm(t, ctx, auth, l2client, rustFile("keccak")) + wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + wasm, _ := readWasmFile(t, rustFile("keccak")) otherAddressSameCode := deployContract(t, ctx, auth, l2client, wasm) arbWasm, err := pgen.NewArbWasm(types.ArbWasmAddress, l2client) @@ -68,6 +81,7 @@ func keccakTest(t *testing.T, jit bool) { Fatal(t, "activate should have failed with ProgramUpToDate", err) } }) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) if programAddress == otherAddressSameCode { Fatal(t, "expected to deploy at two separate program addresses") @@ -141,11 +155,18 @@ func keccakTest(t *testing.T, jit bool) { func TestProgramActivateTwice(t *testing.T) { t.Parallel() - testActivateTwice(t, true) + t.Run("WithDefaultWasmTargets", func(t *testing.T) { + testActivateTwice(t, true) + }) + t.Run("WithAllWasmTargets", func(t *testing.T) { + testActivateTwice(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs(allWasmTargets) + }) + }) } -func testActivateTwice(t *testing.T, jit bool) { - builder, auth, cleanup := setupProgramTest(t, jit) +func testActivateTwice(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { + builder, auth, cleanup := setupProgramTest(t, jit, builderOpts...) ctx := builder.ctx l2info := builder.L2Info l2client := builder.L2.Client @@ -171,6 +192,10 @@ func testActivateTwice(t *testing.T, jit bool) { colors.PrintBlue("keccak program B deployed to ", keccakB) multiAddr := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) + + wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + preimage := []byte("it's time to du-du-du-du d-d-d-d-d-d-d de-duplicate") keccakArgs := []byte{0x01} // keccak the preimage once @@ -194,6 +219,7 @@ func testActivateTwice(t *testing.T, jit bool) { // Calling the contract pre-activation should fail. checkReverts() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // mechanisms for creating calldata activateProgram, _ := util.NewCallParser(pgen.ArbWasmABI, "activateProgram") @@ -216,6 +242,7 @@ func testActivateTwice(t *testing.T, jit bool) { // Ensure the revert also reverted keccak's activation checkReverts() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // Activate keccak program A, then call into B, which should succeed due to being the same codehash args = argsForMulticall(vm.CALL, types.ArbWasmAddress, oneEth, pack(activateProgram(keccakA))) @@ -223,6 +250,7 @@ func testActivateTwice(t *testing.T, jit bool) { tx = l2info.PrepareTxTo("Owner", &multiAddr, 1e9, oneEth, args) ensure(tx, l2client.SendTransaction(ctx, tx)) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 2) validateBlocks(t, 7, jit, builder) } @@ -1834,7 +1862,9 @@ func createMapFromDb(db ethdb.KeyValueStore) (map[string][]byte, error) { } func TestWasmStoreRebuilding(t *testing.T) { - builder, auth, cleanup := setupProgramTest(t, true) + builder, auth, cleanup := setupProgramTest(t, true, func(b *NodeBuilder) { + b.WithExtraArchs(allWasmTargets) + }) ctx := builder.ctx l2info := builder.L2Info l2client := builder.L2.Client @@ -1871,6 +1901,7 @@ func TestWasmStoreRebuilding(t *testing.T) { storeMap, err := createMapFromDb(wasmDb) Require(t, err) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // close nodeB cleanupB() @@ -1927,5 +1958,52 @@ func TestWasmStoreRebuilding(t *testing.T) { } } + checkWasmStoreContent(t, wasmDbAfterRebuild, builder.execConfig.StylusTarget.ExtraArchs, 1) cleanupB() } + +func readModuleHashes(t *testing.T, wasmDb ethdb.KeyValueStore) []common.Hash { + modulesSet := make(map[common.Hash]struct{}) + asmPrefix := []byte{0x00, 'w'} + it := wasmDb.NewIterator(asmPrefix, nil) + defer it.Release() + for it.Next() { + key := it.Key() + if len(key) != rawdb.WasmKeyLen { + t.Fatalf("unexpected activated module key length, len: %d, key: %v", len(key), key) + } + moduleHash := key[rawdb.WasmPrefixLen:] + if len(moduleHash) != common.HashLength { + t.Fatalf("Invalid moduleHash length in key: %v, moduleHash: %v", key, moduleHash) + } + modulesSet[common.BytesToHash(moduleHash)] = struct{}{} + } + modules := make([]common.Hash, 0, len(modulesSet)) + for module := range modulesSet { + modules = append(modules, module) + } + return modules +} + +func checkWasmStoreContent(t *testing.T, wasmDb ethdb.KeyValueStore, targets []string, numModules int) { + modules := readModuleHashes(t, wasmDb) + if len(modules) != numModules { + t.Fatalf("Unexpected number of module hashes found in wasm store, want: %d, have: %d", numModules, len(modules)) + } + for _, module := range modules { + for _, target := range targets { + wasmTarget := ethdb.WasmTarget(target) + if !rawdb.IsSupportedWasmTarget(wasmTarget) { + t.Fatalf("internal test error - unsupported target passed to checkWasmStoreContent: %v", target) + } + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("Failed to read activated asm for target: %v, module: %v", target, module) + } + }() + _ = rawdb.ReadActivatedAsm(wasmDb, wasmTarget, module) + }() + } + } +} diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index cd3904ca060..7b09552e28a 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -95,7 +95,7 @@ func removeStatesFromDb(t *testing.T, bc *core.BlockChain, db ethdb.Database, fr func TestRecreateStateForRPCNoDepthLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -134,7 +134,7 @@ func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { defer cancel() // #nosec G115 depthGasLimit := int64(256 * util.NormalizeL2GasForL1GasInitial(800_000, params.GWei)) - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = depthGasLimit execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -171,7 +171,7 @@ func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { func TestRecreateStateForRPCDepthLimitExceeded(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = int64(200) execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -208,7 +208,7 @@ func TestRecreateStateForRPCMissingBlockParent(t *testing.T) { var headerCacheLimit uint64 = 512 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -256,7 +256,7 @@ func TestRecreateStateForRPCBeyondGenesis(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -294,7 +294,7 @@ func TestRecreateStateForRPCBlockNotFoundWhileRecreating(t *testing.T) { var blockCacheLimit uint64 = 256 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -342,7 +342,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = maxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -483,7 +483,7 @@ func TestSkippingSavingStateAndRecreatingAfterRestart(t *testing.T) { func TestGettingStateForRPCFullNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Caching.SnapshotCache = 0 // disable snapshots execConfig.Caching.BlockAge = 0 // use only Caching.BlockCount to keep only last N blocks in dirties cache, no matter how new they are execConfig.Sequencer.MaxBlockSpeed = 0 @@ -527,7 +527,7 @@ func TestGettingStateForRPCFullNode(t *testing.T) { func TestGettingStateForRPCHybridArchiveNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Caching.Archive = true // For now Archive node should use HashScheme execConfig.Caching.StateScheme = rawdb.HashScheme diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index 106dfc6d463..aa9fbfd72ea 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -1042,7 +1042,7 @@ func elevateL2Basefee(t *testing.T, ctx context.Context, builder *NodeBuilder) { _, err = precompilesgen.NewArbosTest(common.HexToAddress("0x69"), builder.L2.Client) Require(t, err, "failed to deploy ArbosTest") - burnAmount := ExecConfigDefaultTest().RPC.RPCGasCap + burnAmount := ExecConfigDefaultTest(t).RPC.RPCGasCap burnTarget := uint64(5 * l2pricing.InitialSpeedLimitPerSecondV6 * l2pricing.InitialBacklogTolerance) for i := uint64(0); i < (burnTarget+burnAmount)/burnAmount; i++ { burnArbGas := arbosTestAbi.Methods["burnArbGas"] diff --git a/system_tests/validation_mock_test.go b/system_tests/validation_mock_test.go index 88421e4c4b5..2739c7545e6 100644 --- a/system_tests/validation_mock_test.go +++ b/system_tests/validation_mock_test.go @@ -8,9 +8,9 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbnode" @@ -61,8 +61,8 @@ func (s *mockSpawner) WasmModuleRoots() ([]common.Hash, error) { return mockWasmModuleRoots, nil } -func (s *mockSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{"mock"} +func (s *mockSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{"mock"} } func (s *mockSpawner) Launch(entry *validator.ValidationInput, moduleRoot common.Hash) validator.ValidationRun { diff --git a/validator/client/redis/producer.go b/validator/client/redis/producer.go index f98c246d0e6..adc2f34af57 100644 --- a/validator/client/redis/producer.go +++ b/validator/client/redis/producer.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/go-redis/redis/v8" "github.com/offchainlabs/nitro/pubsub" @@ -35,7 +36,7 @@ func (c ValidationClientConfig) Enabled() bool { func (c ValidationClientConfig) Validate() error { for _, arch := range c.StylusArchs { - if !rawdb.Target(arch).IsValid() { + if !rawdb.IsSupportedWasmTarget(ethdb.WasmTarget(arch)) { return fmt.Errorf("Invalid stylus arch: %v", arch) } } @@ -162,10 +163,10 @@ func (c *ValidationClient) Name() string { return c.config.Name } -func (c *ValidationClient) StylusArchs() []rawdb.Target { - stylusArchs := make([]rawdb.Target, 0, len(c.config.StylusArchs)) +func (c *ValidationClient) StylusArchs() []ethdb.WasmTarget { + stylusArchs := make([]ethdb.WasmTarget, 0, len(c.config.StylusArchs)) for _, arch := range c.config.StylusArchs { - stylusArchs = append(stylusArchs, rawdb.Target(arch)) + stylusArchs = append(stylusArchs, ethdb.WasmTarget(arch)) } return stylusArchs } diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 00bd992f46f..3b18ad18513 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" @@ -31,7 +32,7 @@ type ValidationClient struct { stopwaiter.StopWaiter client *rpcclient.RpcClient name string - stylusArchs []rawdb.Target + stylusArchs []ethdb.WasmTarget room atomic.Int32 wasmModuleRoots []common.Hash } @@ -40,7 +41,7 @@ func NewValidationClient(config rpcclient.ClientConfigFetcher, stack *node.Node) return &ValidationClient{ client: rpcclient.NewRpcClient(config, stack), name: "not started", - stylusArchs: []rawdb.Target{"not started"}, + stylusArchs: []ethdb.WasmTarget{"not started"}, } } @@ -67,20 +68,20 @@ func (c *ValidationClient) Start(ctx context.Context) error { if len(name) == 0 { return errors.New("couldn't read name from server") } - var stylusArchs []rawdb.Target + var stylusArchs []ethdb.WasmTarget if err := c.client.CallContext(ctx, &stylusArchs, server_api.Namespace+"_stylusArchs"); err != nil { var rpcError rpc.Error ok := errors.As(err, &rpcError) if !ok || rpcError.ErrorCode() != -32601 { return fmt.Errorf("could not read stylus arch from server: %w", err) } - stylusArchs = []rawdb.Target{rawdb.Target("pre-stylus")} // invalid, will fail if trying to validate block with stylus + stylusArchs = []ethdb.WasmTarget{ethdb.WasmTarget("pre-stylus")} // invalid, will fail if trying to validate block with stylus } else { if len(stylusArchs) == 0 { return fmt.Errorf("could not read stylus archs from validation server") } for _, stylusArch := range stylusArchs { - if stylusArch != rawdb.TargetWavm && stylusArch != rawdb.LocalTarget() && stylusArch != "mock" { + if !rawdb.IsSupportedWasmTarget(ethdb.WasmTarget(stylusArch)) && stylusArch != "mock" { return fmt.Errorf("unsupported stylus architecture: %v", stylusArch) } } @@ -118,11 +119,11 @@ func (c *ValidationClient) WasmModuleRoots() ([]common.Hash, error) { return nil, errors.New("not started") } -func (c *ValidationClient) StylusArchs() []rawdb.Target { +func (c *ValidationClient) StylusArchs() []ethdb.WasmTarget { if c.Started() { return c.stylusArchs } - return []rawdb.Target{"not started"} + return []ethdb.WasmTarget{"not started"} } func (c *ValidationClient) Stop() { diff --git a/validator/interface.go b/validator/interface.go index 81b40ae5cfc..af086291372 100644 --- a/validator/interface.go +++ b/validator/interface.go @@ -4,7 +4,7 @@ import ( "context" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/containers" ) @@ -14,7 +14,7 @@ type ValidationSpawner interface { Start(context.Context) error Stop() Name() string - StylusArchs() []rawdb.Target + StylusArchs() []ethdb.WasmTarget Room() int } diff --git a/validator/server_api/json.go b/validator/server_api/json.go index dbe2bb1fee6..6fe936e17d6 100644 --- a/validator/server_api/json.go +++ b/validator/server_api/json.go @@ -11,7 +11,7 @@ import ( "os" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbcompress" "github.com/offchainlabs/nitro/arbutil" @@ -64,7 +64,7 @@ type InputJSON struct { BatchInfo []BatchInfoJson DelayedMsgB64 string StartState validator.GoGlobalState - UserWasms map[rawdb.Target]map[common.Hash]string + UserWasms map[ethdb.WasmTarget]map[common.Hash]string DebugChain bool } @@ -96,7 +96,7 @@ func ValidationInputToJson(entry *validator.ValidationInput) *InputJSON { DelayedMsgB64: base64.StdEncoding.EncodeToString(entry.DelayedMsg), StartState: entry.StartState, PreimagesB64: jsonPreimagesMap, - UserWasms: make(map[rawdb.Target]map[common.Hash]string), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash]string), DebugChain: entry.DebugChain, } for _, binfo := range entry.BatchInfo { @@ -128,7 +128,7 @@ func ValidationInputFromJson(entry *InputJSON) (*validator.ValidationInput, erro DelayedMsgNr: entry.DelayedMsgNr, StartState: entry.StartState, Preimages: preimages, - UserWasms: make(map[rawdb.Target]map[common.Hash][]byte), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash][]byte), DebugChain: entry.DebugChain, } delayed, err := base64.StdEncoding.DecodeString(entry.DelayedMsgB64) diff --git a/validator/server_arb/validator_spawner.go b/validator/server_arb/validator_spawner.go index eb530703031..6f0d0cee1d8 100644 --- a/validator/server_arb/validator_spawner.go +++ b/validator/server_arb/validator_spawner.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/pflag" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/stopwaiter" @@ -89,8 +90,8 @@ func (s *ArbitratorSpawner) WasmModuleRoots() ([]common.Hash, error) { return s.locator.ModuleRoots(), nil } -func (s *ArbitratorSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{rawdb.TargetWavm} +func (s *ArbitratorSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{rawdb.TargetWavm} } func (s *ArbitratorSpawner) Name() string { diff --git a/validator/server_jit/spawner.go b/validator/server_jit/spawner.go index 92b50b17cba..d77317d2184 100644 --- a/validator/server_jit/spawner.go +++ b/validator/server_jit/spawner.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" @@ -72,8 +73,8 @@ func (v *JitSpawner) WasmModuleRoots() ([]common.Hash, error) { return v.locator.ModuleRoots(), nil } -func (v *JitSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{rawdb.LocalTarget()} +func (v *JitSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{rawdb.LocalTarget()} } func (v *JitSpawner) execute( diff --git a/validator/validation_entry.go b/validator/validation_entry.go index 2c357659ad2..d340993fa2e 100644 --- a/validator/validation_entry.go +++ b/validator/validation_entry.go @@ -2,7 +2,7 @@ package validator import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbutil" ) @@ -17,7 +17,7 @@ type ValidationInput struct { HasDelayedMsg bool DelayedMsgNr uint64 Preimages map[arbutil.PreimageType]map[common.Hash][]byte - UserWasms map[rawdb.Target]map[common.Hash][]byte + UserWasms map[ethdb.WasmTarget]map[common.Hash][]byte BatchInfo []BatchInfo DelayedMsg []byte StartState GoGlobalState diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 74564665336..2b025600cc0 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -111,10 +111,12 @@ func (s *ValidationServer) Start(ctx_in context.Context) { req, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) + requestTokenQueue <- struct{}{} return 0 } if req == nil { - // There's nothing in the queue. + // There's nothing in the queue + requestTokenQueue <- struct{}{} return time.Second } select { @@ -152,9 +154,10 @@ func (s *ValidationServer) Start(ctx_in context.Context) { res, err := valRun.Await(ctx) if err != nil { log.Error("Error validating", "request value", work.req.Value, "error", err) - } - if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { - log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } else { + if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { + log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } } select { case <-ctx.Done(): diff --git a/validator/valnode/validation_api.go b/validator/valnode/validation_api.go index a79ac7fa55e..a10d931dfc2 100644 --- a/validator/valnode/validation_api.go +++ b/validator/valnode/validation_api.go @@ -12,7 +12,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" @@ -45,7 +45,7 @@ func (a *ValidationServerAPI) WasmModuleRoots() ([]common.Hash, error) { return a.spawner.WasmModuleRoots() } -func (a *ValidationServerAPI) StylusArchs() ([]rawdb.Target, error) { +func (a *ValidationServerAPI) StylusArchs() ([]ethdb.WasmTarget, error) { return a.spawner.StylusArchs(), nil } From 925a623477d1ed3fa82cc42e6e90a947378d4d0b Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 18:06:14 -0600 Subject: [PATCH 105/156] cargo_fmt --- arbitrator/prover/src/lib.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arbitrator/prover/src/lib.rs b/arbitrator/prover/src/lib.rs index 20472691c18..e45d12c4926 100644 --- a/arbitrator/prover/src/lib.rs +++ b/arbitrator/prover/src/lib.rs @@ -73,8 +73,13 @@ pub unsafe extern "C" fn arbitrator_load_machine( with_forwarder: bool, ) -> *mut Machine { let debug_chain = debug_chain != 0; - match arbitrator_load_machine_impl(binary_path, library_paths, library_paths_size, debug_chain, with_forwarder) - { + match arbitrator_load_machine_impl( + binary_path, + library_paths, + library_paths_size, + debug_chain, + with_forwarder, + ) { Ok(mach) => mach, Err(err) => { eprintln!("Error loading binary: {:?}", err); From c5f7181a9319f14dfd704c9e08e68508d3323788 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 18:14:05 -0600 Subject: [PATCH 106/156] use import resolvers --- arbitrator/prover/src/host.rs | 9 +- arbitrator/prover/src/machine.rs | 148 ++++++++++++++++++++----------- 2 files changed, 102 insertions(+), 55 deletions(-) diff --git a/arbitrator/prover/src/host.rs b/arbitrator/prover/src/host.rs index 1d0fe658ecc..dc0f0b74b2d 100644 --- a/arbitrator/prover/src/host.rs +++ b/arbitrator/prover/src/host.rs @@ -405,7 +405,7 @@ impl Hostio { } } -pub fn get_impl(module: &str, name: &str) -> Result<(Function, bool)> { +pub fn get_impl(module: &str, name: &str) -> Result { let hostio: Hostio = format!("{module}__{name}").parse()?; let append = |code: &mut Vec| { @@ -414,8 +414,11 @@ pub fn get_impl(module: &str, name: &str) -> Result<(Function, bool)> { Ok(()) }; - let debug = module == "console" || module == "debug"; - Function::new(&[], append, hostio.ty(), &[]).map(|x| (x, debug)) + Function::new(&[], append, hostio.ty(), &[]) +} + +pub fn hostio_module_is_debug(module: &str) -> bool { + module == "console" || module == "debug" } /// Adds internal functions to a module. diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index c045fc72c9b..9f2ba081dc1 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -346,14 +346,95 @@ lazy_static! { }; } +trait ImportResolver: for<'a, 'b> Fn(&'a str, &'b str) -> Result + Sized { + fn compose(self, other: impl ImportResolver) -> impl ImportResolver { + move |module, name| { + self(module, name).or_else(|original_err| { + let mut result = other(module, name); + if result.is_err() { + for err_layer in original_err.chain().rev() { + result = result.wrap_err(format!("{err_layer}")); + } + } + result + }) + } + } + + fn condition(self, condition: bool) -> impl ImportResolver { + move |module, name| { + if condition { + self(module, name) + } else { + bail!("resolver disabled") + } + } + } +} + +impl Fn(&'a str, &'b str) -> Result> ImportResolver for F {} + impl Module { - fn from_binary( + fn make_imports_resolver(import_map: &HashMap) -> impl ImportResolver + '_ { + move |module, name| { + let qualified_name = format!("{module}__{name}"); + + let Some(import) = import_map.get(&qualified_name) else { + bail!("func not found {module} {name}") + }; + let wavm = vec![ + Instruction::simple(Opcode::InitFrame), + Instruction::with_data( + Opcode::CrossModuleCall, + pack_cross_module_call(import.module, import.func), + ), + Instruction::simple(Opcode::Return), + ]; + Ok(Function::new_from_wavm(wavm, import.ty.clone(), vec![])) + } + } + + fn make_forward_resolver(import_map: &HashMap) -> impl ImportResolver + '_ { + move |module, name| { + let qualified_name = format!("{module}__{name}"); + + let Some(import) = import_map.get(&qualified_name) else { + bail!("func not found {module} {name}") + }; + let wavm = vec![ + Instruction::simple(Opcode::InitFrame), + Instruction::with_data( + Opcode::CrossModuleForward, + pack_cross_module_call(import.module, import.func), + ), + Instruction::simple(Opcode::Return), + ]; + Ok(Function::new_from_wavm(wavm, import.ty.clone(), vec![])) + } + } + + fn hostio_resolver(module: &str, name: &str) -> Result { + if host::hostio_module_is_debug(module) { + return Self::notfound_resolver(module, name); + } + host::get_impl(module, name) + } + + fn debug_resolver(module: &str, name: &str) -> Result { + if !host::hostio_module_is_debug(module) { + return Self::notfound_resolver(module, name); + } + host::get_impl(module, name) + } + + fn notfound_resolver(module: &str, name: &str) -> Result { + bail!("import not found {module} {name}") + } + + fn from_binary( bin: &WasmBinary, - available_imports: &HashMap, + import_resolver: R, floating_point_impls: &FloatingPointImpls, - allow_hostapi: bool, - is_forwarder: bool, - debug_funcs: bool, stylus_data: Option, ) -> Result { let mut code = Vec::new(); @@ -366,38 +447,8 @@ impl Module { let module = import.module; let have_ty = &bin.types[import.offset as usize]; - let qualified_name = format!("{module}__{}", import.name); + let func = import_resolver(import.module, import.name)?; - let func = if let Some(import) = available_imports.get(&qualified_name) { - let call = match is_forwarder { - true => Opcode::CrossModuleForward, - false => Opcode::CrossModuleCall, - }; - let wavm = vec![ - Instruction::simple(Opcode::InitFrame), - Instruction::with_data( - call, - pack_cross_module_call(import.module, import.func), - ), - Instruction::simple(Opcode::Return), - ]; - Function::new_from_wavm(wavm, import.ty.clone(), vec![]) - } else if let Ok((hostio, debug)) = host::get_impl(import.module, import.name) { - ensure!( - (debug && debug_funcs) || (!debug && allow_hostapi), - "Host func {} in {} not enabled debug_funcs={debug_funcs} hostapi={allow_hostapi} debug={debug}", - import.name.red(), - import.module.red(), - ); - hostio - } else { - bail!( - "No such import {} in {} for {}", - import.name.red(), - import.module.red(), - bin_name.red() - ) - }; ensure!( &func.ty == have_ty, "Import {} for {} has different function signature than export.\nexpected {} in {}\nbut have {}", @@ -602,11 +653,9 @@ impl Module { ) -> Result { Self::from_binary( bin, - &USER_IMPORTS, + Module::make_imports_resolver(&USER_IMPORTS) + .compose(Module::debug_resolver.condition(debug_funcs)), &HashMap::default(), - false, - false, - debug_funcs, stylus_data, ) } @@ -1377,11 +1426,8 @@ impl Machine { if let Some(lib) = forwarder { modules.push(Module::from_binary( &lib, - &available_imports, + Module::make_forward_resolver(&available_imports), &floating_point_impls, - true, - true, - debug_funcs, None, )?); } @@ -1389,22 +1435,20 @@ impl Machine { for lib in libraries.iter() { modules.push(Module::from_binary( lib, - &available_imports, + Module::hostio_resolver + .compose(Module::debug_resolver.condition(debug_funcs)) + .compose(Module::make_imports_resolver(&available_imports)), &floating_point_impls, - true, - false, - debug_funcs, None, )?); } modules.push(Module::from_binary( &bin, - &available_imports, + Module::make_imports_resolver(&available_imports) + .compose(Module::hostio_resolver.condition(allow_hostapi_from_main)) + .compose(Module::debug_resolver.condition(debug_funcs)), &floating_point_impls, - allow_hostapi_from_main, - false, - debug_funcs, stylus_data, )?); From 06954ca91f186d8bcf58bd3a1bc4611f719ea6e5 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 18:54:08 -0600 Subject: [PATCH 107/156] cargo fmt --- arbitrator/prover/src/machine.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index 9f2ba081dc1..2418d3cc2e6 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -375,7 +375,9 @@ trait ImportResolver: for<'a, 'b> Fn(&'a str, &'b str) -> Result + Siz impl Fn(&'a str, &'b str) -> Result> ImportResolver for F {} impl Module { - fn make_imports_resolver(import_map: &HashMap) -> impl ImportResolver + '_ { + fn make_imports_resolver( + import_map: &HashMap, + ) -> impl ImportResolver + '_ { move |module, name| { let qualified_name = format!("{module}__{name}"); @@ -394,7 +396,9 @@ impl Module { } } - fn make_forward_resolver(import_map: &HashMap) -> impl ImportResolver + '_ { + fn make_forward_resolver( + import_map: &HashMap, + ) -> impl ImportResolver + '_ { move |module, name| { let qualified_name = format!("{module}__{name}"); From 6d0ee874def40855f2be93bf89625bc56854f48b Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 19:18:36 -0600 Subject: [PATCH 108/156] block_validator: nits --- staker/stateless_block_validator.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index 3d92b9df59b..5c92fb40257 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -195,7 +195,7 @@ func newValidationEntry( Number: batchNum, Data: fullBatchInfo.PostedData, }) - clonePreimagesInto(preimages, fullBatchInfo.Preimages) + copyPreimagesInto(preimages, fullBatchInfo.Preimages) return fullBatchInfo.PostedData, nil } @@ -326,10 +326,10 @@ func (v *StatelessBlockValidator) readBatch(ctx context.Context, batchNum uint64 return true, &fullInfo, nil } -func clonePreimagesInto(dest, source map[arbutil.PreimageType]map[common.Hash][]byte) { +func copyPreimagesInto(dest, source map[arbutil.PreimageType]map[common.Hash][]byte) { for piType, piMap := range source { if dest[piType] == nil { - dest[piType] = make(map[common.Hash][]byte, len(source[piType])) + dest[piType] = make(map[common.Hash][]byte, len(piMap)) } for hash, preimage := range piMap { dest[piType][hash] = preimage @@ -350,9 +350,10 @@ func (v *StatelessBlockValidator) ValidationEntryRecord(ctx context.Context, e * return fmt.Errorf("recording failed: pos %d, hash expected %v, got %v", e.Pos, e.End.BlockHash, recording.BlockHash) } if recording.Preimages != nil { - recordingPreimages := make(map[arbutil.PreimageType]map[common.Hash][]byte) - recordingPreimages[arbutil.Keccak256PreimageType] = recording.Preimages - clonePreimagesInto(e.Preimages, recordingPreimages) + recordingPreimages := map[arbutil.PreimageType]map[common.Hash][]byte{ + arbutil.Keccak256PreimageType: recording.Preimages, + } + copyPreimagesInto(e.Preimages, recordingPreimages) } e.UserWasms = recording.UserWasms } From 5591e5a1c4e787c493e4957b3f73248a2ec704c1 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 21:05:35 -0600 Subject: [PATCH 109/156] block_validator: separate reading full batches from just posted data --- arbos/arbostypes/incomingmessage.go | 11 ++++ staker/block_validator.go | 51 ++++++++++--------- staker/stateless_block_validator.go | 79 ++++++++++++++++++----------- 3 files changed, 86 insertions(+), 55 deletions(-) diff --git a/arbos/arbostypes/incomingmessage.go b/arbos/arbostypes/incomingmessage.go index 04ce8ebe2e0..c4c2dc037be 100644 --- a/arbos/arbostypes/incomingmessage.go +++ b/arbos/arbostypes/incomingmessage.go @@ -182,6 +182,17 @@ func (msg *L1IncomingMessage) FillInBatchGasCost(batchFetcher FallibleBatchFetch return nil } +func (msg *L1IncomingMessage) PastBatchesRequired() ([]uint64, error) { + if msg.Header.Kind != L1MessageType_BatchPostingReport { + return nil, nil + } + _, _, _, batchNum, _, _, err := ParseBatchPostingReportMessageFields(bytes.NewReader(msg.L2msg)) + if err != nil { + return nil, fmt.Errorf("failed to parse batch posting report: %w", err) + } + return []uint64{batchNum}, nil +} + func ParseIncomingL1Message(rd io.Reader, batchFetcher FallibleBatchFetcher) (*L1IncomingMessage, error) { var kindBuf [1]byte _, err := rd.Read(kindBuf[:]) diff --git a/staker/block_validator.go b/staker/block_validator.go index 4a5bd740a86..e1b2c75b844 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -58,7 +58,7 @@ type BlockValidator struct { // can only be accessed from creation thread or if holding reorg-write nextCreateBatch *FullBatchInfo nextCreateBatchReread bool - prevBatchCache map[uint64]*FullBatchInfo + prevBatchCache map[uint64][]byte nextCreateStartGS validator.GoGlobalState nextCreatePrevDelayed uint64 @@ -275,7 +275,7 @@ func NewBlockValidator( progressValidationsChan: make(chan struct{}, 1), config: config, fatalErr: fatalErr, - prevBatchCache: make(map[uint64]*FullBatchInfo), + prevBatchCache: make(map[uint64][]byte), } if !config().Dangerous.ResetBlockValidation { validated, err := ret.ReadLastValidatedInfo() @@ -576,12 +576,12 @@ func (v *BlockValidator) createNextValidationEntry(ctx context.Context) (bool, e } if v.nextCreateStartGS.PosInBatch == 0 || v.nextCreateBatchReread { // new batch - found, fullBatchInfo, err := v.readBatch(ctx, v.nextCreateStartGS.Batch) + found, fullBatchInfo, err := v.readFullBatch(ctx, v.nextCreateStartGS.Batch) if !found { return false, err } if v.nextCreateBatch != nil { - v.prevBatchCache[v.nextCreateBatch.Number] = v.nextCreateBatch + v.prevBatchCache[v.nextCreateBatch.Number] = v.nextCreateBatch.PostedData } v.nextCreateBatch = fullBatchInfo // #nosec G115 @@ -589,7 +589,7 @@ func (v *BlockValidator) createNextValidationEntry(ctx context.Context) (bool, e batchCacheLimit := v.config().BatchCacheLimit if len(v.prevBatchCache) > int(batchCacheLimit) { for num := range v.prevBatchCache { - if num < v.nextCreateStartGS.Batch-uint64(batchCacheLimit) { + if num+uint64(batchCacheLimit) < v.nextCreateStartGS.Batch { delete(v.prevBatchCache, num) } } @@ -610,26 +610,29 @@ func (v *BlockValidator) createNextValidationEntry(ctx context.Context) (bool, e return false, fmt.Errorf("illegal batch msg count %d pos %d batch %d", v.nextCreateBatch.MsgCount, pos, endGS.Batch) } chainConfig := v.streamer.ChainConfig() - batchReader := func(batchNum uint64) (*FullBatchInfo, error) { - if batchNum == v.nextCreateBatch.Number { - return v.nextCreateBatch, nil - } - // only batch-posting-reports will get here, and there's only one per batch - if entry, found := v.prevBatchCache[batchNum]; found { + prevBatchNums, err := msg.Message.PastBatchesRequired() + if err != nil { + return false, err + } + prevBatches := make([]validator.BatchInfo, 0, len(prevBatchNums)) + // prevBatchNums are only used for batch reports, each is only used once + for _, batchNum := range prevBatchNums { + data, found := v.prevBatchCache[batchNum] + if found { delete(v.prevBatchCache, batchNum) - return entry, nil - } - found, entry, err := v.readBatch(ctx, batchNum) - if err != nil { - return nil, err - } - if !found { - return nil, fmt.Errorf("batch %d not found", batchNum) + } else { + data, err = v.readPostedBatch(ctx, batchNum) + if err != nil { + return false, err + } } - return entry, nil + prevBatches = append(prevBatches, validator.BatchInfo{ + Number: batchNum, + Data: data, + }) } entry, err := newValidationEntry( - pos, v.nextCreateStartGS, endGS, msg, batchReader, v.nextCreatePrevDelayed, chainConfig, + pos, v.nextCreateStartGS, endGS, msg, v.nextCreateBatch, prevBatches, v.nextCreatePrevDelayed, chainConfig, ) if err != nil { return false, err @@ -1030,7 +1033,7 @@ func (v *BlockValidator) UpdateLatestStaked(count arbutil.MessageIndex, globalSt v.nextCreatePrevDelayed = msg.DelayedMessagesRead v.nextCreateBatchReread = true if v.nextCreateBatch != nil { - v.prevBatchCache[v.nextCreateBatch.Number] = v.nextCreateBatch + v.prevBatchCache[v.nextCreateBatch.Number] = v.nextCreateBatch.PostedData } v.createdA.Store(countUint64) } @@ -1058,7 +1061,7 @@ func (v *BlockValidator) ReorgToBatchCount(count uint64) { defer v.reorgMutex.Unlock() if v.nextCreateStartGS.Batch >= count { v.nextCreateBatchReread = true - v.prevBatchCache = make(map[uint64]*FullBatchInfo) + v.prevBatchCache = make(map[uint64][]byte) } } @@ -1099,7 +1102,7 @@ func (v *BlockValidator) Reorg(ctx context.Context, count arbutil.MessageIndex) v.nextCreateStartGS = buildGlobalState(*res, endPosition) v.nextCreatePrevDelayed = msg.DelayedMessagesRead v.nextCreateBatchReread = true - v.prevBatchCache = make(map[uint64]*FullBatchInfo) + v.prevBatchCache = make(map[uint64][]byte) countUint64 := uint64(count) v.createdA.Store(countUint64) // under the reorg mutex we don't need atomic access diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index 5c92fb40257..60306d712f3 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -116,7 +116,6 @@ const ( type FullBatchInfo struct { Number uint64 - BlockHash common.Hash PostedData []byte MsgCount arbutil.MessageIndex Preimages map[arbutil.PreimageType]map[common.Hash][]byte @@ -179,25 +178,27 @@ func newValidationEntry( start validator.GoGlobalState, end validator.GoGlobalState, msg *arbostypes.MessageWithMetadata, - fullBatchFetcher func(uint64) (*FullBatchInfo, error), + fullBatchInfo *FullBatchInfo, + prevBatches []validator.BatchInfo, prevDelayed uint64, chainConfig *params.ChainConfig, ) (*validationEntry, error) { preimages := make(map[arbutil.PreimageType]map[common.Hash][]byte) - valBatches := make([]validator.BatchInfo, 0) - - valBatchFetcher := func(batchNum uint64) ([]byte, error) { - fullBatchInfo, err := fullBatchFetcher(batchNum) - if err != nil { - return nil, err - } - valBatches = append(valBatches, validator.BatchInfo{ - Number: batchNum, + if fullBatchInfo == nil { + return nil, fmt.Errorf("fullbatchInfo cannot be nil") + } + if fullBatchInfo.Number != start.Batch { + return nil, fmt.Errorf("got wrong batch expected: %d got: %d", start.Batch, fullBatchInfo.Number) + } + valBatches := []validator.BatchInfo{ + { + Number: fullBatchInfo.Number, Data: fullBatchInfo.PostedData, - }) - copyPreimagesInto(preimages, fullBatchInfo.Preimages) - return fullBatchInfo.PostedData, nil + }, } + valBatches = append(valBatches, prevBatches...) + + copyPreimagesInto(preimages, fullBatchInfo.Preimages) hasDelayed := false var delayedNum uint64 @@ -208,14 +209,6 @@ func newValidationEntry( return nil, fmt.Errorf("illegal validation entry delayedMessage %d, previous %d", msg.DelayedMessagesRead, prevDelayed) } - if _, err := valBatchFetcher(start.Batch); err != nil { - return nil, err - } - msg.Message.BatchGasCost = nil - if err := msg.Message.FillInBatchGasCost(valBatchFetcher); err != nil { - return nil, err - } - return &validationEntry{ Stage: ReadyForRecord, Pos: pos, @@ -274,7 +267,19 @@ func NewStatelessBlockValidator( }, nil } -func (v *StatelessBlockValidator) readBatch(ctx context.Context, batchNum uint64) (bool, *FullBatchInfo, error) { +func (v *StatelessBlockValidator) readPostedBatch(ctx context.Context, batchNum uint64) ([]byte, error) { + batchCount, err := v.inboxTracker.GetBatchCount() + if err != nil { + return nil, err + } + if batchCount <= batchNum { + return nil, fmt.Errorf("batch not found: %d", batchNum) + } + postedData, _, err := v.inboxReader.GetSequencerMessageBytes(ctx, batchNum) + return postedData, err +} + +func (v *StatelessBlockValidator) readFullBatch(ctx context.Context, batchNum uint64) (bool, *FullBatchInfo, error) { batchCount, err := v.inboxTracker.GetBatchCount() if err != nil { return false, nil, err @@ -318,7 +323,6 @@ func (v *StatelessBlockValidator) readBatch(ctx context.Context, batchNum uint64 } fullInfo := FullBatchInfo{ Number: batchNum, - BlockHash: batchBlockHash, PostedData: postedData, MsgCount: batchMsgCount, Preimages: preimages, @@ -427,17 +431,30 @@ func (v *StatelessBlockValidator) CreateReadyValidationEntry(ctx context.Context } start := buildGlobalState(*prevResult, startPos) end := buildGlobalState(*result, endPos) - fullBatchReader := func(batchNum uint64) (*FullBatchInfo, error) { - found, fullBlockInfo, err := v.readBatch(ctx, batchNum) + found, fullBatchInfo, err := v.readFullBatch(ctx, start.Batch) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("batch %d not found", startPos.BatchNumber) + } + + prevBatchNums, err := msg.Message.PastBatchesRequired() + if err != nil { + return nil, err + } + prevBatches := make([]validator.BatchInfo, 0, len(prevBatchNums)) + for _, batchNum := range prevBatchNums { + data, err := v.readPostedBatch(ctx, batchNum) if err != nil { return nil, err } - if !found { - return nil, fmt.Errorf("batch %d not found", startPos.BatchNumber) - } - return fullBlockInfo, nil + prevBatches = append(prevBatches, validator.BatchInfo{ + Number: batchNum, + Data: data, + }) } - entry, err := newValidationEntry(pos, start, end, msg, fullBatchReader, prevDelayed, v.streamer.ChainConfig()) + entry, err := newValidationEntry(pos, start, end, msg, fullBatchInfo, prevBatches, prevDelayed, v.streamer.ChainConfig()) if err != nil { return nil, err } From 7553cec93b4d29c6ee68d0192ab586ad3fcac9d6 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 21:19:48 -0600 Subject: [PATCH 110/156] go-ethereum: remove koanf from recordingdb --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 81114dde8a2..c7539d8f7df 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 81114dde8a26bae90c188605c4a36d5919a4a265 +Subproject commit c7539d8f7dfa3070a050220588cbe3fa20e2ce08 From 0b518804ca523207cdb4cb0e96541578033aa0db Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 11 Sep 2024 23:01:27 -0500 Subject: [PATCH 111/156] Geth: pull in change to bubble up snapshot db errors --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 81114dde8a2..4002f12d881 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 81114dde8a26bae90c188605c4a36d5919a4a265 +Subproject commit 4002f12d8816a2851413ec1c0fcda387090c216b From 3342f18c7bf1bb8e3cd23adb8d458d66e12e6f0e Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 11 Sep 2024 23:15:17 -0500 Subject: [PATCH 112/156] Only warn on potentially incompatible WASM module roots --- cmd/nitro/nitro.go | 110 +++++++++++++++++++++++---------------------- 1 file changed, 56 insertions(+), 54 deletions(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index e66d99b56e7..a13a92fdc18 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -437,61 +437,9 @@ func mainImpl() int { // Check that node is compatible with on-chain WASM module root on startup and before any ArbOS upgrades take effect to prevent divergences if nodeConfig.Node.ParentChainReader.Enable && nodeConfig.Validation.Wasm.EnableWasmrootsCheck { - // Fetch current on-chain WASM module root - rollupUserLogic, err := rollupgen.NewRollupUserLogic(rollupAddrs.Rollup, l1Client) + err := checkWasmModuleRootCompatibility(ctx, nodeConfig.Validation.Wasm, l1Client, rollupAddrs) if err != nil { - log.Error("failed to create rollupUserLogic", "err", err) - return 1 - } - moduleRoot, err := rollupUserLogic.WasmModuleRoot(&bind.CallOpts{Context: ctx}) - if err != nil { - log.Error("failed to get on-chain WASM module root", "err", err) - return 1 - } - if (moduleRoot == common.Hash{}) { - log.Error("on-chain WASM module root is zero") - return 1 - } - // Check if the on-chain WASM module root belongs to the set of allowed module roots - allowedWasmModuleRoots := nodeConfig.Validation.Wasm.AllowedWasmModuleRoots - if len(allowedWasmModuleRoots) > 0 { - moduleRootMatched := false - for _, root := range allowedWasmModuleRoots { - bytes, err := hex.DecodeString(strings.TrimPrefix(root, "0x")) - if err == nil { - if common.HexToHash(root) == common.BytesToHash(bytes) { - moduleRootMatched = true - break - } - continue - } - locator, locatorErr := server_common.NewMachineLocator(root) - if locatorErr != nil { - log.Warn("allowed-wasm-module-roots: value not a hex nor valid path:", "value", root, "locatorErr", locatorErr, "decodeErr", err) - continue - } - path := locator.GetMachinePath(moduleRoot) - if _, err := os.Stat(path); err == nil { - moduleRootMatched = true - break - } - } - if !moduleRootMatched { - log.Error("on-chain WASM module root did not match with any of the allowed WASM module roots") - return 1 - } - } else { - // If no allowed module roots were provided in config, check if we have a validator machine directory for the on-chain WASM module root - locator, err := server_common.NewMachineLocator(nodeConfig.Validation.Wasm.RootPath) - if err != nil { - log.Warn("failed to create machine locator. Skipping the check for compatibility with on-chain WASM module root", "err", err) - } else { - path := locator.GetMachinePath(moduleRoot) - if _, err := os.Stat(path); err != nil { - log.Error("unable to find validator machine directory for the on-chain WASM module root", "err", err) - return 1 - } - } + log.Warn("failed to check if node is compatible with on-chain WASM module root", "err", err) } } @@ -1078,3 +1026,57 @@ type NodeConfigFetcher struct { func (f *NodeConfigFetcher) Get() *arbnode.Config { return &f.LiveConfig.Get().Node } + +func checkWasmModuleRootCompatibility(ctx context.Context, wasmConfig valnode.WasmConfig, l1Client *ethclient.Client, rollupAddrs chaininfo.RollupAddresses) error { + // Fetch current on-chain WASM module root + rollupUserLogic, err := rollupgen.NewRollupUserLogic(rollupAddrs.Rollup, l1Client) + if err != nil { + return fmt.Errorf("failed to create RollupUserLogic: %w", err) + } + moduleRoot, err := rollupUserLogic.WasmModuleRoot(&bind.CallOpts{Context: ctx}) + if err != nil { + return fmt.Errorf("failed to get on-chain WASM module root: %w", err) + } + if (moduleRoot == common.Hash{}) { + return errors.New("on-chain WASM module root is zero") + } + // Check if the on-chain WASM module root belongs to the set of allowed module roots + allowedWasmModuleRoots := wasmConfig.AllowedWasmModuleRoots + if len(allowedWasmModuleRoots) > 0 { + moduleRootMatched := false + for _, root := range allowedWasmModuleRoots { + bytes, err := hex.DecodeString(strings.TrimPrefix(root, "0x")) + if err == nil { + if common.HexToHash(root) == common.BytesToHash(bytes) { + moduleRootMatched = true + break + } + continue + } + locator, locatorErr := server_common.NewMachineLocator(root) + if locatorErr != nil { + log.Warn("allowed-wasm-module-roots: value not a hex nor valid path:", "value", root, "locatorErr", locatorErr, "decodeErr", err) + continue + } + path := locator.GetMachinePath(moduleRoot) + if _, err := os.Stat(path); err == nil { + moduleRootMatched = true + break + } + } + if !moduleRootMatched { + return errors.New("on-chain WASM module root did not match with any of the allowed WASM module roots") + } + } else { + // If no allowed module roots were provided in config, check if we have a validator machine directory for the on-chain WASM module root + locator, err := server_common.NewMachineLocator(wasmConfig.RootPath) + if err != nil { + return fmt.Errorf("failed to create machine locator: %w", err) + } + path := locator.GetMachinePath(moduleRoot) + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("unable to find validator machine directory for the on-chain WASM module root: %w", err) + } + } + return nil +} From 5d7a330c86b1f238d30106a3d00d140e582247b7 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Thu, 12 Sep 2024 13:45:43 +0800 Subject: [PATCH 113/156] fix: deploy both erc20 based and eth based templates --- cmd/deploy/deploy.go | 2 -- deploy/deploy.go | 21 +++++++++++++-------- system_tests/common_test.go | 1 - 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/cmd/deploy/deploy.go b/cmd/deploy/deploy.go index d8c0aeeac46..e0388e3d4b6 100644 --- a/cmd/deploy/deploy.go +++ b/cmd/deploy/deploy.go @@ -61,7 +61,6 @@ func main() { authorizevalidators := flag.Uint64("authorizevalidators", 0, "Number of validators to preemptively authorize") txTimeout := flag.Duration("txtimeout", 10*time.Minute, "Timeout when waiting for a transaction to be included in a block") prod := flag.Bool("prod", false, "Whether to configure the rollup for production or testing") - isUsingFeeToken := flag.Bool("isUsingFeeToken", false, "true if the chain uses custom fee token") flag.Parse() l1ChainId := new(big.Int).SetUint64(*l1ChainIdUint) maxDataSize := new(big.Int).SetUint64(*maxDataSizeUint) @@ -190,7 +189,6 @@ func main() { arbnode.GenerateRollupConfig(*prod, moduleRoot, ownerAddress, &chainConfig, chainConfigJson, loserEscrowAddress), nativeToken, maxDataSize, - *isUsingFeeToken, ) if err != nil { flag.Usage() diff --git a/deploy/deploy.go b/deploy/deploy.go index 5e7755cae30..f7ae6d0b597 100644 --- a/deploy/deploy.go +++ b/deploy/deploy.go @@ -31,7 +31,7 @@ func andTxSucceeded(ctx context.Context, l1Reader *headerreader.HeaderReader, tx return nil } -func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, isUsingFeeToken bool) (common.Address, error) { +func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int) (common.Address, error) { client := l1Reader.Client() /// deploy eth based templates @@ -46,7 +46,12 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade if err != nil { return common.Address{}, fmt.Errorf("blob basefee reader deploy error: %w", err) } - seqInboxTemplate, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, isUsingFeeToken) + seqInboxTemplateEthBased, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, false) + err = andTxSucceeded(ctx, l1Reader, tx, err) + if err != nil { + return common.Address{}, fmt.Errorf("sequencer inbox deploy error: %w", err) + } + seqInboxTemplateERC20Based, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, true) err = andTxSucceeded(ctx, l1Reader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("sequencer inbox deploy error: %w", err) @@ -72,7 +77,7 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade ethBasedTemplates := rollupgen.BridgeCreatorBridgeContracts{ Bridge: bridgeTemplate, - SequencerInbox: seqInboxTemplate, + SequencerInbox: seqInboxTemplateEthBased, Inbox: inboxTemplate, RollupEventInbox: rollupEventBridgeTemplate, Outbox: outboxTemplate, @@ -105,7 +110,7 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade erc20BasedTemplates := rollupgen.BridgeCreatorBridgeContracts{ Bridge: erc20BridgeTemplate, - SequencerInbox: seqInboxTemplate, + SequencerInbox: seqInboxTemplateERC20Based, Inbox: erc20InboxTemplate, RollupEventInbox: erc20RollupEventBridgeTemplate, Outbox: erc20OutboxTemplate, @@ -161,8 +166,8 @@ func deployChallengeFactory(ctx context.Context, l1Reader *headerreader.HeaderRe return ospEntryAddr, challengeManagerAddr, nil } -func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, isUsingFeeToken bool) (*rollupgen.RollupCreator, common.Address, common.Address, common.Address, error) { - bridgeCreator, err := deployBridgeCreator(ctx, l1Reader, auth, maxDataSize, isUsingFeeToken) +func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int) (*rollupgen.RollupCreator, common.Address, common.Address, common.Address, error) { + bridgeCreator, err := deployBridgeCreator(ctx, l1Reader, auth, maxDataSize) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("bridge creator deploy error: %w", err) } @@ -234,12 +239,12 @@ func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReade return rollupCreator, rollupCreatorAddress, validatorUtils, validatorWalletCreator, nil } -func DeployOnL1(ctx context.Context, parentChainReader *headerreader.HeaderReader, deployAuth *bind.TransactOpts, batchPosters []common.Address, batchPosterManager common.Address, authorizeValidators uint64, config rollupgen.Config, nativeToken common.Address, maxDataSize *big.Int, isUsingFeeToken bool) (*chaininfo.RollupAddresses, error) { +func DeployOnL1(ctx context.Context, parentChainReader *headerreader.HeaderReader, deployAuth *bind.TransactOpts, batchPosters []common.Address, batchPosterManager common.Address, authorizeValidators uint64, config rollupgen.Config, nativeToken common.Address, maxDataSize *big.Int) (*chaininfo.RollupAddresses, error) { if config.WasmModuleRoot == (common.Hash{}) { return nil, errors.New("no machine specified") } - rollupCreator, _, validatorUtils, validatorWalletCreator, err := deployRollupCreator(ctx, parentChainReader, deployAuth, maxDataSize, isUsingFeeToken) + rollupCreator, _, validatorUtils, validatorWalletCreator, err := deployRollupCreator(ctx, parentChainReader, deployAuth, maxDataSize) if err != nil { return nil, fmt.Errorf("error deploying rollup creator: %w", err) } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 7a78cee309e..6975990be04 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1040,7 +1040,6 @@ func DeployOnTestL1( arbnode.GenerateRollupConfig(prodConfirmPeriodBlocks, wasmModuleRoot, l1info.GetAddress("RollupOwner"), chainConfig, serializedChainConfig, common.Address{}), nativeToken, maxDataSize, - false, ) Require(t, err) l1info.SetContract("Bridge", addresses.Bridge) From e92300cb2c652846295af6de7da692a6b901cf3e Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Thu, 12 Sep 2024 13:47:04 +0800 Subject: [PATCH 114/156] chore: error message --- deploy/deploy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/deploy.go b/deploy/deploy.go index f7ae6d0b597..f2099f976a2 100644 --- a/deploy/deploy.go +++ b/deploy/deploy.go @@ -49,12 +49,12 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade seqInboxTemplateEthBased, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, false) err = andTxSucceeded(ctx, l1Reader, tx, err) if err != nil { - return common.Address{}, fmt.Errorf("sequencer inbox deploy error: %w", err) + return common.Address{}, fmt.Errorf("sequencer inbox eth based deploy error: %w", err) } seqInboxTemplateERC20Based, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, true) err = andTxSucceeded(ctx, l1Reader, tx, err) if err != nil { - return common.Address{}, fmt.Errorf("sequencer inbox deploy error: %w", err) + return common.Address{}, fmt.Errorf("sequencer inbox erc20 based deploy error: %w", err) } inboxTemplate, tx, _, err := bridgegen.DeployInbox(auth, client, maxDataSize) From 4c0c9b36973c026db464918eaa1034324b2ce39e Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 12 Sep 2024 10:34:28 -0500 Subject: [PATCH 115/156] Bump geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 4002f12d881..7e7d4ff4cd4 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 4002f12d8816a2851413ec1c0fcda387090c216b +Subproject commit 7e7d4ff4cd440d5e2e294f65cfe60e1a37ba1bb8 From 374fa8ca43369eac3d0a947b68d177b2d4acd9ef Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 12 Sep 2024 11:17:16 -0500 Subject: [PATCH 116/156] Bump geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 7e7d4ff4cd4..ad31b75098f 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 7e7d4ff4cd440d5e2e294f65cfe60e1a37ba1bb8 +Subproject commit ad31b75098f874c7124e7b29b47db8662d47b9ff From a9b455ca5328cf31a4a243220588ed513ebc8dbf Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 12:19:08 -0600 Subject: [PATCH 117/156] makefile lint --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9ecba1ce2d1..b736b049062 100644 --- a/Makefile +++ b/Makefile @@ -479,7 +479,7 @@ contracts/test/prover/proofs/rust-%.json: $(arbitrator_cases)/rust/$(wasm32_wasi $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin --with-forwarder contracts/test/prover/proofs/go.json: $(arbitrator_cases)/go/testcase.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin $(arbitrator_tests_link_deps) $(arbitrator_cases)/user.wasm - $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm --with-forwarder + $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm --with-forwarder # avoid testing user.wasm in onestepproofs. It can only run as stylus program. contracts/test/prover/proofs/user.json: From d156b8081ebc5ebad47b6634317033fe21f7ad12 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 12:40:39 -0600 Subject: [PATCH 118/156] add traces to workers --- pubsub/consumer.go | 6 ++++++ validator/valnode/redis/consumer.go | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/pubsub/consumer.go b/pubsub/consumer.go index df3695606d3..1fccae56ade 100644 --- a/pubsub/consumer.go +++ b/pubsub/consumer.go @@ -77,6 +77,10 @@ func (c *Consumer[Request, Response]) Start(ctx context.Context) { ) } +func (c *Consumer[Request, Response]) Id() string { + return c.id +} + func (c *Consumer[Request, Response]) StopAndWait() { c.StopWaiter.StopAndWait() c.deleteHeartBeat(c.GetParentContext()) @@ -164,10 +168,12 @@ func (c *Consumer[Request, Response]) SetResult(ctx context.Context, messageID s if err != nil { return fmt.Errorf("marshaling result: %w", err) } + log.Trace("consumer: setting result", "cid", c.id, "messageId", messageID) acquired, err := c.client.SetNX(ctx, messageID, resp, c.cfg.ResponseEntryTimeout).Result() if err != nil || !acquired { return fmt.Errorf("setting result for message: %v, error: %w", messageID, err) } + log.Trace("consumer: xack", "cid", c.id, "messageId", messageID) if _, err := c.client.XAck(ctx, c.redisStream, c.redisGroup, messageID).Result(); err != nil { return fmt.Errorf("acking message: %v, error: %w", messageID, err) } diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 2b025600cc0..90bffbb704c 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -103,11 +103,13 @@ func (s *ValidationServer) Start(ctx_in context.Context) { case <-ready: // Wait until the stream exists and start consuming iteratively. } s.StopWaiter.CallIteratively(func(ctx context.Context) time.Duration { + log.Trace("waiting for request token", "cid", c.Id()) select { case <-ctx.Done(): return 0 case <-requestTokenQueue: } + log.Trace("got request token", "cid", c.Id()) req, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) @@ -115,10 +117,12 @@ func (s *ValidationServer) Start(ctx_in context.Context) { return 0 } if req == nil { + log.Trace("consumed nil", "cid", c.Id()) // There's nothing in the queue requestTokenQueue <- struct{}{} return time.Second } + log.Trace("forwarding work", "cid", c.Id(), "workid", req.ID) select { case <-ctx.Done(): case workQueue <- workUnit{req, moduleRoot}: @@ -144,20 +148,24 @@ func (s *ValidationServer) Start(ctx_in context.Context) { for i := 0; i < workers; i++ { s.StopWaiter.LaunchThread(func(ctx context.Context) { for { + log.Trace("waiting for work", "thread", i) var work workUnit select { case <-ctx.Done(): return case work = <-workQueue: } + log.Trace("got work", "thread", i, "workid", work.req.ID) valRun := s.spawner.Launch(work.req.Value, work.moduleRoot) res, err := valRun.Await(ctx) if err != nil { log.Error("Error validating", "request value", work.req.Value, "error", err) } else { + log.Trace("done work", "thread", i, "workid", work.req.ID) if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) } + log.Trace("set result", "thread", i, "workid", work.req.ID) } select { case <-ctx.Done(): From 1ab23ac4fcb8bf088244254a2db8b56dd2b4dc90 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 12:43:13 -0600 Subject: [PATCH 119/156] redis consumer logs to Debug --- pubsub/consumer.go | 4 ++-- validator/valnode/redis/consumer.go | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pubsub/consumer.go b/pubsub/consumer.go index 1fccae56ade..bd73e729e7d 100644 --- a/pubsub/consumer.go +++ b/pubsub/consumer.go @@ -168,12 +168,12 @@ func (c *Consumer[Request, Response]) SetResult(ctx context.Context, messageID s if err != nil { return fmt.Errorf("marshaling result: %w", err) } - log.Trace("consumer: setting result", "cid", c.id, "messageId", messageID) + log.Debug("consumer: setting result", "cid", c.id, "messageId", messageID) acquired, err := c.client.SetNX(ctx, messageID, resp, c.cfg.ResponseEntryTimeout).Result() if err != nil || !acquired { return fmt.Errorf("setting result for message: %v, error: %w", messageID, err) } - log.Trace("consumer: xack", "cid", c.id, "messageId", messageID) + log.Debug("consumer: xack", "cid", c.id, "messageId", messageID) if _, err := c.client.XAck(ctx, c.redisStream, c.redisGroup, messageID).Result(); err != nil { return fmt.Errorf("acking message: %v, error: %w", messageID, err) } diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 90bffbb704c..49e72477f58 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -103,13 +103,13 @@ func (s *ValidationServer) Start(ctx_in context.Context) { case <-ready: // Wait until the stream exists and start consuming iteratively. } s.StopWaiter.CallIteratively(func(ctx context.Context) time.Duration { - log.Trace("waiting for request token", "cid", c.Id()) + log.Debug("waiting for request token", "cid", c.Id()) select { case <-ctx.Done(): return 0 case <-requestTokenQueue: } - log.Trace("got request token", "cid", c.Id()) + log.Debug("got request token", "cid", c.Id()) req, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) @@ -117,12 +117,12 @@ func (s *ValidationServer) Start(ctx_in context.Context) { return 0 } if req == nil { - log.Trace("consumed nil", "cid", c.Id()) + log.Debug("consumed nil", "cid", c.Id()) // There's nothing in the queue requestTokenQueue <- struct{}{} return time.Second } - log.Trace("forwarding work", "cid", c.Id(), "workid", req.ID) + log.Debug("forwarding work", "cid", c.Id(), "workid", req.ID) select { case <-ctx.Done(): case workQueue <- workUnit{req, moduleRoot}: @@ -135,7 +135,7 @@ func (s *ValidationServer) Start(ctx_in context.Context) { for { select { case <-readyStreams: - log.Trace("At least one stream is ready") + log.Debug("At least one stream is ready") return // Don't block Start if at least one of the stream is ready. case <-time.After(s.config.StreamTimeout): log.Error("Waiting for redis streams timed out") @@ -148,24 +148,24 @@ func (s *ValidationServer) Start(ctx_in context.Context) { for i := 0; i < workers; i++ { s.StopWaiter.LaunchThread(func(ctx context.Context) { for { - log.Trace("waiting for work", "thread", i) + log.Debug("waiting for work", "thread", i) var work workUnit select { case <-ctx.Done(): return case work = <-workQueue: } - log.Trace("got work", "thread", i, "workid", work.req.ID) + log.Debug("got work", "thread", i, "workid", work.req.ID) valRun := s.spawner.Launch(work.req.Value, work.moduleRoot) res, err := valRun.Await(ctx) if err != nil { log.Error("Error validating", "request value", work.req.Value, "error", err) } else { - log.Trace("done work", "thread", i, "workid", work.req.ID) + log.Debug("done work", "thread", i, "workid", work.req.ID) if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) } - log.Trace("set result", "thread", i, "workid", work.req.ID) + log.Debug("set result", "thread", i, "workid", work.req.ID) } select { case <-ctx.Done(): From 370c6f1494fb5c24a1edf25f300fec9b12b505a1 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 13:16:43 -0600 Subject: [PATCH 120/156] node config: always add targets in the same order This prevents an errored change in StylusTargetConfig from a change in the order of targets --- execution/gethexec/node.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 21c2b4beced..355ed7a6470 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "reflect" + "sort" "sync/atomic" "testing" @@ -59,6 +60,11 @@ func (c *StylusTargetConfig) Validate() error { for target := range targetsSet { targets = append(targets, target) } + sort.Slice( + targets, + func(i, j int) bool { + return targets[i] < targets[j] + }) c.wasmTargets = targets return nil } From 44ff879590ca80beb6a3e4b64d826690acab660f Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 12 Sep 2024 14:37:54 -0500 Subject: [PATCH 121/156] Fix setupFastConfirmation not getting called --- staker/fast_confirm.go | 4 ++++ staker/staker.go | 16 +++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/staker/fast_confirm.go b/staker/fast_confirm.go index 88f457f5281..5dc7f012055 100644 --- a/staker/fast_confirm.go +++ b/staker/fast_confirm.go @@ -121,10 +121,12 @@ func (f *FastConfirmSafe) tryFastConfirmation(ctx context.Context, blockHash com return err } if alreadyApproved.Cmp(common.Big1) == 0 { + log.Info("Already approved Safe tx hash for fast confirmation, checking if we can execute the Safe tx", "safeHash", safeTxHash, "nodeHash", nodeHash) _, err = f.checkApprovedHashAndExecTransaction(ctx, fastConfirmCallData, safeTxHash) return err } + log.Info("Approving Safe tx hash to fast confirm", "safeHash", safeTxHash, "nodeHash", nodeHash) auth, err := f.builder.Auth(ctx) if err != nil { return err @@ -231,6 +233,7 @@ func (f *FastConfirmSafe) checkApprovedHashAndExecTransaction(ctx context.Contex if err != nil { return false, err } + log.Info("Executing Safe tx to fast confirm", "safeHash", safeTxHash) _, err = f.safe.ExecTransaction( auth, f.wallet.RollupAddress(), @@ -249,5 +252,6 @@ func (f *FastConfirmSafe) checkApprovedHashAndExecTransaction(ctx context.Contex } return true, nil } + log.Info("Not enough Safe tx approvals yet to fast confirm", "safeHash", safeTxHash) return false, nil } diff --git a/staker/staker.go b/staker/staker.go index 6e93d273119..77ca93e02c1 100644 --- a/staker/staker.go +++ b/staker/staker.go @@ -268,7 +268,6 @@ type Staker struct { inboxReader InboxReaderInterface statelessBlockValidator *StatelessBlockValidator fatalErr chan<- error - enableFastConfirmation bool fastConfirmSafe *FastConfirmSafe } @@ -363,7 +362,10 @@ func (s *Staker) Initialize(ctx context.Context) error { return err } - return s.blockValidator.InitAssumeValid(stakedInfo.AfterState().GlobalState) + err = s.blockValidator.InitAssumeValid(stakedInfo.AfterState().GlobalState) + if err != nil { + return err + } } return s.setupFastConfirmation(ctx) } @@ -390,9 +392,9 @@ func (s *Staker) setupFastConfirmation(ctx context.Context) error { if err != nil { return fmt.Errorf("getting rollup fast confirmer address: %w", err) } + log.Info("Setting up fast confirmation", "wallet", walletAddress, "fastConfirmer", fastConfirmer) if fastConfirmer == walletAddress { // We can directly fast confirm nodes - s.enableFastConfirmation = true return nil } else if fastConfirmer == (common.Address{}) { // No fast confirmer enabled @@ -419,13 +421,12 @@ func (s *Staker) setupFastConfirmation(ctx context.Context) error { if !isOwner { return fmt.Errorf("staker wallet address %v is not an owner of the fast confirm safe %v", walletAddress, fastConfirmer) } - s.enableFastConfirmation = true s.fastConfirmSafe = fastConfirmSafe return nil } func (s *Staker) tryFastConfirmationNodeNumber(ctx context.Context, number uint64, hash common.Hash) error { - if !s.enableFastConfirmation { + if !s.config().EnableFastConfirmation { return nil } nodeInfo, err := s.rollup.LookupNode(ctx, number) @@ -436,7 +437,7 @@ func (s *Staker) tryFastConfirmationNodeNumber(ctx context.Context, number uint6 } func (s *Staker) tryFastConfirmation(ctx context.Context, blockHash common.Hash, sendRoot common.Hash, nodeHash common.Hash) error { - if !s.enableFastConfirmation { + if !s.config().EnableFastConfirmation { return nil } if s.fastConfirmSafe != nil { @@ -446,6 +447,7 @@ func (s *Staker) tryFastConfirmation(ctx context.Context, blockHash common.Hash, if err != nil { return err } + log.Info("Fast confirming node with wallet", "wallet", auth.From, "nodeHash", nodeHash) _, err = s.rollup.FastConfirmNextNode(auth, blockHash, sendRoot, nodeHash) return err } @@ -802,13 +804,13 @@ func (s *Staker) Act(ctx context.Context) (*types.Transaction, error) { confirmedCorrect = stakedOnNode } if confirmedCorrect { + log.Info("trying to fast confirm previous node", "node", firstUnresolvedNode, "nodeHash", nodeInfo.NodeHash) err = s.tryFastConfirmationNodeNumber(ctx, firstUnresolvedNode, nodeInfo.NodeHash) if err != nil { return nil, err } if s.builder.BuildingTransactionCount() > 0 { // Try to fast confirm previous nodes before working on new ones - log.Info("fast confirming previous node", "node", firstUnresolvedNode) return s.wallet.ExecuteTransactions(ctx, s.builder, cfg.gasRefunder) } } From 7c3b4be042313c36bf7c23553327aeb8c4bc53d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Sep 2024 21:28:27 +0000 Subject: [PATCH 122/156] Bump rustix from 0.37.23 to 0.37.27 in /arbitrator/stylus/tests/erc20 Bumps [rustix](https://github.com/bytecodealliance/rustix) from 0.37.23 to 0.37.27. - [Release notes](https://github.com/bytecodealliance/rustix/releases) - [Changelog](https://github.com/bytecodealliance/rustix/blob/main/CHANGELOG.md) - [Commits](https://github.com/bytecodealliance/rustix/compare/v0.37.23...v0.37.27) --- updated-dependencies: - dependency-name: rustix dependency-type: indirect ... Signed-off-by: dependabot[bot] --- arbitrator/stylus/tests/erc20/Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arbitrator/stylus/tests/erc20/Cargo.lock b/arbitrator/stylus/tests/erc20/Cargo.lock index c3e215978de..f5e1e0b15e5 100644 --- a/arbitrator/stylus/tests/erc20/Cargo.lock +++ b/arbitrator/stylus/tests/erc20/Cargo.lock @@ -575,9 +575,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.23" +version = "0.37.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" dependencies = [ "bitflags", "errno", From 7075c7f2e02c8404fadf0243da9282577a2406b6 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 19:21:42 -0600 Subject: [PATCH 123/156] redis producer: delete entry after result exists --- pubsub/producer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pubsub/producer.go b/pubsub/producer.go index 2b1cdb5e3ff..e7f2ac938c2 100644 --- a/pubsub/producer.go +++ b/pubsub/producer.go @@ -234,6 +234,7 @@ func (p *Producer[Request, Response]) checkResponses(ctx context.Context) time.D promise.Produce(resp) responded++ } + p.client.Del(ctx, id) delete(p.promises, id) } var trimmed int64 From f5750e7ad22812cf193aabb959badf4a8b25d0cb Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 19:23:52 -0600 Subject: [PATCH 124/156] producer: debug check responses loop --- pubsub/producer.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pubsub/producer.go b/pubsub/producer.go index e7f2ac938c2..5eec3a4b523 100644 --- a/pubsub/producer.go +++ b/pubsub/producer.go @@ -205,30 +205,33 @@ func setMinIdInt(min *[2]uint64, id string) error { // checkResponses checks iteratively whether response for the promise is ready. func (p *Producer[Request, Response]) checkResponses(ctx context.Context) time.Duration { minIdInt := [2]uint64{math.MaxUint64, math.MaxUint64} + log.Debug("redis producer: check responses starting") p.promisesLock.Lock() defer p.promisesLock.Unlock() responded := 0 errored := 0 + checked := 0 for id, promise := range p.promises { if ctx.Err() != nil { return 0 } + checked++ res, err := p.client.Get(ctx, id).Result() if err != nil { errSetId := setMinIdInt(&minIdInt, id) if errSetId != nil { - log.Error("error setting minId", "err", err) + log.Error("redis producer: error setting minId", "err", err) return p.cfg.CheckResultInterval } if !errors.Is(err, redis.Nil) { - log.Error("Error reading value in redis", "key", id, "error", err) + log.Error("redis producer: Error reading value in redis", "key", id, "error", err) } continue } var resp Response if err := json.Unmarshal([]byte(res), &resp); err != nil { promise.ProduceError(fmt.Errorf("error unmarshalling: %w", err)) - log.Error("Error unmarshaling", "value", res, "error", err) + log.Error("redis producer: Error unmarshaling", "value", res, "error", err) errored++ } else { promise.Produce(resp) @@ -246,7 +249,7 @@ func (p *Producer[Request, Response]) checkResponses(ctx context.Context) time.D } else { trimmed, trimErr = p.client.XTrimMaxLen(ctx, p.redisStream, 0).Result() } - log.Trace("trimming", "id", minId, "trimmed", trimmed, "responded", responded, "errored", errored, "trim-err", trimErr) + log.Debug("trimming", "id", minId, "trimmed", trimmed, "responded", responded, "errored", errored, "trim-err", trimErr, "checked", checked) return p.cfg.CheckResultInterval } From 0975c44a39e207dffb5f1f480112e5ed6da5f012 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 19:29:27 -0600 Subject: [PATCH 125/156] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index c7539d8f7df..9ca65b86101 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit c7539d8f7dfa3070a050220588cbe3fa20e2ce08 +Subproject commit 9ca65b86101b4c87df188923d6eeff4db992520f From b5fa17d9a447d39b08680fdf9807301623aa2d2e Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 20:25:42 -0600 Subject: [PATCH 126/156] fix race in test --- validator/valnode/redis/consumer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 49e72477f58..e0d53ffb2ef 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -146,6 +146,7 @@ func (s *ValidationServer) Start(ctx_in context.Context) { } }) for i := 0; i < workers; i++ { + i := i s.StopWaiter.LaunchThread(func(ctx context.Context) { for { log.Debug("waiting for work", "thread", i) From 6a967acf92c2786933c04d5016e8b9b2bc8458d6 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 13 Sep 2024 19:10:40 +0530 Subject: [PATCH 127/156] Fix Submodule CI Pin check --- .github/workflows/submodule-pin-check.sh | 26 ----------------------- .github/workflows/submodule-pin-check.yml | 23 +++++++++++++++++++- 2 files changed, 22 insertions(+), 27 deletions(-) delete mode 100755 .github/workflows/submodule-pin-check.sh diff --git a/.github/workflows/submodule-pin-check.sh b/.github/workflows/submodule-pin-check.sh deleted file mode 100755 index aecb287ce14..00000000000 --- a/.github/workflows/submodule-pin-check.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -declare -Ar exceptions=( - [contracts]=origin/develop - [nitro-testnode]=origin/master - - #TODO Rachel to check these are the intended branches. - [arbitrator/langs/c]=origin/vm-storage-cache - [arbitrator/tools/wasmer]=origin/adopt-v4.2.8 -) - -divergent=0 -for mod in `git submodule --quiet foreach 'echo $name'`; do - branch=origin/HEAD - if [[ -v exceptions[$mod] ]]; then - branch=${exceptions[$mod]} - fi - - if ! git -C $mod merge-base --is-ancestor HEAD $branch; then - echo $mod diverges from $branch - divergent=1 - fi -done - -exit $divergent - diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index f045f71f68c..5e4be387d93 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -18,11 +18,32 @@ jobs: with: fetch-depth: 0 submodules: true + ref: "${{ github.event.pull_request.merge_commit_sha }}" - name: Check all submodules are ancestors of origin/HEAD or configured branch run: | status_state="pending" - if ${{ github.workspace }}/.github/workflows/submodule-pin-check.sh; then + declare -Ar exceptions=( + [contracts]=origin/develop + [nitro-testnode]=origin/master + + #TODO Rachel to check these are the intended branches. + [arbitrator/langs/c]=origin/vm-storage-cache + [arbitrator/tools/wasmer]=origin/adopt-v4.2.8 + ) + divergent=0 + for mod in `git submodule --quiet foreach 'echo $name'`; do + branch=origin/HEAD + if [[ -v exceptions[$mod] ]]; then + branch=${exceptions[$mod]} + fi + + if ! git -C $mod merge-base --is-ancestor HEAD $branch; then + echo $mod diverges from $branch + divergent=1 + fi + done + if $divergent then status_state="success" else resp="$(curl -sSL --fail-with-body \ From 37f1ec56f432bab835cbecb1e16fa365134583d7 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 13 Sep 2024 21:09:02 +0530 Subject: [PATCH 128/156] test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 5e4be387d93..b2df5bff0c2 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request_target: + pull_request: branches: [ master ] types: [synchronize, opened, reopened] From 3321cc4f998d5184bf4edda8ef0619681efbe672 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 13 Sep 2024 21:09:30 +0530 Subject: [PATCH 129/156] revert test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index b2df5bff0c2..5e4be387d93 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request: + pull_request_target: branches: [ master ] types: [synchronize, opened, reopened] From 55109fc6b386ff1d69e236d04fb1c9f454d0727e Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 13 Sep 2024 21:14:08 +0530 Subject: [PATCH 130/156] fix bash --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 5e4be387d93..76c777baad6 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -43,7 +43,7 @@ jobs: divergent=1 fi done - if $divergent then + if $divergent; then status_state="success" else resp="$(curl -sSL --fail-with-body \ From 2b95a6099b6d9ae39e76e3641870f7ebe2fc9ba5 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 13 Sep 2024 21:14:26 +0530 Subject: [PATCH 131/156] test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 76c777baad6..46926b6430b 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request_target: + pull_request: branches: [ master ] types: [synchronize, opened, reopened] From 421bca1baccc7ed2e87c164f6ef2616243bacc1f Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 13 Sep 2024 21:14:37 +0530 Subject: [PATCH 132/156] revert test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 46926b6430b..76c777baad6 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request: + pull_request_target: branches: [ master ] types: [synchronize, opened, reopened] From 8612635a7efc72b91c1c0c8a54575b4fe51b031c Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 16 Sep 2024 16:42:48 +0530 Subject: [PATCH 133/156] Changes based on PR comments --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 76c777baad6..a6f53e7a25f 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -43,7 +43,7 @@ jobs: divergent=1 fi done - if $divergent; then + if [ $divergent -eq 1 ]; then status_state="success" else resp="$(curl -sSL --fail-with-body \ From 7307caa628fd785a6747c65036b86002f8e3fb35 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 16 Sep 2024 16:45:13 +0530 Subject: [PATCH 134/156] test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index a6f53e7a25f..40fe2d7cb2c 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request_target: + pull_request: branches: [ master ] types: [synchronize, opened, reopened] From b0f166ac36c238064ab6fc1c547fc2a3740015ec Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 16 Sep 2024 16:45:27 +0530 Subject: [PATCH 135/156] revert test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 40fe2d7cb2c..a6f53e7a25f 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request: + pull_request_target: branches: [ master ] types: [synchronize, opened, reopened] From 35671c8804a8c534f1388f89221a4933c946e02a Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 16 Sep 2024 17:33:37 +0530 Subject: [PATCH 136/156] change to 0 --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index a6f53e7a25f..90419b530e9 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -43,7 +43,7 @@ jobs: divergent=1 fi done - if [ $divergent -eq 1 ]; then + if [ $divergent -eq 0 ]; then status_state="success" else resp="$(curl -sSL --fail-with-body \ From 7460b7b2fb4aedea85254c3a6e5e1091a38600af Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 16 Sep 2024 17:33:53 +0530 Subject: [PATCH 137/156] test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 90419b530e9..a3638a49294 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request_target: + pull_request: branches: [ master ] types: [synchronize, opened, reopened] From 7c41e5d657ebfe3cf52a1bd349c057158abce98a Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 16 Sep 2024 17:34:06 +0530 Subject: [PATCH 138/156] revert test the change --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index a3638a49294..90419b530e9 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request: + pull_request_target: branches: [ master ] types: [synchronize, opened, reopened] From 9b8a8f6207ce03ad42e8fbf856d2aba4cdd1529b Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 16 Sep 2024 10:53:02 -0300 Subject: [PATCH 139/156] Fix warn message formatting --- arbnode/seq_coordinator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbnode/seq_coordinator.go b/arbnode/seq_coordinator.go index a582b64ffac..176ace114b8 100644 --- a/arbnode/seq_coordinator.go +++ b/arbnode/seq_coordinator.go @@ -491,7 +491,7 @@ func (c *SeqCoordinator) updateWithLockout(ctx context.Context, nextChosen strin // Before proceeding, first try deleting finalized messages from redis and setting the finalizedMsgCount key finalized, err := c.sync.GetFinalizedMsgCount(ctx) if err != nil { - log.Warn("Error getting finalizedMessageCount from syncMonitor: %w", err) + log.Warn("Error getting finalizedMessageCount from syncMonitor", "err", err) } else if finalized == 0 { log.Warn("SyncMonitor returned zero finalizedMessageCount") } else if err := c.deleteFinalizedMsgsFromRedis(ctx, finalized); err != nil { From f8fee5edcbec3dc8da0ac8ce4d7119ad95fa10c5 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 10 Sep 2024 12:47:02 -0300 Subject: [PATCH 140/156] Make scripts/startup-testnode.bash shellcheck compliant --- scripts/startup-testnode.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/startup-testnode.bash b/scripts/startup-testnode.bash index 701e7ff59ae..5313a9ec5d4 100755 --- a/scripts/startup-testnode.bash +++ b/scripts/startup-testnode.bash @@ -5,9 +5,9 @@ timeout 60 ./nitro-testnode/test-node.bash --init --dev || exit_status=$? -if [ -n "$exit_status" ] && [ $exit_status -ne 0 ] && [ $exit_status -ne 124 ]; then +if [ -n "$exit_status" ] && [ "$exit_status" -ne 0 ] && [ "$exit_status" -ne 124 ]; then echo "Startup failed." - exit $exit_status + exit "$exit_status" fi echo "Startup succeeded." From 6228fd9127eced651576628ff87c97bb090c7b71 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 10 Sep 2024 13:01:16 -0300 Subject: [PATCH 141/156] Make scripts/fuzz.bash shellcheck compliant --- scripts/fuzz.bash | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/scripts/fuzz.bash b/scripts/fuzz.bash index 6271b917b66..a73c208e888 100755 --- a/scripts/fuzz.bash +++ b/scripts/fuzz.bash @@ -2,12 +2,12 @@ set -e -mydir=`dirname $0` +mydir=$(dirname "$0") cd "$mydir" function printusage { - echo Usage: $0 --build \[--binary-path PATH\] - echo " " $0 \ \[--binary-path PATH\] \[--fuzzcache-path PATH\] \[--nitro-path PATH\] \[--duration DURATION\] + echo Usage: "$0" --build \[--binary-path PATH\] + echo " " "$0" \ \[--binary-path PATH\] \[--fuzzcache-path PATH\] \[--nitro-path PATH\] \[--duration DURATION\] echo echo fuzzer names: echo " " FuzzPrecompiles @@ -22,7 +22,6 @@ if [[ $# -eq 0 ]]; then exit fi -fuzz_executable=../target/bin/system_test.fuzz binpath=../target/bin/ fuzzcachepath=../target/var/fuzz-cache nitropath=../ @@ -72,7 +71,7 @@ while [[ $# -gt 0 ]]; do shift ;; FuzzPrecompiles | FuzzStateTransition) - if [[ ! -z "$test_name" ]]; then + if [[ -n "$test_name" ]]; then echo can only run one fuzzer at a time exit 1 fi @@ -81,7 +80,7 @@ while [[ $# -gt 0 ]]; do shift ;; FuzzInboxMultiplexer) - if [[ ! -z "$test_name" ]]; then + if [[ -n "$test_name" ]]; then echo can only run one fuzzer at a time exit 1 fi @@ -102,17 +101,17 @@ fi if $run_build; then for build_group in system_tests arbstate; do - go test -c ${nitropath}/${build_group} -fuzz Fuzz -o "$binpath"/${build_group}.fuzz + go test -c "${nitropath}"/${build_group} -fuzz Fuzz -o "$binpath"/${build_group}.fuzz done fi -if [[ ! -z $test_group ]]; then - timeout "$((60 * duration))" "$binpath"/${test_group}.fuzz -test.run "^$" -test.fuzzcachedir "$fuzzcachepath" -test.fuzz $test_name || exit_status=$? +if [[ -n $test_group ]]; then + timeout "$((60 * duration))" "$binpath"/${test_group}.fuzz -test.run "^$" -test.fuzzcachedir "$fuzzcachepath" -test.fuzz "$test_name" || exit_status=$? fi -if [ -n "$exit_status" ] && [ $exit_status -ne 0 ] && [ $exit_status -ne 124 ]; then +if [ -n "$exit_status" ] && [ "$exit_status" -ne 0 ] && [ "$exit_status" -ne 124 ]; then echo "Fuzzing failed." - exit $exit_status + exit "$exit_status" fi echo "Fuzzing succeeded." From 4df4c39bc02a144420a4297416ba7949b9d62634 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 10 Sep 2024 14:06:00 -0300 Subject: [PATCH 142/156] Make convert-databases.bash shellcheck compliant --- scripts/convert-databases.bash | 43 ++++++++++++++++------------------ 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/scripts/convert-databases.bash b/scripts/convert-databases.bash index 3020b389b46..baddcdcacd9 100755 --- a/scripts/convert-databases.bash +++ b/scripts/convert-databases.bash @@ -33,7 +33,7 @@ printStatus() { } printUsage() { -echo Usage: $0 \[OPTIONS..\] +echo Usage: "$0" \[OPTIONS..\] echo echo OPTIONS: echo "--dbconv dbconv binary path (default: \"$DEFAULT_DBCONV\")" @@ -42,15 +42,15 @@ echo Usage: $0 \[OPTIONS..\] echo "--force remove destination directory if it exists" echo "--skip-existing skip convertion of databases which directories already exist in the destination directory" echo "--clean sets what should be removed in case of error, possible values:" - echo " \"failed\" - remove database which conversion failed (default)" + echo " \"failed\" - remove database which conversion failed (default)" echo " \"none\" - remove nothing, leave unfinished and potentially corrupted databases" echo " \"all\" - remove whole destination directory" } removeDir() { cmd="rm -r \"$1\"" - echo $cmd - eval $cmd + echo "$cmd" + eval "$cmd" return $? } @@ -62,7 +62,7 @@ cleanup() { ;; failed) echo "== Note: removing only failed destination directory" - dstdir=$(echo $dst/$1 | tr -s /) + dstdir=$(echo "$dst"/"$1" | tr -s /) removeDir "$dstdir" ;; none) @@ -127,8 +127,8 @@ if $force && $skip_existing; then exit 1 fi -if [ $clean != "all" ] && [ $clean != "failed" ] && [ $clean != "none" ] ; then - echo Error: Invalid --clean value: $clean +if [ "$clean" != "all" ] && [ "$clean" != "failed" ] && [ "$clean" != "none" ] ; then + echo Error: Invalid --clean value: "$clean" printUsage exit 1 fi @@ -138,8 +138,8 @@ if ! [ -e "$dbconv" ]; then exit 1 fi -if ! [ -n "$dst" ]; then - echo Error: Missing destination directory \(\-\-dst\) +if [ -z "$dst" ]; then + echo "Error: Missing destination directory (--dst)" printUsage exit 1 fi @@ -168,9 +168,8 @@ fi if [ -e "$dst" ] && ! $skip_existing; then if $force; then - echo == Warning! Destination already exists, --force is set, removing all files under path: "$dst" - removeDir "$dst" - if [ $? -ne 0 ]; then + echo "== Warning! Destination already exists, --force is set, removing all files under path: $dst" + if ! removeDir "$dst"; then echo Error: failed to remove "$dst" exit 1 fi @@ -183,14 +182,13 @@ fi convert_result= convert () { srcdir="$src"/$1 - dstdir=$(echo $dst/$1 | tr -s /) - if ! [ -e $dstdir ]; then + dstdir=$(echo "$dst"/"$1" | tr -s /) + if ! [ -e "$dstdir" ]; then echo "== Converting $1 db" cmd="$dbconv --src.db-engine=leveldb --src.data \"$srcdir\" --dst.db-engine=pebble --dst.data \"$dstdir\" --convert --compact" - echo $cmd - eval $cmd - if [ $? -ne 0 ]; then - cleanup $1 + echo "$cmd" + if ! eval "$cmd"; then + cleanup "$1" convert_result="FAILED" return 1 fi @@ -221,9 +219,8 @@ if ! [ -e "$dst"/l2chaindata/ancient ]; then ancient_dst=$(echo "$dst"/l2chaindata/ | tr -s /) echo "== Copying l2chaindata ancients" cmd="cp -r \"$ancient_src\" \"$ancient_dst\"" - echo $cmd - eval $cmd - if [ $? -ne 0 ]; then + echo "$cmd" + if ! eval "$cmd"; then l2chaindata_ancient_status="FAILED (failed to copy)" cleanup "l2chaindata" printStatus @@ -249,7 +246,7 @@ if [ $res -ne 0 ]; then exit 1 fi -if [ -e $src/wasm ]; then +if [ -e "$src"/wasm ]; then convert "wasm" res=$? wasm_status=$convert_result @@ -262,7 +259,7 @@ else wasm_status="not found in source directory" fi -if [ -e $src/classic-msg ]; then +if [ -e "$src"/classic-msg ]; then convert "classic-msg" res=$? classicmsg_status=$convert_result From 6efc51b4acc8f727e5a42d847ce6284a17df26f2 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 10 Sep 2024 14:49:41 -0300 Subject: [PATCH 143/156] Make scripts/build-brotli.sh shellcheck compliant --- scripts/build-brotli.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/build-brotli.sh b/scripts/build-brotli.sh index 7160936baa7..1a23a88ae09 100755 --- a/scripts/build-brotli.sh +++ b/scripts/build-brotli.sh @@ -2,7 +2,7 @@ set -e -mydir=`dirname $0` +mydir=$(dirname "$0") cd "$mydir" BUILD_WASM=false @@ -35,7 +35,7 @@ usage(){ echo "all relative paths are relative to script location" } -while getopts "s:t:c:D:wldhf" option; do +while getopts "n:s:t:c:D:wldhf" option; do case $option in h) usage @@ -62,6 +62,9 @@ while getopts "s:t:c:D:wldhf" option; do s) SOURCE_DIR="$OPTARG" ;; + *) + usage + ;; esac done @@ -74,7 +77,7 @@ if [ ! -d "$TARGET_DIR" ]; then mkdir -p "${TARGET_DIR}lib" ln -s "lib" "${TARGET_DIR}lib64" # Fedora build fi -TARGET_DIR_ABS=`cd -P "$TARGET_DIR"; pwd` +TARGET_DIR_ABS=$(cd -P "$TARGET_DIR"; pwd) if $USE_DOCKER; then @@ -94,9 +97,9 @@ cd "$SOURCE_DIR" if $BUILD_WASM; then mkdir -p buildfiles/build-wasm mkdir -p buildfiles/install-wasm - TEMP_INSTALL_DIR_ABS=`cd -P buildfiles/install-wasm; pwd` + TEMP_INSTALL_DIR_ABS=$(cd -P buildfiles/install-wasm; pwd) cd buildfiles/build-wasm - cmake ../../ -DCMAKE_C_COMPILER=emcc -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS=-fPIC -DCMAKE_INSTALL_PREFIX="$TEMP_INSTALL_DIR_ABS" -DCMAKE_AR=`which emar` -DCMAKE_RANLIB=`which touch` + cmake ../../ -DCMAKE_C_COMPILER=emcc -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS=-fPIC -DCMAKE_INSTALL_PREFIX="$TEMP_INSTALL_DIR_ABS" -DCMAKE_AR="$(which emar)" -DCMAKE_RANLIB="$(which touch)" make -j make install cp -rv "$TEMP_INSTALL_DIR_ABS/lib" "$TARGET_DIR_ABS/lib-wasm" From f7d434344b16c9f259aea77a4ebf5aea8bbca912 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Tue, 10 Sep 2024 15:31:15 -0300 Subject: [PATCH 144/156] ShellCheck CI --- .github/workflows/shellcheck-ci.yml | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/shellcheck-ci.yml diff --git a/.github/workflows/shellcheck-ci.yml b/.github/workflows/shellcheck-ci.yml new file mode 100644 index 00000000000..c86b4bde1c3 --- /dev/null +++ b/.github/workflows/shellcheck-ci.yml @@ -0,0 +1,30 @@ +name: ShellCheck CI +run-name: ShellCheck CI triggered from @${{ github.actor }} of ${{ github.head_ref }} + +on: + workflow_dispatch: + merge_group: + pull_request: + push: + branches: + - master + +jobs: + arbitrator: + name: Run ShellCheck + runs-on: ubuntu-8 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run ShellCheck + uses: ludeeus/action-shellcheck@master + with: + ignore_paths: >- + ./fastcache/** + ./contracts/** + ./safe-smart-account/** + ./go-ethereum/** + ./nitro-testnode/** + ./brotli/** + ./arbitrator/** From 82a2c255093c623800e12ac1f7753ce8a4264f76 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 11 Sep 2024 12:58:05 -0300 Subject: [PATCH 145/156] Rename GitHub action job to shellcheck --- .github/workflows/shellcheck-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/shellcheck-ci.yml b/.github/workflows/shellcheck-ci.yml index c86b4bde1c3..d1c7b58580d 100644 --- a/.github/workflows/shellcheck-ci.yml +++ b/.github/workflows/shellcheck-ci.yml @@ -10,7 +10,7 @@ on: - master jobs: - arbitrator: + shellcheck: name: Run ShellCheck runs-on: ubuntu-8 steps: From ad0cd310dfb866f90a80dd76b14e03dfd22f84a8 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 10:17:28 -0300 Subject: [PATCH 146/156] Adds chainSupportsBlobs to DeployOnL1 --- cmd/deploy/deploy.go | 1 + deploy/deploy.go | 21 ++++++++++++--------- system_tests/common_test.go | 1 + 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/cmd/deploy/deploy.go b/cmd/deploy/deploy.go index e0388e3d4b6..811e54dff54 100644 --- a/cmd/deploy/deploy.go +++ b/cmd/deploy/deploy.go @@ -189,6 +189,7 @@ func main() { arbnode.GenerateRollupConfig(*prod, moduleRoot, ownerAddress, &chainConfig, chainConfigJson, loserEscrowAddress), nativeToken, maxDataSize, + true, ) if err != nil { flag.Usage() diff --git a/deploy/deploy.go b/deploy/deploy.go index f2099f976a2..041e1049f5c 100644 --- a/deploy/deploy.go +++ b/deploy/deploy.go @@ -31,7 +31,7 @@ func andTxSucceeded(ctx context.Context, l1Reader *headerreader.HeaderReader, tx return nil } -func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int) (common.Address, error) { +func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, chainSupportsBlobs bool) (common.Address, error) { client := l1Reader.Client() /// deploy eth based templates @@ -41,10 +41,13 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade return common.Address{}, fmt.Errorf("bridge deploy error: %w", err) } - reader4844, tx, _, err := yulgen.DeployReader4844(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) - if err != nil { - return common.Address{}, fmt.Errorf("blob basefee reader deploy error: %w", err) + var reader4844 common.Address + if chainSupportsBlobs { + reader4844, tx, _, err = yulgen.DeployReader4844(auth, client) + err = andTxSucceeded(ctx, l1Reader, tx, err) + if err != nil { + return common.Address{}, fmt.Errorf("blob basefee reader deploy error: %w", err) + } } seqInboxTemplateEthBased, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, false) err = andTxSucceeded(ctx, l1Reader, tx, err) @@ -166,8 +169,8 @@ func deployChallengeFactory(ctx context.Context, l1Reader *headerreader.HeaderRe return ospEntryAddr, challengeManagerAddr, nil } -func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int) (*rollupgen.RollupCreator, common.Address, common.Address, common.Address, error) { - bridgeCreator, err := deployBridgeCreator(ctx, l1Reader, auth, maxDataSize) +func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, chainSupportsBlobs bool) (*rollupgen.RollupCreator, common.Address, common.Address, common.Address, error) { + bridgeCreator, err := deployBridgeCreator(ctx, l1Reader, auth, maxDataSize, chainSupportsBlobs) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("bridge creator deploy error: %w", err) } @@ -239,12 +242,12 @@ func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReade return rollupCreator, rollupCreatorAddress, validatorUtils, validatorWalletCreator, nil } -func DeployOnL1(ctx context.Context, parentChainReader *headerreader.HeaderReader, deployAuth *bind.TransactOpts, batchPosters []common.Address, batchPosterManager common.Address, authorizeValidators uint64, config rollupgen.Config, nativeToken common.Address, maxDataSize *big.Int) (*chaininfo.RollupAddresses, error) { +func DeployOnL1(ctx context.Context, parentChainReader *headerreader.HeaderReader, deployAuth *bind.TransactOpts, batchPosters []common.Address, batchPosterManager common.Address, authorizeValidators uint64, config rollupgen.Config, nativeToken common.Address, maxDataSize *big.Int, chainSupportsBlobs bool) (*chaininfo.RollupAddresses, error) { if config.WasmModuleRoot == (common.Hash{}) { return nil, errors.New("no machine specified") } - rollupCreator, _, validatorUtils, validatorWalletCreator, err := deployRollupCreator(ctx, parentChainReader, deployAuth, maxDataSize) + rollupCreator, _, validatorUtils, validatorWalletCreator, err := deployRollupCreator(ctx, parentChainReader, deployAuth, maxDataSize, chainSupportsBlobs) if err != nil { return nil, fmt.Errorf("error deploying rollup creator: %w", err) } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 2607ae5c8bd..0895610cfc5 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1040,6 +1040,7 @@ func DeployOnTestL1( arbnode.GenerateRollupConfig(prodConfirmPeriodBlocks, wasmModuleRoot, l1info.GetAddress("RollupOwner"), chainConfig, serializedChainConfig, common.Address{}), nativeToken, maxDataSize, + true, ) Require(t, err) l1info.SetContract("Bridge", addresses.Bridge) From 6a9deba4672e73ff92ca29b92d92e54046a93641 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 10:21:36 -0300 Subject: [PATCH 147/156] Renames DeployOnL1 to DeployOnParentChain --- cmd/deploy/deploy.go | 2 +- deploy/deploy.go | 2 +- system_tests/common_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/deploy/deploy.go b/cmd/deploy/deploy.go index 811e54dff54..c70ceb1d94f 100644 --- a/cmd/deploy/deploy.go +++ b/cmd/deploy/deploy.go @@ -179,7 +179,7 @@ func main() { defer l1Reader.StopAndWait() nativeToken := common.HexToAddress(*nativeTokenAddressString) - deployedAddresses, err := deploycode.DeployOnL1( + deployedAddresses, err := deploycode.DeployOnParentChain( ctx, l1Reader, l1TransactionOpts, diff --git a/deploy/deploy.go b/deploy/deploy.go index 041e1049f5c..c4f7fc891ef 100644 --- a/deploy/deploy.go +++ b/deploy/deploy.go @@ -242,7 +242,7 @@ func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReade return rollupCreator, rollupCreatorAddress, validatorUtils, validatorWalletCreator, nil } -func DeployOnL1(ctx context.Context, parentChainReader *headerreader.HeaderReader, deployAuth *bind.TransactOpts, batchPosters []common.Address, batchPosterManager common.Address, authorizeValidators uint64, config rollupgen.Config, nativeToken common.Address, maxDataSize *big.Int, chainSupportsBlobs bool) (*chaininfo.RollupAddresses, error) { +func DeployOnParentChain(ctx context.Context, parentChainReader *headerreader.HeaderReader, deployAuth *bind.TransactOpts, batchPosters []common.Address, batchPosterManager common.Address, authorizeValidators uint64, config rollupgen.Config, nativeToken common.Address, maxDataSize *big.Int, chainSupportsBlobs bool) (*chaininfo.RollupAddresses, error) { if config.WasmModuleRoot == (common.Hash{}) { return nil, errors.New("no machine specified") } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 0895610cfc5..ed984a07e06 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1030,7 +1030,7 @@ func DeployOnTestL1( nativeToken := common.Address{} maxDataSize := big.NewInt(117964) - addresses, err := deploy.DeployOnL1( + addresses, err := deploy.DeployOnParentChain( ctx, l1Reader, &l1TransactionOpts, From 9630bfbd12294ed3c52422c451ebec8765a18c95 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 10:22:26 -0300 Subject: [PATCH 148/156] Renames l1Reader to parentChainReader on deploy --- deploy/deploy.go | 84 ++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/deploy/deploy.go b/deploy/deploy.go index c4f7fc891ef..bb4b2e65948 100644 --- a/deploy/deploy.go +++ b/deploy/deploy.go @@ -20,23 +20,23 @@ import ( "github.com/offchainlabs/nitro/util/headerreader" ) -func andTxSucceeded(ctx context.Context, l1Reader *headerreader.HeaderReader, tx *types.Transaction, err error) error { +func andTxSucceeded(ctx context.Context, parentChainReader *headerreader.HeaderReader, tx *types.Transaction, err error) error { if err != nil { return fmt.Errorf("error submitting tx: %w", err) } - _, err = l1Reader.WaitForTxApproval(ctx, tx) + _, err = parentChainReader.WaitForTxApproval(ctx, tx) if err != nil { return fmt.Errorf("error executing tx: %w", err) } return nil } -func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, chainSupportsBlobs bool) (common.Address, error) { - client := l1Reader.Client() +func deployBridgeCreator(ctx context.Context, parentChainReader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, chainSupportsBlobs bool) (common.Address, error) { + client := parentChainReader.Client() /// deploy eth based templates bridgeTemplate, tx, _, err := bridgegen.DeployBridge(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("bridge deploy error: %w", err) } @@ -44,36 +44,36 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade var reader4844 common.Address if chainSupportsBlobs { reader4844, tx, _, err = yulgen.DeployReader4844(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("blob basefee reader deploy error: %w", err) } } seqInboxTemplateEthBased, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, false) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("sequencer inbox eth based deploy error: %w", err) } seqInboxTemplateERC20Based, tx, _, err := bridgegen.DeploySequencerInbox(auth, client, maxDataSize, reader4844, true) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("sequencer inbox erc20 based deploy error: %w", err) } inboxTemplate, tx, _, err := bridgegen.DeployInbox(auth, client, maxDataSize) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("inbox deploy error: %w", err) } rollupEventBridgeTemplate, tx, _, err := rollupgen.DeployRollupEventInbox(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("rollup event bridge deploy error: %w", err) } outboxTemplate, tx, _, err := bridgegen.DeployOutbox(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("outbox deploy error: %w", err) } @@ -88,25 +88,25 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade /// deploy ERC20 based templates erc20BridgeTemplate, tx, _, err := bridgegen.DeployERC20Bridge(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("bridge deploy error: %w", err) } erc20InboxTemplate, tx, _, err := bridgegen.DeployERC20Inbox(auth, client, maxDataSize) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("inbox deploy error: %w", err) } erc20RollupEventBridgeTemplate, tx, _, err := rollupgen.DeployERC20RollupEventInbox(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("rollup event bridge deploy error: %w", err) } erc20OutboxTemplate, tx, _, err := bridgegen.DeployERC20Outbox(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("outbox deploy error: %w", err) } @@ -120,7 +120,7 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade } bridgeCreatorAddr, tx, _, err := rollupgen.DeployBridgeCreator(auth, client, ethBasedTemplates, erc20BasedTemplates) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, fmt.Errorf("bridge creator deploy error: %w", err) } @@ -128,40 +128,40 @@ func deployBridgeCreator(ctx context.Context, l1Reader *headerreader.HeaderReade return bridgeCreatorAddr, nil } -func deployChallengeFactory(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts) (common.Address, common.Address, error) { - client := l1Reader.Client() +func deployChallengeFactory(ctx context.Context, parentChainReader *headerreader.HeaderReader, auth *bind.TransactOpts) (common.Address, common.Address, error) { + client := parentChainReader.Client() osp0, tx, _, err := ospgen.DeployOneStepProver0(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, common.Address{}, fmt.Errorf("osp0 deploy error: %w", err) } ospMem, tx, _, err := ospgen.DeployOneStepProverMemory(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, common.Address{}, fmt.Errorf("ospMemory deploy error: %w", err) } ospMath, tx, _, err := ospgen.DeployOneStepProverMath(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, common.Address{}, fmt.Errorf("ospMath deploy error: %w", err) } ospHostIo, tx, _, err := ospgen.DeployOneStepProverHostIo(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, common.Address{}, fmt.Errorf("ospHostIo deploy error: %w", err) } challengeManagerAddr, tx, _, err := challengegen.DeployChallengeManager(auth, client) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, common.Address{}, fmt.Errorf("challenge manager deploy error: %w", err) } ospEntryAddr, tx, _, err := ospgen.DeployOneStepProofEntry(auth, client, osp0, ospMem, ospMath, ospHostIo) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return common.Address{}, common.Address{}, fmt.Errorf("ospEntry deploy error: %w", err) } @@ -169,55 +169,55 @@ func deployChallengeFactory(ctx context.Context, l1Reader *headerreader.HeaderRe return ospEntryAddr, challengeManagerAddr, nil } -func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, chainSupportsBlobs bool) (*rollupgen.RollupCreator, common.Address, common.Address, common.Address, error) { - bridgeCreator, err := deployBridgeCreator(ctx, l1Reader, auth, maxDataSize, chainSupportsBlobs) +func deployRollupCreator(ctx context.Context, parentChainReader *headerreader.HeaderReader, auth *bind.TransactOpts, maxDataSize *big.Int, chainSupportsBlobs bool) (*rollupgen.RollupCreator, common.Address, common.Address, common.Address, error) { + bridgeCreator, err := deployBridgeCreator(ctx, parentChainReader, auth, maxDataSize, chainSupportsBlobs) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("bridge creator deploy error: %w", err) } - ospEntryAddr, challengeManagerAddr, err := deployChallengeFactory(ctx, l1Reader, auth) + ospEntryAddr, challengeManagerAddr, err := deployChallengeFactory(ctx, parentChainReader, auth) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, err } - rollupAdminLogic, tx, _, err := rollupgen.DeployRollupAdminLogic(auth, l1Reader.Client()) - err = andTxSucceeded(ctx, l1Reader, tx, err) + rollupAdminLogic, tx, _, err := rollupgen.DeployRollupAdminLogic(auth, parentChainReader.Client()) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("rollup admin logic deploy error: %w", err) } - rollupUserLogic, tx, _, err := rollupgen.DeployRollupUserLogic(auth, l1Reader.Client()) - err = andTxSucceeded(ctx, l1Reader, tx, err) + rollupUserLogic, tx, _, err := rollupgen.DeployRollupUserLogic(auth, parentChainReader.Client()) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("rollup user logic deploy error: %w", err) } - rollupCreatorAddress, tx, rollupCreator, err := rollupgen.DeployRollupCreator(auth, l1Reader.Client()) - err = andTxSucceeded(ctx, l1Reader, tx, err) + rollupCreatorAddress, tx, rollupCreator, err := rollupgen.DeployRollupCreator(auth, parentChainReader.Client()) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("rollup creator deploy error: %w", err) } - upgradeExecutor, tx, _, err := upgrade_executorgen.DeployUpgradeExecutor(auth, l1Reader.Client()) - err = andTxSucceeded(ctx, l1Reader, tx, err) + upgradeExecutor, tx, _, err := upgrade_executorgen.DeployUpgradeExecutor(auth, parentChainReader.Client()) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("upgrade executor deploy error: %w", err) } - validatorUtils, tx, _, err := rollupgen.DeployValidatorUtils(auth, l1Reader.Client()) - err = andTxSucceeded(ctx, l1Reader, tx, err) + validatorUtils, tx, _, err := rollupgen.DeployValidatorUtils(auth, parentChainReader.Client()) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("validator utils deploy error: %w", err) } - validatorWalletCreator, tx, _, err := rollupgen.DeployValidatorWalletCreator(auth, l1Reader.Client()) - err = andTxSucceeded(ctx, l1Reader, tx, err) + validatorWalletCreator, tx, _, err := rollupgen.DeployValidatorWalletCreator(auth, parentChainReader.Client()) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("validator wallet creator deploy error: %w", err) } - l2FactoriesDeployHelper, tx, _, err := rollupgen.DeployDeployHelper(auth, l1Reader.Client()) - err = andTxSucceeded(ctx, l1Reader, tx, err) + l2FactoriesDeployHelper, tx, _, err := rollupgen.DeployDeployHelper(auth, parentChainReader.Client()) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("deploy helper creator deploy error: %w", err) } @@ -234,7 +234,7 @@ func deployRollupCreator(ctx context.Context, l1Reader *headerreader.HeaderReade validatorWalletCreator, l2FactoriesDeployHelper, ) - err = andTxSucceeded(ctx, l1Reader, tx, err) + err = andTxSucceeded(ctx, parentChainReader, tx, err) if err != nil { return nil, common.Address{}, common.Address{}, common.Address{}, fmt.Errorf("rollup set template error: %w", err) } From e1edcb3845ef797640558e3148f6c9c4876f0169 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 11:35:05 -0300 Subject: [PATCH 149/156] Basic L3 tests --- system_tests/common_test.go | 261 +++++++++++++++++++++++++++++++----- system_tests/l3_test.go | 37 +++++ 2 files changed, 264 insertions(+), 34 deletions(-) create mode 100644 system_tests/l3_test.go diff --git a/system_tests/common_test.go b/system_tests/common_test.go index ed984a07e06..91b421a7c45 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -235,6 +235,7 @@ type NodeBuilder struct { valnodeConfig *valnode.Config L1Info info L2Info info + L3Info info // L1, L2 Node parameters dataDir string @@ -248,6 +249,54 @@ type NodeBuilder struct { // Created nodes L1 *TestClient L2 *TestClient + L3 *TestClient +} + +type NitroConfig struct { + chainConfig *params.ChainConfig + nodeConfig *arbnode.Config + execConfig *gethexec.Config + stackConfig *node.Config + valnodeConfig *valnode.Config + + withProdConfirmPeriodBlocks bool + isSequencer bool +} + +func L3NitroConfigDefaultTest(t *testing.T) *NitroConfig { + chainConfig := ¶ms.ChainConfig{ + ChainID: big.NewInt(333333), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + ArbitrumChainParams: params.ArbitrumDevTestParams(), + Clique: ¶ms.CliqueConfig{ + Period: 0, + Epoch: 0, + }, + } + + valnodeConfig := valnode.TestValidationConfig + return &NitroConfig{ + chainConfig: chainConfig, + nodeConfig: arbnode.ConfigDefaultL2Test(), + execConfig: ExecConfigDefaultTest(t), + stackConfig: testhelpers.CreateStackConfigForTest(t.TempDir()), + valnodeConfig: &valnodeConfig, + + withProdConfirmPeriodBlocks: false, + isSequencer: true, + } } func NewNodeBuilder(ctx context.Context) *NodeBuilder { @@ -337,10 +386,146 @@ func (b *NodeBuilder) BuildL1(t *testing.T) { b.L1Info, b.L1.Client, b.L1.L1Backend, b.L1.Stack = createTestL1BlockChain(t, b.L1Info) locator, err := server_common.NewMachineLocator(b.valnodeConfig.Wasm.RootPath) Require(t, err) - b.addresses, b.initMessage = DeployOnTestL1(t, b.ctx, b.L1Info, b.L1.Client, b.chainConfig, locator.LatestWasmModuleRoot(), b.withProdConfirmPeriodBlocks) + b.addresses, b.initMessage = deployOnParentChain( + t, + b.ctx, + b.L1Info, + b.L1.Client, + &headerreader.TestConfig, + b.chainConfig, + locator.LatestWasmModuleRoot(), + b.withProdConfirmPeriodBlocks, + true, + ) b.L1.cleanup = func() { requireClose(t, b.L1.Stack) } } +func buildOnParentChain( + t *testing.T, + ctx context.Context, + + dataDir string, + + parentChainInfo info, + parentChainTestClient *TestClient, + parentChainId *big.Int, + + chainConfig *params.ChainConfig, + stackConfig *node.Config, + execConfig *gethexec.Config, + nodeConfig *arbnode.Config, + valnodeConfig *valnode.Config, + isSequencer bool, + chainInfo info, + + initMessage *arbostypes.ParsedInitMessage, + addresses *chaininfo.RollupAddresses, +) *TestClient { + if parentChainTestClient == nil { + t.Fatal("must build parent chain before building chain") + } + + chainTestClient := NewTestClient(ctx) + + var chainDb ethdb.Database + var arbDb ethdb.Database + var blockchain *core.BlockChain + _, chainTestClient.Stack, chainDb, arbDb, blockchain = createNonL1BlockChainWithStackConfig( + t, chainInfo, dataDir, chainConfig, initMessage, stackConfig, execConfig) + + var sequencerTxOptsPtr *bind.TransactOpts + var dataSigner signature.DataSignerFunc + if isSequencer { + sequencerTxOpts := parentChainInfo.GetDefaultTransactOpts("Sequencer", ctx) + sequencerTxOptsPtr = &sequencerTxOpts + dataSigner = signature.DataSignerFromPrivateKey(parentChainInfo.GetInfoWithPrivKey("Sequencer").PrivateKey) + } else { + nodeConfig.BatchPoster.Enable = false + nodeConfig.Sequencer = false + nodeConfig.DelayedSequencer.Enable = false + execConfig.Sequencer.Enable = false + } + + var validatorTxOptsPtr *bind.TransactOpts + if nodeConfig.Staker.Enable { + validatorTxOpts := parentChainInfo.GetDefaultTransactOpts("Validator", ctx) + validatorTxOptsPtr = &validatorTxOpts + } + + AddValNodeIfNeeded(t, ctx, nodeConfig, true, "", valnodeConfig.Wasm.RootPath) + + Require(t, execConfig.Validate()) + execConfigToBeUsedInConfigFetcher := execConfig + execConfigFetcher := func() *gethexec.Config { return execConfigToBeUsedInConfigFetcher } + execNode, err := gethexec.CreateExecutionNode(ctx, chainTestClient.Stack, chainDb, blockchain, parentChainTestClient.Client, execConfigFetcher) + Require(t, err) + + fatalErrChan := make(chan error, 10) + chainTestClient.ConsensusNode, err = arbnode.CreateNode( + ctx, chainTestClient.Stack, execNode, arbDb, NewFetcherFromConfig(nodeConfig), blockchain.Config(), parentChainTestClient.Client, + addresses, validatorTxOptsPtr, sequencerTxOptsPtr, dataSigner, fatalErrChan, parentChainId, nil) + Require(t, err) + + err = chainTestClient.ConsensusNode.Start(ctx) + Require(t, err) + + chainTestClient.Client = ClientForStack(t, chainTestClient.Stack) + + StartWatchChanErr(t, ctx, fatalErrChan, chainTestClient.ConsensusNode) + + chainTestClient.ExecNode = getExecNode(t, chainTestClient.ConsensusNode) + chainTestClient.cleanup = func() { chainTestClient.ConsensusNode.StopAndWait() } + + return chainTestClient +} + +func (b *NodeBuilder) BuildL3OnL2(t *testing.T, nitroConfig *NitroConfig) func() { + b.L3Info = NewArbTestInfo(t, nitroConfig.chainConfig.ChainID) + + locator, err := server_common.NewMachineLocator(nitroConfig.valnodeConfig.Wasm.RootPath) + Require(t, err) + + parentChainReaderConfig := headerreader.TestConfig + parentChainReaderConfig.Dangerous.WaitForTxApprovalSafePoll = 0 + addresses, initMessage := deployOnParentChain( + t, + b.ctx, + b.L2Info, + b.L2.Client, + &parentChainReaderConfig, + nitroConfig.chainConfig, + locator.LatestWasmModuleRoot(), + nitroConfig.withProdConfirmPeriodBlocks, + false, + ) + + b.L3 = buildOnParentChain( + t, + b.ctx, + + b.dataDir, + + b.L2Info, + b.L2, + b.chainConfig.ChainID, + + nitroConfig.chainConfig, + nitroConfig.stackConfig, + nitroConfig.execConfig, + nitroConfig.nodeConfig, + nitroConfig.valnodeConfig, + nitroConfig.isSequencer, + b.L3Info, + + initMessage, + addresses, + ) + + return func() { + b.L3.cleanup() + } +} + func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() { if b.L1 == nil { t.Fatal("must build L1 before building L2") @@ -350,7 +535,7 @@ func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() { var l2chainDb ethdb.Database var l2arbDb ethdb.Database var l2blockchain *core.BlockChain - _, b.L2.Stack, l2chainDb, l2arbDb, l2blockchain = createL2BlockChainWithStackConfig( + _, b.L2.Stack, l2chainDb, l2arbDb, l2blockchain = createNonL1BlockChainWithStackConfig( t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) var sequencerTxOptsPtr *bind.TransactOpts @@ -465,7 +650,7 @@ func (b *NodeBuilder) RestartL2Node(t *testing.T) { } b.L2.cleanup() - l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) + l2info, stack, chainDb, arbDb, blockchain := createNonL1BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) execConfigFetcher := func() *gethexec.Config { return b.execConfig } execNode, err := gethexec.CreateExecutionNode(b.ctx, stack, chainDb, blockchain, nil, execConfigFetcher) @@ -1004,60 +1189,68 @@ func getInitMessage(ctx context.Context, t *testing.T, l1client client, addresse return initMessage } -func DeployOnTestL1( - t *testing.T, ctx context.Context, l1info info, l1client client, chainConfig *params.ChainConfig, wasmModuleRoot common.Hash, prodConfirmPeriodBlocks bool, +func deployOnParentChain( + t *testing.T, + ctx context.Context, + parentChainInfo info, + parentChainClient client, + parentChainReaderConfig *headerreader.Config, + chainConfig *params.ChainConfig, + wasmModuleRoot common.Hash, + prodConfirmPeriodBlocks bool, + chainSupportsBlobs bool, ) (*chaininfo.RollupAddresses, *arbostypes.ParsedInitMessage) { - l1info.GenerateAccount("RollupOwner") - l1info.GenerateAccount("Sequencer") - l1info.GenerateAccount("Validator") - l1info.GenerateAccount("User") - - SendWaitTestTransactions(t, ctx, l1client, []*types.Transaction{ - l1info.PrepareTx("Faucet", "RollupOwner", 30000, big.NewInt(9223372036854775807), nil), - l1info.PrepareTx("Faucet", "Sequencer", 30000, big.NewInt(9223372036854775807), nil), - l1info.PrepareTx("Faucet", "Validator", 30000, big.NewInt(9223372036854775807), nil), - l1info.PrepareTx("Faucet", "User", 30000, big.NewInt(9223372036854775807), nil)}) - - l1TransactionOpts := l1info.GetDefaultTransactOpts("RollupOwner", ctx) + parentChainInfo.GenerateAccount("RollupOwner") + parentChainInfo.GenerateAccount("Sequencer") + parentChainInfo.GenerateAccount("Validator") + parentChainInfo.GenerateAccount("User") + + SendWaitTestTransactions(t, ctx, parentChainClient, []*types.Transaction{ + parentChainInfo.PrepareTx("Faucet", "RollupOwner", parentChainInfo.TransferGas, big.NewInt(9223372036854775807), nil), + parentChainInfo.PrepareTx("Faucet", "Sequencer", parentChainInfo.TransferGas, big.NewInt(9223372036854775807), nil), + parentChainInfo.PrepareTx("Faucet", "Validator", parentChainInfo.TransferGas, big.NewInt(9223372036854775807), nil), + parentChainInfo.PrepareTx("Faucet", "User", parentChainInfo.TransferGas, big.NewInt(9223372036854775807), nil)}) + + parentChainTransactionOpts := parentChainInfo.GetDefaultTransactOpts("RollupOwner", ctx) serializedChainConfig, err := json.Marshal(chainConfig) Require(t, err) - arbSys, _ := precompilesgen.NewArbSys(types.ArbSysAddress, l1client) - l1Reader, err := headerreader.New(ctx, l1client, func() *headerreader.Config { return &headerreader.TestConfig }, arbSys) + arbSys, _ := precompilesgen.NewArbSys(types.ArbSysAddress, parentChainClient) + parentChainReader, err := headerreader.New(ctx, parentChainClient, func() *headerreader.Config { return parentChainReaderConfig }, arbSys) Require(t, err) - l1Reader.Start(ctx) - defer l1Reader.StopAndWait() + parentChainReader.Start(ctx) + defer parentChainReader.StopAndWait() nativeToken := common.Address{} maxDataSize := big.NewInt(117964) addresses, err := deploy.DeployOnParentChain( ctx, - l1Reader, - &l1TransactionOpts, - []common.Address{l1info.GetAddress("Sequencer")}, - l1info.GetAddress("RollupOwner"), + parentChainReader, + &parentChainTransactionOpts, + []common.Address{parentChainInfo.GetAddress("Sequencer")}, + parentChainInfo.GetAddress("RollupOwner"), 0, - arbnode.GenerateRollupConfig(prodConfirmPeriodBlocks, wasmModuleRoot, l1info.GetAddress("RollupOwner"), chainConfig, serializedChainConfig, common.Address{}), + arbnode.GenerateRollupConfig(prodConfirmPeriodBlocks, wasmModuleRoot, parentChainInfo.GetAddress("RollupOwner"), chainConfig, serializedChainConfig, common.Address{}), nativeToken, maxDataSize, - true, + chainSupportsBlobs, ) Require(t, err) - l1info.SetContract("Bridge", addresses.Bridge) - l1info.SetContract("SequencerInbox", addresses.SequencerInbox) - l1info.SetContract("Inbox", addresses.Inbox) - l1info.SetContract("UpgradeExecutor", addresses.UpgradeExecutor) - initMessage := getInitMessage(ctx, t, l1client, addresses) + parentChainInfo.SetContract("Bridge", addresses.Bridge) + parentChainInfo.SetContract("SequencerInbox", addresses.SequencerInbox) + parentChainInfo.SetContract("Inbox", addresses.Inbox) + parentChainInfo.SetContract("UpgradeExecutor", addresses.UpgradeExecutor) + initMessage := getInitMessage(ctx, t, parentChainClient, addresses) return addresses, initMessage } func createL2BlockChain( t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { - return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, execConfig) + return createNonL1BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, execConfig) } -func createL2BlockChainWithStackConfig( +func createNonL1BlockChainWithStackConfig( t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { if l2info == nil { diff --git a/system_tests/l3_test.go b/system_tests/l3_test.go new file mode 100644 index 00000000000..5737ed73c49 --- /dev/null +++ b/system_tests/l3_test.go @@ -0,0 +1,37 @@ +package arbtest + +import ( + "context" + "math/big" + "testing" +) + +func TestBasicL3(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + cleanupL1AndL2 := builder.Build(t) + defer cleanupL1AndL2() + + l3NodeNitroConfig := L3NitroConfigDefaultTest(t) + l3NodeNitroConfig.nodeConfig.ParentChainReader.Enable = true + cleanupL3 := builder.BuildL3OnL2(t, l3NodeNitroConfig) + defer cleanupL3() + + builder.L3Info.GenerateAccount("User2") + tx := builder.L3Info.PrepareTx("Owner", "User2", builder.L3Info.TransferGas, big.NewInt(1e12), nil) + + err := builder.L3.Client.SendTransaction(ctx, tx) + Require(t, err) + + _, err = builder.L3.EnsureTxSucceeded(tx) + Require(t, err) + + l2balance, err := builder.L3.Client.BalanceAt(ctx, builder.L3Info.GetAddress("User2"), nil) + Require(t, err) + if l2balance.Cmp(big.NewInt(1e12)) != 0 { + t.Fatal("Unexpected balance:", l2balance) + } +} From 88ad39c40c1a2d333316e8c6af4ecc76e9e2fdb9 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 13:10:36 -0300 Subject: [PATCH 150/156] Uses buildOnParentChain on BuildL2OnL1 --- system_tests/common_test.go | 65 ++++++++++--------------------------- 1 file changed, 17 insertions(+), 48 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 91b421a7c45..f89fea27335 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -527,59 +527,28 @@ func (b *NodeBuilder) BuildL3OnL2(t *testing.T, nitroConfig *NitroConfig) func() } func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() { - if b.L1 == nil { - t.Fatal("must build L1 before building L2") - } - b.L2 = NewTestClient(b.ctx) - - var l2chainDb ethdb.Database - var l2arbDb ethdb.Database - var l2blockchain *core.BlockChain - _, b.L2.Stack, l2chainDb, l2arbDb, l2blockchain = createNonL1BlockChainWithStackConfig( - t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) - - var sequencerTxOptsPtr *bind.TransactOpts - var dataSigner signature.DataSignerFunc - if b.isSequencer { - sequencerTxOpts := b.L1Info.GetDefaultTransactOpts("Sequencer", b.ctx) - sequencerTxOptsPtr = &sequencerTxOpts - dataSigner = signature.DataSignerFromPrivateKey(b.L1Info.GetInfoWithPrivKey("Sequencer").PrivateKey) - } else { - b.nodeConfig.BatchPoster.Enable = false - b.nodeConfig.Sequencer = false - b.nodeConfig.DelayedSequencer.Enable = false - b.execConfig.Sequencer.Enable = false - } - - var validatorTxOptsPtr *bind.TransactOpts - if b.nodeConfig.Staker.Enable { - validatorTxOpts := b.L1Info.GetDefaultTransactOpts("Validator", b.ctx) - validatorTxOptsPtr = &validatorTxOpts - } - - AddValNodeIfNeeded(t, b.ctx, b.nodeConfig, true, "", b.valnodeConfig.Wasm.RootPath) - - Require(t, b.execConfig.Validate()) - execConfig := b.execConfig - execConfigFetcher := func() *gethexec.Config { return execConfig } - execNode, err := gethexec.CreateExecutionNode(b.ctx, b.L2.Stack, l2chainDb, l2blockchain, b.L1.Client, execConfigFetcher) - Require(t, err) + b.L2 = buildOnParentChain( + t, + b.ctx, - fatalErrChan := make(chan error, 10) - b.L2.ConsensusNode, err = arbnode.CreateNode( - b.ctx, b.L2.Stack, execNode, l2arbDb, NewFetcherFromConfig(b.nodeConfig), l2blockchain.Config(), b.L1.Client, - b.addresses, validatorTxOptsPtr, sequencerTxOptsPtr, dataSigner, fatalErrChan, big.NewInt(1337), nil) - Require(t, err) + b.dataDir, - err = b.L2.ConsensusNode.Start(b.ctx) - Require(t, err) + b.L1Info, + b.L1, + big.NewInt(1337), - b.L2.Client = ClientForStack(t, b.L2.Stack) + b.chainConfig, + b.l2StackConfig, + b.execConfig, + b.nodeConfig, + b.valnodeConfig, + b.isSequencer, + b.L2Info, - StartWatchChanErr(t, b.ctx, fatalErrChan, b.L2.ConsensusNode) + b.initMessage, + b.addresses, + ) - b.L2.ExecNode = getExecNode(t, b.L2.ConsensusNode) - b.L2.cleanup = func() { b.L2.ConsensusNode.StopAndWait() } return func() { b.L2.cleanup() if b.L1 != nil && b.L1.cleanup != nil { From 479f85a1636ae746bb6e66c678455b0b4c08635e Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 13:13:22 -0300 Subject: [PATCH 151/156] Rename l1Client to parentChainClient in getInitMessage --- system_tests/common_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index f89fea27335..9f2a2bb2be4 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1143,8 +1143,8 @@ func createTestL1BlockChain(t *testing.T, l1info info) (info, *ethclient.Client, return l1info, l1Client, l1backend, stack } -func getInitMessage(ctx context.Context, t *testing.T, l1client client, addresses *chaininfo.RollupAddresses) *arbostypes.ParsedInitMessage { - bridge, err := arbnode.NewDelayedBridge(l1client, addresses.Bridge, addresses.DeployedAt) +func getInitMessage(ctx context.Context, t *testing.T, parentChainClient client, addresses *chaininfo.RollupAddresses) *arbostypes.ParsedInitMessage { + bridge, err := arbnode.NewDelayedBridge(parentChainClient, addresses.Bridge, addresses.DeployedAt) Require(t, err) deployedAtBig := arbmath.UintToBig(addresses.DeployedAt) messages, err := bridge.LookupMessagesInRange(ctx, deployedAtBig, deployedAtBig, nil) From c1bd83e437f9479bbcf93a60bfd72f13dde33703 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 13:14:41 -0300 Subject: [PATCH 152/156] Rename l2info to info in createNonL1BlockChainWithStackConfig --- system_tests/common_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 9f2a2bb2be4..160815f4bb9 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1220,10 +1220,10 @@ func createL2BlockChain( } func createNonL1BlockChainWithStackConfig( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, execConfig *gethexec.Config, + t *testing.T, info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { - if l2info == nil { - l2info = NewArbTestInfo(t, chainConfig.ChainID) + if info == nil { + info = NewArbTestInfo(t, chainConfig.ChainID) } if stackConfig == nil { stackConfig = testhelpers.CreateStackConfigForTest(dataDir) @@ -1243,7 +1243,7 @@ func createNonL1BlockChainWithStackConfig( arbDb, err := stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) - initReader := statetransfer.NewMemoryInitDataReader(&l2info.ArbInitData) + initReader := statetransfer.NewMemoryInitDataReader(&info.ArbInitData) if initMessage == nil { serializedChainConfig, err := json.Marshal(chainConfig) Require(t, err) @@ -1258,7 +1258,7 @@ func createNonL1BlockChainWithStackConfig( blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) - return l2info, stack, chainDb, arbDb, blockchain + return info, stack, chainDb, arbDb, blockchain } func ClientForStack(t *testing.T, backend *node.Node) *ethclient.Client { From 5852df193419b66e1641821afbe2d466b4ee5441 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 15:13:54 -0300 Subject: [PATCH 153/156] Adds l3Config to NodeBuilder --- system_tests/common_test.go | 26 ++++++++++++++------------ system_tests/l3_test.go | 4 +--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 160815f4bb9..2eba898e95b 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -233,6 +233,7 @@ type NodeBuilder struct { l1StackConfig *node.Config l2StackConfig *node.Config valnodeConfig *valnode.Config + l3Config *NitroConfig L1Info info L2Info info L3Info info @@ -289,7 +290,7 @@ func L3NitroConfigDefaultTest(t *testing.T) *NitroConfig { valnodeConfig := valnode.TestValidationConfig return &NitroConfig{ chainConfig: chainConfig, - nodeConfig: arbnode.ConfigDefaultL2Test(), + nodeConfig: arbnode.ConfigDefaultL1Test(), execConfig: ExecConfigDefaultTest(t), stackConfig: testhelpers.CreateStackConfigForTest(t.TempDir()), valnodeConfig: &valnodeConfig, @@ -322,6 +323,7 @@ func (b *NodeBuilder) DefaultConfig(t *testing.T, withL1 bool) *NodeBuilder { cp := valnode.TestValidationConfig b.valnodeConfig = &cp b.execConfig = ExecConfigDefaultTest(t) + b.l3Config = L3NitroConfigDefaultTest(t) return b } @@ -479,10 +481,10 @@ func buildOnParentChain( return chainTestClient } -func (b *NodeBuilder) BuildL3OnL2(t *testing.T, nitroConfig *NitroConfig) func() { - b.L3Info = NewArbTestInfo(t, nitroConfig.chainConfig.ChainID) +func (b *NodeBuilder) BuildL3OnL2(t *testing.T) func() { + b.L3Info = NewArbTestInfo(t, b.l3Config.chainConfig.ChainID) - locator, err := server_common.NewMachineLocator(nitroConfig.valnodeConfig.Wasm.RootPath) + locator, err := server_common.NewMachineLocator(b.l3Config.valnodeConfig.Wasm.RootPath) Require(t, err) parentChainReaderConfig := headerreader.TestConfig @@ -493,9 +495,9 @@ func (b *NodeBuilder) BuildL3OnL2(t *testing.T, nitroConfig *NitroConfig) func() b.L2Info, b.L2.Client, &parentChainReaderConfig, - nitroConfig.chainConfig, + b.l3Config.chainConfig, locator.LatestWasmModuleRoot(), - nitroConfig.withProdConfirmPeriodBlocks, + b.l3Config.withProdConfirmPeriodBlocks, false, ) @@ -509,12 +511,12 @@ func (b *NodeBuilder) BuildL3OnL2(t *testing.T, nitroConfig *NitroConfig) func() b.L2, b.chainConfig.ChainID, - nitroConfig.chainConfig, - nitroConfig.stackConfig, - nitroConfig.execConfig, - nitroConfig.nodeConfig, - nitroConfig.valnodeConfig, - nitroConfig.isSequencer, + b.l3Config.chainConfig, + b.l3Config.stackConfig, + b.l3Config.execConfig, + b.l3Config.nodeConfig, + b.l3Config.valnodeConfig, + b.l3Config.isSequencer, b.L3Info, initMessage, diff --git a/system_tests/l3_test.go b/system_tests/l3_test.go index 5737ed73c49..6ed10e31933 100644 --- a/system_tests/l3_test.go +++ b/system_tests/l3_test.go @@ -15,9 +15,7 @@ func TestBasicL3(t *testing.T) { cleanupL1AndL2 := builder.Build(t) defer cleanupL1AndL2() - l3NodeNitroConfig := L3NitroConfigDefaultTest(t) - l3NodeNitroConfig.nodeConfig.ParentChainReader.Enable = true - cleanupL3 := builder.BuildL3OnL2(t, l3NodeNitroConfig) + cleanupL3 := builder.BuildL3OnL2(t) defer cleanupL3() builder.L3Info.GenerateAccount("User2") From 467ec77ab125b7cc40808888a14f2c5ecf93d871 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Thu, 5 Sep 2024 16:35:54 -0300 Subject: [PATCH 154/156] Build2ndNodeOnL3 --- system_tests/common_test.go | 151 ++++++++++++++++++++++++++---------- system_tests/l3_test.go | 34 ++++++-- 2 files changed, 135 insertions(+), 50 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 2eba898e95b..f894eb065fb 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -238,13 +238,15 @@ type NodeBuilder struct { L2Info info L3Info info - // L1, L2 Node parameters + // L1, L2, L3 Node parameters dataDir string isSequencer bool takeOwnership bool withL1 bool addresses *chaininfo.RollupAddresses + l3Addresses *chaininfo.RollupAddresses initMessage *arbostypes.ParsedInitMessage + l3InitMessage *arbostypes.ParsedInitMessage withProdConfirmPeriodBlocks bool // Created nodes @@ -489,7 +491,7 @@ func (b *NodeBuilder) BuildL3OnL2(t *testing.T) func() { parentChainReaderConfig := headerreader.TestConfig parentChainReaderConfig.Dangerous.WaitForTxApprovalSafePoll = 0 - addresses, initMessage := deployOnParentChain( + b.l3Addresses, b.l3InitMessage = deployOnParentChain( t, b.ctx, b.L2Info, @@ -519,8 +521,8 @@ func (b *NodeBuilder) BuildL3OnL2(t *testing.T) func() { b.l3Config.isSequencer, b.L3Info, - initMessage, - addresses, + b.l3InitMessage, + b.l3Addresses, ) return func() { @@ -646,13 +648,25 @@ func (b *NodeBuilder) RestartL2Node(t *testing.T) { b.L2Info = l2info } -func (b *NodeBuilder) Build2ndNode(t *testing.T, params *SecondNodeParams) (*TestClient, func()) { - if b.L2 == nil { - t.Fatal("builder did not previously build a L2 Node") - } - if b.withL1 && b.L1 == nil { - t.Fatal("builder did not previously build a L1 Node") - } +func build2ndNode( + t *testing.T, + ctx context.Context, + + firstNodeStackConfig *node.Config, + firsNodeExecConfig *gethexec.Config, + firstNodeNodeConfig *arbnode.Config, + firstNodeInfo info, + firstNodeTestClient *TestClient, + valnodeConfig *valnode.Config, + + parentChainTestClient *TestClient, + parentChainInfo info, + + params *SecondNodeParams, + + addresses *chaininfo.RollupAddresses, + initMessage *arbostypes.ParsedInitMessage, +) (*TestClient, func()) { if params.nodeConfig == nil { params.nodeConfig = arbnode.ConfigDefaultL1NonSequencerTest() } @@ -660,18 +674,18 @@ func (b *NodeBuilder) Build2ndNode(t *testing.T, params *SecondNodeParams) (*Tes params.nodeConfig.DataAvailability = *params.dasConfig } if params.stackConfig == nil { - params.stackConfig = b.l2StackConfig + params.stackConfig = firstNodeStackConfig // should use different dataDir from the previously used ones params.stackConfig.DataDir = t.TempDir() } if params.initData == nil { - params.initData = &b.L2Info.ArbInitData + params.initData = &firstNodeInfo.ArbInitData } if params.execConfig == nil { - params.execConfig = b.execConfig + params.execConfig = firsNodeExecConfig } if params.addresses == nil { - params.addresses = b.addresses + params.addresses = addresses } if params.execConfig.RPC.MaxRecreateStateDepth == arbitrum.UninitializedMaxRecreateStateDepth { if params.execConfig.Caching.Archive { @@ -680,16 +694,69 @@ func (b *NodeBuilder) Build2ndNode(t *testing.T, params *SecondNodeParams) (*Tes params.execConfig.RPC.MaxRecreateStateDepth = arbitrum.DefaultNonArchiveNodeMaxRecreateStateDepth } } - if b.nodeConfig.BatchPoster.Enable && params.nodeConfig.BatchPoster.Enable && params.nodeConfig.BatchPoster.RedisUrl == "" { + if firstNodeNodeConfig.BatchPoster.Enable && params.nodeConfig.BatchPoster.Enable && params.nodeConfig.BatchPoster.RedisUrl == "" { t.Fatal("The batch poster must use Redis when enabled for multiple nodes") } - l2 := NewTestClient(b.ctx) - l2.Client, l2.ConsensusNode = - Create2ndNodeWithConfig(t, b.ctx, b.L2.ConsensusNode, b.L1.Stack, b.L1Info, params.initData, params.nodeConfig, params.execConfig, params.stackConfig, b.valnodeConfig, params.addresses, b.initMessage) - l2.ExecNode = getExecNode(t, l2.ConsensusNode) - l2.cleanup = func() { l2.ConsensusNode.StopAndWait() } - return l2, func() { l2.cleanup() } + testClient := NewTestClient(ctx) + testClient.Client, testClient.ConsensusNode = + Create2ndNodeWithConfig(t, ctx, firstNodeTestClient.ConsensusNode, parentChainTestClient.Stack, parentChainInfo, params.initData, params.nodeConfig, params.execConfig, params.stackConfig, valnodeConfig, params.addresses, initMessage) + testClient.ExecNode = getExecNode(t, testClient.ConsensusNode) + testClient.cleanup = func() { testClient.ConsensusNode.StopAndWait() } + return testClient, func() { testClient.cleanup() } +} + +func (b *NodeBuilder) Build2ndNode(t *testing.T, params *SecondNodeParams) (*TestClient, func()) { + if b.L2 == nil { + t.Fatal("builder did not previously built an L2 Node") + } + if b.withL1 && b.L1 == nil { + t.Fatal("builder did not previously built an L1 Node") + } + return build2ndNode( + t, + b.ctx, + + b.l2StackConfig, + b.execConfig, + b.nodeConfig, + b.L2Info, + b.L2, + b.valnodeConfig, + + b.L1, + b.L1Info, + + params, + + b.addresses, + b.initMessage, + ) +} + +func (b *NodeBuilder) Build2ndNodeOnL3(t *testing.T, params *SecondNodeParams) (*TestClient, func()) { + if b.L3 == nil { + t.Fatal("builder did not previously built an L3 Node") + } + return build2ndNode( + t, + b.ctx, + + b.l3Config.stackConfig, + b.l3Config.execConfig, + b.l3Config.nodeConfig, + b.L3Info, + b.L3, + b.l3Config.valnodeConfig, + + b.L2, + b.L2Info, + + params, + + b.l3Addresses, + b.l3InitMessage, + ) } func (b *NodeBuilder) BridgeBalance(t *testing.T, account string, amount *big.Int) (*types.Transaction, *types.Receipt) { @@ -1303,9 +1370,9 @@ func Create2ndNodeWithConfig( t *testing.T, ctx context.Context, first *arbnode.Node, - l1stack *node.Node, - l1info *BlockchainTestInfo, - l2InitData *statetransfer.ArbosInitializationInfo, + parentChainStack *node.Node, + parentChainInfo *BlockchainTestInfo, + chainInitData *statetransfer.ArbosInitializationInfo, nodeConfig *arbnode.Config, execConfig *gethexec.Config, stackConfig *node.Config, @@ -1320,34 +1387,34 @@ func Create2ndNodeWithConfig( execConfig = ExecConfigDefaultNonSequencerTest(t) } feedErrChan := make(chan error, 10) - l1rpcClient := l1stack.Attach() - l1client := ethclient.NewClient(l1rpcClient) + parentChainRpcClient := parentChainStack.Attach() + parentChainClient := ethclient.NewClient(parentChainRpcClient) if stackConfig == nil { stackConfig = testhelpers.CreateStackConfigForTest(t.TempDir()) } - l2stack, err := node.New(stackConfig) + chainStack, err := node.New(stackConfig) Require(t, err) - l2chainData, err := l2stack.OpenDatabaseWithExtraOptions("l2chaindata", 0, 0, "l2chaindata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("l2chaindata")) + chainData, err := chainStack.OpenDatabaseWithExtraOptions("l2chaindata", 0, 0, "l2chaindata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("l2chaindata")) Require(t, err) - wasmData, err := l2stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) + wasmData, err := chainStack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) - l2arbDb, err := l2stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) + arbDb, err := chainStack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) - initReader := statetransfer.NewMemoryInitDataReader(l2InitData) + initReader := statetransfer.NewMemoryInitDataReader(chainInitData) - dataSigner := signature.DataSignerFromPrivateKey(l1info.GetInfoWithPrivKey("Sequencer").PrivateKey) - sequencerTxOpts := l1info.GetDefaultTransactOpts("Sequencer", ctx) - validatorTxOpts := l1info.GetDefaultTransactOpts("Validator", ctx) + dataSigner := signature.DataSignerFromPrivateKey(parentChainInfo.GetInfoWithPrivKey("Sequencer").PrivateKey) + sequencerTxOpts := parentChainInfo.GetDefaultTransactOpts("Sequencer", ctx) + validatorTxOpts := parentChainInfo.GetDefaultTransactOpts("Validator", ctx) firstExec := getExecNode(t, first) chainConfig := firstExec.ArbInterface.BlockChain().Config() - coreCacheConfig := gethexec.DefaultCacheConfigFor(l2stack, &execConfig.Caching) - l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) + coreCacheConfig := gethexec.DefaultCacheConfigFor(chainStack, &execConfig.Caching) + blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) AddValNodeIfNeeded(t, ctx, nodeConfig, true, "", valnodeConfig.Wasm.RootPath) @@ -1355,19 +1422,19 @@ func Create2ndNodeWithConfig( Require(t, execConfig.Validate()) Require(t, nodeConfig.Validate()) configFetcher := func() *gethexec.Config { return execConfig } - currentExec, err := gethexec.CreateExecutionNode(ctx, l2stack, l2chainDb, l2blockchain, l1client, configFetcher) + currentExec, err := gethexec.CreateExecutionNode(ctx, chainStack, chainDb, blockchain, parentChainClient, configFetcher) Require(t, err) - currentNode, err := arbnode.CreateNode(ctx, l2stack, currentExec, l2arbDb, NewFetcherFromConfig(nodeConfig), l2blockchain.Config(), l1client, addresses, &validatorTxOpts, &sequencerTxOpts, dataSigner, feedErrChan, big.NewInt(1337), nil) + currentNode, err := arbnode.CreateNode(ctx, chainStack, currentExec, arbDb, NewFetcherFromConfig(nodeConfig), blockchain.Config(), parentChainClient, addresses, &validatorTxOpts, &sequencerTxOpts, dataSigner, feedErrChan, big.NewInt(1337), nil) Require(t, err) err = currentNode.Start(ctx) Require(t, err) - l2client := ClientForStack(t, l2stack) + chainClient := ClientForStack(t, chainStack) StartWatchChanErr(t, ctx, feedErrChan, currentNode) - return l2client, currentNode + return chainClient, currentNode } func GetBalance(t *testing.T, ctx context.Context, client *ethclient.Client, account common.Address) *big.Int { diff --git a/system_tests/l3_test.go b/system_tests/l3_test.go index 6ed10e31933..97eabcee78a 100644 --- a/system_tests/l3_test.go +++ b/system_tests/l3_test.go @@ -4,9 +4,12 @@ import ( "context" "math/big" "testing" + "time" + + "github.com/offchainlabs/nitro/arbnode" ) -func TestBasicL3(t *testing.T) { +func TestSimpleL3(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -15,19 +18,34 @@ func TestBasicL3(t *testing.T) { cleanupL1AndL2 := builder.Build(t) defer cleanupL1AndL2() - cleanupL3 := builder.BuildL3OnL2(t) - defer cleanupL3() + cleanupL3FirstNode := builder.BuildL3OnL2(t) + defer cleanupL3FirstNode() + firstNodeTestClient := builder.L3 + + secondNodeNodeConfig := arbnode.ConfigDefaultL1NonSequencerTest() + secondNodeTestClient, cleanupL3SecondNode := builder.Build2ndNodeOnL3(t, &SecondNodeParams{nodeConfig: secondNodeNodeConfig}) + defer cleanupL3SecondNode() - builder.L3Info.GenerateAccount("User2") - tx := builder.L3Info.PrepareTx("Owner", "User2", builder.L3Info.TransferGas, big.NewInt(1e12), nil) + accountName := "User2" + builder.L3Info.GenerateAccount(accountName) + tx := builder.L3Info.PrepareTx("Owner", accountName, builder.L3Info.TransferGas, big.NewInt(1e12), nil) - err := builder.L3.Client.SendTransaction(ctx, tx) + err := firstNodeTestClient.Client.SendTransaction(ctx, tx) Require(t, err) - _, err = builder.L3.EnsureTxSucceeded(tx) + // Checks that first node has the correct balance + _, err = firstNodeTestClient.EnsureTxSucceeded(tx) Require(t, err) + l2balance, err := firstNodeTestClient.Client.BalanceAt(ctx, builder.L3Info.GetAddress(accountName), nil) + Require(t, err) + if l2balance.Cmp(big.NewInt(1e12)) != 0 { + t.Fatal("Unexpected balance:", l2balance) + } - l2balance, err := builder.L3.Client.BalanceAt(ctx, builder.L3Info.GetAddress("User2"), nil) + // Checks that second node has the correct balance + _, err = WaitForTx(ctx, secondNodeTestClient.Client, tx.Hash(), time.Second*15) + Require(t, err) + l2balance, err = secondNodeTestClient.Client.BalanceAt(ctx, builder.L3Info.GetAddress(accountName), nil) Require(t, err) if l2balance.Cmp(big.NewInt(1e12)) != 0 { t.Fatal("Unexpected balance:", l2balance) From cb8c8f92281b9631f133466da37ac3fc40ac7c8e Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 24 Sep 2024 09:35:51 -0500 Subject: [PATCH 155/156] Arbitrum Nitro v3.2.0 --- .golangci.yml | 1 + Makefile | 31 +- arbitrator/arbutil/src/evm/api.rs | 2 +- arbitrator/arbutil/src/evm/mod.rs | 3 + arbitrator/arbutil/src/evm/req.rs | 5 +- arbitrator/arbutil/src/hostios.rs | 64 -- arbitrator/arbutil/src/lib.rs | 1 - arbitrator/jit/src/machine.rs | 2 + arbitrator/jit/src/program.rs | 93 ++- arbitrator/prover/Cargo.toml | 4 - arbitrator/prover/build.rs | 89 --- arbitrator/prover/src/binary.rs | 21 +- arbitrator/prover/src/host.rs | 9 +- arbitrator/prover/src/lib.rs | 12 +- arbitrator/prover/src/machine.rs | 269 +++---- arbitrator/prover/src/main.rs | 3 - arbitrator/prover/src/programs/mod.rs | 100 +-- arbitrator/prover/src/test.rs | 2 +- arbitrator/prover/src/value.rs | 12 +- arbitrator/prover/test-cases/forward-test.wat | 32 + .../prover/test-cases/forward/forward.wat | 8 + .../prover/test-cases/forward/target.wat | 27 + arbitrator/stylus/src/host.rs | 89 +-- arbitrator/stylus/src/lib.rs | 13 +- arbitrator/stylus/src/native.rs | 200 ++++-- arbitrator/stylus/src/test/api.rs | 2 +- arbitrator/stylus/src/test/mod.rs | 1 - arbitrator/stylus/src/test/native.rs | 4 +- arbitrator/tools/module_roots/src/main.rs | 1 - arbitrator/wasm-libraries/Cargo.lock | 668 +----------------- arbitrator/wasm-libraries/Cargo.toml | 1 + arbitrator/wasm-libraries/forward/.gitignore | 1 + arbitrator/wasm-libraries/forward/Cargo.toml | 8 + arbitrator/wasm-libraries/forward/src/main.rs | 207 ++++++ .../wasm-libraries/user-host-trait/src/lib.rs | 20 +- .../wasm-libraries/user-host/src/link.rs | 19 +- .../wasm-libraries/user-test/src/program.rs | 2 +- arbitrator/wasm-testsuite/src/main.rs | 2 - arbos/arbosState/arbosstate.go | 58 +- arbos/programs/native.go | 33 +- arbos/programs/programs.go | 6 +- arbos/programs/wasm.go | 11 +- arbos/programs/wasm_api.go | 4 +- arbos/programs/wasmstorehelper.go | 11 +- cmd/chaininfo/arbitrum_chain_info.json | 4 +- cmd/nitro/init.go | 8 +- cmd/nitro/nitro.go | 4 + contracts | 2 +- execution/gethexec/executionengine.go | 4 +- execution/gethexec/wasmstorerebuilder.go | 2 +- go-ethereum | 2 +- precompiles/ArbWasm.go | 8 +- system_tests/block_validator_test.go | 14 + validator/server_arb/machine.go | 2 +- 54 files changed, 935 insertions(+), 1266 deletions(-) delete mode 100644 arbitrator/arbutil/src/hostios.rs delete mode 100644 arbitrator/prover/build.rs create mode 100644 arbitrator/prover/test-cases/forward-test.wat create mode 100644 arbitrator/prover/test-cases/forward/forward.wat create mode 100644 arbitrator/prover/test-cases/forward/target.wat create mode 100644 arbitrator/wasm-libraries/forward/.gitignore create mode 100644 arbitrator/wasm-libraries/forward/Cargo.toml create mode 100644 arbitrator/wasm-libraries/forward/src/main.rs diff --git a/.golangci.yml b/.golangci.yml index 05946701379..8b10d7116ce 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -44,6 +44,7 @@ linters-settings: gosec: excludes: - G404 # checks that random numbers are securely generated + - G115 govet: enable-all: true diff --git a/Makefile b/Makefile index b736b049062..0a71d64f12c 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ replay_wasm=$(output_latest)/replay.wasm arb_brotli_files = $(wildcard arbitrator/brotli/src/*.* arbitrator/brotli/src/*/*.* arbitrator/brotli/*.toml arbitrator/brotli/*.rs) .make/cbrotli-lib .make/cbrotli-wasm arbitrator_generated_header=$(output_root)/include/arbitrator.h -arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, wasi_stub host_io soft-float arbcompress user_host program_exec) +arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, forward wasi_stub host_io soft-float arbcompress user_host program_exec) arbitrator_stylus_lib=$(output_root)/lib/libstylus.a prover_bin=$(output_root)/bin/prover arbitrator_jit=$(output_root)/bin/jit @@ -75,12 +75,16 @@ arbitrator_test_wasms=$(patsubst %.wat,%.wasm, $(arbitrator_tests_wat)) $(patsub arbitrator_tests_link_info = $(shell cat $(arbitrator_cases)/link.txt | xargs) arbitrator_tests_link_deps = $(patsubst %,$(arbitrator_cases)/%.wasm, $(arbitrator_tests_link_info)) +arbitrator_tests_forward_wats = $(wildcard $(arbitrator_cases)/forward/*.wat) +arbitrator_tests_forward_deps = $(arbitrator_tests_forward_wats:wat=wasm) + WASI_SYSROOT?=/opt/wasi-sdk/wasi-sysroot arbitrator_wasm_lib_flags=$(patsubst %, -l %, $(arbitrator_wasm_libs)) rust_arbutil_files = $(wildcard arbitrator/arbutil/src/*.* arbitrator/arbutil/src/*/*.* arbitrator/arbutil/*.toml arbitrator/caller-env/src/*.* arbitrator/caller-env/src/*/*.* arbitrator/caller-env/*.toml) .make/cbrotli-lib +prover_direct_includes = $(patsubst %,$(output_latest)/%.wasm, forward forward_stub) prover_dir = arbitrator/prover/ rust_prover_files = $(wildcard $(prover_dir)/src/*.* $(prover_dir)/src/*/*.* $(prover_dir)/*.toml $(prover_dir)/*.rs) $(rust_arbutil_files) $(prover_direct_includes) $(arb_brotli_files) @@ -88,9 +92,12 @@ wasm_lib = arbitrator/wasm-libraries wasm_lib_cargo = $(wasm_lib)/.cargo/config.toml wasm_lib_deps = $(wildcard $(wasm_lib)/$(1)/*.toml $(wasm_lib)/$(1)/src/*.rs $(wasm_lib)/$(1)/*.rs) $(wasm_lib_cargo) $(rust_arbutil_files) $(arb_brotli_files) .make/machines wasm_lib_go_abi = $(call wasm_lib_deps,go-abi) +wasm_lib_forward = $(call wasm_lib_deps,forward) wasm_lib_user_host_trait = $(call wasm_lib_deps,user-host-trait) wasm_lib_user_host = $(call wasm_lib_deps,user-host) $(wasm_lib_user_host_trait) +forward_dir = $(wasm_lib)/forward + stylus_files = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(wasm_lib_user_host_trait) $(rust_prover_files) jit_dir = arbitrator/jit @@ -274,6 +281,7 @@ clean: rm -f arbitrator/wasm-libraries/soft-float/*.o rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.o rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.a + rm -f arbitrator/wasm-libraries/forward/*.wat rm -rf arbitrator/stylus/tests/*/target/ arbitrator/stylus/tests/*/*.wasm @rm -rf contracts/build contracts/cache solgen/go/ @rm -f .make/* @@ -391,7 +399,7 @@ $(output_latest)/host_io.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,host-io) $( cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package host-io install arbitrator/wasm-libraries/$(wasm32_wasi)/host_io.wasm $@ -$(output_latest)/user_host.wasm: $(DEP_PREDICATE) $(wasm_lib_user_host) $(rust_prover_files) .make/machines +$(output_latest)/user_host.wasm: $(DEP_PREDICATE) $(wasm_lib_user_host) $(rust_prover_files) $(output_latest)/forward_stub.wasm .make/machines cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package user-host install arbitrator/wasm-libraries/$(wasm32_wasi)/user_host.wasm $@ @@ -407,9 +415,17 @@ $(output_latest)/arbcompress.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,brotli) cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package arbcompress install arbitrator/wasm-libraries/$(wasm32_wasi)/arbcompress.wasm $@ +$(output_latest)/forward.wasm: $(DEP_PREDICATE) $(wasm_lib_forward) .make/machines + cargo run --manifest-path $(forward_dir)/Cargo.toml -- --path $(forward_dir)/forward.wat + wat2wasm $(wasm_lib)/forward/forward.wat -o $@ + +$(output_latest)/forward_stub.wasm: $(DEP_PREDICATE) $(wasm_lib_forward) .make/machines + cargo run --manifest-path $(forward_dir)/Cargo.toml -- --path $(forward_dir)/forward_stub.wat --stub + wat2wasm $(wasm_lib)/forward/forward_stub.wat -o $@ + $(output_latest)/machine.wavm.br: $(DEP_PREDICATE) $(prover_bin) $(arbitrator_wasm_libs) $(replay_wasm) - $(prover_bin) $(replay_wasm) --generate-binaries $(output_latest) --with-forwarder \ - $(patsubst %,-l $(output_latest)/%.wasm, soft-float wasi_stub host_io user_host arbcompress program_exec) + $(prover_bin) $(replay_wasm) --generate-binaries $(output_latest) \ + $(patsubst %,-l $(output_latest)/%.wasm, forward soft-float wasi_stub host_io user_host arbcompress program_exec) $(arbitrator_cases)/%.wasm: $(arbitrator_cases)/%.wat wat2wasm $< -o $@ @@ -476,10 +492,10 @@ target/testdata/preimages.bin: python3 scripts/create-test-preimages.py $@ contracts/test/prover/proofs/rust-%.json: $(arbitrator_cases)/rust/$(wasm32_wasi)/%.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin - $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin --with-forwarder + $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin contracts/test/prover/proofs/go.json: $(arbitrator_cases)/go/testcase.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin $(arbitrator_tests_link_deps) $(arbitrator_cases)/user.wasm - $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm --with-forwarder + $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm # avoid testing user.wasm in onestepproofs. It can only run as stylus program. contracts/test/prover/proofs/user.json: @@ -492,6 +508,9 @@ contracts/test/prover/proofs/read-inboxmsg-10.json: contracts/test/prover/proofs/global-state.json: echo "[]" > $@ +contracts/test/prover/proofs/forward-test.json: $(arbitrator_cases)/forward-test.wasm $(arbitrator_tests_forward_deps) $(prover_bin) + $(prover_bin) $< -o $@ --allow-hostapi $(patsubst %,-l %, $(arbitrator_tests_forward_deps)) + contracts/test/prover/proofs/link.json: $(arbitrator_cases)/link.wasm $(arbitrator_tests_link_deps) $(prover_bin) $(prover_bin) $< -o $@ --allow-hostapi --stylus-modules $(arbitrator_tests_link_deps) --require-success diff --git a/arbitrator/arbutil/src/evm/api.rs b/arbitrator/arbutil/src/evm/api.rs index 093e7f2984d..9d4c78c0ded 100644 --- a/arbitrator/arbutil/src/evm/api.rs +++ b/arbitrator/arbutil/src/evm/api.rs @@ -77,7 +77,7 @@ pub trait EvmApi: Send + 'static { /// Reads the 32-byte value in the EVM state trie at offset `key`. /// Returns the value and the access cost in gas. /// Analogous to `vm.SLOAD`. - fn get_bytes32(&mut self, key: Bytes32) -> (Bytes32, u64); + fn get_bytes32(&mut self, key: Bytes32, evm_api_gas_to_use: u64) -> (Bytes32, u64); /// Stores the given value at the given key in Stylus VM's cache of the EVM state trie. /// Note that the actual values only get written after calls to `set_trie_slots`. diff --git a/arbitrator/arbutil/src/evm/mod.rs b/arbitrator/arbutil/src/evm/mod.rs index 1671e670729..36dadd906a9 100644 --- a/arbitrator/arbutil/src/evm/mod.rs +++ b/arbitrator/arbutil/src/evm/mod.rs @@ -74,9 +74,12 @@ pub const GASPRICE_GAS: u64 = GAS_QUICK_STEP; // vm.GasQuickStep (see jump_table.go) pub const ORIGIN_GAS: u64 = GAS_QUICK_STEP; +pub const ARBOS_VERSION_STYLUS_CHARGING_FIXES: u64 = 32; + #[derive(Clone, Copy, Debug, Default)] #[repr(C)] pub struct EvmData { + pub arbos_version: u64, pub block_basefee: Bytes32, pub chainid: u64, pub block_coinbase: Bytes20, diff --git a/arbitrator/arbutil/src/evm/req.rs b/arbitrator/arbutil/src/evm/req.rs index 287db357f32..1cfceda6b7d 100644 --- a/arbitrator/arbutil/src/evm/req.rs +++ b/arbitrator/arbutil/src/evm/req.rs @@ -7,7 +7,6 @@ use crate::{ storage::{StorageCache, StorageWord}, user::UserOutcomeKind, }, - pricing::EVM_API_INK, Bytes20, Bytes32, }; use eyre::{bail, eyre, Result}; @@ -99,13 +98,13 @@ impl> EvmApiRequestor { } impl> EvmApi for EvmApiRequestor { - fn get_bytes32(&mut self, key: Bytes32) -> (Bytes32, u64) { + fn get_bytes32(&mut self, key: Bytes32, evm_api_gas_to_use: u64) -> (Bytes32, u64) { let cache = &mut self.storage_cache; let mut cost = cache.read_gas(); let value = cache.entry(key).or_insert_with(|| { let (res, _, gas) = self.handler.request(EvmApiMethod::GetBytes32, key); - cost = cost.saturating_add(gas).saturating_add(EVM_API_INK); + cost = cost.saturating_add(gas).saturating_add(evm_api_gas_to_use); StorageWord::known(res.try_into().unwrap()) }); (value.value, cost) diff --git a/arbitrator/arbutil/src/hostios.rs b/arbitrator/arbutil/src/hostios.rs deleted file mode 100644 index 05bcb94f73b..00000000000 --- a/arbitrator/arbutil/src/hostios.rs +++ /dev/null @@ -1,64 +0,0 @@ -use std::fmt::Display; - -pub enum ParamType { - I32, - I64, -} - -impl Display for ParamType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - use ParamType::*; - match self { - I32 => write!(f, "i32"), - I64 => write!(f, "i64"), - } - } -} - -use ParamType::*; - -/// order matters! -pub const HOSTIOS: [(&str, &[ParamType], &[ParamType]); 42] = [ - ("read_args", &[I32], &[]), - ("write_result", &[I32, I32], &[]), - ("exit_early", &[I32], &[]), - ("storage_load_bytes32", &[I32, I32], &[]), - ("storage_cache_bytes32", &[I32, I32], &[]), - ("storage_flush_cache", &[I32], &[]), - ("transient_load_bytes32", &[I32, I32], &[]), - ("transient_store_bytes32", &[I32, I32], &[]), - ("call_contract", &[I32, I32, I32, I32, I64, I32], &[I32]), - ("delegate_call_contract", &[I32, I32, I32, I64, I32], &[I32]), - ("static_call_contract", &[I32, I32, I32, I64, I32], &[I32]), - ("create1", &[I32, I32, I32, I32, I32], &[]), - ("create2", &[I32, I32, I32, I32, I32, I32], &[]), - ("read_return_data", &[I32, I32, I32], &[I32]), - ("return_data_size", &[], &[I32]), - ("emit_log", &[I32, I32, I32], &[]), - ("account_balance", &[I32, I32], &[]), - ("account_code", &[I32, I32, I32, I32], &[I32]), - ("account_code_size", &[I32], &[I32]), - ("account_codehash", &[I32, I32], &[]), - ("evm_gas_left", &[], &[I64]), - ("evm_ink_left", &[], &[I64]), - ("block_basefee", &[I32], &[]), - ("chainid", &[], &[I64]), - ("block_coinbase", &[I32], &[]), - ("block_gas_limit", &[], &[I64]), - ("block_number", &[], &[I64]), - ("block_timestamp", &[], &[I64]), - ("contract_address", &[I32], &[]), - ("math_div", &[I32, I32], &[]), - ("math_mod", &[I32, I32], &[]), - ("math_pow", &[I32, I32], &[]), - ("math_add_mod", &[I32, I32, I32], &[]), - ("math_mul_mod", &[I32, I32, I32], &[]), - ("msg_reentrant", &[], &[I32]), - ("msg_sender", &[I32], &[]), - ("msg_value", &[I32], &[]), - ("native_keccak256", &[I32, I32, I32], &[]), - ("tx_gas_price", &[I32], &[]), - ("tx_ink_price", &[], &[I32]), - ("tx_origin", &[I32], &[]), - ("pay_for_memory_grow", &[I32], &[]), -]; diff --git a/arbitrator/arbutil/src/lib.rs b/arbitrator/arbutil/src/lib.rs index 8d8c1d0fca9..9c48a9fefc2 100644 --- a/arbitrator/arbutil/src/lib.rs +++ b/arbitrator/arbutil/src/lib.rs @@ -6,7 +6,6 @@ pub mod color; pub mod crypto; pub mod evm; pub mod format; -pub mod hostios; pub mod math; pub mod operator; pub mod pricing; diff --git a/arbitrator/jit/src/machine.rs b/arbitrator/jit/src/machine.rs index 2a3c5c56169..02523f740aa 100644 --- a/arbitrator/jit/src/machine.rs +++ b/arbitrator/jit/src/machine.rs @@ -129,7 +129,9 @@ pub fn create(opts: &Opts, env: WasmEnv) -> (Instance, FunctionEnv, Sto "send_response" => func!(program::send_response), "create_stylus_config" => func!(program::create_stylus_config), "create_evm_data" => func!(program::create_evm_data), + "create_evm_data_v2" => func!(program::create_evm_data_v2), "activate" => func!(program::activate), + "activate_v2" => func!(program::activate_v2), }, }; diff --git a/arbitrator/jit/src/program.rs b/arbitrator/jit/src/program.rs index c608a3cf852..084afe96bc9 100644 --- a/arbitrator/jit/src/program.rs +++ b/arbitrator/jit/src/program.rs @@ -16,8 +16,45 @@ use prover::{ programs::{config::PricingParams, prelude::*}, }; -/// activates a user program +const DEFAULT_STYLUS_ARBOS_VERSION: u64 = 31; + pub fn activate( + env: WasmEnvMut, + wasm_ptr: GuestPtr, + wasm_size: u32, + pages_ptr: GuestPtr, + asm_estimate_ptr: GuestPtr, + init_cost_ptr: GuestPtr, + cached_init_cost_ptr: GuestPtr, + stylus_version: u16, + debug: u32, + codehash: GuestPtr, + module_hash_ptr: GuestPtr, + gas_ptr: GuestPtr, + err_buf: GuestPtr, + err_buf_len: u32, +) -> Result { + activate_v2( + env, + wasm_ptr, + wasm_size, + pages_ptr, + asm_estimate_ptr, + init_cost_ptr, + cached_init_cost_ptr, + stylus_version, + DEFAULT_STYLUS_ARBOS_VERSION, + debug, + codehash, + module_hash_ptr, + gas_ptr, + err_buf, + err_buf_len, + ) +} + +/// activates a user program +pub fn activate_v2( mut env: WasmEnvMut, wasm_ptr: GuestPtr, wasm_size: u32, @@ -25,7 +62,8 @@ pub fn activate( asm_estimate_ptr: GuestPtr, init_cost_ptr: GuestPtr, cached_init_cost_ptr: GuestPtr, - version: u16, + stylus_version: u16, + arbos_version_for_gas: u64, debug: u32, codehash: GuestPtr, module_hash_ptr: GuestPtr, @@ -40,7 +78,15 @@ pub fn activate( let page_limit = mem.read_u16(pages_ptr); let gas_left = &mut mem.read_u64(gas_ptr); - match Module::activate(&wasm, codehash, version, page_limit, debug, gas_left) { + match Module::activate( + &wasm, + codehash, + stylus_version, + arbos_version_for_gas, + page_limit, + debug, + gas_left, + ) { Ok((module, data)) => { mem.write_u64(gas_ptr, *gas_left); mem.write_u16(pages_ptr, data.footprint); @@ -222,9 +268,47 @@ pub fn create_stylus_config( Ok(res as u64) } -/// Creates an `EvmData` handler from its component parts. pub fn create_evm_data( + env: WasmEnvMut, + block_basefee_ptr: GuestPtr, + chainid: u64, + block_coinbase_ptr: GuestPtr, + block_gas_limit: u64, + block_number: u64, + block_timestamp: u64, + contract_address_ptr: GuestPtr, + module_hash_ptr: GuestPtr, + msg_sender_ptr: GuestPtr, + msg_value_ptr: GuestPtr, + tx_gas_price_ptr: GuestPtr, + tx_origin_ptr: GuestPtr, + cached: u32, + reentrant: u32, +) -> Result { + create_evm_data_v2( + env, + DEFAULT_STYLUS_ARBOS_VERSION, + block_basefee_ptr, + chainid, + block_coinbase_ptr, + block_gas_limit, + block_number, + block_timestamp, + contract_address_ptr, + module_hash_ptr, + msg_sender_ptr, + msg_value_ptr, + tx_gas_price_ptr, + tx_origin_ptr, + cached, + reentrant, + ) +} + +/// Creates an `EvmData` handler from its component parts. +pub fn create_evm_data_v2( mut env: WasmEnvMut, + arbos_version: u64, block_basefee_ptr: GuestPtr, chainid: u64, block_coinbase_ptr: GuestPtr, @@ -243,6 +327,7 @@ pub fn create_evm_data( let (mut mem, _) = env.jit_env(); let evm_data = EvmData { + arbos_version, block_basefee: mem.read_bytes32(block_basefee_ptr), cached: cached != 0, chainid, diff --git a/arbitrator/prover/Cargo.toml b/arbitrator/prover/Cargo.toml index a634dba19eb..54756477657 100644 --- a/arbitrator/prover/Cargo.toml +++ b/arbitrator/prover/Cargo.toml @@ -47,10 +47,6 @@ enum-iterator = "2.0.1" criterion = { version = "0.5.0", features = ["html_reports"] } rand = "0.8.4" -[build-dependencies] -wasmer = { path = "../tools/wasmer/lib/api" } -arbutil = { path = "../arbutil/" } - [[bench]] name = "merkle_bench" harness = false diff --git a/arbitrator/prover/build.rs b/arbitrator/prover/build.rs deleted file mode 100644 index db0560b2c54..00000000000 --- a/arbitrator/prover/build.rs +++ /dev/null @@ -1,89 +0,0 @@ -use arbutil::hostios::HOSTIOS; -use std::{env, fmt::Write, fs, path::Path}; - -pub fn gen_forwarder(out_path: &Path) { - let mut wat = String::new(); - macro_rules! wln { - ($($text:tt)*) => { - writeln!(wat, $($text)*).unwrap(); - }; - } - let s = " "; - - wln!("(module"); - - macro_rules! group { - ($list:expr, $kind:expr) => { - (!$list.is_empty()) - .then(|| { - format!( - " ({} {})", - $kind, - $list - .iter() - .map(|x| x.to_string()) - .collect::>() - .join(" ") - ) - }) - .unwrap_or_default() - }; - } - - wln!("{s};; symbols to re-export"); - for (name, ins, outs) in HOSTIOS { - let params = group!(ins, "param"); - let result = group!(outs, "result"); - wln!(r#"{s}(import "user_host" "{name}" (func $_{name}{params}{result}))"#); - } - wln!(); - - wln!("{s};; reserved offsets for future user_host imports"); - for i in HOSTIOS.len()..512 { - wln!("{s}(func $reserved_{i} unreachable)"); - } - wln!(); - - wln!( - "{s};; allows user_host to request a trap\n\ - {s}(global $trap (mut i32) (i32.const 0))\n\ - {s}(func $check\n\ - {s}{s}global.get $trap ;; see if set\n\ - {s}{s}(global.set $trap (i32.const 0)) ;; reset the flag\n\ - {s}{s}(if (then (unreachable)))\n\ - {s})\n\ - {s}(func (export \"forward__set_trap\")\n\ - {s}{s}(global.set $trap (i32.const 1))\n\ - {s})\n" - ); - - wln!("{s};; user linkage"); - for (name, ins, outs) in HOSTIOS { - let params = group!(ins, "param"); - let result = group!(outs, "result"); - wln!("{s}(func (export \"vm_hooks__{name}\"){params}{result}"); - - for i in 0..ins.len() { - wln!("{s}{s}local.get {i}"); - } - - wln!( - "{s}{s}call $_{name}\n\ - {s}{s}call $check\n\ - {s})" - ); - } - - wln!(")"); - eprintln!("{}", &wat); - - let wasm = wasmer::wat2wasm(wat.as_bytes()).unwrap(); - - fs::write(out_path, wasm.as_ref()).unwrap(); -} - -fn main() { - let out_dir = env::var("OUT_DIR").unwrap(); - let forwarder_path = Path::new(&out_dir).join("forwarder.wasm"); - gen_forwarder(&forwarder_path); -} diff --git a/arbitrator/prover/src/binary.rs b/arbitrator/prover/src/binary.rs index b1d819deb26..2260f6bf488 100644 --- a/arbitrator/prover/src/binary.rs +++ b/arbitrator/prover/src/binary.rs @@ -9,7 +9,9 @@ use crate::{ }, value::{ArbValueType, FunctionType, IntegerValType, Value}, }; -use arbutil::{math::SaturatingSum, Bytes32, Color, DebugColor}; +use arbutil::{ + evm::ARBOS_VERSION_STYLUS_CHARGING_FIXES, math::SaturatingSum, Bytes32, Color, DebugColor, +}; use eyre::{bail, ensure, eyre, Result, WrapErr}; use fnv::{FnvHashMap as HashMap, FnvHashSet as HashSet}; use nom::{ @@ -471,14 +473,10 @@ pub fn parse<'a>(input: &'a [u8], path: &'_ Path) -> Result> { let export = export.rsplit("__").take(1); exports.extend(export); } - // forwarder is allowed to re-export the same name - // TODO: is there a real problem if import_name == export_name but module names don't? - if path != Path::new("forwarder") { - for import in &binary.imports { - let name = import.name; - if exports.contains(name) { - bail!("binary exports an import with the same name {}", name.red()); - } + for import in &binary.imports { + let name = import.name; + if exports.contains(name) { + bail!("binary exports an import with the same name {}", name.red()); } } @@ -645,6 +643,7 @@ impl<'a> WasmBinary<'a> { /// Parses and instruments a user wasm pub fn parse_user( wasm: &'a [u8], + arbos_version_for_gas: u64, page_limit: u16, compile: &CompileConfig, codehash: &Bytes32, @@ -682,6 +681,10 @@ impl<'a> WasmBinary<'a> { limit!(65536, code.expr.len(), "opcodes in func body"); } + if arbos_version_for_gas >= ARBOS_VERSION_STYLUS_CHARGING_FIXES { + limit!(513, bin.imports.len(), "imports") + } + let table_entries = bin.tables.iter().map(|x| x.initial).saturating_sum(); limit!(4096, table_entries, "table entries"); diff --git a/arbitrator/prover/src/host.rs b/arbitrator/prover/src/host.rs index dc0f0b74b2d..1d0fe658ecc 100644 --- a/arbitrator/prover/src/host.rs +++ b/arbitrator/prover/src/host.rs @@ -405,7 +405,7 @@ impl Hostio { } } -pub fn get_impl(module: &str, name: &str) -> Result { +pub fn get_impl(module: &str, name: &str) -> Result<(Function, bool)> { let hostio: Hostio = format!("{module}__{name}").parse()?; let append = |code: &mut Vec| { @@ -414,11 +414,8 @@ pub fn get_impl(module: &str, name: &str) -> Result { Ok(()) }; - Function::new(&[], append, hostio.ty(), &[]) -} - -pub fn hostio_module_is_debug(module: &str) -> bool { - module == "console" || module == "debug" + let debug = module == "console" || module == "debug"; + Function::new(&[], append, hostio.ty(), &[]).map(|x| (x, debug)) } /// Adds internal functions to a module. diff --git a/arbitrator/prover/src/lib.rs b/arbitrator/prover/src/lib.rs index e45d12c4926..0f537478eb7 100644 --- a/arbitrator/prover/src/lib.rs +++ b/arbitrator/prover/src/lib.rs @@ -70,16 +70,10 @@ pub unsafe extern "C" fn arbitrator_load_machine( library_paths: *const *const c_char, library_paths_size: isize, debug_chain: usize, - with_forwarder: bool, ) -> *mut Machine { let debug_chain = debug_chain != 0; - match arbitrator_load_machine_impl( - binary_path, - library_paths, - library_paths_size, - debug_chain, - with_forwarder, - ) { + match arbitrator_load_machine_impl(binary_path, library_paths, library_paths_size, debug_chain) + { Ok(mach) => mach, Err(err) => { eprintln!("Error loading binary: {:?}", err); @@ -93,7 +87,6 @@ unsafe fn arbitrator_load_machine_impl( library_paths: *const *const c_char, library_paths_size: isize, debug_chain: bool, - with_forwarder: bool, ) -> Result<*mut Machine> { let binary_path = cstr_to_string(binary_path); let binary_path = Path::new(&binary_path); @@ -114,7 +107,6 @@ unsafe fn arbitrator_load_machine_impl( Default::default(), Default::default(), get_empty_preimage_resolver(), - with_forwarder, )?; Ok(Box::into_raw(Box::new(mach))) } diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index 2418d3cc2e6..4ece1f7bf26 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -19,7 +19,7 @@ use crate::{ IBinOpType, IRelOpType, IUnOpType, Instruction, Opcode, }, }; -use arbutil::{crypto, hostios::HOSTIOS, math, Bytes32, Color, DebugColor, PreimageType}; +use arbutil::{crypto, math, Bytes32, Color, DebugColor, PreimageType}; use brotli::Dictionary; #[cfg(feature = "native")] use c_kzg::BYTES_PER_BLOB; @@ -330,115 +330,36 @@ pub struct Module { pub(crate) extra_hash: Arc, } -#[cfg(feature = "native")] -static FORWARDER_WASM: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); - lazy_static! { static ref USER_IMPORTS: HashMap = { let mut imports = HashMap::default(); - // 0-513 are internal - for (index, (name, ins, outs)) in HOSTIOS.iter().enumerate() { - let import = AvailableImport::new(FunctionType::new(ins.iter().map(|x|x.into()).collect::>(), outs.iter().map(|x|x.into()).collect::>()), 1, (index + 514).try_into().unwrap()); - imports.insert(format!("vm_hooks__{name}"), import); - } - imports - }; -} + let forward = include_bytes!("../../../target/machines/latest/forward_stub.wasm"); + let forward = binary::parse(forward, Path::new("forward")).unwrap(); -trait ImportResolver: for<'a, 'b> Fn(&'a str, &'b str) -> Result + Sized { - fn compose(self, other: impl ImportResolver) -> impl ImportResolver { - move |module, name| { - self(module, name).or_else(|original_err| { - let mut result = other(module, name); - if result.is_err() { - for err_layer in original_err.chain().rev() { - result = result.wrap_err(format!("{err_layer}")); - } - } - result - }) - } - } - - fn condition(self, condition: bool) -> impl ImportResolver { - move |module, name| { - if condition { - self(module, name) - } else { - bail!("resolver disabled") + for (name, &(export, kind)) in &forward.exports { + if kind == ExportKind::Func { + let ty = match forward.get_function(FunctionIndex::from_u32(export)) { + Ok(ty) => ty, + Err(error) => panic!("failed to read export {name}: {error:?}"), + }; + let import = AvailableImport::new(ty, 1, export); + imports.insert(name.to_owned(), import); } } - } + imports + }; } -impl Fn(&'a str, &'b str) -> Result> ImportResolver for F {} - impl Module { - fn make_imports_resolver( - import_map: &HashMap, - ) -> impl ImportResolver + '_ { - move |module, name| { - let qualified_name = format!("{module}__{name}"); - - let Some(import) = import_map.get(&qualified_name) else { - bail!("func not found {module} {name}") - }; - let wavm = vec![ - Instruction::simple(Opcode::InitFrame), - Instruction::with_data( - Opcode::CrossModuleCall, - pack_cross_module_call(import.module, import.func), - ), - Instruction::simple(Opcode::Return), - ]; - Ok(Function::new_from_wavm(wavm, import.ty.clone(), vec![])) - } - } - - fn make_forward_resolver( - import_map: &HashMap, - ) -> impl ImportResolver + '_ { - move |module, name| { - let qualified_name = format!("{module}__{name}"); - - let Some(import) = import_map.get(&qualified_name) else { - bail!("func not found {module} {name}") - }; - let wavm = vec![ - Instruction::simple(Opcode::InitFrame), - Instruction::with_data( - Opcode::CrossModuleForward, - pack_cross_module_call(import.module, import.func), - ), - Instruction::simple(Opcode::Return), - ]; - Ok(Function::new_from_wavm(wavm, import.ty.clone(), vec![])) - } - } + const FORWARDING_PREFIX: &'static str = "arbitrator_forward__"; - fn hostio_resolver(module: &str, name: &str) -> Result { - if host::hostio_module_is_debug(module) { - return Self::notfound_resolver(module, name); - } - host::get_impl(module, name) - } - - fn debug_resolver(module: &str, name: &str) -> Result { - if !host::hostio_module_is_debug(module) { - return Self::notfound_resolver(module, name); - } - host::get_impl(module, name) - } - - fn notfound_resolver(module: &str, name: &str) -> Result { - bail!("import not found {module} {name}") - } - - fn from_binary( + fn from_binary( bin: &WasmBinary, - import_resolver: R, + available_imports: &HashMap, floating_point_impls: &FloatingPointImpls, + allow_hostapi: bool, + debug_funcs: bool, stylus_data: Option, ) -> Result { let mut code = Vec::new(); @@ -450,18 +371,56 @@ impl Module { for import in &bin.imports { let module = import.module; let have_ty = &bin.types[import.offset as usize]; + // allow_hostapi is only set for system modules like the + // forwarder. We restrict stripping the prefix for user modules. + let (forward, import_name) = + if allow_hostapi && import.name.starts_with(Self::FORWARDING_PREFIX) { + (true, &import.name[Self::FORWARDING_PREFIX.len()..]) + } else { + (false, import.name) + }; - let func = import_resolver(import.module, import.name)?; + let qualified_name = format!("{module}__{import_name}"); + let func = if let Some(import) = available_imports.get(&qualified_name) { + let call = match forward { + true => Opcode::CrossModuleForward, + false => Opcode::CrossModuleCall, + }; + let wavm = vec![ + Instruction::simple(Opcode::InitFrame), + Instruction::with_data( + call, + pack_cross_module_call(import.module, import.func), + ), + Instruction::simple(Opcode::Return), + ]; + Function::new_from_wavm(wavm, import.ty.clone(), vec![]) + } else if let Ok((hostio, debug)) = host::get_impl(import.module, import_name) { + ensure!( + (debug && debug_funcs) || (!debug && allow_hostapi), + "Host func {} in {} not enabled debug_funcs={debug_funcs} hostapi={allow_hostapi} debug={debug}", + import_name.red(), + import.module.red(), + ); + hostio + } else { + bail!( + "No such import {} in {} for {}", + import_name.red(), + import.module.red(), + bin_name.red() + ) + }; ensure!( &func.ty == have_ty, "Import {} for {} has different function signature than export.\nexpected {} in {}\nbut have {}", - import.name.red(), bin_name.red(), func.ty.red(), module.red(), have_ty.red(), + import_name.red(), bin_name.red(), func.ty.red(), module.red(), have_ty.red(), ); func_type_idxs.push(import.offset); code.push(func); - host_call_hooks.push(Some((import.module.into(), import.name.into()))); + host_call_hooks.push(Some((import.module.into(), import_name.into()))); } func_type_idxs.extend(bin.functions.iter()); @@ -657,9 +616,10 @@ impl Module { ) -> Result { Self::from_binary( bin, - Module::make_imports_resolver(&USER_IMPORTS) - .compose(Module::debug_resolver.condition(debug_funcs)), + &USER_IMPORTS, &HashMap::default(), + false, + debug_funcs, stylus_data, ) } @@ -1270,7 +1230,6 @@ impl Machine { global_state: GlobalState, inbox_contents: HashMap<(InboxIdentifier, u64), Vec>, preimage_resolver: PreimageResolver, - with_forwarder: bool, ) -> Result { let bin_source = file_bytes(binary_path)?; let bin = parse(&bin_source, binary_path) @@ -1296,7 +1255,6 @@ impl Machine { inbox_contents, preimage_resolver, None, - with_forwarder, ) } @@ -1326,7 +1284,6 @@ impl Machine { HashMap::default(), Arc::new(|_, _, _| panic!("tried to read preimage")), Some(stylus_data), - false, )?; let footprint: u32 = stylus_data.footprint.into(); @@ -1374,7 +1331,6 @@ impl Machine { inbox_contents: HashMap<(InboxIdentifier, u64), Vec>, preimage_resolver: PreimageResolver, stylus_data: Option, - with_forwarder: bool, ) -> Result { use ArbValueType::*; @@ -1382,19 +1338,25 @@ impl Machine { let mut modules = vec![Module::default()]; let mut available_imports = HashMap::default(); let mut floating_point_impls = HashMap::default(); - - let forwarder = if with_forwarder { - #[cfg(not(feature = "native"))] - bail!("forwarder not supported without native"); - - #[cfg(feature = "native")] - Some(parse(FORWARDER_WASM, Path::new("forwarder"))?) - } else { - None - }; + let main_module_index = u32::try_from(modules.len() + libraries.len())?; + + // make the main module's exports available to libraries + for (name, &(export, kind)) in &bin.exports { + if kind == ExportKind::Func { + let index: usize = export.try_into()?; + if let Some(index) = index.checked_sub(bin.imports.len()) { + let ty: usize = bin.functions[index].try_into()?; + let ty = bin.types[ty].clone(); + available_imports.insert( + format!("env__wavm_guest_call__{name}"), + AvailableImport::new(ty, main_module_index, export), + ); + } + } + } // collect all the library exports in advance so they can use each other's - for (index, lib) in forwarder.iter().chain(libraries.iter()).enumerate() { + for (index, lib) in libraries.iter().enumerate() { let module = 1 + index as u32; // off by one due to the entry point for (name, &(export, kind)) in &lib.exports { if kind == ExportKind::Func { @@ -1402,57 +1364,54 @@ impl Machine { Ok(ty) => ty, Err(error) => bail!("failed to read export {name}: {error}"), }; - let import = AvailableImport::new(ty.clone(), module, export); + let import = AvailableImport::new(ty, module, export); available_imports.insert(name.to_owned(), import); - if let Ok(op) = name.parse::() { - let mut sig = op.signature(); - // wavm codegen takes care of effecting this type change at callsites - for ty in sig.inputs.iter_mut().chain(sig.outputs.iter_mut()) { - if *ty == F32 { - *ty = I32; - } else if *ty == F64 { - *ty = I64; - } - } - ensure!( - ty == sig, - "Wrong type for floating point impl {} expecting {} but got {}", - name.red(), - sig.red(), - ty.red() - ); - floating_point_impls.insert(op, (module, export)); - } } } } - if let Some(lib) = forwarder { - modules.push(Module::from_binary( - &lib, - Module::make_forward_resolver(&available_imports), - &floating_point_impls, - None, - )?); - } - - for lib in libraries.iter() { - modules.push(Module::from_binary( + for lib in libraries { + let module = Module::from_binary( lib, - Module::hostio_resolver - .compose(Module::debug_resolver.condition(debug_funcs)) - .compose(Module::make_imports_resolver(&available_imports)), + &available_imports, &floating_point_impls, + true, + debug_funcs, None, - )?); + )?; + for (name, &func) in &*module.func_exports { + let ty = module.func_types[func as usize].clone(); + if let Ok(op) = name.parse::() { + let mut sig = op.signature(); + // wavm codegen takes care of effecting this type change at callsites + for ty in sig.inputs.iter_mut().chain(sig.outputs.iter_mut()) { + if *ty == F32 { + *ty = I32; + } else if *ty == F64 { + *ty = I64; + } + } + ensure!( + ty == sig, + "Wrong type for floating point impl {} expecting {} but got {}", + name.red(), + sig.red(), + ty.red() + ); + floating_point_impls.insert(op, (modules.len() as u32, func)); + } + } + modules.push(module); } + // Shouldn't be necessary, but to be safe, don't allow the main binary to import its own guest calls + available_imports.retain(|_, i| i.module as usize != modules.len()); modules.push(Module::from_binary( &bin, - Module::make_imports_resolver(&available_imports) - .compose(Module::hostio_resolver.condition(allow_hostapi_from_main)) - .compose(Module::debug_resolver.condition(debug_funcs)), + &available_imports, &floating_point_impls, + allow_hostapi_from_main, + debug_funcs, stylus_data, )?); diff --git a/arbitrator/prover/src/main.rs b/arbitrator/prover/src/main.rs index 442b35992dc..dba32e0e727 100644 --- a/arbitrator/prover/src/main.rs +++ b/arbitrator/prover/src/main.rs @@ -86,8 +86,6 @@ struct Opts { skip_until_host_io: bool, #[structopt(long)] max_steps: Option, - #[structopt(short, long)] - with_forwarder: bool, } fn file_with_stub_header(path: &Path, headerlength: usize) -> Result> { @@ -213,7 +211,6 @@ fn main() -> Result<()> { global_state, inbox_contents, preimage_resolver, - opts.with_forwarder, )?; for path in &opts.stylus_modules { diff --git a/arbitrator/prover/src/programs/mod.rs b/arbitrator/prover/src/programs/mod.rs index a5df2e31a80..a35308e7ff4 100644 --- a/arbitrator/prover/src/programs/mod.rs +++ b/arbitrator/prover/src/programs/mod.rs @@ -8,7 +8,7 @@ use crate::{ programs::config::CompileConfig, value::{FunctionType as ArbFunctionType, Value}, }; -use arbutil::{math::SaturatingSum, Bytes32, Color}; +use arbutil::{evm::ARBOS_VERSION_STYLUS_CHARGING_FIXES, math::SaturatingSum, Bytes32, Color}; use eyre::{bail, eyre, Report, Result, WrapErr}; use fnv::FnvHashMap as HashMap; use std::fmt::Debug; @@ -418,58 +418,64 @@ impl Module { pub fn activate( wasm: &[u8], codehash: &Bytes32, - version: u16, + stylus_version: u16, + arbos_version_for_gas: u64, // must only be used for activation gas page_limit: u16, debug: bool, gas: &mut u64, ) -> Result<(Self, StylusData)> { - // converts a number of microseconds to gas - // TODO: collapse to a single value after finalizing factors - let us_to_gas = |us: u64| { - let fudge = 2; - let sync_rate = 1_000_000 / 2; - let speed = 7_000_000; - us.saturating_mul(fudge * speed) / sync_rate - }; - - macro_rules! pay { - ($us:expr) => { - let amount = us_to_gas($us); - if *gas < amount { - *gas = 0; - bail!("out of gas"); - } - *gas -= amount; + let compile = CompileConfig::version(stylus_version, debug); + let (bin, stylus_data) = + WasmBinary::parse_user(wasm, arbos_version_for_gas, page_limit, &compile, codehash) + .wrap_err("failed to parse wasm")?; + + if arbos_version_for_gas > 0 { + // converts a number of microseconds to gas + // TODO: collapse to a single value after finalizing factors + let us_to_gas = |us: u64| { + let fudge = 2; + let sync_rate = 1_000_000 / 2; + let speed = 7_000_000; + us.saturating_mul(fudge * speed) / sync_rate }; - } - - // pay for wasm - let wasm_len = wasm.len() as u64; - pay!(wasm_len.saturating_mul(31_733 / 100_000)); - - let compile = CompileConfig::version(version, debug); - let (bin, stylus_data) = WasmBinary::parse_user(wasm, page_limit, &compile, codehash) - .wrap_err("failed to parse wasm")?; - // pay for funcs - let funcs = bin.functions.len() as u64; - pay!(funcs.saturating_mul(17_263) / 100_000); - - // pay for data - let data = bin.datas.iter().map(|x| x.data.len()).saturating_sum() as u64; - pay!(data.saturating_mul(17_376) / 100_000); - - // pay for elements - let elems = bin.elements.iter().map(|x| x.range.len()).saturating_sum() as u64; - pay!(elems.saturating_mul(17_376) / 100_000); - - // pay for memory - let mem = bin.memories.first().map(|x| x.initial).unwrap_or_default(); - pay!(mem.saturating_mul(2217)); - - // pay for code - let code = bin.codes.iter().map(|x| x.expr.len()).saturating_sum() as u64; - pay!(code.saturating_mul(535) / 1_000); + macro_rules! pay { + ($us:expr) => { + let amount = us_to_gas($us); + if *gas < amount { + *gas = 0; + bail!("out of gas"); + } + *gas -= amount; + }; + } + + // pay for wasm + if arbos_version_for_gas >= ARBOS_VERSION_STYLUS_CHARGING_FIXES { + let wasm_len = wasm.len() as u64; + pay!(wasm_len.saturating_mul(31_733) / 100_000); + } + + // pay for funcs + let funcs = bin.functions.len() as u64; + pay!(funcs.saturating_mul(17_263) / 100_000); + + // pay for data + let data = bin.datas.iter().map(|x| x.data.len()).saturating_sum() as u64; + pay!(data.saturating_mul(17_376) / 100_000); + + // pay for elements + let elems = bin.elements.iter().map(|x| x.range.len()).saturating_sum() as u64; + pay!(elems.saturating_mul(17_376) / 100_000); + + // pay for memory + let mem = bin.memories.first().map(|x| x.initial).unwrap_or_default(); + pay!(mem.saturating_mul(2217)); + + // pay for code + let code = bin.codes.iter().map(|x| x.expr.len()).saturating_sum() as u64; + pay!(code.saturating_mul(535) / 1_000); + } let module = Self::from_user_binary(&bin, compile.debug.debug_funcs, Some(stylus_data)) .wrap_err("failed to build user module")?; diff --git a/arbitrator/prover/src/test.rs b/arbitrator/prover/src/test.rs index 2f32aff0125..97170441ffa 100644 --- a/arbitrator/prover/src/test.rs +++ b/arbitrator/prover/src/test.rs @@ -57,7 +57,7 @@ pub fn reject_ambiguous_imports() { #[test] pub fn test_compress() -> Result<()> { - let data = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); + let data = include_bytes!("../../../target/machines/latest/forward_stub.wasm"); let mut last = vec![]; for dict in [Dictionary::Empty, Dictionary::StylusProgram] { diff --git a/arbitrator/prover/src/value.rs b/arbitrator/prover/src/value.rs index f2f54404f04..6afffdf7a05 100644 --- a/arbitrator/prover/src/value.rs +++ b/arbitrator/prover/src/value.rs @@ -2,7 +2,7 @@ // For license information, see https://github.com/nitro/blob/master/LICENSE use crate::binary::FloatType; -use arbutil::{hostios::ParamType, Bytes32, Color}; +use arbutil::{Bytes32, Color}; use digest::Digest; use eyre::{bail, ErrReport, Result}; use serde::{Deserialize, Serialize}; @@ -77,16 +77,6 @@ impl From for ValType { } } -impl From<&ParamType> for ArbValueType { - fn from(ty: &ParamType) -> Self { - use ParamType as V; - match ty { - V::I32 => Self::I32, - V::I64 => Self::I64, - } - } -} - #[cfg(feature = "native")] pub fn parser_type(ty: &wasmer::Type) -> wasmer::wasmparser::ValType { match ty { diff --git a/arbitrator/prover/test-cases/forward-test.wat b/arbitrator/prover/test-cases/forward-test.wat new file mode 100644 index 00000000000..b9beff0d825 --- /dev/null +++ b/arbitrator/prover/test-cases/forward-test.wat @@ -0,0 +1,32 @@ + +(module + (import "forward" "add" (func $add (param i32 i32) (result i32))) + (import "forward" "sub" (func $sub (param i32 i32) (result i32))) + (import "forward" "mul" (func $mul (param i32 i32) (result i32))) + (func $start + ;; this address will update each time a forwarded call is made + i32.const 0xa4b + i32.const 805 + i32.store + + i32.const 11 + i32.const 5 + call $sub + + i32.const 3 + i32.const -2 + call $mul + + call $add + (if + (then (unreachable))) + + ;; check that the address has changed + i32.const 0xa4b + i32.load + i32.const 808 + i32.ne + (if + (then (unreachable)))) + (start $start) + (memory 1)) diff --git a/arbitrator/prover/test-cases/forward/forward.wat b/arbitrator/prover/test-cases/forward/forward.wat new file mode 100644 index 00000000000..ff55953e626 --- /dev/null +++ b/arbitrator/prover/test-cases/forward/forward.wat @@ -0,0 +1,8 @@ + +(module + (import "target" "arbitrator_forward__add" (func $add (param i32 i32) (result i32))) + (import "target" "arbitrator_forward__sub" (func $sub (param i32 i32) (result i32))) + (import "target" "arbitrator_forward__mul" (func $mul (param i32 i32) (result i32))) + (export "forward__add" (func $add)) + (export "forward__sub" (func $sub)) + (export "forward__mul" (func $mul))) diff --git a/arbitrator/prover/test-cases/forward/target.wat b/arbitrator/prover/test-cases/forward/target.wat new file mode 100644 index 00000000000..0779eb753d5 --- /dev/null +++ b/arbitrator/prover/test-cases/forward/target.wat @@ -0,0 +1,27 @@ + +(module + (import "env" "wavm_caller_load8" (func $load (param i32) (result i32))) + (import "env" "wavm_caller_store8" (func $store (param i32 i32))) + (func (export "target__add") (param i32 i32) (result i32) + call $write_caller + local.get 0 + local.get 1 + i32.add) + (func (export "target__sub") (param i32 i32) (result i32) + call $write_caller + local.get 0 + local.get 1 + i32.sub) + (func (export "target__mul") (param i32 i32) (result i32) + call $write_caller + local.get 0 + local.get 1 + i32.mul) + (func $write_caller (export "write_caller") + ;; increment the value at address 0xa4b in the caller + i32.const 0xa4b + i32.const 0xa4b + call $load + i32.const 1 + i32.add + call $store)) diff --git a/arbitrator/stylus/src/host.rs b/arbitrator/stylus/src/host.rs index fbe5657c5f4..1afc1b4e51e 100644 --- a/arbitrator/stylus/src/host.rs +++ b/arbitrator/stylus/src/host.rs @@ -13,6 +13,7 @@ use arbutil::{ }; use caller_env::GuestPtr; use eyre::Result; +use prover::value::Value; use std::{ fmt::Display, mem::{self, MaybeUninit}, @@ -440,76 +441,26 @@ pub(crate) fn pay_for_memory_grow>( hostio!(env, pay_for_memory_grow(pages)) } -pub(crate) mod console { - use super::*; - - pub(crate) fn log_txt>( - mut env: WasmEnvMut, - ptr: GuestPtr, - len: u32, - ) -> MaybeEscape { - hostio!(env, console_log_text(ptr, len)) - } - - pub(crate) fn log_i32>( - mut env: WasmEnvMut, - value: u32, - ) -> MaybeEscape { - hostio!(env, console_log(value)) - } - - pub(crate) fn tee_i32>( - mut env: WasmEnvMut, - value: u32, - ) -> Result { - hostio!(env, console_tee(value)) - } - - pub(crate) fn log_i64>( - mut env: WasmEnvMut, - value: u64, - ) -> MaybeEscape { - hostio!(env, console_log(value)) - } - - pub(crate) fn tee_i64>( - mut env: WasmEnvMut, - value: u64, - ) -> Result { - hostio!(env, console_tee(value)) - } - - pub(crate) fn log_f32>( - mut env: WasmEnvMut, - value: f32, - ) -> MaybeEscape { - hostio!(env, console_log(value)) - } - - pub(crate) fn tee_f32>( - mut env: WasmEnvMut, - value: f32, - ) -> Result { - hostio!(env, console_tee(value)) - } - - pub(crate) fn log_f64>( - mut env: WasmEnvMut, - value: f64, - ) -> MaybeEscape { - hostio!(env, console_log(value)) - } - - pub(crate) fn tee_f64>( - mut env: WasmEnvMut, - value: f64, - ) -> Result { - hostio!(env, console_tee(value)) - } +pub(crate) fn console_log_text>( + mut env: WasmEnvMut, + ptr: GuestPtr, + len: u32, +) -> MaybeEscape { + hostio!(env, console_log_text(ptr, len)) } -pub(crate) mod debug { - use super::*; +pub(crate) fn console_log, T: Into>( + mut env: WasmEnvMut, + value: T, +) -> MaybeEscape { + hostio!(env, console_log(value)) +} - pub(crate) fn null_host>(_: WasmEnvMut) {} +pub(crate) fn console_tee, T: Into + Copy>( + mut env: WasmEnvMut, + value: T, +) -> Result { + hostio!(env, console_tee(value)) } + +pub(crate) fn null_host>(_: WasmEnvMut) {} diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs index a252b60a01d..052bfc7229b 100644 --- a/arbitrator/stylus/src/lib.rs +++ b/arbitrator/stylus/src/lib.rs @@ -139,7 +139,8 @@ impl RustBytes { pub unsafe extern "C" fn stylus_activate( wasm: GoSliceData, page_limit: u16, - version: u16, + stylus_version: u16, + arbos_version_for_gas: u64, debug: bool, output: *mut RustBytes, codehash: *const Bytes32, @@ -153,7 +154,15 @@ pub unsafe extern "C" fn stylus_activate( let codehash = &*codehash; let gas = &mut *gas; - let (module, info) = match native::activate(wasm, codehash, version, page_limit, debug, gas) { + let (module, info) = match native::activate( + wasm, + codehash, + stylus_version, + arbos_version_for_gas, + page_limit, + debug, + gas, + ) { Ok(val) => val, Err(err) => return output.write_err(err), }; diff --git a/arbitrator/stylus/src/native.rs b/arbitrator/stylus/src/native.rs index 7a82314fbca..516c6602e73 100644 --- a/arbitrator/stylus/src/native.rs +++ b/arbitrator/stylus/src/native.rs @@ -33,7 +33,7 @@ use std::{ ops::{Deref, DerefMut}, }; use wasmer::{ - AsStoreMut, Function, FunctionEnv, Imports, Instance, Memory, Module, Pages, Store, Target, + imports, AsStoreMut, Function, FunctionEnv, Instance, Memory, Module, Pages, Store, Target, TypedFunction, Value, WasmTypeList, }; use wasmer_vm::VMExtern; @@ -151,58 +151,68 @@ impl> NativeInstance { fn from_module(module: Module, mut store: Store, env: WasmEnv) -> Result { let debug_funcs = env.compile.debug.debug_funcs; let func_env = FunctionEnv::new(&mut store, env); - let mut imports = Imports::new(); macro_rules! func { - ($rust_mod:path, $func:ident) => {{ - use $rust_mod as rust_mod; - Function::new_typed_with_env(&mut store, &func_env, rust_mod::$func) - }}; - } - macro_rules! define_imports { - ($($wasm_mod:literal => $rust_mod:path { $( $import:ident ),* $(,)? },)* $(,)?) => { - $( - $( - define_imports!(@@imports $wasm_mod, func!($rust_mod, $import), $import, "arbitrator_forward__"); - )* - )* - }; - (@@imports $wasm_mod:literal, $func:expr, $import:ident, $($p:expr),*) => { - define_imports!(@imports $wasm_mod, $func, $import, $($p),*, ""); - }; - (@imports $wasm_mod:literal, $func:expr, $import:ident, $($p:expr),*) => { - $( - imports.define($wasm_mod, concat!($p, stringify!($import)), $func); - )* + ($func:expr) => { + Function::new_typed_with_env(&mut store, &func_env, $func) }; } - define_imports!( - "vm_hooks" => host { - read_args, write_result, exit_early, - storage_load_bytes32, storage_cache_bytes32, storage_flush_cache, transient_load_bytes32, transient_store_bytes32, - call_contract, delegate_call_contract, static_call_contract, create1, create2, read_return_data, return_data_size, - emit_log, - account_balance, account_code, account_codehash, account_code_size, - evm_gas_left, evm_ink_left, - block_basefee, chainid, block_coinbase, block_gas_limit, block_number, block_timestamp, - contract_address, - math_div, math_mod, math_pow, math_add_mod, math_mul_mod, - msg_reentrant, msg_sender, msg_value, - tx_gas_price, tx_ink_price, tx_origin, - pay_for_memory_grow, - native_keccak256, + let mut imports = imports! { + "vm_hooks" => { + "read_args" => func!(host::read_args), + "write_result" => func!(host::write_result), + "exit_early" => func!(host::exit_early), + "storage_load_bytes32" => func!(host::storage_load_bytes32), + "storage_cache_bytes32" => func!(host::storage_cache_bytes32), + "storage_flush_cache" => func!(host::storage_flush_cache), + "transient_load_bytes32" => func!(host::transient_load_bytes32), + "transient_store_bytes32" => func!(host::transient_store_bytes32), + "call_contract" => func!(host::call_contract), + "delegate_call_contract" => func!(host::delegate_call_contract), + "static_call_contract" => func!(host::static_call_contract), + "create1" => func!(host::create1), + "create2" => func!(host::create2), + "read_return_data" => func!(host::read_return_data), + "return_data_size" => func!(host::return_data_size), + "emit_log" => func!(host::emit_log), + "account_balance" => func!(host::account_balance), + "account_code" => func!(host::account_code), + "account_codehash" => func!(host::account_codehash), + "account_code_size" => func!(host::account_code_size), + "evm_gas_left" => func!(host::evm_gas_left), + "evm_ink_left" => func!(host::evm_ink_left), + "block_basefee" => func!(host::block_basefee), + "chainid" => func!(host::chainid), + "block_coinbase" => func!(host::block_coinbase), + "block_gas_limit" => func!(host::block_gas_limit), + "block_number" => func!(host::block_number), + "block_timestamp" => func!(host::block_timestamp), + "contract_address" => func!(host::contract_address), + "math_div" => func!(host::math_div), + "math_mod" => func!(host::math_mod), + "math_pow" => func!(host::math_pow), + "math_add_mod" => func!(host::math_add_mod), + "math_mul_mod" => func!(host::math_mul_mod), + "msg_reentrant" => func!(host::msg_reentrant), + "msg_sender" => func!(host::msg_sender), + "msg_value" => func!(host::msg_value), + "tx_gas_price" => func!(host::tx_gas_price), + "tx_ink_price" => func!(host::tx_ink_price), + "tx_origin" => func!(host::tx_origin), + "pay_for_memory_grow" => func!(host::pay_for_memory_grow), + "native_keccak256" => func!(host::native_keccak256), }, - ); + }; if debug_funcs { - define_imports!( - "console" => host::console { - log_txt, - log_i32, log_i64, log_f32, log_f64, - tee_i32, tee_i64, tee_f32, tee_f64, - }, - "debug" => host::debug { - null_host, - }, - ); + imports.define("console", "log_txt", func!(host::console_log_text)); + imports.define("console", "log_i32", func!(host::console_log::)); + imports.define("console", "log_i64", func!(host::console_log::)); + imports.define("console", "log_f32", func!(host::console_log::)); + imports.define("console", "log_f64", func!(host::console_log::)); + imports.define("console", "tee_i32", func!(host::console_tee::)); + imports.define("console", "tee_i64", func!(host::console_tee::)); + imports.define("console", "tee_f32", func!(host::console_tee::)); + imports.define("console", "tee_f64", func!(host::console_tee::)); + imports.define("debug", "null_host", func!(host::null_host)); } let instance = Instance::new(&mut store, &module, &imports)?; let exports = &instance.exports; @@ -341,8 +351,86 @@ impl> StartlessMachine for NativeInstance { } pub fn module(wasm: &[u8], compile: CompileConfig, target: Target) -> Result> { - let store = compile.store(target); + let mut store = compile.store(target); let module = Module::new(&store, wasm)?; + macro_rules! stub { + (u8 <- $($types:tt)+) => { + Function::new_typed(&mut store, $($types)+ -> u8 { panic!("incomplete import") }) + }; + (u32 <- $($types:tt)+) => { + Function::new_typed(&mut store, $($types)+ -> u32 { panic!("incomplete import") }) + }; + (u64 <- $($types:tt)+) => { + Function::new_typed(&mut store, $($types)+ -> u64 { panic!("incomplete import") }) + }; + (f32 <- $($types:tt)+) => { + Function::new_typed(&mut store, $($types)+ -> f32 { panic!("incomplete import") }) + }; + (f64 <- $($types:tt)+) => { + Function::new_typed(&mut store, $($types)+ -> f64 { panic!("incomplete import") }) + }; + ($($types:tt)+) => { + Function::new_typed(&mut store, $($types)+ -> () { panic!("incomplete import") }) + }; + } + let mut imports = imports! { + "vm_hooks" => { + "read_args" => stub!(|_: u32|), + "write_result" => stub!(|_: u32, _: u32|), + "exit_early" => stub!(|_: u32|), + "storage_load_bytes32" => stub!(|_: u32, _: u32|), + "storage_cache_bytes32" => stub!(|_: u32, _: u32|), + "storage_flush_cache" => stub!(|_: u32|), + "transient_load_bytes32" => stub!(|_: u32, _: u32|), + "transient_store_bytes32" => stub!(|_: u32, _: u32|), + "call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u32, _: u64, _: u32|), + "delegate_call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u64, _: u32|), + "static_call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u64, _: u32|), + "create1" => stub!(|_: u32, _: u32, _: u32, _: u32, _: u32|), + "create2" => stub!(|_: u32, _: u32, _: u32, _: u32, _: u32, _: u32|), + "read_return_data" => stub!(u32 <- |_: u32, _: u32, _: u32|), + "return_data_size" => stub!(u32 <- ||), + "emit_log" => stub!(|_: u32, _: u32, _: u32|), + "account_balance" => stub!(|_: u32, _: u32|), + "account_code" => stub!(u32 <- |_: u32, _: u32, _: u32, _: u32|), + "account_codehash" => stub!(|_: u32, _: u32|), + "account_code_size" => stub!(u32 <- |_: u32|), + "evm_gas_left" => stub!(u64 <- ||), + "evm_ink_left" => stub!(u64 <- ||), + "block_basefee" => stub!(|_: u32|), + "chainid" => stub!(u64 <- ||), + "block_coinbase" => stub!(|_: u32|), + "block_gas_limit" => stub!(u64 <- ||), + "block_number" => stub!(u64 <- ||), + "block_timestamp" => stub!(u64 <- ||), + "contract_address" => stub!(|_: u32|), + "math_div" => stub!(|_: u32, _: u32|), + "math_mod" => stub!(|_: u32, _: u32|), + "math_pow" => stub!(|_: u32, _: u32|), + "math_add_mod" => stub!(|_: u32, _: u32, _: u32|), + "math_mul_mod" => stub!(|_: u32, _: u32, _: u32|), + "msg_reentrant" => stub!(u32 <- ||), + "msg_sender" => stub!(|_: u32|), + "msg_value" => stub!(|_: u32|), + "tx_gas_price" => stub!(|_: u32|), + "tx_ink_price" => stub!(u32 <- ||), + "tx_origin" => stub!(|_: u32|), + "pay_for_memory_grow" => stub!(|_: u16|), + "native_keccak256" => stub!(|_: u32, _: u32, _: u32|), + }, + }; + if compile.debug.debug_funcs { + imports.define("console", "log_txt", stub!(|_: u32, _: u32|)); + imports.define("console", "log_i32", stub!(|_: u32|)); + imports.define("console", "log_i64", stub!(|_: u64|)); + imports.define("console", "log_f32", stub!(|_: f32|)); + imports.define("console", "log_f64", stub!(|_: f64|)); + imports.define("console", "tee_i32", stub!(u32 <- |_: u32|)); + imports.define("console", "tee_i64", stub!(u64 <- |_: u64|)); + imports.define("console", "tee_f32", stub!(f32 <- |_: f32|)); + imports.define("console", "tee_f64", stub!(f64 <- |_: f64|)); + imports.define("debug", "null_host", stub!(||)); + } let module = module.serialize()?; Ok(module.to_vec()) @@ -351,13 +439,21 @@ pub fn module(wasm: &[u8], compile: CompileConfig, target: Target) -> Result Result<(ProverModule, StylusData)> { - let (module, stylus_data) = - ProverModule::activate(wasm, codehash, version, page_limit, debug, gas)?; + let (module, stylus_data) = ProverModule::activate( + wasm, + codehash, + stylus_version, + arbos_version_for_gas, + page_limit, + debug, + gas, + )?; Ok((module, stylus_data)) } diff --git a/arbitrator/stylus/src/test/api.rs b/arbitrator/stylus/src/test/api.rs index 5d9f625e5e4..66d600a6f78 100644 --- a/arbitrator/stylus/src/test/api.rs +++ b/arbitrator/stylus/src/test/api.rs @@ -68,7 +68,7 @@ impl TestEvmApi { } impl EvmApi for TestEvmApi { - fn get_bytes32(&mut self, key: Bytes32) -> (Bytes32, u64) { + fn get_bytes32(&mut self, key: Bytes32, _evm_api_gas_to_use: u64) -> (Bytes32, u64) { let storage = &mut self.storage.lock(); let storage = storage.get_mut(&self.program).unwrap(); let value = storage.get(&key).cloned().unwrap_or_default(); diff --git a/arbitrator/stylus/src/test/mod.rs b/arbitrator/stylus/src/test/mod.rs index 42c94465643..00c9c62ae4b 100644 --- a/arbitrator/stylus/src/test/mod.rs +++ b/arbitrator/stylus/src/test/mod.rs @@ -164,7 +164,6 @@ fn new_test_machine(path: &str, compile: &CompileConfig) -> Result { HashMap::default(), Arc::new(|_, _, _| panic!("tried to read preimage")), Some(stylus_data), - false, )?; mach.set_ink(u64::MAX); mach.set_stack(u32::MAX); diff --git a/arbitrator/stylus/src/test/native.rs b/arbitrator/stylus/src/test/native.rs index 503e5875fea..9669932a03c 100644 --- a/arbitrator/stylus/src/test/native.rs +++ b/arbitrator/stylus/src/test/native.rs @@ -381,7 +381,7 @@ fn test_storage() -> Result<()> { let (mut native, mut evm) = TestInstance::new_with_evm(filename, &compile, config)?; run_native(&mut native, &store_args, ink)?; - assert_eq!(evm.get_bytes32(key.into()).0, Bytes32(value)); + assert_eq!(evm.get_bytes32(key.into(), 0).0, Bytes32(value)); assert_eq!(run_native(&mut native, &load_args, ink)?, value); let mut machine = Machine::from_user_path(Path::new(filename), &compile)?; @@ -465,7 +465,7 @@ fn test_calls() -> Result<()> { run_native(&mut native, &args, ink)?; for (key, value) in slots { - assert_eq!(evm.get_bytes32(key).0, value); + assert_eq!(evm.get_bytes32(key, 0).0, value); } Ok(()) } diff --git a/arbitrator/tools/module_roots/src/main.rs b/arbitrator/tools/module_roots/src/main.rs index fc22bed5fa7..32e47648476 100644 --- a/arbitrator/tools/module_roots/src/main.rs +++ b/arbitrator/tools/module_roots/src/main.rs @@ -39,7 +39,6 @@ fn main() -> Result<()> { GlobalState::default(), HashMap::default(), Arc::new(|_, _, _| panic!("tried to read preimage")), - false, )?; let mut stylus = vec![]; diff --git a/arbitrator/wasm-libraries/Cargo.lock b/arbitrator/wasm-libraries/Cargo.lock index b234424f690..7620ff538b8 100644 --- a/arbitrator/wasm-libraries/Cargo.lock +++ b/arbitrator/wasm-libraries/Cargo.lock @@ -2,21 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "addr2line" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" -dependencies = [ - "gimli 0.29.0", -] - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - [[package]] name = "ahash" version = "0.7.8" @@ -106,21 +91,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" -[[package]] -name = "backtrace" -version = "0.3.73" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" -dependencies = [ - "addr2line", - "cc", - "cfg-if 1.0.0", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - [[package]] name = "bincode" version = "1.3.3" @@ -217,12 +187,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.7.0" @@ -239,15 +203,6 @@ dependencies = [ "rand_pcg", ] -[[package]] -name = "cc" -version = "1.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" -dependencies = [ - "shlex", -] - [[package]] name = "cfg-if" version = "0.1.10" @@ -281,19 +236,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" -[[package]] -name = "corosensei" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80128832c58ea9cbd041d2a759ec449224487b2c1e400453d99d244eead87a8e" -dependencies = [ - "autocfg", - "cfg-if 1.0.0", - "libc", - "scopeguard", - "windows-sys 0.33.0", -] - [[package]] name = "cpufeatures" version = "0.2.12" @@ -303,123 +245,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cranelift-bforest" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2ab4512dfd3a6f4be184403a195f76e81a8a9f9e6c898e19d2dc3ce20e0115" -dependencies = [ - "cranelift-entity", -] - -[[package]] -name = "cranelift-codegen" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98b022ed2a5913a38839dfbafe6cf135342661293b08049843362df4301261dc" -dependencies = [ - "arrayvec", - "bumpalo", - "cranelift-bforest", - "cranelift-codegen-meta", - "cranelift-codegen-shared", - "cranelift-egraph", - "cranelift-entity", - "cranelift-isle", - "gimli 0.26.2", - "log", - "regalloc2", - "smallvec", - "target-lexicon", -] - -[[package]] -name = "cranelift-codegen-meta" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "639307b45434ad112a98f8300c0f0ab085cbefcd767efcdef9ef19d4c0756e74" -dependencies = [ - "cranelift-codegen-shared", -] - -[[package]] -name = "cranelift-codegen-shared" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "278e52e29c53fcf32431ef08406c295699a70306d05a0715c5b1bf50e33a9ab7" - -[[package]] -name = "cranelift-egraph" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624b54323b06e675293939311943ba82d323bb340468ce1889be5da7932c8d73" -dependencies = [ - "cranelift-entity", - "fxhash", - "hashbrown 0.12.3", - "indexmap 1.9.3", - "log", - "smallvec", -] - -[[package]] -name = "cranelift-entity" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a59bcbca89c3f1b70b93ab3cbba5e5e0cbf3e63dadb23c7525cb142e21a9d4c" - -[[package]] -name = "cranelift-frontend" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d70abacb8cfef3dc8ff7e8836e9c1d70f7967dfdac824a4cd5e30223415aca6" -dependencies = [ - "cranelift-codegen", - "log", - "smallvec", - "target-lexicon", -] - -[[package]] -name = "cranelift-isle" -version = "0.91.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "393bc73c451830ff8dbb3a07f61843d6cb41a084f9996319917c0b291ed785bb" - -[[package]] -name = "crossbeam-deque" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-queue" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" - [[package]] name = "crunchy" version = "0.2.2" @@ -505,19 +330,6 @@ dependencies = [ "syn 2.0.72", ] -[[package]] -name = "dashmap" -version = "5.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" -dependencies = [ - "cfg-if 1.0.0", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - [[package]] name = "derivative" version = "2.2.0" @@ -644,33 +456,26 @@ dependencies = [ "once_cell", ] -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "forward" +version = "0.1.0" +dependencies = [ + "eyre", + "structopt", +] + [[package]] name = "funty" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -692,23 +497,6 @@ dependencies = [ "wasi", ] -[[package]] -name = "gimli" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" -dependencies = [ - "fallible-iterator", - "indexmap 1.9.3", - "stable_deref_trait", -] - -[[package]] -name = "gimli" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" - [[package]] name = "hashbrown" version = "0.12.3" @@ -807,15 +595,6 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" -[[package]] -name = "js-sys" -version = "0.3.70" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" -dependencies = [ - "wasm-bindgen", -] - [[package]] name = "keccak" version = "0.1.5" @@ -853,12 +632,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "log" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" - [[package]] name = "lru" version = "0.12.4" @@ -868,57 +641,12 @@ dependencies = [ "hashbrown 0.14.5", ] -[[package]] -name = "mach" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" -dependencies = [ - "libc", -] - -[[package]] -name = "mach2" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" -dependencies = [ - "libc", -] - [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" -[[package]] -name = "memmap2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" -dependencies = [ - "libc", -] - -[[package]] -name = "memmap2" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d28bba84adfe6646737845bc5ebbfa2c08424eb1c37e94a1fd2a82adb56a872" -dependencies = [ - "libc", -] - -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - [[package]] name = "memory_units" version = "0.4.0" @@ -931,15 +659,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" -dependencies = [ - "adler", -] - [[package]] name = "more-asserts" version = "0.2.2" @@ -1072,15 +791,6 @@ dependencies = [ "syn 2.0.72", ] -[[package]] -name = "object" -version = "0.36.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" -dependencies = [ - "memchr", -] - [[package]] name = "once_cell" version = "1.19.0" @@ -1122,12 +832,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "pin-project-lite" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" - [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -1208,7 +912,6 @@ dependencies = [ "smallvec", "static_assertions", "structopt", - "wasmer", "wasmer-types", "wasmparser", "wat", @@ -1273,26 +976,6 @@ dependencies = [ "rand_core", ] -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "redox_syscall" version = "0.5.3" @@ -1302,30 +985,6 @@ dependencies = [ "bitflags 2.6.0", ] -[[package]] -name = "regalloc2" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300d4fbfb40c1c66a78ba3ddd41c1110247cf52f97b87d0f2fc9209bd49b030c" -dependencies = [ - "fxhash", - "log", - "slice-group-by", - "smallvec", -] - -[[package]] -name = "region" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b6ebd13bc009aef9cd476c1310d49ac354d36e240cf1bd753290f3dc7199a7" -dependencies = [ - "bitflags 1.3.2", - "libc", - "mach2", - "windows-sys 0.52.0", -] - [[package]] name = "rend" version = "0.4.2" @@ -1416,12 +1075,6 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" -[[package]] -name = "self_cell" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d369a96f978623eb3dc28807c4852d6cc617fed53da5d3c400feff1ef34a714a" - [[package]] name = "semver" version = "1.0.23" @@ -1437,17 +1090,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-wasm-bindgen" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b4c031cd0d9014307d82b8abf653c0290fbdaeb4c02d00c63cf52f728628bf" -dependencies = [ - "js-sys", - "serde", - "wasm-bindgen", -] - [[package]] name = "serde_derive" version = "1.0.204" @@ -1539,22 +1181,6 @@ dependencies = [ "keccak", ] -[[package]] -name = "shared-buffer" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6c99835bad52957e7aa241d3975ed17c1e5f8c92026377d117a606f36b84b16" -dependencies = [ - "bytes", - "memmap2 0.6.2", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "simdutf8" version = "0.1.4" @@ -1567,12 +1193,6 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" -[[package]] -name = "slice-group-by" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" - [[package]] name = "smallvec" version = "1.13.2" @@ -1582,12 +1202,6 @@ dependencies = [ "serde", ] -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - [[package]] name = "static_assertions" version = "1.1.0" @@ -1734,37 +1348,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tracing" -version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.72", -] - -[[package]] -name = "tracing-core" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" -dependencies = [ - "once_cell", -] - [[package]] name = "typenum" version = "1.17.0" @@ -1862,147 +1445,13 @@ dependencies = [ "wee_alloc", ] -[[package]] -name = "wasm-bindgen" -version = "0.2.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" -dependencies = [ - "cfg-if 1.0.0", - "once_cell", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.72", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.72", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" - [[package]] name = "wasm-encoder" -version = "0.32.0" +version = "0.215.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba64e81215916eaeb48fee292f29401d69235d62d8b8fd92a7b2844ec5ae5f7" -dependencies = [ - "leb128", -] - -[[package]] -name = "wasmer" -version = "4.2.8" +checksum = "4fb56df3e06b8e6b77e37d2969a50ba51281029a9aeb3855e76b7f49b6418847" dependencies = [ - "bytes", - "cfg-if 1.0.0", - "derivative", - "indexmap 1.9.3", - "js-sys", - "more-asserts", - "rustc-demangle", - "serde", - "serde-wasm-bindgen", - "shared-buffer", - "target-lexicon", - "thiserror", - "tracing", - "wasm-bindgen", - "wasmer-compiler", - "wasmer-compiler-cranelift", - "wasmer-derive", - "wasmer-types", - "wasmer-vm", - "wat", - "winapi", -] - -[[package]] -name = "wasmer-compiler" -version = "4.2.8" -dependencies = [ - "backtrace", - "bytes", - "cfg-if 1.0.0", - "enum-iterator 0.7.0", - "enumset", - "lazy_static", "leb128", - "memmap2 0.5.10", - "more-asserts", - "region", - "rkyv", - "self_cell", - "shared-buffer", - "smallvec", - "thiserror", - "wasmer-types", - "wasmer-vm", - "wasmparser", - "winapi", -] - -[[package]] -name = "wasmer-compiler-cranelift" -version = "4.2.8" -dependencies = [ - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "gimli 0.26.2", - "more-asserts", - "rayon", - "smallvec", - "target-lexicon", - "tracing", - "wasmer-compiler", - "wasmer-types", -] - -[[package]] -name = "wasmer-derive" -version = "4.2.8" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", ] [[package]] @@ -2019,32 +1468,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "wasmer-vm" -version = "4.2.8" -dependencies = [ - "backtrace", - "cc", - "cfg-if 1.0.0", - "corosensei", - "crossbeam-queue", - "dashmap", - "derivative", - "enum-iterator 0.7.0", - "fnv", - "indexmap 1.9.3", - "lazy_static", - "libc", - "mach", - "memoffset", - "more-asserts", - "region", - "scopeguard", - "thiserror", - "wasmer-types", - "winapi", -] - [[package]] name = "wasmparser" version = "0.121.2" @@ -2058,10 +1481,11 @@ dependencies = [ [[package]] name = "wast" -version = "64.0.0" +version = "215.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a259b226fd6910225aa7baeba82f9d9933b6d00f2ce1b49b80fa4214328237cc" +checksum = "1ff1d00d893593249e60720be04a7c1f42f1c4dc3806a2869f4e66ab61eb54cb" dependencies = [ + "bumpalo", "leb128", "memchr", "unicode-width", @@ -2070,9 +1494,9 @@ dependencies = [ [[package]] name = "wat" -version = "1.0.71" +version = "1.215.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53253d920ab413fca1c7dc2161d601c79b4fdf631d0ba51dd4343bf9b556c3f6" +checksum = "670bf4d9c8cf76ae242d70ded47c546525b6dafaa6871f9bcb065344bf2b4e3d" dependencies = [ "wast", ] @@ -2111,28 +1535,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43dbb096663629518eb1dfa72d80243ca5a6aca764cae62a2df70af760a9be75" -dependencies = [ - "windows_aarch64_msvc 0.33.0", - "windows_i686_gnu 0.33.0", - "windows_i686_msvc 0.33.0", - "windows_x86_64_gnu 0.33.0", - "windows_x86_64_msvc 0.33.0", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -2140,13 +1542,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", + "windows_aarch64_msvc", + "windows_i686_gnu", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", + "windows_i686_msvc", + "windows_x86_64_gnu", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.52.6", + "windows_x86_64_msvc", ] [[package]] @@ -2155,24 +1557,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_msvc" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd761fd3eb9ab8cc1ed81e56e567f02dd82c4c837e48ac3b2181b9ffc5060807" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_i686_gnu" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab0cf703a96bab2dc0c02c0fa748491294bf9b7feb27e1f4f96340f208ada0e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -2185,24 +1575,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_msvc" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cfdbe89cc9ad7ce618ba34abc34bbb6c36d99e96cae2245b7943cd75ee773d0" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_x86_64_gnu" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4dd9b0c0e9ece7bb22e84d70d01b71c6d6248b81a3c60d11869451b4cb24784" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -2215,12 +1593,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_msvc" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1e4aa646495048ec7f3ffddc411e1d829c026a2ec62b39da15c1055e406eaa" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/arbitrator/wasm-libraries/Cargo.toml b/arbitrator/wasm-libraries/Cargo.toml index 2fd585ee7df..837df8f4da5 100644 --- a/arbitrator/wasm-libraries/Cargo.toml +++ b/arbitrator/wasm-libraries/Cargo.toml @@ -7,5 +7,6 @@ members = [ "user-host-trait", "user-test", "program-exec", + "forward", ] resolver = "2" diff --git a/arbitrator/wasm-libraries/forward/.gitignore b/arbitrator/wasm-libraries/forward/.gitignore new file mode 100644 index 00000000000..40da2042b75 --- /dev/null +++ b/arbitrator/wasm-libraries/forward/.gitignore @@ -0,0 +1 @@ +**.wat diff --git a/arbitrator/wasm-libraries/forward/Cargo.toml b/arbitrator/wasm-libraries/forward/Cargo.toml new file mode 100644 index 00000000000..73ed9d8827b --- /dev/null +++ b/arbitrator/wasm-libraries/forward/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "forward" +version = "0.1.0" +edition = "2021" + +[dependencies] +eyre = "0.6.5" +structopt = "0.3.26" diff --git a/arbitrator/wasm-libraries/forward/src/main.rs b/arbitrator/wasm-libraries/forward/src/main.rs new file mode 100644 index 00000000000..f978a8723b0 --- /dev/null +++ b/arbitrator/wasm-libraries/forward/src/main.rs @@ -0,0 +1,207 @@ +// Copyright 2022-2024, Offchain Labs, Inc. +// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE + +use eyre::Result; +use std::{fs::File, io::Write, path::PathBuf}; +use structopt::StructOpt; + +/// order matters! +const HOSTIOS: [[&str; 3]; 42] = [ + ["read_args", "i32", ""], + ["write_result", "i32 i32", ""], + ["exit_early", "i32", ""], + ["storage_load_bytes32", "i32 i32", ""], + ["storage_cache_bytes32", "i32 i32", ""], + ["storage_flush_cache", "i32", ""], + ["transient_load_bytes32", "i32 i32", ""], + ["transient_store_bytes32", "i32 i32", ""], + ["call_contract", "i32 i32 i32 i32 i64 i32", "i32"], + ["delegate_call_contract", "i32 i32 i32 i64 i32", "i32"], + ["static_call_contract", "i32 i32 i32 i64 i32", "i32"], + ["create1", "i32 i32 i32 i32 i32", ""], + ["create2", "i32 i32 i32 i32 i32 i32", ""], + ["read_return_data", "i32 i32 i32", "i32"], + ["return_data_size", "", "i32"], + ["emit_log", "i32 i32 i32", ""], + ["account_balance", "i32 i32", ""], + ["account_code", "i32 i32 i32 i32", "i32"], + ["account_code_size", "i32", "i32"], + ["account_codehash", "i32 i32", ""], + ["evm_gas_left", "", "i64"], + ["evm_ink_left", "", "i64"], + ["block_basefee", "i32", ""], + ["chainid", "", "i64"], + ["block_coinbase", "i32", ""], + ["block_gas_limit", "", "i64"], + ["block_number", "", "i64"], + ["block_timestamp", "", "i64"], + ["contract_address", "i32", ""], + ["math_div", "i32 i32", ""], + ["math_mod", "i32 i32", ""], + ["math_pow", "i32 i32", ""], + ["math_add_mod", "i32 i32 i32", ""], + ["math_mul_mod", "i32 i32 i32", ""], + ["msg_reentrant", "", "i32"], + ["msg_sender", "i32", ""], + ["msg_value", "i32", ""], + ["native_keccak256", "i32 i32 i32", ""], + ["tx_gas_price", "i32", ""], + ["tx_ink_price", "", "i32"], + ["tx_origin", "i32", ""], + ["pay_for_memory_grow", "i32", ""], +]; + +#[derive(StructOpt)] +#[structopt(name = "arbitrator-prover")] +struct Opts { + #[structopt(long)] + path: PathBuf, + #[structopt(long)] + stub: bool, +} + +fn main() -> Result<()> { + let opts = Opts::from_args(); + let file = &mut File::options() + .create(true) + .write(true) + .truncate(true) + .open(opts.path)?; + + match opts.stub { + true => forward_stub(file), + false => forward(file), + } +} + +fn forward(file: &mut File) -> Result<()> { + macro_rules! wln { + ($($text:tt)*) => { + writeln!(file, $($text)*)?; + }; + } + let s = " "; + + wln!( + ";; Copyright 2022-2023, Offchain Labs, Inc.\n\ + ;; For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE\n\ + ;; This file is auto-generated.\n\ + \n\ + (module" + ); + + macro_rules! group { + ($list:expr, $kind:expr) => { + (!$list.is_empty()) + .then(|| format!(" ({} {})", $kind, $list)) + .unwrap_or_default() + }; + } + + wln!("{s};; symbols to re-export"); + for [name, ins, outs] in HOSTIOS { + let params = group!(ins, "param"); + let result = group!(outs, "result"); + wln!( + r#"{s}(import "user_host" "arbitrator_forward__{name}" (func ${name}{params}{result}))"# + ); + } + wln!(); + + wln!("{s};; reserved offsets for future user_host imports"); + for i in HOSTIOS.len()..512 { + wln!("{s}(func $reserved_{i} unreachable)"); + } + wln!(); + + wln!( + "{s};; allows user_host to request a trap\n\ + {s}(global $trap (mut i32) (i32.const 0))\n\ + {s}(func $check\n\ + {s}{s}global.get $trap ;; see if set\n\ + {s}{s}(global.set $trap (i32.const 0)) ;; reset the flag\n\ + {s}{s}(if (then (unreachable)))\n\ + {s})\n\ + {s}(func (export \"forward__set_trap\")\n\ + {s}{s}(global.set $trap (i32.const 1))\n\ + {s})\n" + ); + + wln!("{s};; user linkage"); + for [name, ins, outs] in HOSTIOS { + let params = group!(ins, "param"); + let result = group!(outs, "result"); + wln!("{s}(func (export \"vm_hooks__{name}\"){params}{result}"); + + let gets = (1 + ins.len()) / 4; + for i in 0..gets { + wln!("{s}{s}local.get {i}"); + } + + wln!( + "{s}{s}call ${name}\n\ + {s}{s}call $check\n\ + {s})" + ); + } + + wln!(")"); + Ok(()) +} + +fn forward_stub(file: &mut File) -> Result<()> { + macro_rules! wln { + ($($text:tt)*) => { + writeln!(file, $($text)*)?; + }; + } + let s = " "; + + wln!( + ";; Copyright 2022-2023, Offchain Labs, Inc.\n\ + ;; For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE\n\ + ;; This file is auto-generated.\n\ + \n\ + (module" + ); + + macro_rules! group { + ($list:expr, $kind:expr) => { + (!$list.is_empty()) + .then(|| format!(" ({} {})", $kind, $list)) + .unwrap_or_default() + }; + } + + wln!("{s};; stubs for the symbols we re-export"); + for [name, ins, outs] in HOSTIOS { + let params = group!(ins, "param"); + let result = group!(outs, "result"); + wln!("{s}(func ${name}{params}{result} unreachable)"); + } + wln!(); + + wln!("{s};; reserved offsets for future user_host imports"); + for i in HOSTIOS.len()..512 { + wln!("{s}(func $reserved_{i} unreachable)"); + } + wln!(); + + wln!( + "{s};; allows user_host to request a trap\n\ + {s}(global $trap (mut i32) (i32.const 0))\n\ + {s}(func $check unreachable)\n\ + {s};; stub for the forward__set_trap function\n\ + {s}(func $forward__set_trap unreachable)" + ); + + wln!("{s};; user linkage"); + for [name, ins, outs] in HOSTIOS { + let params = group!(ins, "param"); + let result = group!(outs, "result"); + wln!("{s}(func (export \"vm_hooks__{name}\"){params}{result} unreachable)"); + } + + wln!(")"); + Ok(()) +} diff --git a/arbitrator/wasm-libraries/user-host-trait/src/lib.rs b/arbitrator/wasm-libraries/user-host-trait/src/lib.rs index 37af85c382c..12a6bdbed2a 100644 --- a/arbitrator/wasm-libraries/user-host-trait/src/lib.rs +++ b/arbitrator/wasm-libraries/user-host-trait/src/lib.rs @@ -8,7 +8,7 @@ use arbutil::{ api::{DataReader, EvmApi}, storage::StorageCache, user::UserOutcomeKind, - EvmData, + EvmData, ARBOS_VERSION_STYLUS_CHARGING_FIXES, }, pricing::{self, EVM_API_INK, HOSTIO_INK, PTR_INK}, Bytes20, Bytes32, @@ -143,11 +143,20 @@ pub trait UserHost: GasMeteredMachine { /// [`SLOAD`]: https://www.evm.codes/#54 fn storage_load_bytes32(&mut self, key: GuestPtr, dest: GuestPtr) -> Result<(), Self::Err> { self.buy_ink(HOSTIO_INK + 2 * PTR_INK)?; - self.require_gas(evm::COLD_SLOAD_GAS + EVM_API_INK + StorageCache::REQUIRED_ACCESS_GAS)?; // cache-miss case + let arbos_version = self.evm_data().arbos_version; + // require for cache-miss case, preserve wrong behavior for old arbos + let evm_api_gas_to_use = if arbos_version < ARBOS_VERSION_STYLUS_CHARGING_FIXES { + EVM_API_INK + } else { + self.pricing().ink_to_gas(EVM_API_INK) + }; + self.require_gas( + evm::COLD_SLOAD_GAS + StorageCache::REQUIRED_ACCESS_GAS + evm_api_gas_to_use, + )?; let key = self.read_bytes32(key)?; - let (value, gas_cost) = self.evm_api().get_bytes32(key); + let (value, gas_cost) = self.evm_api().get_bytes32(key, evm_api_gas_to_use); self.buy_gas(gas_cost)?; self.write_bytes32(dest, value)?; trace!("storage_load_bytes32", self, key, value) @@ -185,7 +194,10 @@ pub trait UserHost: GasMeteredMachine { self.require_gas(evm::SSTORE_SENTRY_GAS)?; // see operations_acl_arbitrum.go let gas_left = self.gas_left()?; - self.evm_api().flush_storage_cache(clear, gas_left)?; + let gas_cost = self.evm_api().flush_storage_cache(clear, gas_left)?; + if self.evm_data().arbos_version >= ARBOS_VERSION_STYLUS_CHARGING_FIXES { + self.buy_gas(gas_cost)?; + } trace!("storage_flush_cache", self, [be!(clear as u8)], &[]) } diff --git a/arbitrator/wasm-libraries/user-host/src/link.rs b/arbitrator/wasm-libraries/user-host/src/link.rs index 428611167d6..f4c402fd97b 100644 --- a/arbitrator/wasm-libraries/user-host/src/link.rs +++ b/arbitrator/wasm-libraries/user-host/src/link.rs @@ -37,14 +37,15 @@ struct MemoryLeaf([u8; 32]); /// /// pages_ptr: starts pointing to max allowed pages, returns number of pages used #[no_mangle] -pub unsafe extern "C" fn programs__activate( +pub unsafe extern "C" fn programs__activate_v2( wasm_ptr: GuestPtr, wasm_size: usize, pages_ptr: GuestPtr, asm_estimate_ptr: GuestPtr, init_cost_ptr: GuestPtr, cached_init_cost_ptr: GuestPtr, - version: u16, + stylus_version: u16, + arbos_version_for_gas: u64, debug: u32, codehash: GuestPtr, module_hash_ptr: GuestPtr, @@ -58,7 +59,15 @@ pub unsafe extern "C" fn programs__activate( let page_limit = STATIC_MEM.read_u16(pages_ptr); let gas_left = &mut STATIC_MEM.read_u64(gas_ptr); - match Module::activate(&wasm, codehash, version, page_limit, debug, gas_left) { + match Module::activate( + &wasm, + codehash, + stylus_version, + arbos_version_for_gas, + page_limit, + debug, + gas_left, + ) { Ok((module, data)) => { STATIC_MEM.write_u64(gas_ptr, *gas_left); STATIC_MEM.write_u16(pages_ptr, data.footprint); @@ -242,7 +251,8 @@ pub unsafe extern "C" fn programs__create_stylus_config( /// Creates an `EvmData` handler from its component parts. /// #[no_mangle] -pub unsafe extern "C" fn programs__create_evm_data( +pub unsafe extern "C" fn programs__create_evm_data_v2( + arbos_version: u64, block_basefee_ptr: GuestPtr, chainid: u64, block_coinbase_ptr: GuestPtr, @@ -259,6 +269,7 @@ pub unsafe extern "C" fn programs__create_evm_data( reentrant: u32, ) -> u64 { let evm_data = EvmData { + arbos_version, block_basefee: read_bytes32(block_basefee_ptr), cached: cached != 0, chainid, diff --git a/arbitrator/wasm-libraries/user-test/src/program.rs b/arbitrator/wasm-libraries/user-test/src/program.rs index c56ea52ad06..85b522ee740 100644 --- a/arbitrator/wasm-libraries/user-test/src/program.rs +++ b/arbitrator/wasm-libraries/user-test/src/program.rs @@ -102,7 +102,7 @@ impl Program { pub struct MockEvmApi; impl EvmApi for MockEvmApi { - fn get_bytes32(&mut self, key: Bytes32) -> (Bytes32, u64) { + fn get_bytes32(&mut self, key: Bytes32, _evm_api_gas_to_use: u64) -> (Bytes32, u64) { let value = KEYS.lock().get(&key).cloned().unwrap_or_default(); (value, 2100) // pretend worst case } diff --git a/arbitrator/wasm-testsuite/src/main.rs b/arbitrator/wasm-testsuite/src/main.rs index cb861e3a357..2144dcf992d 100644 --- a/arbitrator/wasm-testsuite/src/main.rs +++ b/arbitrator/wasm-testsuite/src/main.rs @@ -343,7 +343,6 @@ fn main() -> eyre::Result<()> { GlobalState::default(), HashMap::default(), machine::get_empty_preimage_resolver(), - false, ); if let Err(error) = &mech { @@ -430,7 +429,6 @@ fn main() -> eyre::Result<()> { GlobalState::default(), HashMap::default(), machine::get_empty_preimage_resolver(), - false, ) .expect_err(&format!("failed to reject invalid module {}", filename)); } diff --git a/arbos/arbosState/arbosstate.go b/arbos/arbosState/arbosstate.go index 91c2207aaef..f53d9c892ab 100644 --- a/arbos/arbosState/arbosstate.go +++ b/arbos/arbosState/arbosstate.go @@ -41,28 +41,29 @@ import ( // persisted beyond the end of the test.) type ArbosState struct { - arbosVersion uint64 // version of the ArbOS storage format and semantics - maxArbosVersionSupported uint64 // maximum ArbOS version supported by this code - maxDebugArbosVersionSupported uint64 // maximum ArbOS version supported by this code in debug mode - upgradeVersion storage.StorageBackedUint64 // version we're planning to upgrade to, or 0 if not planning to upgrade - upgradeTimestamp storage.StorageBackedUint64 // when to do the planned upgrade - networkFeeAccount storage.StorageBackedAddress - l1PricingState *l1pricing.L1PricingState - l2PricingState *l2pricing.L2PricingState - retryableState *retryables.RetryableState - addressTable *addressTable.AddressTable - chainOwners *addressSet.AddressSet - sendMerkle *merkleAccumulator.MerkleAccumulator - programs *programs.Programs - blockhashes *blockhash.Blockhashes - chainId storage.StorageBackedBigInt - chainConfig storage.StorageBackedBytes - genesisBlockNum storage.StorageBackedUint64 - infraFeeAccount storage.StorageBackedAddress - brotliCompressionLevel storage.StorageBackedUint64 // brotli compression level used for pricing - backingStorage *storage.Storage - Burner burn.Burner -} + arbosVersion uint64 // version of the ArbOS storage format and semantics + upgradeVersion storage.StorageBackedUint64 // version we're planning to upgrade to, or 0 if not planning to upgrade + upgradeTimestamp storage.StorageBackedUint64 // when to do the planned upgrade + networkFeeAccount storage.StorageBackedAddress + l1PricingState *l1pricing.L1PricingState + l2PricingState *l2pricing.L2PricingState + retryableState *retryables.RetryableState + addressTable *addressTable.AddressTable + chainOwners *addressSet.AddressSet + sendMerkle *merkleAccumulator.MerkleAccumulator + programs *programs.Programs + blockhashes *blockhash.Blockhashes + chainId storage.StorageBackedBigInt + chainConfig storage.StorageBackedBytes + genesisBlockNum storage.StorageBackedUint64 + infraFeeAccount storage.StorageBackedAddress + brotliCompressionLevel storage.StorageBackedUint64 // brotli compression level used for pricing + backingStorage *storage.Storage + Burner burn.Burner +} + +const MaxArbosVersionSupported uint64 = params.ArbosVersion_StylusChargingFixes +const MaxDebugArbosVersionSupported uint64 = params.ArbosVersion_StylusChargingFixes var ErrUninitializedArbOS = errors.New("ArbOS uninitialized") var ErrAlreadyInitialized = errors.New("ArbOS is already initialized") @@ -78,8 +79,6 @@ func OpenArbosState(stateDB vm.StateDB, burner burn.Burner) (*ArbosState, error) } return &ArbosState{ arbosVersion, - 31, - 31, backingStorage.OpenStorageBackedUint64(uint64(upgradeVersionOffset)), backingStorage.OpenStorageBackedUint64(uint64(upgradeTimestampOffset)), backingStorage.OpenStorageBackedAddress(uint64(networkFeeAccountOffset)), @@ -332,6 +331,9 @@ func (state *ArbosState) UpgradeArbosVersion( ensure(params.UpgradeToVersion(2)) ensure(params.Save()) + case 32: + // no change state needed + default: return fmt.Errorf( "the chain is upgrading to unsupported ArbOS version %v, %w", @@ -416,14 +418,6 @@ func (state *ArbosState) RetryableState() *retryables.RetryableState { return state.retryableState } -func (state *ArbosState) MaxArbosVersionSupported() uint64 { - return state.maxArbosVersionSupported -} - -func (state *ArbosState) MaxDebugArbosVersionSupported() uint64 { - return state.maxDebugArbosVersionSupported -} - func (state *ArbosState) L1PricingState() *l1pricing.L1PricingState { return state.l1PricingState } diff --git a/arbos/programs/native.go b/arbos/programs/native.go index e834888b470..b13cfef2cf2 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -51,11 +51,12 @@ func activateProgram( codehash common.Hash, wasm []byte, page_limit uint16, - version uint16, + stylusVersion uint16, + arbosVersionForGas uint64, debug bool, burner burn.Burner, ) (*activationInfo, error) { - info, asmMap, err := activateProgramInternal(db, program, codehash, wasm, page_limit, version, debug, burner.GasLeft()) + info, asmMap, err := activateProgramInternal(db, program, codehash, wasm, page_limit, stylusVersion, arbosVersionForGas, debug, burner.GasLeft()) if err != nil { return nil, err } @@ -69,7 +70,8 @@ func activateProgramInternal( codehash common.Hash, wasm []byte, page_limit uint16, - version uint16, + stylusVersion uint16, + arbosVersionForGas uint64, debug bool, gasLeft *uint64, ) (*activationInfo, map[ethdb.WasmTarget][]byte, error) { @@ -81,7 +83,8 @@ func activateProgramInternal( status_mod := userStatus(C.stylus_activate( goSlice(wasm), u16(page_limit), - u16(version), + u16(stylusVersion), + u64(arbosVersionForGas), cbool(debug), output, &codeHash, @@ -100,6 +103,7 @@ func activateProgramInternal( } return nil, nil, err } + hash := moduleHash.toHash() targets := db.Database().WasmTargets() type result struct { target ethdb.WasmTarget @@ -116,7 +120,7 @@ func activateProgramInternal( output := &rustBytes{} status_asm := C.stylus_compile( goSlice(wasm), - u16(version), + u16(stylusVersion), cbool(debug), goSlice([]byte(target)), output, @@ -140,10 +144,17 @@ func activateProgramInternal( } } if err != nil { - return nil, nil, fmt.Errorf("compilation failed for one or more targets: %w", err) + log.Error( + "Compilation failed for one or more targets despite activation succeeding", + "address", addressForLogging, + "codeHash", codeHash, + "moduleHash", hash, + "targets", targets, + "err", err, + ) + panic(fmt.Sprintf("Compilation of %v failed for one or more targets despite activation succeeding: %v", addressForLogging, err)) } - hash := moduleHash.toHash() info := &activationInfo{ moduleHash: hash, initGas: uint16(stylusData.init_cost), @@ -168,9 +179,12 @@ func getLocalAsm(statedb vm.StateDB, moduleHash common.Hash, addressForLogging c return nil, fmt.Errorf("failed to reactivate program address: %v err: %w", addressForLogging, err) } - unlimitedGas := uint64(0xffffffffffff) + // don't charge gas + zeroArbosVersion := uint64(0) + zeroGas := uint64(0) + // we know program is activated, so it must be in correct version and not use too much memory - info, asmMap, err := activateProgramInternal(statedb, addressForLogging, codeHash, wasm, pagelimit, program.version, debugMode, &unlimitedGas) + info, asmMap, err := activateProgramInternal(statedb, addressForLogging, codeHash, wasm, pagelimit, program.version, zeroArbosVersion, debugMode, &zeroGas) if err != nil { log.Error("failed to reactivate program", "address", addressForLogging, "expected moduleHash", moduleHash, "err", err) return nil, fmt.Errorf("failed to reactivate program address: %v err: %w", addressForLogging, err) @@ -391,6 +405,7 @@ func (params *ProgParams) encode() C.StylusConfig { func (data *EvmData) encode() C.EvmData { return C.EvmData{ + arbos_version: u64(data.arbosVersion), block_basefee: hashToBytes32(data.blockBasefee), chainid: u64(data.chainId), block_coinbase: addressToBytes20(data.blockCoinbase), diff --git a/arbos/programs/programs.go b/arbos/programs/programs.go index 12102bac84f..80fc139a9d6 100644 --- a/arbos/programs/programs.go +++ b/arbos/programs/programs.go @@ -82,7 +82,7 @@ func (p Programs) CacheManagers() *addressSet.AddressSet { return p.cacheManagers } -func (p Programs) ActivateProgram(evm *vm.EVM, address common.Address, runMode core.MessageRunMode, debugMode bool) ( +func (p Programs) ActivateProgram(evm *vm.EVM, address common.Address, arbosVersion uint64, runMode core.MessageRunMode, debugMode bool) ( uint16, common.Hash, common.Hash, *big.Int, bool, error, ) { statedb := evm.StateDB @@ -116,7 +116,7 @@ func (p Programs) ActivateProgram(evm *vm.EVM, address common.Address, runMode c // require the program's footprint not exceed the remaining memory budget pageLimit := am.SaturatingUSub(params.PageLimit, statedb.GetStylusPagesOpen()) - info, err := activateProgram(statedb, address, codeHash, wasm, pageLimit, stylusVersion, debugMode, burner) + info, err := activateProgram(statedb, address, codeHash, wasm, pageLimit, stylusVersion, arbosVersion, debugMode, burner) if err != nil { return 0, codeHash, common.Hash{}, nil, true, err } @@ -222,6 +222,7 @@ func (p Programs) CallProgram( } evmData := &EvmData{ + arbosVersion: evm.Context.ArbOSVersion, blockBasefee: common.BigToHash(evm.Context.BaseFee), chainId: evm.ChainConfig().ChainID.Uint64(), blockCoinbase: evm.Context.Coinbase, @@ -517,6 +518,7 @@ func (p Programs) progParams(version uint16, debug bool, params *StylusParams) * } type EvmData struct { + arbosVersion uint64 blockBasefee common.Hash chainId uint64 blockCoinbase common.Address diff --git a/arbos/programs/wasm.go b/arbos/programs/wasm.go index f7191dca8f5..8f79944fbe9 100644 --- a/arbos/programs/wasm.go +++ b/arbos/programs/wasm.go @@ -36,7 +36,7 @@ type rustConfig byte type rustModule byte type rustEvmData byte -//go:wasmimport programs activate +//go:wasmimport programs activate_v2 func programActivate( wasm_ptr unsafe.Pointer, wasm_size uint32, @@ -44,7 +44,8 @@ func programActivate( asm_estimation_ptr unsafe.Pointer, init_gas_ptr unsafe.Pointer, cached_init_gas_ptr unsafe.Pointer, - version uint32, + stylusVersion uint32, + arbosVersion uint64, debug uint32, codehash unsafe.Pointer, module_hash_ptr unsafe.Pointer, @@ -59,7 +60,8 @@ func activateProgram( codehash common.Hash, wasm []byte, pageLimit u16, - version u16, + stylusVersion u16, + arbosVersion uint64, debug bool, burner burn.Burner, ) (*activationInfo, error) { @@ -79,7 +81,8 @@ func activateProgram( unsafe.Pointer(&asmEstimate), unsafe.Pointer(&initGas), unsafe.Pointer(&cachedInitGas), - uint32(version), + uint32(stylusVersion), + arbosVersion, debugMode, arbutil.SliceToUnsafePointer(codehash[:]), arbutil.SliceToUnsafePointer(moduleHash[:]), diff --git a/arbos/programs/wasm_api.go b/arbos/programs/wasm_api.go index d7bac056c09..a4ebc1f778f 100644 --- a/arbos/programs/wasm_api.go +++ b/arbos/programs/wasm_api.go @@ -20,8 +20,9 @@ func createStylusConfig(version uint32, max_depth uint32, ink_price uint32, debu type evmDataHandler uint64 -//go:wasmimport programs create_evm_data +//go:wasmimport programs create_evm_data_v2 func createEvmData( + arbosVersion uint64, blockBaseFee unsafe.Pointer, chainid uint64, blockCoinbase unsafe.Pointer, @@ -45,6 +46,7 @@ func (params *ProgParams) createHandler() stylusConfigHandler { func (data *EvmData) createHandler() evmDataHandler { return createEvmData( + data.arbosVersion, arbutil.SliceToUnsafePointer(data.blockBasefee[:]), data.chainId, arbutil.SliceToUnsafePointer(data.blockCoinbase[:]), diff --git a/arbos/programs/wasmstorehelper.go b/arbos/programs/wasmstorehelper.go index 434820dd9c9..c2d1aa65b0d 100644 --- a/arbos/programs/wasmstorehelper.go +++ b/arbos/programs/wasmstorehelper.go @@ -17,12 +17,12 @@ import ( // SaveActiveProgramToWasmStore is used to save active stylus programs to wasm store during rebuilding func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash common.Hash, code []byte, time uint64, debugMode bool, rebuildingStartBlockTime uint64) error { - params, err := p.Params() + progParams, err := p.Params() if err != nil { return err } - program, err := p.getActiveProgram(codeHash, time, params) + program, err := p.getActiveProgram(codeHash, time, progParams) if err != nil { // The program is not active so return early log.Info("program is not active, getActiveProgram returned error, hence do not include in rebuilding", "err", err) @@ -56,10 +56,13 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash return fmt.Errorf("failed to reactivate program while rebuilding wasm store: %w", err) } - unlimitedGas := uint64(0xffffffffffff) + // don't charge gas + zeroArbosVersion := uint64(0) + zeroGas := uint64(0) + // We know program is activated, so it must be in correct version and not use too much memory // Empty program address is supplied because we dont have access to this during rebuilding of wasm store - info, asmMap, err := activateProgramInternal(statedb, common.Address{}, codeHash, wasm, params.PageLimit, program.version, debugMode, &unlimitedGas) + info, asmMap, err := activateProgramInternal(statedb, common.Address{}, codeHash, wasm, progParams.PageLimit, program.version, zeroArbosVersion, debugMode, &zeroGas) if err != nil { log.Error("failed to reactivate program while rebuilding wasm store", "expected moduleHash", moduleHash, "err", err) return fmt.Errorf("failed to reactivate program while rebuilding wasm store: %w", err) diff --git a/cmd/chaininfo/arbitrum_chain_info.json b/cmd/chaininfo/arbitrum_chain_info.json index 524433a7b58..f862c6dfbfa 100644 --- a/cmd/chaininfo/arbitrum_chain_info.json +++ b/cmd/chaininfo/arbitrum_chain_info.json @@ -164,7 +164,7 @@ "EnableArbOS": true, "AllowDebugPrecompiles": true, "DataAvailabilityCommittee": false, - "InitialArbOSVersion": 31, + "InitialArbOSVersion": 32, "InitialChainOwner": "0x0000000000000000000000000000000000000000", "GenesisBlockNum": 0 } @@ -196,7 +196,7 @@ "EnableArbOS": true, "AllowDebugPrecompiles": true, "DataAvailabilityCommittee": true, - "InitialArbOSVersion": 31, + "InitialArbOSVersion": 32, "InitialChainOwner": "0x0000000000000000000000000000000000000000", "GenesisBlockNum": 0 } diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index fc59f2d231a..07c74cb802f 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -354,12 +354,12 @@ func validateBlockChain(blockChain *core.BlockChain, chainConfig *params.ChainCo } // Make sure we don't allow accidentally downgrading ArbOS if chainConfig.DebugMode() { - if currentArbosState.ArbOSVersion() > currentArbosState.MaxDebugArbosVersionSupported() { - return fmt.Errorf("attempted to launch node in debug mode with ArbOS version %v on ArbOS state with version %v", currentArbosState.MaxDebugArbosVersionSupported(), currentArbosState.ArbOSVersion()) + if currentArbosState.ArbOSVersion() > arbosState.MaxDebugArbosVersionSupported { + return fmt.Errorf("attempted to launch node in debug mode with ArbOS version %v on ArbOS state with version %v", arbosState.MaxDebugArbosVersionSupported, currentArbosState.ArbOSVersion()) } } else { - if currentArbosState.ArbOSVersion() > currentArbosState.MaxArbosVersionSupported() { - return fmt.Errorf("attempted to launch node with ArbOS version %v on ArbOS state with version %v", currentArbosState.MaxArbosVersionSupported(), currentArbosState.ArbOSVersion()) + if currentArbosState.ArbOSVersion() > arbosState.MaxArbosVersionSupported { + return fmt.Errorf("attempted to launch node with ArbOS version %v on ArbOS state with version %v", arbosState.MaxArbosVersionSupported, currentArbosState.ArbOSVersion()) } } diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index a13a92fdc18..1078f448088 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -472,6 +472,10 @@ func mainImpl() int { if nodeConfig.BlocksReExecutor.Enable && l2BlockChain != nil { blocksReExecutor = blocksreexecutor.New(&nodeConfig.BlocksReExecutor, l2BlockChain, fatalErrChan) if nodeConfig.Init.ThenQuit { + if err := gethexec.PopulateStylusTargetCache(&nodeConfig.Execution.StylusTarget); err != nil { + log.Error("error populating stylus target cache", "err", err) + return 1 + } success := make(chan struct{}) blocksReExecutor.Start(ctx, success) deferFuncs = append(deferFuncs, func() { blocksReExecutor.StopAndWait() }) diff --git a/contracts b/contracts index 23fc7962829..7396313311a 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit 23fc79628292aa5d604d449fed48937ae7faeb2f +Subproject commit 7396313311ab17cb30e2eef27cccf96f0a9e8f7f diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 8594d5867df..8d6484e3c90 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -150,7 +150,7 @@ func (s *ExecutionEngine) MarkFeedStart(to arbutil.MessageIndex) { } } -func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { +func PopulateStylusTargetCache(targetConfig *StylusTargetConfig) error { localTarget := rawdb.LocalTarget() targets := targetConfig.WasmTargets() var nativeSet bool @@ -186,7 +186,7 @@ func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusT if rustCacheSize != 0 { programs.ResizeWasmLruCache(rustCacheSize) } - if err := populateStylusTargetCache(targetConfig); err != nil { + if err := PopulateStylusTargetCache(targetConfig); err != nil { return fmt.Errorf("error populating stylus target cache: %w", err) } return nil diff --git a/execution/gethexec/wasmstorerebuilder.go b/execution/gethexec/wasmstorerebuilder.go index 698ba3ec8a4..e3eb8e9268e 100644 --- a/execution/gethexec/wasmstorerebuilder.go +++ b/execution/gethexec/wasmstorerebuilder.go @@ -63,7 +63,7 @@ func RebuildWasmStore(ctx context.Context, wasmStore ethdb.KeyValueStore, chainD var err error var stateDb *state.StateDB - if err := populateStylusTargetCache(targetConfig); err != nil { + if err := PopulateStylusTargetCache(targetConfig); err != nil { return fmt.Errorf("error populating stylus target cache: %w", err) } diff --git a/go-ethereum b/go-ethereum index 9ca65b86101..17cd0016754 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 9ca65b86101b4c87df188923d6eeff4db992520f +Subproject commit 17cd00167543a5a2b0b083e32820051100154c2f diff --git a/precompiles/ArbWasm.go b/precompiles/ArbWasm.go index 9f42cacb5ab..bc24c8a6e83 100644 --- a/precompiles/ArbWasm.go +++ b/precompiles/ArbWasm.go @@ -5,6 +5,8 @@ package precompiles import ( "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/vm" + gethparams "github.com/ethereum/go-ethereum/params" "github.com/offchainlabs/nitro/arbos/programs" "github.com/offchainlabs/nitro/arbos/util" "github.com/offchainlabs/nitro/util/arbmath" @@ -32,12 +34,13 @@ func (con ArbWasm) ActivateProgram(c ctx, evm mech, value huge, program addr) (u debug := evm.ChainConfig().DebugMode() runMode := c.txProcessor.RunMode() programs := c.State.Programs() + arbosVersion := c.State.ArbOSVersion() // charge a fixed cost up front to begin activation if err := c.Burn(1659168); err != nil { return 0, nil, err } - version, codeHash, moduleHash, dataFee, takeAllGas, err := programs.ActivateProgram(evm, program, runMode, debug) + version, codeHash, moduleHash, dataFee, takeAllGas, err := programs.ActivateProgram(evm, program, arbosVersion, runMode, debug) if takeAllGas { _ = c.BurnOut() } @@ -133,6 +136,9 @@ func (con ArbWasm) MinInitGas(c ctx, _ mech) (uint64, uint64, error) { params, err := c.State.Programs().Params() init := uint64(params.MinInitGas) * programs.MinInitGasUnits cached := uint64(params.MinCachedInitGas) * programs.MinCachedGasUnits + if c.State.ArbOSVersion() < gethparams.ArbosVersion_StylusChargingFixes { + return 0, 0, vm.ErrExecutionReverted + } return init, cached, err } diff --git a/system_tests/block_validator_test.go b/system_tests/block_validator_test.go index cca5572f4e7..9125c3921e8 100644 --- a/system_tests/block_validator_test.go +++ b/system_tests/block_validator_test.go @@ -284,6 +284,20 @@ func TestBlockValidatorSimpleOnchain(t *testing.T) { testBlockValidatorSimple(t, opts) } +func TestBlockValidatorSimpleJITOnchainWithPublishedMachine(t *testing.T) { + cr, err := github.LatestConsensusRelease(context.Background()) + Require(t, err) + machPath := populateMachineDir(t, cr) + opts := Options{ + dasModeString: "onchain", + workloadLoops: 1, + workload: ethSend, + arbitrator: false, + wasmRootDir: machPath, + } + testBlockValidatorSimple(t, opts) +} + func TestBlockValidatorSimpleOnchainWithPublishedMachine(t *testing.T) { cr, err := github.LatestConsensusRelease(context.Background()) Require(t, err) diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index e77e071d708..adca9695e22 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -92,7 +92,7 @@ func LoadSimpleMachine(wasm string, libraries []string, debugChain bool) (*Arbit cWasm := C.CString(wasm) cLibraries := CreateCStringList(libraries) debug := usize(arbmath.BoolToUint32(debugChain)) - mach := C.arbitrator_load_machine(cWasm, cLibraries, C.long(len(libraries)), debug, false) + mach := C.arbitrator_load_machine(cWasm, cLibraries, C.long(len(libraries)), debug) C.free(unsafe.Pointer(cWasm)) FreeCStringList(cLibraries, len(libraries)) if mach == nil { From 4fa04c0636da911cc702f91b63a9ba2dc7bc1963 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 24 Sep 2024 09:41:12 -0500 Subject: [PATCH 156/156] Re-enable G115 linter --- .golangci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 8b10d7116ce..05946701379 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -44,7 +44,6 @@ linters-settings: gosec: excludes: - G404 # checks that random numbers are securely generated - - G115 govet: enable-all: true