Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add directory locking mechanism #500

Merged
merged 5 commits into from
Jan 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/MIGRATING.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
- CosmWasm gas values were reduced by a factor of 1000, so each instruction now
consumes 150 CosmWasm gas instead of 150000. This should be taken into account
when converting between CosmWasm gas and Cosmos SDK gas.
- A new lockfile called `exclusive.lock` in the base directory ensures that no
two `VM` instances operate on the same directory in parallel. This was
unsupported before already but now leads to an error early on. When doing
parallel testing, use a different directory for each instance.

## Renamings

Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.21
require (
github.com/google/btree v1.0.0
github.com/stretchr/testify v1.8.1
golang.org/x/sys v0.16.0
)

require (
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
30 changes: 28 additions & 2 deletions internal/api/lib.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,13 @@ import "C"

import (
"fmt"
"os"
"runtime"
"strings"
"syscall"

"golang.org/x/sys/unix"

"github.com/CosmWasm/wasmvm/types"
)

Expand All @@ -32,12 +35,33 @@ type (
)

type Cache struct {
ptr *C.cache_t
ptr *C.cache_t
lockfile os.File
}

type Querier = types.Querier

func InitCache(dataDir string, supportedCapabilities []string, cacheSize uint32, instanceMemoryLimit uint32) (Cache, error) {
// libwasmvm would create this directory too but we need it earlier for the lockfile
err := os.MkdirAll(dataDir, 0o755)
if err != nil {
return Cache{}, fmt.Errorf("Could not create base directory")
}

lockfile, err := os.OpenFile(dataDir+"/exclusive.lock", os.O_WRONLY|os.O_CREATE, 0o666)
if err != nil {
return Cache{}, fmt.Errorf("Could not open exclusive.lock")
}
_, err = lockfile.WriteString("This is a lockfile that prevent two VM instances to operate on the same directory in parallel.\nSee codebase at github.com/CosmWasm/wasmvm for more information.\nSafety first – brought to you by Confio ❤️\n")
if err != nil {
return Cache{}, fmt.Errorf("Error writing to exclusive.lock")
}

err = unix.Flock(int(lockfile.Fd()), unix.LOCK_EX|unix.LOCK_NB)
if err != nil {
return Cache{}, fmt.Errorf("Could not lock exclusive.lock. Is a different VM running in the same directory already?")
}

dataDirBytes := []byte(dataDir)
supportedCapabilitiesBytes := []byte(strings.Join(supportedCapabilities, ","))

Expand All @@ -52,11 +76,13 @@ func InitCache(dataDir string, supportedCapabilities []string, cacheSize uint32,
if err != nil {
return Cache{}, errorWithMessage(err, errmsg)
}
return Cache{ptr: ptr}, nil
return Cache{ptr: ptr, lockfile: *lockfile}, nil
}

func ReleaseCache(cache Cache) {
C.release_cache(cache.ptr)

cache.lockfile.Close() // Also releases the file lock
}

func StoreCode(cache Cache, wasm []byte) ([]byte, error) {
Expand Down
44 changes: 43 additions & 1 deletion internal/api/lib_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,49 @@ func TestInitCacheErrorsForBrokenDir(t *testing.T) {
// On Unix we should not have permission to create this.
cannotBeCreated := "/foo:bar"
_, err := InitCache(cannotBeCreated, TESTING_CAPABILITIES, TESTING_CACHE_SIZE, TESTING_MEMORY_LIMIT)
require.ErrorContains(t, err, "Error creating state directory")
require.ErrorContains(t, err, "Could not create base directory")
}

func TestInitLockingPreventsConcurrentAccess(t *testing.T) {
tmpdir, err := os.MkdirTemp("", "wasmvm-testing")
require.NoError(t, err)
defer os.RemoveAll(tmpdir)

cache1, err1 := InitCache(tmpdir, TESTING_CAPABILITIES, TESTING_CACHE_SIZE, TESTING_MEMORY_LIMIT)
require.NoError(t, err1)

_, err2 := InitCache(tmpdir, TESTING_CAPABILITIES, TESTING_CACHE_SIZE, TESTING_MEMORY_LIMIT)
require.ErrorContains(t, err2, "Could not lock exclusive.lock")

ReleaseCache(cache1)

// Now we can try again
cache3, err3 := InitCache(tmpdir, TESTING_CAPABILITIES, TESTING_CACHE_SIZE, TESTING_MEMORY_LIMIT)
require.NoError(t, err3)
ReleaseCache(cache3)
}

func TestInitLockingAllowsMultipleInstancesInDifferentDirs(t *testing.T) {
tmpdir1, err := os.MkdirTemp("", "wasmvm-testing1")
require.NoError(t, err)
tmpdir2, err := os.MkdirTemp("", "wasmvm-testing2")
require.NoError(t, err)
tmpdir3, err := os.MkdirTemp("", "wasmvm-testing3")
require.NoError(t, err)
defer os.RemoveAll(tmpdir1)
defer os.RemoveAll(tmpdir2)
defer os.RemoveAll(tmpdir3)

cache1, err1 := InitCache(tmpdir1, TESTING_CAPABILITIES, TESTING_CACHE_SIZE, TESTING_MEMORY_LIMIT)
require.NoError(t, err1)
cache2, err2 := InitCache(tmpdir2, TESTING_CAPABILITIES, TESTING_CACHE_SIZE, TESTING_MEMORY_LIMIT)
require.NoError(t, err2)
cache3, err3 := InitCache(tmpdir3, TESTING_CAPABILITIES, TESTING_CACHE_SIZE, TESTING_MEMORY_LIMIT)
require.NoError(t, err3)

ReleaseCache(cache1)
ReleaseCache(cache2)
ReleaseCache(cache3)
}

func TestInitCacheEmptyCapabilities(t *testing.T) {
Expand Down
3 changes: 2 additions & 1 deletion lib.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ func NewVM(dataDir string, supportedCapabilities []string, memoryLimit uint32, p
return &VM{cache: cache, printDebug: printDebug}, nil
}

// Cleanup should be called when no longer using this to free resources on the rust-side
// Cleanup should be called when no longer using this instances.
// It frees resources in libwasmvm (the Rust part) and releases a lock in the base directory.
func (vm *VM) Cleanup() {
api.ReleaseCache(vm.cache)
}
Expand Down
2 changes: 1 addition & 1 deletion types/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ type DenomUnit struct {
type DecCoin struct {
// An amount in the base denom of the distributed token.
//
// Some chains have choosen atto (10^-18) for their token's base denomination. If we used `Decimal` here, we could only store 340282366920938463463.374607431768211455atoken which is 340.28 TOKEN.
// Some chains have chosen atto (10^-18) for their token's base denomination. If we used `Decimal` here, we could only store 340282366920938463463.374607431768211455atoken which is 340.28 TOKEN.
Amount string `json:"amount"`
Denom string `json:"denom"`
}
Expand Down