Skip to content

Commit

Permalink
Dot file locking. (#179)
Browse files Browse the repository at this point in the history
  • Loading branch information
ncruces authored Oct 30, 2024
1 parent 1ad1608 commit b2e8636
Show file tree
Hide file tree
Showing 14 changed files with 178 additions and 25 deletions.
8 changes: 8 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ on:
- '**.wasm.bz2'
pull_request:
branches: [ "main" ]
paths:
- '**.go'
- '**.wasm'
- '**.wasm.bz2'
workflow_dispatch:

jobs:
Expand Down Expand Up @@ -52,6 +56,10 @@ jobs:
run: go test -v -tags sqlite3_flock ./...
if: matrix.os == 'macos-latest'

- name: Test dot locks
run: go test -v -tags sqlite3_dotlk ./...
if: matrix.os == 'macos-latest'

- name: Test no shared memory
run: go test -v -tags sqlite3_noshm ./...
if: matrix.os == 'ubuntu-latest'
Expand Down
2 changes: 1 addition & 1 deletion internal/util/mmap.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//go:build unix && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
//go:build unix && !(sqlite3_noshm || sqlite3_nosys)

package util

Expand Down
2 changes: 1 addition & 1 deletion internal/util/mmap_other.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//go:build !unix || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
//go:build !unix || sqlite3_noshm || sqlite3_nosys

package util

Expand Down
19 changes: 10 additions & 9 deletions vfs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,19 @@ POSIX advisory locks, which SQLite uses on Unix, are
On Linux and macOS, this package uses
[OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html)
to synchronize access to database files.
OFD locks are fully compatible with POSIX advisory locks.

This package can also use
[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2),
albeit with reduced concurrency (`BEGIN IMMEDIATE` behaves like `BEGIN EXCLUSIVE`).
On BSD, macOS, and illumos, BSD locks are fully compatible with POSIX advisory locks;
on Linux and z/OS, they are fully functional, but incompatible;
elsewhere, they are very likely broken.
BSD locks are the default on BSD and illumos,
but you can opt into them with the `sqlite3_flock` build tag.

On Windows, this package uses `LockFileEx` and `UnlockFileEx`,
like SQLite.

You can also opt into a cross platform locking implementation
with the `sqlite3_dotlk` build tag.

Otherwise, file locking is not supported, and you must use
[`nolock=1`](https://sqlite.org/uri.html#urinolock)
(or [`immutable=1`](https://sqlite.org/uri.html#uriimmutable))
Expand Down Expand Up @@ -86,8 +85,8 @@ The implementation is compatible with SQLite's
### Build Tags

The VFS can be customized with a few build tags:
- `sqlite3_flock` forces the use of BSD locks; it can be used on z/OS to enable locking,
and elsewhere to test BSD locks.
- `sqlite3_flock` forces the use of BSD locks.
- `sqlite3_dotlk` forces the use of dot-file locks.
- `sqlite3_nosys` prevents importing [`x/sys`](https://pkg.go.dev/golang.org/x/sys);
disables locking _and_ shared memory on all platforms.
- `sqlite3_noshm` disables shared memory on all platforms.
Expand All @@ -96,17 +95,19 @@ The VFS can be customized with a few build tags:
> The default configuration of this package is compatible with the standard
> [Unix and Windows SQLite VFSes](https://sqlite.org/vfs.html#multiple_vfses);
> `sqlite3_flock` builds are compatible with the
> [`unix-flock` VFS](https://sqlite.org/compile.html#enable_locking_style).
> [`unix-flock` VFS](https://sqlite.org/compile.html#enable_locking_style);
> `sqlite3_dotlk` builds are compatible with the
> [`unix-dotfile` VFS](https://sqlite.org/compile.html#enable_locking_style).
> If incompatible file locking is used, accessing databases concurrently with
> _other_ SQLite libraries will eventually corrupt data.
### Custom VFSes

- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum)
wraps a VFS to offer encryption at rest.
- [`github.com/ncruces/go-sqlite3/vfs/memdb`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb)
implements an in-memory VFS.
- [`github.com/ncruces/go-sqlite3/vfs/readervfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs)
implements a VFS for immutable databases.
- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum)
wraps a VFS to offer encryption at rest.
- [`github.com/ncruces/go-sqlite3/vfs/xts`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/xts)
wraps a VFS to offer encryption at rest.
13 changes: 7 additions & 6 deletions vfs/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ func testSymlinks(path string) error {

func (vfsOS) Delete(path string, syncDir bool) error {
err := os.Remove(path)
if errors.Is(err, fs.ErrNotExist) {
return _IOERR_DELETE_NOENT
}
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return _IOERR_DELETE_NOENT
}
return err
}
if runtime.GOOS != "windows" && syncDir {
Expand Down Expand Up @@ -151,6 +151,7 @@ func (f *vfsFile) Close() error {
if f.shm != nil {
f.shm.Close()
}
f.Unlock(LOCK_NONE)
return f.File.Close()
}

Expand Down Expand Up @@ -206,10 +207,10 @@ func (f *vfsFile) HasMoved() (bool, error) {
return false, err
}
pi, err := os.Stat(f.Name())
if errors.Is(err, fs.ErrNotExist) {
return true, nil
}
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return true, nil
}
return false, err
}
return !os.SameFile(fi, pi), nil
Expand Down
2 changes: 1 addition & 1 deletion vfs/lock.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
//go:build ((linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && !sqlite3_nosys) || sqlite3_flock || sqlite3_dotlk

package vfs

Expand Down
2 changes: 1 addition & 1 deletion vfs/lock_other.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//go:build !(linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || sqlite3_nosys
//go:build !(((linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && !sqlite3_nosys) || sqlite3_flock || sqlite3_dotlk)

package vfs

Expand Down
2 changes: 1 addition & 1 deletion vfs/os_bsd.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
//go:build ((freebsd || openbsd || netbsd || dragonfly || illumos) && !(sqlite3_dotlk || sqlite3_nosys)) || sqlite3_flock

package vfs

Expand Down
143 changes: 143 additions & 0 deletions vfs/os_dotlk.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
//go:build sqlite3_dotlk

package vfs

import (
"errors"
"io/fs"
"os"
"sync"
)

var (
// +checklocks:vfsDotLocksMtx
vfsDotLocks = map[string]*vfsDotLocker{}
vfsDotLocksMtx sync.Mutex
)

type vfsDotLocker struct {
shared int // +checklocks:vfsDotLocksMtx
pending *os.File // +checklocks:vfsDotLocksMtx
reserved *os.File // +checklocks:vfsDotLocksMtx
}

func osGetSharedLock(file *os.File) _ErrorCode {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()

name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
err := os.Mkdir(name+".lock", 0777)
if errors.Is(err, fs.ErrExist) {
return _BUSY // Another process has the lock.
}
if err != nil {
return _IOERR_LOCK
}
locker = &vfsDotLocker{}
vfsDotLocks[name] = locker
}

if locker.pending != nil {
return _BUSY
}
locker.shared++
return _OK
}

func osGetReservedLock(file *os.File) _ErrorCode {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()

name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
return _IOERR_LOCK
}

if locker.reserved != nil && locker.reserved != file {
return _BUSY
}
locker.reserved = file
return _OK
}

func osGetExclusiveLock(file *os.File, _ *LockLevel) _ErrorCode {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()

name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
return _IOERR_LOCK
}

if locker.pending != nil && locker.pending != file {
return _BUSY
}
locker.pending = file
if locker.shared > 1 {
return _BUSY
}
return _OK
}

func osDowngradeLock(file *os.File, _ LockLevel) _ErrorCode {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()

name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
return _IOERR_UNLOCK
}

if locker.reserved == file {
locker.reserved = nil
}
if locker.pending == file {
locker.pending = nil
}
return _OK
}

func osReleaseLock(file *os.File, state LockLevel) _ErrorCode {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()

name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
return _IOERR_UNLOCK
}

if locker.shared == 1 {
err := os.Remove(name + ".lock")
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return _IOERR_UNLOCK
}
delete(vfsDotLocks, name)
}

if locker.reserved == file {
locker.reserved = nil
}
if locker.pending == file {
locker.pending = nil
}
locker.shared--
return _OK
}

func osCheckReservedLock(file *os.File) (bool, _ErrorCode) {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()

name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
return false, _OK
}
return locker.reserved != nil, _OK
}
2 changes: 1 addition & 1 deletion vfs/os_ofd.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//go:build (linux || darwin) && !(sqlite3_flock || sqlite3_nosys)
//go:build (linux || darwin) && !(sqlite3_flock || sqlite3_dotlk || sqlite3_nosys)

package vfs

Expand Down
2 changes: 1 addition & 1 deletion vfs/os_windows.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//go:build !sqlite3_nosys
//go:build !(sqlite3_dotlk || sqlite3_nosys)

package vfs

Expand Down
2 changes: 1 addition & 1 deletion vfs/shm.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//go:build (darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
//go:build (darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_dotlk || sqlite3_noshm || sqlite3_nosys)

package vfs

Expand Down
2 changes: 1 addition & 1 deletion vfs/shm_bsd.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_dotlk || sqlite3_noshm || sqlite3_nosys)

package vfs

Expand Down
2 changes: 1 addition & 1 deletion vfs/shm_other.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_dotlk || sqlite3_noshm || sqlite3_nosys

package vfs

Expand Down

0 comments on commit b2e8636

Please sign in to comment.