Share memory by copying. (#180)

ncruces · Oct 31, 2024 · 17f7840 · 17f7840
1 parent b2e8636
commit 17f7840
Show file tree

Hide file tree

Showing 13 changed files with 307 additions and 34 deletions.
diff --git a/.github/workflows/cross.sh b/.github/workflows/cross.sh
@@ -17,11 +17,13 @@ echo aix           ; GOOS=aix       GOARCH=ppc64 go build .
 echo js            ; GOOS=js        GOARCH=wasm  go build .
 echo wasip1        ; GOOS=wasip1    GOARCH=wasm  go build .
 echo linux-flock   ; GOOS=linux     GOARCH=amd64 go build -tags sqlite3_flock .
-echo linux-noshm   ; GOOS=linux     GOARCH=amd64 go build -tags sqlite3_noshm .
+echo linux-dotlk   ; GOOS=linux     GOARCH=amd64 go build -tags sqlite3_dotlk .
 echo linux-nosys   ; GOOS=linux     GOARCH=amd64 go build -tags sqlite3_nosys .
 echo darwin-flock  ; GOOS=darwin    GOARCH=amd64 go build -tags sqlite3_flock .
-echo darwin-noshm  ; GOOS=darwin    GOARCH=amd64 go build -tags sqlite3_noshm .
+echo darwin-dotlk  ; GOOS=darwin    GOARCH=amd64 go build -tags sqlite3_dotlk .
 echo darwin-nosys  ; GOOS=darwin    GOARCH=amd64 go build -tags sqlite3_nosys .
+echo windows-dotlk ; GOOS=windows   GOARCH=amd64 go build -tags sqlite3_dotlk .
 echo windows-nosys ; GOOS=windows   GOARCH=amd64 go build -tags sqlite3_nosys .
+echo freebsd-dotlk ; GOOS=freebsd   GOARCH=amd64 go build -tags sqlite3_dotlk .
 echo freebsd-nosys ; GOOS=freebsd   GOARCH=amd64 go build -tags sqlite3_nosys .
-echo solaris-flock ; GOOS=solaris   GOARCH=amd64 go build -tags sqlite3_flock .
+echo solaris-dotlk ; GOOS=solaris   GOARCH=amd64 go build -tags sqlite3_dotlk .
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -60,10 +60,6 @@ jobs:
       run: go test -v -tags sqlite3_dotlk ./...
       if: matrix.os == 'macos-latest'
 
-    - name: Test no shared memory
-      run: go test -v -tags sqlite3_noshm ./...
-      if: matrix.os == 'ubuntu-latest'
-
     - name: Test no locks
       run: go test -v -tags sqlite3_nosys ./...
       if: matrix.os == 'ubuntu-latest'

diff --git a/driver/example_test.go b/driver/example_test.go
@@ -1,4 +1,4 @@
-//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
+//go:build ((linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && !sqlite3_nosys) || sqlite3_flock || sqlite3_dotlk
 
 package driver_test
 

diff --git a/internal/util/mmap.go b/internal/util/mmap.go
@@ -1,4 +1,4 @@
-//go:build unix && !(sqlite3_noshm || sqlite3_nosys)
+//go:build unix && !sqlite3_nosys
 
 package util
 

diff --git a/internal/util/mmap_other.go b/internal/util/mmap_other.go
@@ -1,4 +1,4 @@
-//go:build !unix || sqlite3_noshm || sqlite3_nosys
+//go:build !unix || sqlite3_nosys
 
 package util
 

diff --git a/tests/bradfitz/sql_test.go b/tests/bradfitz/sql_test.go
@@ -1,4 +1,4 @@
-//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
+//go:build ((linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && !sqlite3_nosys) || sqlite3_flock || sqlite3_dotlk
 
 package bradfitz
 

diff --git a/vfs/README.md b/vfs/README.md
@@ -15,8 +15,7 @@ The main differences are [file locking](#file-locking) and [WAL mode](#write-ahe
 
 POSIX advisory locks, which SQLite uses on Unix, are
 [broken by design](https://github.com/sqlite/sqlite/blob/b74eb0/src/os_unix.c#L1073-L1161).
-
-On Linux and macOS, this package uses
+Instead, on Linux and macOS, this package uses
 [OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html)
 to synchronize access to database files.
 
@@ -45,7 +44,7 @@ to check if your build supports file locking.
 
 ### Write-Ahead Logging
 
-On little-endian Unix, this package uses `mmap` to implement
+On Unix, this package may use `mmap` to implement
 [shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
 like SQLite.
 
@@ -54,6 +53,9 @@ a WAL database can only be accessed by a single proccess.
 Other processes that attempt to access a database locked with BSD locks,
 will fail with the [`SQLITE_PROTOCOL`](https://sqlite.org/rescode.html#protocol) error code.
 
+You can also opt into a cross platform, in-process, memory sharing implementation
+with the `sqlite3_dotlk` build tag.
+
 Otherwise, [WAL support is limited](https://sqlite.org/wal.html#noshm),
 and `EXCLUSIVE` locking mode must be set to create, read, and write WAL databases.
 To use `EXCLUSIVE` locking mode with the
@@ -66,7 +68,7 @@ to check if your build supports shared memory.
 
 ### Batch-Atomic Write
 
-On 64-bit Linux, this package supports
+On Linux, this package may support
 [batch-atomic writes](https://sqlite.org/cgi/src/technote/714)
 on the F2FS filesystem.
 
@@ -87,9 +89,7 @@ The implementation is compatible with SQLite's
 The VFS can be customized with a few build tags:
 - `sqlite3_flock` forces the use of BSD locks.
 - `sqlite3_dotlk` forces the use of dot-file locks.
-- `sqlite3_nosys` prevents importing [`x/sys`](https://pkg.go.dev/golang.org/x/sys);
-  disables locking _and_ shared memory on all platforms.
-- `sqlite3_noshm` disables shared memory on all platforms.
+- `sqlite3_nosys` prevents importing [`x/sys`](https://pkg.go.dev/golang.org/x/sys).
 
 > [!IMPORTANT]
 > The default configuration of this package is compatible with the standard

diff --git a/vfs/adiantum/example_test.go b/vfs/adiantum/example_test.go
@@ -1,4 +1,4 @@
-//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
+//go:build ((linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && !sqlite3_nosys) || sqlite3_flock || sqlite3_dotlk
 
 package adiantum_test
 

diff --git a/vfs/shm.go b/vfs/shm.go
@@ -1,4 +1,4 @@
-//go:build (darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_dotlk || sqlite3_noshm || sqlite3_nosys)
+//go:build ((darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !sqlite3_nosys) || sqlite3_flock || sqlite3_dotlk
 
 package vfs
 

diff --git a/vfs/shm_bsd.go b/vfs/shm_bsd.go
@@ -1,4 +1,4 @@
-//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_dotlk || sqlite3_noshm || sqlite3_nosys)
+//go:build ((freebsd || openbsd || netbsd || dragonfly || illumos) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_dotlk || sqlite3_nosys)) || sqlite3_flock
 
 package vfs
 
@@ -23,9 +23,9 @@ type vfsShmFile struct {
 	// +checklocks:vfsShmFilesMtx
 	refs int
 
-	// +checklocks:lockMtx
-	lock    [_SHM_NLOCK]int16
-	lockMtx sync.Mutex
+	// +checklocks:Mutex
+	lock [_SHM_NLOCK]int16
+	sync.Mutex
 }
 
 var (
@@ -54,7 +54,7 @@ func (s *vfsShm) Close() error {
 	s.shmLock(0, _SHM_NLOCK, _SHM_UNLOCK)
 
 	// Decrease reference count.
-	if s.vfsShmFile.refs > 1 {
+	if s.vfsShmFile.refs > 0 {
 		s.vfsShmFile.refs--
 		s.vfsShmFile = nil
 		return nil
@@ -119,7 +119,6 @@ func (s *vfsShm) shmOpen() (rc _ErrorCode) {
 	s.vfsShmFile = &vfsShmFile{
 		File: f,
 		info: fi,
-		refs: 1,
 	}
 	f = nil // Don't close the file.
 	for i, g := range vfsShmFiles {
@@ -174,8 +173,8 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
 }
 
 func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode {
-	s.lockMtx.Lock()
-	defer s.lockMtx.Unlock()
+	s.Lock()
+	defer s.Unlock()
 
 	switch {
 	case flags&_SHM_UNLOCK != 0:
@@ -234,8 +233,7 @@ func (s *vfsShm) shmUnmap(delete bool) {
 	for _, r := range s.regions {
 		r.Unmap()
 	}
-	clear(s.regions)
-	s.regions = s.regions[:0]
+	s.regions = nil
 
 	// Close the file.
 	if delete {
@@ -245,7 +243,7 @@ func (s *vfsShm) shmUnmap(delete bool) {
 }
 
 func (s *vfsShm) shmBarrier() {
-	s.lockMtx.Lock()
+	s.Lock()
 	//lint:ignore SA2001 memory barrier.
-	s.lockMtx.Unlock()
+	s.Unlock()
 }