Skip to content

Commit

Permalink
add freelist interface unit tests
Browse files Browse the repository at this point in the history 
adding more unit tests for better coverage of the interface and some
implementation details.

Signed-off-by: Thomas Jungblut <[email protected]>
  • Loading branch information
@tjungblu
tjungblu committed Aug 8, 2024
1 parent f777377 commit e457dac
Show file tree
Hide file tree
Showing 3 changed files with 355 additions and 0 deletions.
12 changes: 12 additions & 0 deletions internal/freelist/array_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"reflect"
"testing"

"github.com/stretchr/testify/require"

"go.etcd.io/bbolt/internal/common"
)

Expand Down Expand Up @@ -50,3 +52,13 @@ func TestFreelistArray_allocate(t *testing.T) {
t.Fatalf("exp=%v; got=%v", exp, f.freePageIds())
}
}

func TestInvalidArrayAllocation(t *testing.T) {
f := NewArrayFreelist()
// page 0 and 1 are reserved for meta pages, so they should never be free pages.
ids := []common.Pgid{1}
f.Init(ids)
require.Panics(t, func() {
f.Allocate(common.Txid(1), 1)
})
}
333 changes: 333 additions & 0 deletions internal/freelist/freelist_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
package freelist

import (
"fmt"
"math"
"math/rand"
"os"
"reflect"
"slices"
"sort"
"testing"
"testing/quick"
"unsafe"

"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -34,6 +38,55 @@ func TestFreelist_free_overflow(t *testing.T) {
}
}

// Ensure that double freeing a page is causing a panic
func TestFreelist_free_double_free_panics(t *testing.T) {
f := newTestFreelist()
f.Free(100, common.NewPage(12, 0, 0, 3))
require.Panics(t, func() {
f.Free(100, common.NewPage(12, 0, 0, 3))
})
}

// Ensure that attempting to free the meta page panics
func TestFreelist_free_meta_panics(t *testing.T) {
f := newTestFreelist()
require.Panics(t, func() {
f.Free(100, common.NewPage(0, 0, 0, 0))
})
require.Panics(t, func() {
f.Free(100, common.NewPage(1, 0, 0, 0))
})
}

func TestFreelist_free_freelist(t *testing.T) {
f := newTestFreelist()
f.Free(100, common.NewPage(12, common.FreelistPageFlag, 0, 0))
pp := f.pendingPageIds()[100]
require.Equal(t, []common.Pgid{12}, pp.ids)
require.Equal(t, []common.Txid{0}, pp.alloctx)
}

func TestFreelist_free_freelist_alloctx(t *testing.T) {
f := newTestFreelist()
f.Free(100, common.NewPage(12, common.FreelistPageFlag, 0, 0))
f.Rollback(100)
require.Empty(t, f.freePageIds())
require.Empty(t, f.pendingPageIds())
require.False(t, f.Freed(12))

f.Free(101, common.NewPage(12, common.FreelistPageFlag, 0, 0))
require.True(t, f.Freed(12))
if exp := []common.Pgid{12}; !reflect.DeepEqual(exp, f.pendingPageIds()[101].ids) {
t.Fatalf("exp=%v; got=%v", exp, f.pendingPageIds()[101].ids)
}
f.ReleasePendingPages()
require.True(t, f.Freed(12))
require.Empty(t, f.pendingPageIds())
if exp := common.Pgids([]common.Pgid{12}); !reflect.DeepEqual(exp, f.freePageIds()) {
t.Fatalf("exp=%v; got=%v", exp, f.freePageIds())
}
}

// Ensure that a transaction's free pages can be released.
func TestFreelist_release(t *testing.T) {
f := newTestFreelist()
Expand Down Expand Up @@ -220,6 +273,30 @@ func TestFreeList_reload(t *testing.T) {
require.Equal(t, []common.Pgid{10, 11, 12}, f2.pendingPageIds()[5].ids)
}

// Ensure that the txIDx swap, less and len are properly implemented
func TestTxidSorting(t *testing.T) {
require.NoError(t, quick.Check(func(a []uint64) bool {
var txids []common.Txid
for _, txid := range a {
txids = append(txids, common.Txid(txid))
}

sort.Sort(txIDx(txids))

var r []uint64
for _, txid := range txids {
r = append(r, uint64(txid))
}

if !slices.IsSorted(r) {
t.Errorf("txids were not sorted correctly=%v", txids)
return false
}

return true
}, nil))
}

// Ensure that a freelist can deserialize from a freelist page.
func TestFreelist_read(t *testing.T) {
// Create a page.
Expand All @@ -243,6 +320,18 @@ func TestFreelist_read(t *testing.T) {
}
}

// Ensure that we never read a non-freelist page
func TestFreelist_read_panics(t *testing.T) {
buf := make([]byte, 4096)
page := common.LoadPage(buf)
page.SetFlags(common.BranchPageFlag)
page.SetCount(2)
f := newTestFreelist()
require.Panics(t, func() {
f.Read(page)
})
}

// Ensure that a freelist can serialize into a freelist page.
func TestFreelist_write(t *testing.T) {
// Create a freelist and write it to a page.
Expand All @@ -266,6 +355,250 @@ func TestFreelist_write(t *testing.T) {
}
}

func TestFreelist_E2E_HappyPath(t *testing.T) {
f := newTestFreelist()
f.Init([]common.Pgid{})
requirePages(t, f, common.Pgids{}, common.Pgids{})

allocated := f.Allocate(common.Txid(1), 5)
require.Equal(t, common.Pgid(0), allocated)
// tx.go may now allocate more space, and eventually we need to delete a page again
f.Free(common.Txid(2), common.NewPage(5, common.LeafPageFlag, 0, 0))
f.Free(common.Txid(2), common.NewPage(3, common.LeafPageFlag, 0, 0))
f.Free(common.Txid(2), common.NewPage(8, common.LeafPageFlag, 0, 0))
// the above will only mark the pages as pending, so free pages should not return anything
requirePages(t, f, common.Pgids{}, common.Pgids{3, 5, 8})

// someone wants to do a read on top of the next tx id
f.AddReadonlyTXID(common.Txid(3))
// this should free the above pages for tx 2 entirely
f.ReleasePendingPages()
requirePages(t, f, common.Pgids{3, 5, 8}, common.Pgids{})

// no span of two pages available should yield a zero-page result
require.Equal(t, common.Pgid(0), f.Allocate(common.Txid(4), 2))
// we should be able to allocate those pages independently however,
// map and array differ in the order they return the pages
expectedPgids := map[common.Pgid]struct{}{3: {}, 5: {}, 8: {}}
for i := 0; i < 3; i++ {
allocated = f.Allocate(common.Txid(4), 1)
require.Contains(t, expectedPgids, allocated, "expected to find pgid %d", allocated)
require.False(t, f.Freed(allocated))
delete(expectedPgids, allocated)
}
require.Emptyf(t, expectedPgids, "unexpectedly more than one page was still found")
// no more free pages to allocate
require.Equal(t, common.Pgid(0), f.Allocate(common.Txid(4), 1))
}

func TestFreelist_E2E_MultiSpanOverflows(t *testing.T) {
f := newTestFreelist()
f.Init([]common.Pgid{})
f.Free(common.Txid(10), common.NewPage(20, common.LeafPageFlag, 0, 1))
f.Free(common.Txid(10), common.NewPage(25, common.LeafPageFlag, 0, 2))
f.Free(common.Txid(10), common.NewPage(35, common.LeafPageFlag, 0, 3))
f.Free(common.Txid(10), common.NewPage(39, common.LeafPageFlag, 0, 2))
f.Free(common.Txid(10), common.NewPage(45, common.LeafPageFlag, 0, 4))
requirePages(t, f, common.Pgids{}, common.Pgids{20, 21, 25, 26, 27, 35, 36, 37, 38, 39, 40, 41, 45, 46, 47, 48, 49})
f.ReleasePendingPages()
requirePages(t, f, common.Pgids{20, 21, 25, 26, 27, 35, 36, 37, 38, 39, 40, 41, 45, 46, 47, 48, 49}, common.Pgids{})

// that sequence, regardless of implementation, should always yield the same blocks of pages
allocSequence := []int{7, 5, 3, 2}
expectedSpanStarts := []common.Pgid{35, 45, 25, 20}
for i, pageNums := range allocSequence {
allocated := f.Allocate(common.Txid(11), pageNums)
require.Equal(t, expectedSpanStarts[i], allocated)
// ensure all pages in that span are not considered free anymore
for i := 0; i < pageNums; i++ {
require.False(t, f.Freed(allocated+common.Pgid(i)))
}
}
}

func TestFreelist_E2E_Rollbacks(t *testing.T) {
freelist := newTestFreelist()
freelist.Init([]common.Pgid{})
freelist.Free(common.Txid(2), common.NewPage(5, common.LeafPageFlag, 0, 1))
freelist.Free(common.Txid(2), common.NewPage(8, common.LeafPageFlag, 0, 0))
requirePages(t, freelist, common.Pgids{}, common.Pgids{5, 6, 8})
freelist.Rollback(common.Txid(2))
requirePages(t, freelist, common.Pgids{}, common.Pgids{})

// unknown transaction should not trigger anything
freelist.Free(common.Txid(4), common.NewPage(13, common.LeafPageFlag, 0, 3))
requirePages(t, freelist, common.Pgids{}, common.Pgids{13, 14, 15, 16})
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{13, 14, 15, 16}, common.Pgids{})
freelist.Rollback(common.Txid(1337))
requirePages(t, freelist, common.Pgids{13, 14, 15, 16}, common.Pgids{})
}

func TestFreelist_E2E_RollbackPanics(t *testing.T) {
freelist := newTestFreelist()
freelist.Init([]common.Pgid{5})
requirePages(t, freelist, common.Pgids{5}, common.Pgids{})

_ = freelist.Allocate(common.Txid(5), 1)
require.Panics(t, func() {
// depending on the verification level, either should panic
freelist.Free(common.Txid(5), common.NewPage(5, common.LeafPageFlag, 0, 0))
freelist.Rollback(5)
})
}

func TestFreelist_E2E_ReadOnlyTxTracking(t *testing.T) {
freelist := newTestFreelist()
freelist.Init([]common.Pgid{})
freelist.Free(common.Txid(10), common.NewPage(10, common.LeafPageFlag, 0, 2))
freelist.Free(common.Txid(11), common.NewPage(20, common.LeafPageFlag, 0, 2))
freelist.Free(common.Txid(12), common.NewPage(30, common.LeafPageFlag, 0, 2))
requirePages(t, freelist, common.Pgids{}, common.Pgids{10, 11, 12, 20, 21, 22, 30, 31, 32})

freelist.AddReadonlyTXID(11)
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{10, 11, 12}, common.Pgids{20, 21, 22, 30, 31, 32})

// this should be a no-op, as we still have a read TX open with id 11
freelist.AddReadonlyTXID(12)
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{10, 11, 12}, common.Pgids{20, 21, 22, 30, 31, 32})

// now 12 should be the latest
freelist.RemoveReadonlyTXID(11)
freelist.AddReadonlyTXID(12)
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{10, 11, 12, 20, 21, 22}, common.Pgids{30, 31, 32})

// 12 was registered twice, so we also have to remove it twice to have an effect
freelist.RemoveReadonlyTXID(12)
freelist.AddReadonlyTXID(13)
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{10, 11, 12, 20, 21, 22}, common.Pgids{30, 31, 32})

freelist.RemoveReadonlyTXID(12)
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{10, 11, 12, 20, 21, 22, 30, 31, 32}, common.Pgids{})
}

// tests the reloading from another physical page
func TestFreelist_E2E_Reload(t *testing.T) {
freelist := newTestFreelist()
freelist.Init([]common.Pgid{})
freelist.Free(common.Txid(2), common.NewPage(5, common.LeafPageFlag, 0, 1))
freelist.Free(common.Txid(2), common.NewPage(8, common.LeafPageFlag, 0, 0))
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{5, 6, 8}, common.Pgids{})
buf := make([]byte, 4096)
p := common.LoadPage(buf)
freelist.Write(p)

freelist.Free(common.Txid(3), common.NewPage(3, common.LeafPageFlag, 0, 1))
freelist.Free(common.Txid(3), common.NewPage(10, common.LeafPageFlag, 0, 2))
requirePages(t, freelist, common.Pgids{5, 6, 8}, common.Pgids{3, 4, 10, 11, 12})

otherBuf := make([]byte, 4096)
px := common.LoadPage(otherBuf)
freelist.Write(px)

loadFreeList := newTestFreelist()
loadFreeList.Init([]common.Pgid{})
loadFreeList.Read(px)
requirePages(t, loadFreeList, common.Pgids{3, 4, 5, 6, 8, 10, 11, 12}, common.Pgids{})
// restore the original freelist again
loadFreeList.Reload(p)
requirePages(t, loadFreeList, common.Pgids{5, 6, 8}, common.Pgids{})

// reload another page with different free pages to test we are deduplicating the free pages with the pending ones correctly
freelist = newTestFreelist()
freelist.Init([]common.Pgid{})
freelist.Free(common.Txid(5), common.NewPage(5, common.LeafPageFlag, 0, 4))
freelist.Reload(p)
requirePages(t, freelist, common.Pgids{}, common.Pgids{5, 6, 7, 8, 9})
}

// tests the loading and reloading from physical pages
func TestFreelist_E2E_SerDe_HappyPath(t *testing.T) {
freelist := newTestFreelist()
freelist.Init([]common.Pgid{})
freelist.Free(common.Txid(2), common.NewPage(5, common.LeafPageFlag, 0, 1))
freelist.Free(common.Txid(2), common.NewPage(8, common.LeafPageFlag, 0, 0))
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{5, 6, 8}, common.Pgids{})

freelist.Free(common.Txid(3), common.NewPage(3, common.LeafPageFlag, 0, 1))
freelist.Free(common.Txid(3), common.NewPage(10, common.LeafPageFlag, 0, 2))
requirePages(t, freelist, common.Pgids{5, 6, 8}, common.Pgids{3, 4, 10, 11, 12})

buf := make([]byte, 4096)
p := common.LoadPage(buf)
require.Equal(t, 80, freelist.EstimatedWritePageSize())
freelist.Write(p)

loadFreeList := newTestFreelist()
loadFreeList.Init([]common.Pgid{})
loadFreeList.Read(p)
requirePages(t, loadFreeList, common.Pgids{3, 4, 5, 6, 8, 10, 11, 12}, common.Pgids{})
}

// tests the loading of a freelist against other implementations with various sizes
func TestFreelist_E2E_SerDe_AcrossImplementations(t *testing.T) {
testSizes := []int{0, 1, 10, 100, 1000, math.MaxUint16, math.MaxUint16 + 1, math.MaxUint16 * 2}
for _, size := range testSizes {
t.Run(fmt.Sprintf("n=%d", size), func(t *testing.T) {
freelist := newTestFreelist()
expectedFreePgids := common.Pgids{}
for i := 0; i < size; i++ {
pgid := common.Pgid(i + 2)
freelist.Free(common.Txid(1), common.NewPage(pgid, common.LeafPageFlag, 0, 0))
expectedFreePgids = append(expectedFreePgids, pgid)
}
freelist.ReleasePendingPages()
requirePages(t, freelist, expectedFreePgids, common.Pgids{})
buf := make([]byte, freelist.EstimatedWritePageSize())
p := common.LoadPage(buf)
freelist.Write(p)

for n, loadFreeList := range map[string]Interface{
"hashmap": NewHashMapFreelist(),
"array": NewArrayFreelist(),
} {
t.Run(n, func(t *testing.T) {
loadFreeList.Read(p)
requirePages(t, loadFreeList, expectedFreePgids, common.Pgids{})
})
}
})
}
}

func requirePages(t *testing.T, f Interface, freePageIds common.Pgids, pendingPageIds common.Pgids) {
require.Equal(t, f.FreeCount()+f.PendingCount(), f.Count())
require.Equalf(t, freePageIds, f.freePageIds(), "unexpected free pages")
require.Equal(t, len(freePageIds), f.FreeCount())

pp := allPendingPages(f.pendingPageIds())
require.Equalf(t, pendingPageIds, pp, "unexpected pending pages")
require.Equal(t, len(pp), f.PendingCount())

for _, pgid := range f.freePageIds() {
require.Truef(t, f.Freed(pgid), "expected free page to return true on Freed")
}

for _, pgid := range pp {
require.Truef(t, f.Freed(pgid), "expected pending page to return true on Freed")
}
}

func allPendingPages(p map[common.Txid]*txPending) common.Pgids {
pgids := common.Pgids{}
for _, pending := range p {
pgids = append(pgids, pending.ids...)
}
sort.Sort(pgids)
return pgids
}

func Benchmark_FreelistRelease10K(b *testing.B) { benchmark_FreelistRelease(b, 10000) }
func Benchmark_FreelistRelease100K(b *testing.B) { benchmark_FreelistRelease(b, 100000) }
func Benchmark_FreelistRelease1000K(b *testing.B) { benchmark_FreelistRelease(b, 1000000) }
Expand Down
Loading

0 comments on commit e457dac

Please sign in to comment.