Skip to content

Commit

Permalink
Expose bitfield operations (#9)
Browse files Browse the repository at this point in the history
* expose bitfield operations from storage

* add test for bitfields

* fix error message

* bitfield get never errors

* review by @mafintosh
chm-diederichs authored Jun 26, 2024
1 parent 38cc34f commit 77a74a5
Showing 2 changed files with 134 additions and 1 deletion.
47 changes: 46 additions & 1 deletion index.js
Original file line number Diff line number Diff line change
@@ -98,6 +98,14 @@ class WriteBatch {
return this._deleteRange(DATA.TREE, start, end)
}

putBitfieldPage (index, page) {
this.write.tryPut(encodeDataIndex(this.storage.dataPointer, DATA.BITFIELD, index), page)
}

deleteBitfieldPage (index) {
this.write.tryDelete(encodeDataIndex(this.storage.dataPointer, DATA.BITFIELD, index))
}

_deleteRange (type, start, end) {
const s = encodeDataIndex(this.storage.dataPointer, type, start)
const e = encodeDataIndex(this.storage.dataPointer, type, end === -1 ? Infinity : end)
@@ -150,6 +158,11 @@ class ReadBatch {
return node
}

async getBitfieldPage (index) {
const key = encodeDataIndex(this.storage.dataPointer, DATA.BITFIELD, index)
return this._get(key, null)
}

async _has (key) {
return (await this.read.get(key)) !== null
}
@@ -304,6 +317,13 @@ class HypercoreStorage {
return s
}

createBitfieldPageStream (opts = {}) {
const r = encodeIndexRange(this.dataPointer, DATA.BITFIELD, opts)
const s = this.db.iterator(r)
s._readableState.map = mapStreamBitfieldPage
return s
}

getHead () {
const b = this.createReadBatch()
const p = b.getHead()
@@ -325,12 +345,25 @@ class HypercoreStorage {
return p
}

async peakLastTreeNode (opts = {}) {
getBitfieldPage (index) {
const b = this.createReadBatch()
const p = b.getBitfieldPage(index)
b.tryFlush()
return p
}

async peakLastTreeNode () {
const last = await this.db.peek(encodeIndexRange(this.dataPointer, DATA.TREE, { reverse: true }))
if (last === null) return null
return c.decode(m.TreeNode, last.value)
}

async peakLastBitfieldPage () {
const last = await this.db.peek(encodeIndexRange(this.dataPointer, DATA.BITFIELD, { reverse: true }))
if (last === null) return null
return mapStreamBitfieldPage(last)
}

close () {
return this.db.close()
}
@@ -340,6 +373,18 @@ function mapStreamTreeNode (data) {
return c.decode(m.TreeNode, data.value)
}

function mapStreamBitfieldPage (data) {
const state = { start: 0, end: data.key.byteLength, buffer: data.key }

UINT.decode(state) // TL.DATA
UINT.decode(state) // pointer
UINT.decode(state) // DATA.BITFIELD

const index = UINT.decode(state)

return { index, page: data.value }
}

function mapOnlyDiscoveryKey (data) {
return data.key.subarray(1)
}
88 changes: 88 additions & 0 deletions test/basic.js
Original file line number Diff line number Diff line change
@@ -400,6 +400,94 @@ test('make two cores', async function (t) {
t.unlike(c1.corePointer, c2.corePointer)
})

test('bitfield pages', async function (t) {
const c = await getCore(t)

const empty = Buffer.alloc(4096)
const full = Buffer.alloc(4096, 0xff)

{
const b = c.createWriteBatch()

b.putBitfieldPage(0, empty)
b.putBitfieldPage(1, full)
b.putBitfieldPage(10244243, empty)
b.putBitfieldPage(10244244, full)

await b.flush()
}

{
const b = c.createReadBatch()

const page1 = b.getBitfieldPage(0)
const page2 = b.getBitfieldPage(1)
const page3 = b.getBitfieldPage(10244243)
const page4 = b.getBitfieldPage(10244244)
const pageNull = b.getBitfieldPage(2)
b.tryFlush()

t.alike(await page1, empty)
t.alike(await page2, full)
t.alike(await page3, empty)
t.alike(await page4, full)
t.alike(await pageNull, null)
}

t.alike(await c.peakLastBitfieldPage(), { index: 10244244, page: full })

{
const pages = []
for await (const page of c.createBitfieldPageStream()) {
pages.push(page)
}

t.alike(pages, [
{ index: 0, page: empty },
{ index: 1, page: full },
{ index: 10244243, page: empty },
{ index: 10244244, page: full }
])
}

{
const b = c.createWriteBatch()

b.deleteBitfieldPage(0)
b.deleteBitfieldPage(1)
b.deleteBitfieldPage(10244243)
b.deleteBitfieldPage(10244244)

await b.flush()
}

{
const b = c.createReadBatch()

const page1 = b.getBitfieldPage(0)
const page2 = b.getBitfieldPage(1)
const page3 = b.getBitfieldPage(10244243)
const page4 = b.getBitfieldPage(10244244)
b.tryFlush()

t.alike(await page1, null)
t.alike(await page2, null)
t.alike(await page3, null)
t.alike(await page4, null)
}

t.alike(await c.peakLastBitfieldPage(), null)

{
const pages = []
for await (const page of c.createBitfieldPageStream()) {
pages.push(page)
}

t.alike(pages, [])
}
})

test('make lots of cores in parallel', async function (t) {
const s = await getStorage(t)
const promises = []

0 comments on commit 77a74a5

Please sign in to comment.