Skip to content

Commit

Permalink
/go/libraries/doltcore/{doltdb,env}: add paginated tag visitor
Browse files Browse the repository at this point in the history
  • Loading branch information
coffeegoddd committed Jan 17, 2025
1 parent 91f667f commit 5001979
Show file tree
Hide file tree
Showing 3 changed files with 262 additions and 0 deletions.
23 changes: 23 additions & 0 deletions go/libraries/doltcore/doltdb/doltdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,29 @@ func (ddb *DoltDB) ResolveTag(ctx context.Context, tagRef ref.TagRef) (*Tag, err
return NewTag(ctx, tagRef.GetPath(), ds, ddb.vrw, ddb.ns)
}

// ResolveTagMeta takes a TagRef and returns the corresponding TagMeta object.
func (ddb *DoltDB) ResolveTagMeta(ctx context.Context, tagRef ref.TagRef) (*datas.TagMeta, error) {
ds, err := ddb.db.GetDataset(ctx, tagRef.String())
if err != nil {
return nil, ErrTagNotFound
}

if !ds.HasHead() {
return nil, ErrTagNotFound
}

if !ds.IsTag() {
return nil, fmt.Errorf("tagRef head is not a tag")
}

meta, _, err := ds.HeadTag()
if err != nil {
return nil, err
}

return meta, nil
}

// ResolveWorkingSet takes a WorkingSetRef and returns the corresponding WorkingSet object.
func (ddb *DoltDB) ResolveWorkingSet(ctx context.Context, workingSetRef ref.WorkingSetRef) (*WorkingSet, error) {
ds, err := ddb.db.GetDataset(ctx, workingSetRef.String())
Expand Down
110 changes: 110 additions & 0 deletions go/libraries/doltcore/env/actions/tag.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,3 +138,113 @@ func IterResolvedTags(ctx context.Context, ddb *doltdb.DoltDB, cb func(tag *dolt

return nil
}

type TagRefWithMeta struct {
TagRef ref.TagRef
Meta *datas.TagMeta
}

type RefPageToken struct {
TagName string
}

const DefaultPageSize = 100

// IterResolvedTagsPaginated iterates over tags in dEnv.DoltDB from newest to oldest, resolving the tag to a commit and calling cb().
// Returns a next page token if there are more results available.
func IterResolvedTagsPaginated(ctx context.Context, ddb *doltdb.DoltDB, pageToken *RefPageToken, cb func(tag *doltdb.Tag) (stop bool, err error)) (*RefPageToken, error) {
tagRefs, err := ddb.GetTags(ctx)
if err != nil {
return nil, err
}

// for each tag, get the meta
tagMetas := make([]*TagRefWithMeta, 0, len(tagRefs))
for _, r := range tagRefs {
tr, ok := r.(ref.TagRef)
if !ok {
return nil, fmt.Errorf("DoltDB.GetTags() returned non-tag DoltRef")
}
meta, err := ddb.ResolveTagMeta(ctx, tr)
if err != nil {
return nil, err
}
tagMetas = append(tagMetas, &TagRefWithMeta{TagRef: tr, Meta: meta})
}

// sort by meta timestamp
sort.Slice(tagMetas, func(i, j int) bool {
return tagMetas[i].Meta.Timestamp > tagMetas[j].Meta.Timestamp
})

// find starting index based on page token
startIdx := 0
if pageToken != nil && pageToken.TagName != "" {
for i, tm := range tagMetas {
if tm.TagRef.GetPath() == pageToken.TagName {
startIdx = i + 1 // start after the token
break
}
}
}

// get page of results
endIdx := startIdx + DefaultPageSize
if endIdx > len(tagMetas) {
endIdx = len(tagMetas)
}

pageTagMetas := tagMetas[startIdx:endIdx]

// resolve tags for this page
for _, tm := range pageTagMetas {
tag, err := ddb.ResolveTag(ctx, tm.TagRef)
if err != nil {
return nil, err
}

stop, err := cb(tag)
if err != nil {
return nil, err
}
if stop {
break
}
}

// return next page token if there are more results
var nextPageToken *RefPageToken
if endIdx < len(tagMetas) {
lastTag := pageTagMetas[len(pageTagMetas)-1]
nextPageToken = &RefPageToken{
TagName: lastTag.TagRef.GetPath(),
}
}

return nextPageToken, nil
}

// VisitResolvedTag iterates over tags in ddb until the given tag name is found, then calls cb() with the resolved tag.
func VisitResolvedTag(ctx context.Context, ddb *doltdb.DoltDB, tagName string, cb func(tag *doltdb.Tag) error) error {
tagRefs, err := ddb.GetTags(ctx)
if err != nil {
return err
}

for _, r := range tagRefs {
tr, ok := r.(ref.TagRef)
if !ok {
return fmt.Errorf("DoltDB.GetTags() returned non-tag DoltRef")
}

if tr.GetPath() == tagName {
tag, err := ddb.ResolveTag(ctx, tr)
if err != nil {
return err
}
return cb(tag)
}
}

return doltdb.ErrTagNotFound
}
129 changes: 129 additions & 0 deletions go/libraries/doltcore/env/actions/tag_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
package actions

import (
"context"
"fmt"
"testing"
"time"

"github.com/stretchr/testify/require"

"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/store/types"
)

const (
testHomeDir = "/user/bheni"
workingDir = "/user/bheni/datasets/addresses"
credsDir = "creds"

configFile = "config.json"
GlobalConfigFile = "config_global.json"
)

func testHomeDirFunc() (string, error) {
return testHomeDir, nil
}

func createTestEnv() (*env.DoltEnv, *filesys.InMemFS) {
initialDirs := []string{testHomeDir, workingDir}
initialFiles := map[string][]byte{}

fs := filesys.NewInMemFS(initialDirs, initialFiles, workingDir)
dEnv := env.Load(context.Background(), testHomeDirFunc, fs, doltdb.InMemDoltDB, "test")

return dEnv, fs
}

func TestVisitResolvedTag(t *testing.T) {
dEnv, _ := createTestEnv()
ctx := context.Background()

// Initialize repo
err := dEnv.InitRepo(ctx, types.Format_Default, "test user", "[email protected]", "main")
require.NoError(t, err)

// Create a tag
tagName := "test-tag"
tagMsg := "test tag message"
err = CreateTag(ctx, dEnv, tagName, "main", TagProps{TaggerName: "test user", TaggerEmail: "[email protected]", Description: tagMsg})
require.NoError(t, err)

// Visit the tag and verify its properties
var foundTag *doltdb.Tag
err = VisitResolvedTag(ctx, dEnv.DoltDB, tagName, func(tag *doltdb.Tag) error {
foundTag = tag
return nil
})
require.NoError(t, err)
require.NotNil(t, foundTag)
require.Equal(t, tagName, foundTag.Name)
require.Equal(t, tagMsg, foundTag.Meta.Description)

// Test visiting non-existent tag
err = VisitResolvedTag(ctx, dEnv.DoltDB, "non-existent-tag", func(tag *doltdb.Tag) error {
return nil
})
require.Equal(t, doltdb.ErrTagNotFound, err)
}

func TestIterResolvedTagsPaginated(t *testing.T) {
dEnv, _ := createTestEnv()
ctx := context.Background()

// Initialize repo
err := dEnv.InitRepo(ctx, types.Format_Default, "test user", "[email protected]", "main")
require.NoError(t, err)

expectedTagNames := make([]string, DefaultPageSize*2)
// Create multiple tags with different timestamps
tagNames := make([]string, DefaultPageSize*2)
for i := range tagNames {
tagName := fmt.Sprintf("tag-%d", i)
err = CreateTag(ctx, dEnv, tagName, "main", TagProps{
TaggerName: "test user",
TaggerEmail: "[email protected]",
Description: fmt.Sprintf("test tag %s", tagName),
})
time.Sleep(2 * time.Millisecond)
require.NoError(t, err)
tagNames[i] = tagName
expectedTagNames[len(expectedTagNames)-i-1] = tagName
}

// Test first page
var foundTags []string
pageToken, err := IterResolvedTagsPaginated(ctx, dEnv.DoltDB, nil, func(tag *doltdb.Tag) (bool, error) {
foundTags = append(foundTags, tag.Name)
return false, nil
})
require.NoError(t, err)
require.NotNil(t, pageToken) // Should have next page
require.Equal(t, DefaultPageSize, len(foundTags)) // Default page size tags returned

// Test second page
var secondPageTags []string
nextPageToken, err := IterResolvedTagsPaginated(ctx, dEnv.DoltDB, pageToken, func(tag *doltdb.Tag) (bool, error) {
secondPageTags = append(secondPageTags, tag.Name)
return false, nil
})
require.NoError(t, err)
require.Nil(t, nextPageToken) // Should be no more pages
require.Equal(t, DefaultPageSize, len(secondPageTags)) // Remaining tags

// Verify all tags were found
allFoundTags := append(foundTags, secondPageTags...)
require.Equal(t, len(tagNames), len(allFoundTags))
require.Equal(t, expectedTagNames, allFoundTags)

// Test early termination
var earlyTermTags []string
_, err = IterResolvedTagsPaginated(ctx, dEnv.DoltDB, nil, func(tag *doltdb.Tag) (bool, error) {
earlyTermTags = append(earlyTermTags, tag.Name)
return true, nil // Stop after first tag
})
require.NoError(t, err)
require.Equal(t, 1, len(earlyTermTags))
}

0 comments on commit 5001979

Please sign in to comment.