Fix: recuisively search file by default, fix file ingestion with edit

Signed-off-by: Daishan Peng <[email protected]>

pkg/cmd/askdir.go

@@ -40,7 +40,7 @@ func (s *ClientAskDir) Run(cmd *cobra.Command, args []string) error {
 	ingestOpts := &client.IngestPathsOpts{
 		IgnoreExtensions: strings.Split(s.IgnoreExtensions, ","),
 		Concurrency:      s.Concurrency,
-		Recursive:        s.Recursive,
+		Recursive:        !s.NoRecursive,
 	}
 
 	retrieveOpts := &datastore.RetrieveOpts{

pkg/cmd/ingest.go

@@ -23,12 +23,12 @@ type ClientIngest struct {
 type ClientIngestOpts struct {
 	IgnoreExtensions string `usage:"Comma-separated list of file extensions to ignore" env:"KNOW_INGEST_IGNORE_EXTENSIONS"`
 	Concurrency      int    `usage:"Number of concurrent ingestion processes" short:"c" default:"10" env:"KNOW_INGEST_CONCURRENCY"`
-	Recursive        bool   `usage:"Recursively ingest directories" short:"r" default:"false" env:"KNOW_INGEST_RECURSIVE"`
+	NoRecursive      bool   `usage:"Don't recursively ingest directories" default:"false" env:"KNOW_NO_INGEST_RECURSIVE"`
 }
 
 func (s *ClientIngest) Customize(cmd *cobra.Command) {
 	cmd.Use = "ingest [--dataset <dataset-id>] <path>"
-	cmd.Short = "Ingest a file/directory into a dataset (non-recursive)"
+	cmd.Short = "Ingest a file/directory into a dataset"
 	cmd.Args = cobra.ExactArgs(1)
 }
 
@@ -44,7 +44,7 @@ func (s *ClientIngest) Run(cmd *cobra.Command, args []string) error {
 	ingestOpts := &client.IngestPathsOpts{
 		IgnoreExtensions: strings.Split(s.IgnoreExtensions, ","),
 		Concurrency:      s.Concurrency,
-		Recursive:        s.Recursive,
+		Recursive:        !s.NoRecursive,
 		TextSplitterOpts: &s.TextSplitterOpts,
 	}
 

pkg/datastore/document.go

@@ -17,7 +17,7 @@ func (s *Datastore) DeleteDocument(ctx context.Context, documentID, datasetID st
 	}
 
 	// Remove from VectorStore
-	if err := s.Vectorstore.RemoveDocument(ctx, documentID, datasetID); err != nil {
+	if err := s.Vectorstore.RemoveDocument(ctx, documentID, datasetID, nil, nil); err != nil {
 		return fmt.Errorf("failed to remove document from VectorStore: %w", err)
 	}
 

pkg/datastore/files.go

@@ -21,7 +21,7 @@ func (s *Datastore) DeleteFile(ctx context.Context, datasetID, fileID string) er
 
 	// Remove owned documents from VectorStore and Database
 	for _, doc := range file.Documents {
-		if err := s.Vectorstore.RemoveDocument(ctx, doc.ID, datasetID); err != nil {
+		if err := s.Vectorstore.RemoveDocument(ctx, doc.ID, datasetID, nil, nil); err != nil {
 			return fmt.Errorf("failed to remove document from VectorStore: %w", err)
 		}
 

pkg/datastore/ingest.go

@@ -92,7 +92,7 @@ func (s *Datastore) Ingest(ctx context.Context, datasetID string, content []byte
 	ingestionFlow.FillDefaults(filetype, opts.TextSplitterOpts)
 
 	// Mandatory Transformation: Add filename to metadata
-	em := &transformers.ExtraMetadata{Metadata: map[string]any{"filename": filename}}
+	em := &transformers.ExtraMetadata{Metadata: map[string]any{"filename": filename, "absPath": opts.FileMetadata.AbsolutePath}}
 	ingestionFlow.Transformations = append(ingestionFlow.Transformations, em)
 
 	docs, err := ingestionFlow.Run(ctx, bytes.NewReader(content))
@@ -105,6 +105,14 @@ func (s *Datastore) Ingest(ctx context.Context, datasetID string, content []byte
 		return nil, nil
 	}
 
+	// Before adding doc, we need to remove the existing documents for duplicates or old contents
+	where := map[string]string{
+		"absPath": opts.FileMetadata.AbsolutePath,
+	}
+	if err := s.Vectorstore.RemoveDocument(ctx, "", datasetID, where, nil); err != nil {
+		return nil, err
+	}
+
 	// Add documents to VectorStore -> This generates the embeddings
 	slog.Debug("Ingesting documents", "count", len(docs))
 	docIDs, err := s.Vectorstore.AddDocuments(ctx, docs, datasetID)

pkg/vectorstore/chromem/chromem.go

@@ -14,7 +14,7 @@ import (
 	"github.com/philippgille/chromem-go"
 )
 
-// EmbeddingParallelThread can be set as an environment variable to control the number of parallel API calls to create embedding for documents. Default is 100
+// VsChromemEmbeddingParallelThread can be set as an environment variable to control the number of parallel API calls to create embedding for documents. Default is 100
 const VsChromemEmbeddingParallelThread = "VS_CHROMEM_EMBEDDING_PARALLEL_THREAD"
 
 type Store struct {
@@ -145,10 +145,10 @@ func (s *Store) RemoveCollection(_ context.Context, collection string) error {
 	return s.db.DeleteCollection(collection)
 }
 
-func (s *Store) RemoveDocument(ctx context.Context, documentID string, collection string) error {
+func (s *Store) RemoveDocument(ctx context.Context, documentID string, collection string, where, whereDocument map[string]string) error {
 	col := s.db.GetCollection(collection, s.embeddingFunc)
 	if col == nil {
 		return fmt.Errorf("%w: %q", errors.ErrCollectionNotFound, collection)
 	}
-	return col.Delete(ctx, nil, nil, documentID)
+	return col.Delete(ctx, where, whereDocument, documentID)
 }

pkg/vectorstore/vectorstores.go

@@ -9,5 +9,5 @@ type VectorStore interface {
 	AddDocuments(ctx context.Context, docs []Document, collection string) ([]string, error)                      // @return documentIDs, error
 	SimilaritySearch(ctx context.Context, query string, numDocuments int, collection string) ([]Document, error) //nolint:lll
 	RemoveCollection(ctx context.Context, collection string) error
-	RemoveDocument(ctx context.Context, documentID string, collection string) error
+	RemoveDocument(ctx context.Context, documentID string, collection string, where, whereDocument map[string]string) error
 }