Merge pull request #832 from bjwswang/main

feat: configure chunksize and chunkoverlap in knowledgebase

api/app-node/documentloader/v1alpha1/document_loader_types.go

@@ -28,10 +28,10 @@ type DocumentLoaderSpec struct {
 	// CommonSpec
 	v1alpha1.CommonSpec `json:",inline"`
 	// ChunkSize for text splitter
-	// +kubebuilder:default=2048
+	// +kubebuilder:default=512
 	ChunkSize int `json:"chunkSize,omitempty"`
 	// ChunkOverlap for text splitter
-	// +kubebuilder:default=200
+	// +kubebuilder:default=100
 	ChunkOverlap int `json:"chunkOverlap,omitempty"`
 	// FileExtName the type of documents, can be .pdf, .txt, .mp3, etc ...
 	FileExtName string `json:"fileExtName,omitempty"`

api/base/v1alpha1/knowledgebase.go

@@ -10,6 +10,19 @@ const (
 	UpdateSourceFileAnnotationKey = Group + "/update-source-file-time"
 )
 
+func (kb *KnowledgeBase) EmbeddingOptions() EmbeddingOptions {
+	options := kb.Spec.EmbeddingOptions
+	if kb.Spec.EmbeddingOptions.ChunkSize == 0 {
+		// default 1024
+		options.ChunkSize = 1024
+	}
+	if kb.Spec.EmbeddingOptions.ChunkOverlap == 0 {
+		// default 100
+		options.ChunkOverlap = 100
+	}
+	return options
+}
+
 func (kb *KnowledgeBase) VectorStoreCollectionName() string {
 	return kb.Namespace + "_" + kb.Name
 }

api/base/v1alpha1/knowledgebase_types.go

@@ -27,13 +27,23 @@ type KnowledgeBaseSpec struct {
 	// Embedder defines the embedder to embedding files
 	Embedder *TypedObjectReference `json:"embedder,omitempty"`
 
-	// TODO: add EmbedderOptions
-
 	// VectorStore defines the vectorstore to store results
 	VectorStore *TypedObjectReference `json:"vectorStore,omitempty"`
 
 	// FileGroups included files Grouped by VersionedDataset
 	FileGroups []FileGroup `json:"fileGroups,omitempty"`
+
+	// Embedding Options
+	EmbeddingOptions `json:",inline"`
+}
+
+type EmbeddingOptions struct {
+	// ChunkSize for text splitter
+	// +kubebuilder:default=1024
+	ChunkSize int `json:"chunkSize,omitempty"`
+	// ChunkOverlap for text splitter
+	// +kubebuilder:default=100
+	ChunkOverlap int `json:"chunkOverlap,omitempty"`
 }
 
 type FileGroupDetail struct {