add vector index mgr to remove vector index type dependency

Signed-off-by: xianliang.li <[email protected]>

cmd/tools/migration/mmap/mmap_230_240.go

@@ -9,8 +9,8 @@ import (
 	"github.com/milvus-io/milvus/internal/metastore/model"
 	"github.com/milvus-io/milvus/internal/rootcoord"
 	"github.com/milvus-io/milvus/internal/tso"
+	"github.com/milvus-io/milvus/internal/util/vecindexmgr"
 	"github.com/milvus-io/milvus/pkg/common"
-	"github.com/milvus-io/milvus/pkg/util/indexparamcheck"
 )
 
 // In Milvus 2.3.x, querynode.MmapDirPath is used to enable mmap and save mmap files.
@@ -84,7 +84,7 @@ func (m *MmapMigration) MigrateIndexCoordCollection(ctx context.Context) {
 
 	alteredIndexes := make([]*model.Index, 0)
 	for _, index := range fieldIndexes {
-		if !indexparamcheck.IsVectorMmapIndex(getIndexType(index.IndexParams)) {
+		if !vecindexmgr.GetVecIndexMgrInstance().IsMMapSupported(getIndexType(index.IndexParams)) {
 			continue
 		}
 		fmt.Printf("migrate index, collection:%v, indexId: %v, indexName: %s\n", index.CollectionID, index.IndexID, index.IndexName)

internal/core/src/index/Index.h

@@ -22,6 +22,7 @@
 #include "common/EasyAssert.h"
 #include "knowhere/comp/index_param.h"
 #include "knowhere/dataset.h"
+#include "knowhere/index/index_factory.h"
 #include "common/Tracer.h"
 #include "common/Types.h"
 #include "index/Meta.h"
@@ -62,23 +63,8 @@ class IndexBase {
     virtual const bool
     HasRawData() const = 0;
 
-    bool
-    IsMmapSupported() const {
-        return index_type_ == knowhere::IndexEnum::INDEX_HNSW ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_IVFPQ ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_IVFSQ8 ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_IDMAP ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP ||
-               index_type_ ==
-                   knowhere::IndexEnum::INDEX_SPARSE_INVERTED_INDEX ||
-               index_type_ == knowhere::IndexEnum::INDEX_SPARSE_WAND ||
-               // support mmap for bitmap/hybrid index
-               index_type_ == milvus::index::BITMAP_INDEX_TYPE ||
-               index_type_ == milvus::index::HYBRID_INDEX_TYPE;
-    }
+    virtual bool
+    IsMmapSupported() const = 0;
 
     const IndexType&
     Type() const {

internal/core/src/index/ScalarIndex.h

@@ -133,6 +133,12 @@ class ScalarIndex : public IndexBase {
         PanicInfo(Unsupported, "pattern match is not supported");
     }
 
+    virtual bool
+    IsMmapSupported() const {
+        return index_type_ == milvus::index::BITMAP_INDEX_TYPE ||
+               index_type_ == milvus::index::HYBRID_INDEX_TYPE;
+    }
+
     virtual int64_t
     Size() = 0;
 

internal/core/src/index/VectorIndex.h

@@ -115,6 +115,12 @@ class VectorIndex : public IndexBase {
             err_msg);
     }
 
+    virtual bool
+    IsMmapSupported() const {
+        return knowhere::IndexFactory::Instance().FeatureCheck(
+            index_type_, knowhere::feature::MMAP);
+    }
+
     knowhere::Json
     PrepareSearchParams(const SearchInfo& search_info) const {
         knowhere::Json search_cfg = search_info.search_params_;

internal/core/src/segcore/vector_index_c.cpp

@@ -0,0 +1,40 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include "segcore/vector_index_c.h"
+
+#include "common/Types.h"
+#include "common/EasyAssert.h"
+#include "knowhere/utils.h"
+#include "knowhere/config.h"
+#include "knowhere/version.h"
+#include "index/Meta.h"
+#include "index/IndexFactory.h"
+#include "pb/index_cgo_msg.pb.h"
+
+int
+GetIndexListSize() {
+    return knowhere::IndexFactory::Instance().GetIndexFeatures().size();
+}
+
+void
+GetIndexFeatures(void* index_key_list, uint64_t* index_feature_list) {
+    auto features = knowhere::IndexFactory::Instance().GetIndexFeatures();
+    int idx = 0;
+
+    const char** index_keys = (const char**)index_key_list;
+    uint64_t* index_features = (uint64_t*)index_feature_list;
+    for (auto it = features.begin(); it != features.end(); ++it) {
+        index_keys[idx] = it->first.c_str();
+        index_features[idx] = it->second;
+        idx++;
+    }
+}

internal/core/src/segcore/vector_index_c.h

@@ -0,0 +1,28 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include <stdint.h>
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include "common/type_c.h"
+
+int
+GetIndexListSize();
+
+void
+GetIndexFeatures(void* index_key_list, uint64_t* index_feature_list);
+
+#ifdef __cplusplus
+}
+#endif

internal/core/thirdparty/knowhere/CMakeLists.txt

@@ -14,8 +14,8 @@
 # Update KNOWHERE_VERSION for the first occurrence
 milvus_add_pkg_config("knowhere")
 set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "")
-set( KNOWHERE_VERSION d0d7eefb )
-set( GIT_REPOSITORY  "https://github.com/zilliztech/knowhere.git")
+set( KNOWHERE_VERSION add_vector_index_mgr_bak )
+set( GIT_REPOSITORY  "https://github.com/foxspy/knowhere.git")
 message(STATUS "Knowhere repo: ${GIT_REPOSITORY}")
 message(STATUS "Knowhere version: ${KNOWHERE_VERSION}")
 

internal/core/unittest/test_c_api.cpp

@@ -45,6 +45,7 @@
 #include "exec/expression/Expr.h"
 #include "segcore/load_index_c.h"
 #include "test_utils/c_api_test_utils.h"
+#include "segcore/vector_index_c.h"
 
 namespace chrono = std::chrono;
 
@@ -712,6 +713,22 @@ TEST(CApiTest, MultiDeleteGrowingSegment) {
     DeleteSegment(segment);
 }
 
+TEST(CApiTest, GetIndexListSizeAndFeatures) {
+    int size = GetIndexListSize();
+    ASSERT_GT(size, 0);
+
+    std::vector<const char*> index_keys(size);
+    std::vector<uint64_t> index_features(size);
+
+    GetIndexFeatures(index_keys.data(), index_features.data());
+
+    for (int i = 0; i < size; i++) {
+        ASSERT_NE(index_keys[i], nullptr);
+        ASSERT_GT(strlen(index_keys[i]), 0);
+        ASSERT_GT(index_features[i], 0);
+    }
+}
+
 TEST(CApiTest, MultiDeleteSealedSegment) {
     auto collection = NewCollection(get_default_schema_config());
     CSegmentInterface segment;

internal/datacoord/import_task.go

@@ -17,11 +17,11 @@
 package datacoord
 
 import (
-	"github.com/milvus-io/milvus/pkg/util/funcutil"
-	"github.com/milvus-io/milvus/pkg/util/paramtable"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/milvus-io/milvus/internal/proto/datapb"
+	"github.com/milvus-io/milvus/pkg/util/funcutil"
+	"github.com/milvus-io/milvus/pkg/util/paramtable"
 	"github.com/milvus-io/milvus/pkg/util/timerecord"
 )
 

internal/datacoord/index_meta.go

@@ -34,6 +34,7 @@ import (
 	"github.com/milvus-io/milvus/internal/metastore/model"
 	"github.com/milvus-io/milvus/internal/proto/indexpb"
 	"github.com/milvus-io/milvus/internal/proto/workerpb"
+	"github.com/milvus-io/milvus/internal/util/vecindexmgr"
 	"github.com/milvus-io/milvus/pkg/common"
 	"github.com/milvus-io/milvus/pkg/log"
 	"github.com/milvus-io/milvus/pkg/metrics"
@@ -980,7 +981,7 @@ func (m *indexMeta) AreAllDiskIndex(collectionID int64, schema *schemapb.Collect
 	})
 	vectorFieldsWithDiskIndex := lo.Filter(vectorFields, func(field *schemapb.FieldSchema, _ int) bool {
 		if indexType, ok := fieldIndexTypes[field.FieldID]; ok {
-			return indexparamcheck.IsDiskIndex(indexType)
+			return vecindexmgr.GetVecIndexMgrInstance().IsDiskVecIndex(indexType)
 		}
 		return false
 	})

internal/datacoord/index_service.go

@@ -28,6 +28,7 @@ import (
 	"github.com/milvus-io/milvus/internal/metastore/model"
 	"github.com/milvus-io/milvus/internal/proto/datapb"
 	"github.com/milvus-io/milvus/internal/proto/indexpb"
+	"github.com/milvus-io/milvus/internal/util/vecindexmgr"
 	"github.com/milvus-io/milvus/pkg/log"
 	"github.com/milvus-io/milvus/pkg/metrics"
 	"github.com/milvus-io/milvus/pkg/util/funcutil"
@@ -231,10 +232,10 @@ func (s *Server) CreateIndex(ctx context.Context, req *indexpb.CreateIndexReques
 			metrics.IndexRequestCounter.WithLabelValues(metrics.FailLabel).Inc()
 			return merr.Status(err), nil
 		}
-		if GetIndexType(req.GetIndexParams()) == indexparamcheck.IndexDISKANN && !s.indexNodeManager.ClientSupportDisk() {
+		if vecindexmgr.GetVecIndexMgrInstance().IsDiskANN(GetIndexType(req.IndexParams)) && !s.indexNodeManager.ClientSupportDisk() {
 			errMsg := "all IndexNodes do not support disk indexes, please verify"
 			log.Warn(errMsg)
-			err = merr.WrapErrIndexNotSupported(indexparamcheck.IndexDISKANN)
+			err = merr.WrapErrIndexNotSupported(GetIndexType(req.IndexParams))
 			metrics.IndexRequestCounter.WithLabelValues(metrics.FailLabel).Inc()
 			return merr.Status(err), nil
 		}

internal/datacoord/task_index.go

@@ -146,7 +146,7 @@ func (it *indexBuildTask) PreCheck(ctx context.Context, dependency *taskSchedule
 	}
 	indexParams := dependency.meta.indexMeta.GetIndexParams(segIndex.CollectionID, segIndex.IndexID)
 	indexType := GetIndexType(indexParams)
-	if isFlatIndex(indexType) || segIndex.NumRows < Params.DataCoordCfg.MinSegmentNumRowsToEnableIndex.GetAsInt64() {
+	if isNoTrainIndex(indexType) || segIndex.NumRows < Params.DataCoordCfg.MinSegmentNumRowsToEnableIndex.GetAsInt64() {
 		log.Ctx(ctx).Info("segment does not need index really", zap.Int64("taskID", it.taskID),
 			zap.Int64("segmentID", segIndex.SegmentID), zap.Int64("num rows", segIndex.NumRows))
 		it.SetStartTime(time.Now())

internal/datacoord/task_scheduler_test.go

@@ -39,7 +39,6 @@ import (
 	"github.com/milvus-io/milvus/internal/proto/indexpb"
 	"github.com/milvus-io/milvus/internal/proto/workerpb"
 	"github.com/milvus-io/milvus/pkg/common"
-	"github.com/milvus-io/milvus/pkg/util/indexparamcheck"
 	"github.com/milvus-io/milvus/pkg/util/merr"
 	"github.com/milvus-io/milvus/pkg/util/paramtable"
 )
@@ -1470,7 +1469,7 @@ func (s *taskSchedulerSuite) Test_indexTaskWithMvOptionalScalarField() {
 				},
 				{
 					Key:   common.IndexTypeKey,
-					Value: indexparamcheck.IndexHNSW,
+					Value: "HNSW",
 				},
 			},
 		},
@@ -1523,7 +1522,7 @@ func (s *taskSchedulerSuite) Test_indexTaskWithMvOptionalScalarField() {
 							},
 							{
 								Key:   common.IndexTypeKey,
-								Value: indexparamcheck.IndexHNSW,
+								Value: "HNSW",
 							},
 						},
 					},
@@ -1624,7 +1623,7 @@ func (s *taskSchedulerSuite) Test_indexTaskWithMvOptionalScalarField() {
 	resetMetaFunc := func() {
 		mt.indexMeta.buildID2SegmentIndex[buildID].IndexState = commonpb.IndexState_Unissued
 		mt.indexMeta.segmentIndexes[segID][indexID].IndexState = commonpb.IndexState_Unissued
-		mt.indexMeta.indexes[collID][indexID].IndexParams[1].Value = indexparamcheck.IndexHNSW
+		mt.indexMeta.indexes[collID][indexID].IndexParams[1].Value = "HNSW"
 		mt.collections[collID].Schema.Fields[0].DataType = schemapb.DataType_FloatVector
 		mt.collections[collID].Schema.Fields[1].IsPartitionKey = true
 		mt.collections[collID].Schema.Fields[1].DataType = schemapb.DataType_VarChar

internal/datacoord/util.go

@@ -30,11 +30,11 @@ import (
 	"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
 	"github.com/milvus-io/milvus/internal/proto/datapb"
 	"github.com/milvus-io/milvus/internal/proto/indexpb"
+	"github.com/milvus-io/milvus/internal/util/vecindexmgr"
 	"github.com/milvus-io/milvus/pkg/common"
 	"github.com/milvus-io/milvus/pkg/log"
 	"github.com/milvus-io/milvus/pkg/metrics"
 	"github.com/milvus-io/milvus/pkg/util/funcutil"
-	"github.com/milvus-io/milvus/pkg/util/indexparamcheck"
 	"github.com/milvus-io/milvus/pkg/util/merr"
 	"github.com/milvus-io/milvus/pkg/util/paramtable"
 	"github.com/milvus-io/milvus/pkg/util/tsoutil"
@@ -203,16 +203,16 @@ func GetIndexType(indexParams []*commonpb.KeyValuePair) string {
 	return invalidIndex
 }
 
-func isFlatIndex(indexType string) bool {
-	return indexType == indexparamcheck.IndexFaissIDMap || indexType == indexparamcheck.IndexFaissBinIDMap
+func isNoTrainIndex(indexType string) bool {
+	return vecindexmgr.GetVecIndexMgrInstance().IsNoTrainIndex(indexType)
 }
 
 func isOptionalScalarFieldSupported(indexType string) bool {
-	return indexType == indexparamcheck.IndexHNSW
+	return vecindexmgr.GetVecIndexMgrInstance().IsMvSupported(indexType)
 }
 
 func isDiskANNIndex(indexType string) bool {
-	return indexType == indexparamcheck.IndexDISKANN
+	return vecindexmgr.GetVecIndexMgrInstance().IsDiskANN(indexType)
 }
 
 func parseBuildIDFromFilePath(key string) (UniqueID, error) {

internal/indexnode/task_index.go

@@ -33,10 +33,10 @@ import (
 	"github.com/milvus-io/milvus/internal/proto/workerpb"
 	"github.com/milvus-io/milvus/internal/storage"
 	"github.com/milvus-io/milvus/internal/util/indexcgowrapper"
+	"github.com/milvus-io/milvus/internal/util/vecindexmgr"
 	"github.com/milvus-io/milvus/pkg/common"
 	"github.com/milvus-io/milvus/pkg/log"
 	"github.com/milvus-io/milvus/pkg/metrics"
-	"github.com/milvus-io/milvus/pkg/util/indexparamcheck"
 	"github.com/milvus-io/milvus/pkg/util/indexparams"
 	"github.com/milvus-io/milvus/pkg/util/merr"
 	"github.com/milvus-io/milvus/pkg/util/metautil"
@@ -210,7 +210,7 @@ func (it *indexBuildTask) Execute(ctx context.Context) error {
 		zap.Int32("currentIndexVersion", it.req.GetCurrentIndexVersion()))
 
 	indexType := it.newIndexParams[common.IndexTypeKey]
-	if indexType == indexparamcheck.IndexDISKANN {
+	if vecindexmgr.GetVecIndexMgrInstance().IsDiskANN(indexType) {
 		// check index node support disk index
 		if !Params.IndexNodeCfg.EnableDisk.GetAsBool() {
 			log.Warn("IndexNode don't support build disk index",

internal/proxy/cgo_util_test.go

@@ -20,7 +20,6 @@ import (
 	"testing"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
-	"github.com/milvus-io/milvus/pkg/util/indexparamcheck"
 )
 
 func Test_CheckVecIndexWithDataTypeExist(t *testing.T) {
@@ -29,25 +28,25 @@ func Test_CheckVecIndexWithDataTypeExist(t *testing.T) {
 		dataType  schemapb.DataType
 		want      bool
 	}{
-		{indexparamcheck.IndexHNSW, schemapb.DataType_FloatVector, true},
-		{indexparamcheck.IndexHNSW, schemapb.DataType_BinaryVector, false},
-		{indexparamcheck.IndexHNSW, schemapb.DataType_Float16Vector, true},
+		{"HNSW", schemapb.DataType_FloatVector, true},
+		{"HNSW", schemapb.DataType_BinaryVector, false},
+		{"HNSW", schemapb.DataType_Float16Vector, true},
 
-		{indexparamcheck.IndexSparseWand, schemapb.DataType_SparseFloatVector, true},
-		{indexparamcheck.IndexSparseWand, schemapb.DataType_FloatVector, false},
-		{indexparamcheck.IndexSparseWand, schemapb.DataType_Float16Vector, false},
+		{"SPARSE_WAND", schemapb.DataType_SparseFloatVector, true},
+		{"SPARSE_WAND", schemapb.DataType_FloatVector, false},
+		{"SPARSE_WAND", schemapb.DataType_Float16Vector, false},
 
-		{indexparamcheck.IndexGpuBF, schemapb.DataType_FloatVector, true},
-		{indexparamcheck.IndexGpuBF, schemapb.DataType_Float16Vector, false},
-		{indexparamcheck.IndexGpuBF, schemapb.DataType_BinaryVector, false},
+		{"GPU_BRUTE_FORCE", schemapb.DataType_FloatVector, true},
+		{"GPU_BRUTE_FORCE", schemapb.DataType_Float16Vector, false},
+		{"GPU_BRUTE_FORCE", schemapb.DataType_BinaryVector, false},
 
-		{indexparamcheck.IndexFaissBinIvfFlat, schemapb.DataType_BinaryVector, true},
-		{indexparamcheck.IndexFaissBinIvfFlat, schemapb.DataType_FloatVector, false},
+		{"BIN_IVF_FLAT", schemapb.DataType_BinaryVector, true},
+		{"BIN_IVF_FLAT", schemapb.DataType_FloatVector, false},
 
-		{indexparamcheck.IndexDISKANN, schemapb.DataType_FloatVector, true},
-		{indexparamcheck.IndexDISKANN, schemapb.DataType_Float16Vector, true},
-		{indexparamcheck.IndexDISKANN, schemapb.DataType_BFloat16Vector, true},
-		{indexparamcheck.IndexDISKANN, schemapb.DataType_BinaryVector, false},
+		{"DISKANN", schemapb.DataType_FloatVector, true},
+		{"DISKANN", schemapb.DataType_Float16Vector, true},
+		{"DISKANN", schemapb.DataType_BFloat16Vector, true},
+		{"DISKANN", schemapb.DataType_BinaryVector, false},
 	}
 
 	for _, test := range cases {