add index option check

Signed-off-by: xianliang.li <[email protected]>

internal/core/src/index/Index.h

@@ -22,6 +22,7 @@
 #include "common/EasyAssert.h"
 #include "knowhere/comp/index_param.h"
 #include "knowhere/dataset.h"
+#include "knowhere/index/index_factory.h"
 #include "common/Tracer.h"
 #include "common/Types.h"
 #include "index/Meta.h"
@@ -64,20 +65,10 @@ class IndexBase {
 
     bool
     IsMmapSupported() const {
-        return index_type_ == knowhere::IndexEnum::INDEX_HNSW ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_IVFPQ ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_IVFSQ8 ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_IDMAP ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP ||
-               index_type_ ==
-                   knowhere::IndexEnum::INDEX_SPARSE_INVERTED_INDEX ||
-               index_type_ == knowhere::IndexEnum::INDEX_SPARSE_WAND ||
-               // support mmap for bitmap/hybrid index
-               index_type_ == milvus::index::BITMAP_INDEX_TYPE ||
-               index_type_ == milvus::index::HYBRID_INDEX_TYPE;
+        return  knowhere::IndexFactory::Instance().FeatureCheck(index_type_, knowhere::feature::MMAP) ||
+                    // support mmap for bitmap/hybrid index
+                       index_type_ == milvus::index::BITMAP_INDEX_TYPE ||
+                       index_type_ == milvus::index::HYBRID_INDEX_TYPE;
     }
 
     const IndexType&

internal/core/src/segcore/vector_index_c.cpp

@@ -0,0 +1,36 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include "segcore/vector_index_c.h"
+
+#include "knowhere/utils.h"
+#include "index/Meta.h"
+#include "index/IndexFactory.h"
+
+int
+GetIndexListSize() {
+    return knowhere::IndexFactory::Instance().GetIndexFeatures().size();
+}
+
+void
+GetIndexFeatures(void* index_key_list, uint64_t* index_feature_list) {
+    auto features = knowhere::IndexFactory::Instance().GetIndexFeatures();
+    int idx = 0;
+
+    const char** index_keys = (const char**)index_key_list;
+    uint64_t* index_features = (uint64_t*)index_feature_list;
+    for (auto it = features.begin(); it != features.end(); ++it) {
+        index_keys[idx] = it->first.c_str();
+        index_features[idx] = it->second;
+        idx++;
+    }
+}
+

internal/core/src/segcore/vector_index_c.h

@@ -0,0 +1,28 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include <stdint.h>
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+
+int
+GetIndexListSize();
+
+void
+GetIndexFeatures(void* index_key_list, uint64_t* index_feature_list);
+
+
+#ifdef __cplusplus
+}
+#endif

internal/core/thirdparty/knowhere/CMakeLists.txt

@@ -14,8 +14,8 @@
 # Update KNOWHERE_VERSION for the first occurrence
 milvus_add_pkg_config("knowhere")
 set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "")
-set( KNOWHERE_VERSION d20907f5 )
-set( GIT_REPOSITORY  "https://github.com/zilliztech/knowhere.git")
+set( KNOWHERE_VERSION add_vector_index_mgr )
+set( GIT_REPOSITORY  "https://github.com/foxspy/knowhere.git")
 message(STATUS "Knowhere repo: ${GIT_REPOSITORY}")
 message(STATUS "Knowhere version: ${KNOWHERE_VERSION}")
 
@@ -60,5 +60,3 @@ endif()
 # get prometheus COMPILE_OPTIONS
 get_property( var DIRECTORY "${knowhere_SOURCE_DIR}" PROPERTY COMPILE_OPTIONS )
 message( STATUS "knowhere src compile options: ${var}" )
-
-set( KNOWHERE_INCLUDE_DIR ${knowhere_SOURCE_DIR}/include CACHE INTERNAL "Path to knowhere include directory" )

internal/datacoord/index_service.go

@@ -222,10 +222,10 @@ func (s *Server) CreateIndex(ctx context.Context, req *indexpb.CreateIndexReques
 			metrics.IndexRequestCounter.WithLabelValues(metrics.FailLabel).Inc()
 			return merr.Status(err), nil
 		}
-		if GetIndexType(req.GetIndexParams()) == indexparamcheck.IndexDISKANN && !s.indexNodeManager.ClientSupportDisk() {
+		if indexparamcheck.GetVecIndexMgrInstance().IsDiskANN(GetIndexType(req.IndexParams)) && !s.indexNodeManager.ClientSupportDisk() {
 			errMsg := "all IndexNodes do not support disk indexes, please verify"
 			log.Warn(errMsg)
-			err = merr.WrapErrIndexNotSupported(indexparamcheck.IndexDISKANN)
+			err = merr.WrapErrIndexNotSupported(GetIndexType(req.IndexParams))
 			metrics.IndexRequestCounter.WithLabelValues(metrics.FailLabel).Inc()
 			return merr.Status(err), nil
 		}

internal/datacoord/task_index.go

@@ -132,7 +132,7 @@ func (it *indexBuildTask) PreCheck(ctx context.Context, dependency *taskSchedule
 	}
 	indexParams := dependency.meta.indexMeta.GetIndexParams(segIndex.CollectionID, segIndex.IndexID)
 	indexType := GetIndexType(indexParams)
-	if isFlatIndex(indexType) || segIndex.NumRows < Params.DataCoordCfg.MinSegmentNumRowsToEnableIndex.GetAsInt64() {
+	if isBruteForce(indexType) || segIndex.NumRows < Params.DataCoordCfg.MinSegmentNumRowsToEnableIndex.GetAsInt64() {
 		log.Ctx(ctx).Info("segment does not need index really", zap.Int64("taskID", it.taskID),
 			zap.Int64("segmentID", segIndex.SegmentID), zap.Int64("num rows", segIndex.NumRows))
 		it.SetState(indexpb.JobState_JobStateFinished, "fake finished index success")

internal/datacoord/util.go

@@ -196,16 +196,16 @@ func GetIndexType(indexParams []*commonpb.KeyValuePair) string {
 	return invalidIndex
 }
 
-func isFlatIndex(indexType string) bool {
-	return indexType == indexparamcheck.IndexFaissIDMap || indexType == indexparamcheck.IndexFaissBinIDMap
+func isBruteForce(indexType string) bool {
+	return indexparamcheck.GetVecIndexMgrInstance().IsBruteForce(indexType)
 }
 
 func isOptionalScalarFieldSupported(indexType string) bool {
-	return indexType == indexparamcheck.IndexHNSW
+	return indexparamcheck.GetVecIndexMgrInstance().IsMvSupported(indexType)
 }
 
 func isDiskANNIndex(indexType string) bool {
-	return indexType == indexparamcheck.IndexDISKANN
+	return indexparamcheck.GetVecIndexMgrInstance().IsDiskANN(indexType)
 }
 
 func parseBuildIDFromFilePath(key string) (UniqueID, error) {

internal/indexnode/task_index.go

@@ -208,7 +208,7 @@ func (it *indexBuildTask) Execute(ctx context.Context) error {
 		zap.Int32("currentIndexVersion", it.req.GetCurrentIndexVersion()))
 
 	indexType := it.newIndexParams[common.IndexTypeKey]
-	if indexType == indexparamcheck.IndexDISKANN {
+	if indexparamcheck.GetVecIndexMgrInstance().IsDiskANN(indexType) {
 		// check index node support disk index
 		if !Params.IndexNodeCfg.EnableDisk.GetAsBool() {
 			log.Warn("IndexNode don't support build disk index",

internal/proxy/task_index.go

@@ -297,7 +297,7 @@ func (cit *createIndexTask) parseIndexParams() error {
 		if !exist {
 			return fmt.Errorf("IndexType not specified")
 		}
-		if indexType == indexparamcheck.IndexDISKANN {
+		if indexparamcheck.GetVecIndexMgrInstance().IsDiskANN(indexType) {
 			err := indexparams.FillDiskIndexParams(Params, indexParamsMap)
 			if err != nil {
 				return err

internal/querynodev2/segments/index_attr_cache.go

@@ -60,7 +60,7 @@ func (c *IndexAttrCache) GetIndexResourceUsage(indexInfo *querypb.FieldIndexInfo
 	if err != nil {
 		return 0, 0, fmt.Errorf("index type not exist in index params")
 	}
-	if indexType == indexparamcheck.IndexDISKANN {
+	if indexparamcheck.GetVecIndexMgrInstance().IsDiskANN(indexType) {
 		neededMemSize := indexInfo.IndexSize / UsedDiskMemoryRatio
 		neededDiskSize := indexInfo.IndexSize - neededMemSize
 		return uint64(neededMemSize), uint64(neededDiskSize), nil

internal/querynodev2/segments/segment.go

@@ -1110,7 +1110,7 @@ func (s *LocalSegment) LoadIndex(ctx context.Context, indexInfo *querypb.FieldIn
 	delete(indexParams, common.MmapEnabledKey)
 
 	// some build params also exist in indexParams, which are useless during loading process
-	if indexParams["index_type"] == indexparamcheck.IndexDISKANN {
+	if indexparamcheck.GetVecIndexMgrInstance().IsDiskANN(indexParams["index_type"]) {
 		if err := indexparams.SetDiskIndexLoadParams(paramtable.Get(), indexParams, indexInfo.GetNumRows()); err != nil {
 			return err
 		}

pkg/util/indexparamcheck/index_type.go

@@ -53,32 +53,20 @@ const (
 )
 
 func IsGpuIndex(indexType IndexType) bool {
-	return indexType == IndexGpuBF ||
-		indexType == IndexRaftIvfFlat ||
-		indexType == IndexRaftIvfPQ ||
-		indexType == IndexRaftCagra
+	return GetVecIndexMgrInstance().IsGPUVecIndex(indexType)
 }
 
 // IsVectorMmapIndex check if the vector index can be mmaped
 func IsVectorMmapIndex(indexType IndexType) bool {
-	return indexType == IndexFaissIDMap ||
-		indexType == IndexFaissIvfFlat ||
-		indexType == IndexFaissIvfPQ ||
-		indexType == IndexFaissIvfSQ8 ||
-		indexType == IndexFaissBinIDMap ||
-		indexType == IndexFaissBinIvfFlat ||
-		indexType == IndexHNSW ||
-		indexType == IndexScaNN ||
-		indexType == IndexSparseInverted ||
-		indexType == IndexSparseWand
+	return GetVecIndexMgrInstance().IsMMapSupported(indexType)
 }
 
 func IsOffsetCacheSupported(indexType IndexType) bool {
 	return indexType == IndexBitmap
 }
 
 func IsDiskIndex(indexType IndexType) bool {
-	return indexType == IndexDISKANN
+	return GetVecIndexMgrInstance().IsDiskANN(indexType)
 }
 
 func IsScalarMmapIndex(indexType IndexType) bool {