enhance: decouple knowhere and milvus

client/index/disk_ann.go

@@ -33,6 +33,7 @@ func NewDiskANNIndex(metricType MetricType) Index {
 	return &diskANNIndex{
 		baseIndex: baseIndex{
 			metricType: metricType,
+			indexType:  DISKANN,
 		},
 	}
 }

client/index/flat.go

@@ -33,6 +33,7 @@ func NewFlatIndex(metricType MetricType) Index {
 	return flatIndex{
 		baseIndex: baseIndex{
 			metricType: metricType,
+			indexType:  Flat,
 		},
 	}
 }
@@ -54,6 +55,7 @@ func NewBinFlatIndex(metricType MetricType) Index {
 	return binFlatIndex{
 		baseIndex: baseIndex{
 			metricType: metricType,
+			indexType:  BinFlat,
 		},
 	}
 }

cmd/tools/migration/mmap/mmap_230_240.go

@@ -9,8 +9,8 @@ import (
 	"github.com/milvus-io/milvus/internal/metastore/model"
 	"github.com/milvus-io/milvus/internal/rootcoord"
 	"github.com/milvus-io/milvus/internal/tso"
+	"github.com/milvus-io/milvus/internal/util/indexparamcheck"
 	"github.com/milvus-io/milvus/pkg/common"
-	"github.com/milvus-io/milvus/pkg/util/indexparamcheck"
 )
 
 // In Milvus 2.3.x, querynode.MmapDirPath is used to enable mmap and save mmap files.

configs/milvus.yaml

@@ -1056,3 +1056,23 @@ streaming:
     backoffMultiplier: 2 # The multiplier of balance task trigger backoff, 2 by default
   txn:
     defaultKeepaliveTimeout: 10s # The default keepalive timeout for wal txn, 10s by default
+
+knowhere:
+  enable: true
+  HNSW:
+    build:
+      efConstruction : 360
+      M: 30
+    search:
+      ef: 30
+  DISKANN:
+    build:
+      max_degree: 56
+      search_list_size: 100
+      pq_code_budget_gb_ratio: 0.125
+      search_cache_budget_gb_ratio: 0.1
+      beam_width_ratio: 4
+    load:
+      cacheRatio: 0.1
+    search:
+      beamRatio: 4.0

internal/core/src/index/Index.h

@@ -22,6 +22,7 @@
 #include "common/EasyAssert.h"
 #include "knowhere/comp/index_param.h"
 #include "knowhere/dataset.h"
+#include "knowhere/index/index_factory.h"
 #include "common/Tracer.h"
 #include "common/Types.h"
 #include "index/Meta.h"
@@ -62,23 +63,8 @@ class IndexBase {
     virtual const bool
     HasRawData() const = 0;
 
-    bool
-    IsMmapSupported() const {
-        return index_type_ == knowhere::IndexEnum::INDEX_HNSW ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_IVFPQ ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_IVFSQ8 ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_IDMAP ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP ||
-               index_type_ ==
-                   knowhere::IndexEnum::INDEX_SPARSE_INVERTED_INDEX ||
-               index_type_ == knowhere::IndexEnum::INDEX_SPARSE_WAND ||
-               // support mmap for bitmap/hybrid index
-               index_type_ == milvus::index::BITMAP_INDEX_TYPE ||
-               index_type_ == milvus::index::HYBRID_INDEX_TYPE;
-    }
+    virtual bool
+    IsMmapSupported() const = 0;
 
     const IndexType&
     Type() const {

internal/core/src/index/ScalarIndex.h

@@ -133,6 +133,12 @@ class ScalarIndex : public IndexBase {
         PanicInfo(Unsupported, "pattern match is not supported");
     }
 
+    virtual bool
+    IsMmapSupported() const {
+        return index_type_ == milvus::index::BITMAP_INDEX_TYPE ||
+               index_type_ == milvus::index::HYBRID_INDEX_TYPE;
+    }
+
     virtual int64_t
     Size() = 0;
 

internal/core/src/index/VectorIndex.h

@@ -115,6 +115,12 @@ class VectorIndex : public IndexBase {
             err_msg);
     }
 
+    virtual bool
+    IsMmapSupported() const {
+        return knowhere::IndexFactory::Instance().FeatureCheck(
+            index_type_, knowhere::feature::MMAP);
+    }
+
     knowhere::Json
     PrepareSearchParams(const SearchInfo& search_info) const {
         knowhere::Json search_cfg = search_info.search_params_;

internal/core/src/segcore/vector_index_c.cpp

@@ -0,0 +1,113 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include "segcore/vector_index_c.h"
+
+#include "common/Types.h"
+#include "common/EasyAssert.h"
+#include "knowhere/utils.h"
+#include "knowhere/config.h"
+#include "knowhere/version.h"
+#include "index/Meta.h"
+#include "index/IndexFactory.h"
+#include "pb/index_cgo_msg.pb.h"
+
+CStatus
+ValidateIndexParams(const char* index_type,
+                    enum CDataType data_type,
+                    const uint8_t* serialized_index_params,
+                    const uint64_t length) {
+    try {
+        auto index_params =
+            std::make_unique<milvus::proto::indexcgo::IndexParams>();
+        auto res =
+            index_params->ParseFromArray(serialized_index_params, length);
+        AssertInfo(res, "Unmarshall index params failed");
+
+        knowhere::Json json;
+
+        for (size_t i = 0; i < index_params->params_size(); i++) {
+            auto& param = index_params->params(i);
+            json[param.key()] = param.value();
+        }
+
+        milvus::DataType dataType(static_cast<milvus::DataType>(data_type));
+
+        knowhere::Status status;
+        std::string error_msg;
+        if (dataType == milvus::DataType::VECTOR_BINARY) {
+            status = knowhere::IndexStaticFaced<knowhere::bin1>::ConfigCheck(
+                index_type,
+                knowhere::Version::GetCurrentVersion().VersionNumber(),
+                json,
+                error_msg);
+        } else if (dataType == milvus::DataType::VECTOR_FLOAT) {
+            status = knowhere::IndexStaticFaced<knowhere::fp32>::ConfigCheck(
+                index_type,
+                knowhere::Version::GetCurrentVersion().VersionNumber(),
+                json,
+                error_msg);
+        } else if (dataType == milvus::DataType::VECTOR_BFLOAT16) {
+            status = knowhere::IndexStaticFaced<knowhere::bf16>::ConfigCheck(
+                index_type,
+                knowhere::Version::GetCurrentVersion().VersionNumber(),
+                json,
+                error_msg);
+        } else if (dataType == milvus::DataType::VECTOR_FLOAT16) {
+            status = knowhere::IndexStaticFaced<knowhere::fp16>::ConfigCheck(
+                index_type,
+                knowhere::Version::GetCurrentVersion().VersionNumber(),
+                json,
+                error_msg);
+        } else if (dataType == milvus::DataType::VECTOR_SPARSE_FLOAT) {
+            status = knowhere::IndexStaticFaced<knowhere::fp32>::ConfigCheck(
+                index_type,
+                knowhere::Version::GetCurrentVersion().VersionNumber(),
+                json,
+                error_msg);
+        } else {
+            status = knowhere::Status::invalid_args;
+        }
+        CStatus cStatus;
+        if (status == knowhere::Status::success) {
+            cStatus.error_code = milvus::Success;
+            cStatus.error_msg = "";
+        } else {
+            cStatus.error_code = milvus::ConfigInvalid;
+            cStatus.error_msg = strdup(error_msg.c_str());
+        }
+        return cStatus;
+    } catch (std::exception& e) {
+        auto cStatus = CStatus();
+        cStatus.error_code = milvus::UnexpectedError;
+        cStatus.error_msg = strdup(e.what());
+        return cStatus;
+    }
+}
+
+int
+GetIndexListSize() {
+    return knowhere::IndexFactory::Instance().GetIndexFeatures().size();
+}
+
+void
+GetIndexFeatures(void* index_key_list, uint64_t* index_feature_list) {
+    auto features = knowhere::IndexFactory::Instance().GetIndexFeatures();
+    int idx = 0;
+
+    const char** index_keys = (const char**)index_key_list;
+    uint64_t* index_features = (uint64_t*)index_feature_list;
+    for (auto it = features.begin(); it != features.end(); ++it) {
+        index_keys[idx] = it->first.c_str();
+        index_features[idx] = it->second;
+        idx++;
+    }
+}

internal/core/src/segcore/vector_index_c.h

@@ -0,0 +1,34 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include <stdint.h>
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include "common/type_c.h"
+
+CStatus
+ValidateIndexParams(const char* index_type,
+                    enum CDataType data_type,
+                    const uint8_t* index_params,
+                    const uint64_t length);
+
+int
+GetIndexListSize();
+
+void
+GetIndexFeatures(void* index_key_list, uint64_t* index_feature_list);
+
+#ifdef __cplusplus
+}
+#endif

internal/core/thirdparty/knowhere/CMakeLists.txt

@@ -14,8 +14,8 @@
 # Update KNOWHERE_VERSION for the first occurrence
 milvus_add_pkg_config("knowhere")
 set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "")
-set( KNOWHERE_VERSION b2011400 )
-set( GIT_REPOSITORY  "https://github.com/zilliztech/knowhere.git")
+set( KNOWHERE_VERSION add_vector_index_mgr_bak )
+set( GIT_REPOSITORY  "https://github.com/foxspy/knowhere.git")
 message(STATUS "Knowhere repo: ${GIT_REPOSITORY}")
 message(STATUS "Knowhere version: ${KNOWHERE_VERSION}")
 

internal/datacoord/compaction_task_clustering_test.go

@@ -24,8 +24,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/milvus-io/milvus/pkg/util/metautil"
-
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/suite"
 	"go.uber.org/atomic"
@@ -41,6 +39,7 @@ import (
 	"github.com/milvus-io/milvus/internal/storage"
 	"github.com/milvus-io/milvus/pkg/common"
 	"github.com/milvus-io/milvus/pkg/util/merr"
+	"github.com/milvus-io/milvus/pkg/util/metautil"
 )
 
 func TestClusteringCompactionTaskSuite(t *testing.T) {

internal/datacoord/import_task.go

@@ -17,11 +17,11 @@
 package datacoord
 
 import (
-	"github.com/milvus-io/milvus/pkg/util/funcutil"
-	"github.com/milvus-io/milvus/pkg/util/paramtable"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/milvus-io/milvus/internal/proto/datapb"
+	"github.com/milvus-io/milvus/pkg/util/funcutil"
+	"github.com/milvus-io/milvus/pkg/util/paramtable"
 	"github.com/milvus-io/milvus/pkg/util/timerecord"
 )
 

internal/datacoord/index_meta.go

@@ -34,10 +34,10 @@ import (
 	"github.com/milvus-io/milvus/internal/metastore/model"
 	"github.com/milvus-io/milvus/internal/proto/indexpb"
 	"github.com/milvus-io/milvus/internal/proto/workerpb"
+	"github.com/milvus-io/milvus/internal/util/indexparamcheck"
 	"github.com/milvus-io/milvus/pkg/common"
 	"github.com/milvus-io/milvus/pkg/log"
 	"github.com/milvus-io/milvus/pkg/metrics"
-	"github.com/milvus-io/milvus/pkg/util/indexparamcheck"
 	"github.com/milvus-io/milvus/pkg/util/indexparams"
 	"github.com/milvus-io/milvus/pkg/util/timerecord"
 	"github.com/milvus-io/milvus/pkg/util/typeutil"

internal/datacoord/index_service.go

@@ -28,10 +28,10 @@ import (
 	"github.com/milvus-io/milvus/internal/metastore/model"
 	"github.com/milvus-io/milvus/internal/proto/datapb"
 	"github.com/milvus-io/milvus/internal/proto/indexpb"
+	"github.com/milvus-io/milvus/internal/util/indexparamcheck"
 	"github.com/milvus-io/milvus/pkg/log"
 	"github.com/milvus-io/milvus/pkg/metrics"
 	"github.com/milvus-io/milvus/pkg/util/funcutil"
-	"github.com/milvus-io/milvus/pkg/util/indexparamcheck"
 	"github.com/milvus-io/milvus/pkg/util/merr"
 	"github.com/milvus-io/milvus/pkg/util/metautil"
 	"github.com/milvus-io/milvus/pkg/util/paramtable"
@@ -231,10 +231,10 @@ func (s *Server) CreateIndex(ctx context.Context, req *indexpb.CreateIndexReques
 			metrics.IndexRequestCounter.WithLabelValues(metrics.FailLabel).Inc()
 			return merr.Status(err), nil
 		}
-		if GetIndexType(req.GetIndexParams()) == indexparamcheck.IndexDISKANN && !s.indexNodeManager.ClientSupportDisk() {
+		if indexparamcheck.GetVecIndexMgrInstance().IsDiskANN(GetIndexType(req.IndexParams)) && !s.indexNodeManager.ClientSupportDisk() {
 			errMsg := "all IndexNodes do not support disk indexes, please verify"
 			log.Warn(errMsg)
-			err = merr.WrapErrIndexNotSupported(indexparamcheck.IndexDISKANN)
+			err = merr.WrapErrIndexNotSupported(GetIndexType(req.IndexParams))
 			metrics.IndexRequestCounter.WithLabelValues(metrics.FailLabel).Inc()
 			return merr.Status(err), nil
 		}

internal/datacoord/index_service_test.go

@@ -42,9 +42,9 @@ import (
 	"github.com/milvus-io/milvus/internal/proto/indexpb"
 	"github.com/milvus-io/milvus/internal/proto/workerpb"
 	"github.com/milvus-io/milvus/internal/storage"
+	"github.com/milvus-io/milvus/internal/util/indexparamcheck"
 	"github.com/milvus-io/milvus/internal/util/sessionutil"
 	"github.com/milvus-io/milvus/pkg/common"
-	"github.com/milvus-io/milvus/pkg/util/indexparamcheck"
 	"github.com/milvus-io/milvus/pkg/util/merr"
 )
 
@@ -620,13 +620,13 @@ func TestServer_AlterIndex(t *testing.T) {
 	s.stateCode.Store(commonpb.StateCode_Healthy)
 
 	t.Run("mmap_unsupported", func(t *testing.T) {
-		indexParams[0].Value = indexparamcheck.IndexRaftCagra
+		indexParams[0].Value = "GPU_CAGRA"
 
 		resp, err := s.AlterIndex(ctx, req)
 		assert.NoError(t, err)
 		assert.ErrorIs(t, merr.CheckRPCCall(resp, err), merr.ErrParameterInvalid)
 
-		indexParams[0].Value = indexparamcheck.IndexFaissIvfFlat
+		indexParams[0].Value = "IVF_FLAT"
 	})
 
 	t.Run("param_value_invalied", func(t *testing.T) {