Restore the MVCC functionality.

.github/workflows/publish-builder.yaml

@@ -50,7 +50,16 @@ jobs:
         id: extracter
         run: |
           echo "::set-output name=version::$(date +%Y%m%d)"
-          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+          echo "::set-output name=sha_short::$(git rev-parse --short=7 HEAD)"
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            milvusdb/milvus-env
+          tags: |
+            type=raw,enable=true,value=${{ matrix.os }}-{{date 'YYYYMMDD'}}-{{sha}}
+            type=raw,enable=true,value=${{ matrix.os }}-latest
       # - name: Setup upterm session
       #   uses: lhotari/action-upterm@v1
       - name: Set up QEMU
@@ -59,25 +68,24 @@ jobs:
           platforms: arm64
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
-      - name: Docker Build
-        if: success() && github.event_name == 'pull_request' && github.repository == 'milvus-io/milvus'
-        shell: bash
-        run: |
-          docker buildx ls
-          docker buildx build --platform linux/amd64,linux/arm64 -t milvusdb/milvus-env:${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/cpu/${OS_NAME}/Dockerfile .      
-      - name: Docker Build&Push
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
-        shell: bash
-        run: |
-          docker buildx ls
-          docker login -u ${{ secrets.DOCKERHUB_USER }} \
-                       -p ${{ secrets.DOCKERHUB_TOKEN }}
-          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/cpu/${OS_NAME}/Dockerfile .
-          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:${OS_NAME}-latest -f build/docker/builder/cpu/${OS_NAME}/Dockerfile .
+      - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          file: build/docker/builder/cpu/${{ matrix.os }}/Dockerfile
       - name: Bump Builder Version
         uses: ./.github/actions/bump-builder-version
         if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
         with:
           tag: "${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}"
           type: cpu
-          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
+          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}

.github/workflows/publish-gpu-builder.yaml

@@ -50,33 +50,38 @@ jobs:
         id: extracter
         run: |
           echo "::set-output name=version::$(date +%Y%m%d)"
-          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+          echo "::set-output name=sha_short::$(git rev-parse --short=7 HEAD)"
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            milvusdb/milvus-env
+          tags: |
+            type=raw,enable=true,value=${{ matrix.os }}-{{date 'YYYYMMDD'}}-{{sha}}
+            type=raw,enable=true,value=${{ matrix.os }}-latest
       # - name: Setup upterm session
       #   uses: lhotari/action-upterm@v1
-      - name: Docker Build
-        if: success() && github.event_name == 'pull_request' && github.repository == 'milvus-io/milvus'
-        shell: bash
-        run: |
-          docker info
-          docker build -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
-      - name: Docker Build&Push
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
-        shell: bash
-        run: |
-          docker info
-          docker login -u ${{ secrets.DOCKERHUB_USER }} \
-                       -p ${{ secrets.DOCKERHUB_TOKEN }}
-          # Building the first image
-          docker build -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
-          docker push milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}
-
-          # Building the second image
-          docker build -t milvusdb/milvus-env:gpu-${OS_NAME}-latest -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
-          docker push milvusdb/milvus-env:gpu-${OS_NAME}-latest
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          file: build/docker/builder/gpu/${{ matrix.os }}/Dockerfile
       - name: Bump Builder Version
         if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
         uses: ./.github/actions/bump-builder-version
         with:
           tag: "${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}"
           type: gpu
-          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
+          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}

Makefile

@@ -447,6 +447,7 @@ generate-mockery-datanode: getdeps
 	$(INSTALL_PATH)/mockery --name=MetaCache --dir=$(PWD)/internal/datanode/metacache --output=$(PWD)/internal/datanode/metacache --filename=mock_meta_cache.go --with-expecter --structname=MockMetaCache --outpkg=metacache --inpackage
 	$(INSTALL_PATH)/mockery --name=SyncManager --dir=$(PWD)/internal/datanode/syncmgr --output=$(PWD)/internal/datanode/syncmgr --filename=mock_sync_manager.go --with-expecter --structname=MockSyncManager --outpkg=syncmgr --inpackage
 	$(INSTALL_PATH)/mockery --name=MetaWriter --dir=$(PWD)/internal/datanode/syncmgr --output=$(PWD)/internal/datanode/syncmgr --filename=mock_meta_writer.go --with-expecter --structname=MockMetaWriter --outpkg=syncmgr --inpackage
+	$(INSTALL_PATH)/mockery --name=Serializer --dir=$(PWD)/internal/datanode/syncmgr --output=$(PWD)/internal/datanode/syncmgr --filename=mock_serializer.go --with-expecter --structname=MockSerializer --outpkg=syncmgr --inpackage
 	$(INSTALL_PATH)/mockery --name=WriteBuffer --dir=$(PWD)/internal/datanode/writebuffer --output=$(PWD)/internal/datanode/writebuffer --filename=mock_write_buffer.go --with-expecter --structname=MockWriteBuffer --outpkg=writebuffer --inpackage
 	$(INSTALL_PATH)/mockery --name=BufferManager --dir=$(PWD)/internal/datanode/writebuffer --output=$(PWD)/internal/datanode/writebuffer --filename=mock_mananger.go --with-expecter --structname=MockBufferManager --outpkg=writebuffer --inpackage
 	$(INSTALL_PATH)/mockery --name=BinlogIO --dir=$(PWD)/internal/datanode/io --output=$(PWD)/internal/datanode/io --filename=mock_binlogio.go --with-expecter --structname=MockBinlogIO --outpkg=io --inpackage

ci/jenkins/PublishImages.groovy

@@ -14,7 +14,7 @@ pipeline {
 
     options {
         timestamps()
-        timeout(time: 100, unit: 'MINUTES')
+        timeout(time: 200, unit: 'MINUTES')
         // parallelsAlwaysFailFast()
         disableConcurrentBuilds()
     }

cmd/milvus/run.go

@@ -11,6 +11,7 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/milvus-io/milvus/pkg/log"
+	"github.com/milvus-io/milvus/pkg/util/hardware"
 	"github.com/milvus-io/milvus/pkg/util/metricsinfo"
 )
 
@@ -36,6 +37,7 @@ func (c *run) execute(args []string, flags *flag.FlagSet) {
 
 	c.printBanner(flags.Output())
 	c.injectVariablesToEnv()
+	c.printHardwareInfo(flags.Output())
 	lock, err := createPidFile(flags.Output(), filename, runtimeDir)
 	if err != nil {
 		panic(err)
@@ -59,6 +61,14 @@ func (c *run) printBanner(w io.Writer) {
 	fmt.Fprintln(w)
 }
 
+func (c *run) printHardwareInfo(w io.Writer) {
+	totalMem := hardware.GetMemoryCount()
+	usedMem := hardware.GetUsedMemoryCount()
+	fmt.Fprintf(w, "TotalMem: %d\n", totalMem)
+	fmt.Fprintf(w, "UsedMem: %d\n", usedMem)
+	fmt.Fprintln(w)
+}
+
 func (c *run) injectVariablesToEnv() {
 	// inject in need
 

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/klauspost/compress v1.16.7
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
-	github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20231228051838-b5442d755fa4
+	github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20231229025438-39bce6abb18f
 	github.com/minio/minio-go/v7 v7.0.61
 	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/client_model v0.3.0

go.sum

@@ -583,8 +583,8 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
 github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20231228051838-b5442d755fa4 h1:nxIohfJOCMbixFAC3q4Lclmv0xg/8q6D8T7D8l258To=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20231228051838-b5442d755fa4/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20231229025438-39bce6abb18f h1:8lNcRqhQgUROtmtiIEdpQHGW82KMI5oASVKxkaZ/tBg=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20231229025438-39bce6abb18f/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
 github.com/milvus-io/milvus-storage/go v0.0.0-20231109072809-1cd7b0866092 h1:UYJ7JB+QlMOoFHNdd8mUa3/lV63t9dnBX7ILXmEEWPY=
 github.com/milvus-io/milvus-storage/go v0.0.0-20231109072809-1cd7b0866092/go.mod h1:GPETMcTZq1gLY1WA6Na5kiNAKnq8SEMMiVKUZrM3sho=
 github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=

internal/core/run_clang_format.sh

@@ -7,7 +7,7 @@ fi
 CorePath=$1
 
 formatThis() {
-    find "$1" | grep -E "(*\.cpp|*\.h|*\.cc)$" | grep -v "gen_tools/templates" | grep -v "\.pb\." | xargs clang-format-10 -i
+    find "$1" | grep -E "(*\.cpp|*\.h|*\.cc)$" | grep -v "gen_tools/templates" | grep -v "\.pb\." | grep -v "tantivy-binding.h" | xargs clang-format-10 -i
 }
 
 formatThis "${CorePath}/src"

internal/core/src/exec/expression/CompareExpr.h

@@ -24,6 +24,7 @@
 #include "common/Vector.h"
 #include "exec/expression/Expr.h"
 #include "segcore/SegmentInterface.h"
+#include "simd/interface.h"
 
 namespace milvus {
 namespace exec {
@@ -41,7 +42,7 @@ using ChunkDataAccessor = std::function<const number(int)>;
 template <typename T, typename U, proto::plan::OpType op>
 struct CompareElementFunc {
     void
-    operator()(const T* left, const U* right, size_t size, bool* res) {
+    operator_base(const T* left, const U* right, size_t size, bool* res) {
         for (int i = 0; i < size; ++i) {
             if constexpr (op == proto::plan::OpType::Equal) {
                 res[i] = left[i] == right[i];
@@ -63,6 +64,24 @@ struct CompareElementFunc {
             }
         }
     }
+
+    void
+    operator()(const T* left, const U* right, size_t size, bool* res) {
+#if defined(USE_DYNAMIC_SIMD)
+        if constexpr (std::is_same_v<T, U>) {
+            milvus::simd::compare_col_func<T>(
+                static_cast<milvus::simd::CompareType>(op),
+                left,
+                right,
+                size,
+                res);
+        } else {
+            operator_base(left, right, size, res);
+        }
+#else
+        operator_base(left, right, size, res);
+#endif
+    }
 };
 
 class PhyCompareFilterExpr : public Expr {

internal/core/src/index/InvertedIndexTantivy.cpp

@@ -71,8 +71,20 @@ BinarySet
 InvertedIndexTantivy<T>::Upload(const Config& config) {
     finish();
 
-    for (const auto& entry : std::filesystem::directory_iterator(path_)) {
-        disk_file_manager_->AddFile(entry.path());
+    boost::filesystem::path p(path_);
+    boost::filesystem::directory_iterator end_iter;
+
+    for (boost::filesystem::directory_iterator iter(p); iter != end_iter;
+         iter++) {
+        if (boost::filesystem::is_directory(*iter)) {
+            LOG_WARN("{} is a directory", iter->path().string());
+        } else {
+            LOG_INFO("trying to add index file: {}", iter->path().string());
+            AssertInfo(disk_file_manager_->AddFile(iter->path().string()),
+                       "failed to add index file: {}",
+                       iter->path().string());
+            LOG_INFO("index file: {} added", iter->path().string());
+        }
     }
 
     BinarySet ret;

internal/core/src/index/VectorDiskIndex.cpp

@@ -134,7 +134,11 @@ template <typename T>
 BinarySet
 VectorDiskAnnIndex<T>::Upload(const Config& config) {
     BinarySet ret;
-    index_.Serialize(ret);
+    auto stat = index_.Serialize(ret);
+    if (stat != knowhere::Status::success) {
+        PanicInfo(ErrorCode::UnexpectedError,
+                  "failed to serialize index, " + KnowhereStatusString(stat));
+    }
     auto remote_paths_to_size = file_manager_->GetRemotePathsToFileSize();
     for (auto& file : remote_paths_to_size) {
         ret.Append(file.first, nullptr, file.second);

internal/core/src/query/visitors/ExecExprVisitor.cpp

@@ -2632,30 +2632,9 @@ ExecExprVisitor::ExecTermVisitorImplTemplate(TermExpr& expr_raw) -> BitsetType {
         return index->In(n, terms.data());
     };
 
-#if defined(USE_DYNAMIC_SIMD)
-    std::function<bool(MayConstRef<T> x)> elem_func;
-    if (n <= milvus::simd::TERM_EXPR_IN_SIZE_THREAD) {
-        elem_func = [&terms, &term_set, n](MayConstRef<T> x) {
-            if constexpr (std::is_integral<T>::value ||
-                          std::is_floating_point<T>::value) {
-                return milvus::simd::find_term_func<T>(terms.data(), n, x);
-            } else {
-                // For string type, simd performance not better than set mode
-                static_assert(std::is_same<T, std::string>::value ||
-                              std::is_same<T, std::string_view>::value);
-                return term_set.find(x) != term_set.end();
-            }
-        };
-    } else {
-        elem_func = [&term_set, n](MayConstRef<T> x) {
-            return term_set.find(x) != term_set.end();
-        };
-    }
-#else
     auto elem_func = [&term_set](MayConstRef<T> x) {
         return term_set.find(x) != term_set.end();
     };
-#endif
 
     auto default_skip_index_func = [&](const SkipIndex& skipIndex,
                                        FieldId fieldId,

internal/core/src/segcore/SegmentGrowingImpl.h

@@ -281,6 +281,16 @@ class SegmentGrowingImpl : public SegmentGrowing {
     void
     check_search(const query::Plan* plan) const override {
         Assert(plan);
+        auto& metric_str = plan->plan_node_->search_info_.metric_type_;
+        auto searched_field_id = plan->plan_node_->search_info_.field_id_;
+        auto index_meta =
+            index_meta_->GetFieldIndexMeta(FieldId(searched_field_id));
+        if (metric_str.empty()) {
+            metric_str = index_meta.GeMetricType();
+        } else {
+            AssertInfo(metric_str == index_meta.GeMetricType(),
+                       "metric type not match");
+        }
     }
 
     const ConcurrentVector<Timestamp>&

internal/core/src/segcore/SegmentInterface.cpp

@@ -67,11 +67,12 @@ SegmentInternalInterface::FillTargetEntry(const query::Plan* plan,
 std::unique_ptr<SearchResult>
 SegmentInternalInterface::Search(
     const query::Plan* plan,
-    const query::PlaceholderGroup* placeholder_group) const {
+    const query::PlaceholderGroup* placeholder_group,
+    Timestamp timestamp) const {
     std::shared_lock lck(mutex_);
     milvus::tracer::AddEvent("obtained_segment_lock_mutex");
     check_search(plan);
-    query::ExecPlanNodeVisitor visitor(*this, 1L << 63, placeholder_group);
+    query::ExecPlanNodeVisitor visitor(*this, timestamp, placeholder_group);
     auto results = std::make_unique<SearchResult>();
     *results = visitor.get_moved_result(*plan->plan_node_);
     results->segment_ = (void*)this;

internal/core/src/segcore/SegmentInterface.h

@@ -54,7 +54,8 @@ class SegmentInterface {
 
     virtual std::unique_ptr<SearchResult>
     Search(const query::Plan* Plan,
-           const query::PlaceholderGroup* placeholder_group) const = 0;
+           const query::PlaceholderGroup* placeholder_group,
+           Timestamp timestamp) const = 0;
 
     virtual std::unique_ptr<proto::segcore::RetrieveResults>
     Retrieve(const query::RetrievePlan* Plan,
@@ -136,7 +137,8 @@ class SegmentInternalInterface : public SegmentInterface {
 
     std::unique_ptr<SearchResult>
     Search(const query::Plan* Plan,
-           const query::PlaceholderGroup* placeholder_group) const override;
+           const query::PlaceholderGroup* placeholder_group,
+           Timestamp timestamp) const override;
 
     void
     FillPrimaryKeys(const query::Plan* plan,