introduce thirdparty milvus-storage

Signed-off-by: shaoting-huang <[email protected]>

Makefile

@@ -87,7 +87,7 @@ build-go:
 	@echo "Building Milvus ..."
 	@source $(PWD)/scripts/setenv.sh && \
 		mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
-		CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CFLAGS="$(CGO_CFLAGS) -I $(CONAN_INCLUDE_DIRS_ARROW)" GO111MODULE=on $(GO) build -pgo=$(PGO_PATH)/default.pgo -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
+		CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CFLAGS="$(CGO_CFLAGS)" GO111MODULE=on $(GO) build -pgo=$(PGO_PATH)/default.pgo -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
 		-tags $(MILVUS_GO_BUILD_TAGS) -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null
 
 milvus-gpu: build-cpp-gpu print-gpu-build-info

go.mod

@@ -68,7 +68,6 @@ require (
 	github.com/greatroar/blobloom v0.0.0-00010101000000-000000000000
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/jolestar/go-commons-pool/v2 v2.1.2
-	github.com/milvus-io/milvus-storage/go v0.0.0-20250115020452-cd5091f31b1c
 	github.com/milvus-io/milvus/pkg v0.0.2-0.20241126032235-cb6542339e84
 	github.com/pkg/errors v0.9.1
 	github.com/remeh/sizedwaitgroup v1.0.0
@@ -278,3 +277,5 @@ replace (
 )
 
 exclude github.com/apache/pulsar-client-go/oauth2 v0.0.0-20211108044248-fe3b7c4e445b
+
+replace github.com/milvus-io/milvus-storage/go => github.com/shaoting-huang/milvus-storage/go v0.0.0-20250116120101-8c3ff6fe5727

go.sum

@@ -632,8 +632,6 @@ github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZz
 github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
 github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250102080446-c3ba3d26a90f h1:So6RKU5wqP/8EaKogicJP8gZ2SrzzS/JprusBaE3RKc=
 github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250102080446-c3ba3d26a90f/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
-github.com/milvus-io/milvus-storage/go v0.0.0-20250115020452-cd5091f31b1c h1:86YuY56+ZZB3lcBJhiXH5kWIz5m1bsObm+dAb8m6ZcY=
-github.com/milvus-io/milvus-storage/go v0.0.0-20250115020452-cd5091f31b1c/go.mod h1:lwKREs0IkySkGeAORlnNvlHj3F3DdOEbvz8xQV3ElTk=
 github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE=
 github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk=
 github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=

internal/storage/serde.go

@@ -32,7 +32,6 @@ import (
 	"google.golang.org/protobuf/proto"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
-	"github.com/milvus-io/milvus-storage/go/packed"
 	"github.com/milvus-io/milvus/pkg/common"
 )
 
@@ -992,28 +991,3 @@ func newSimpleArrowRecord(r arrow.Record, schema map[FieldID]schemapb.DataType,
 		field2Col: field2Col,
 	}
 }
-
-var _ RecordWriter = (*PackedRecordWriter)(nil)
-
-type PackedRecordWriter struct {
-	w packed.PackedWriter
-}
-
-func (crw *PackedRecordWriter) GetWrittenUncompressed() uint64 {
-	// TODO: add interface for packed record writer
-	return 0
-}
-
-func (prw *PackedRecordWriter) Write(r Record) error {
-	return prw.Write(r)
-}
-
-func (prw *PackedRecordWriter) Close() {
-	if prw != nil {
-		prw.Close()
-	}
-}
-
-func NewPackedRecordWriter(path string, schema *arrow.Schema, bufferSize int) (*packed.PackedWriter, error) {
-	return packed.NewPackedWriter(path, schema, bufferSize)
-}