aaa

Signed-off-by: zhenshan.cao <[email protected]>
czs007 · Apr 6, 2022 · 38fd734 · 38fd734
1 parent cead9ba
commit 38fd734
Show file tree

Hide file tree

Showing 5 changed files with 382 additions and 14 deletions.
diff --git a/knowhere/utils/Bitset.cpp b/knowhere/utils/Bitset.cpp
@@ -67,7 +67,7 @@ ConcurrentBitset::operator&=(const BitsetView& view) {
 
 std::shared_ptr<ConcurrentBitset>
 ConcurrentBitset::operator&(const ConcurrentBitset& bitset) const {
-auto result_bitset = std::make_shared<ConcurrentBitset>(bitset.count());
+auto result_bitset = std::make_shared<ConcurrentBitset>(bitset.size());
 
 auto result_8 = result_bitset->mutable_data();
 auto result_64 = reinterpret_cast<uint64_t*>(result_8);
@@ -97,7 +97,7 @@ return result_bitset;
 
 std::shared_ptr<ConcurrentBitset>
 ConcurrentBitset::operator&(const BitsetView& view) const {
-auto result_bitset = std::make_shared<ConcurrentBitset>(view.count());
+auto result_bitset = std::make_shared<ConcurrentBitset>(view.size());
 
 auto result_8 = result_bitset->mutable_data();
 auto result_64 = reinterpret_cast<uint64_t*>(result_8);
@@ -177,7 +177,7 @@ return *this;
 
 std::shared_ptr<ConcurrentBitset>
 ConcurrentBitset::operator|(const ConcurrentBitset& bitset) const {
-auto result_bitset = std::make_shared<ConcurrentBitset>(bitset.count());
+auto result_bitset = std::make_shared<ConcurrentBitset>(bitset.size());
 std::cout<<"1111111"<<std::endl;
 
 auto result_8 = result_bitset->mutable_data();
@@ -213,7 +213,7 @@ return result_bitset;
 
 std::shared_ptr<ConcurrentBitset>
 ConcurrentBitset::operator|(const BitsetView& view) const {
-auto result_bitset = std::make_shared<ConcurrentBitset>(view.count());
+auto result_bitset = std::make_shared<ConcurrentBitset>(view.size());
 
 auto result_8 = result_bitset->mutable_data();
 auto result_64 = reinterpret_cast<uint64_t*>(result_8);

diff --git a/knowhere/utils/Bitset.h b/knowhere/utils/Bitset.h
@@ -76,24 +76,20 @@ class ConcurrentBitset {
 
     inline bool
     test(id_type_t id) const {
-        unsigned char mask = (unsigned char)(0x01) << (id & 0x07);
-        return (bitset_[id >> 3] & mask);
+        uint8_t mask = (uint8_t)(0x01) << (id & 0x07);
+        return bitset_[id >> 3] & mask;
     }
 
     inline void
     set(id_type_t id) {
-	uint8_t mask = ~uint8_t(0);
-//        unsigned char mask = (unsigned char)(0x01) << (id & 0x07);
-        //bitset_[id >> 3].fetch_or(mask);
-        bitset_[id >> 3] = mask;
+        uint8_t mask = (uint8_t)(0x01) << (id & 0x07);
+        bitset_[id >> 3] |= mask;
     }
 
     inline void
     clear(id_type_t id) {
-//	uint8_t mask = ~uint8_t(0);
-        //unsigned char mask = (unsigned char)(0x01) << (id & 0x07);
-//        bitset_[id >> 3].fetch_and(~mask);
-        bitset_[id >> 3] = 0;
+        uint8_t mask = (uint8_t)(0x01) << (id & 0x07);
+        bitset_[id >> 3] &= ~mask;
     }
 
     size_t

diff --git a/knowhere/utils/Bitset2.cpp b/knowhere/utils/Bitset2.cpp
@@ -0,0 +1,239 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include <cstring>
+#include <string>
+#include <memory>
+#include "Bitset2.h"
+#include "BitsetView.h"
+
+namespace faiss {
+
+ConcurrentBitset2&
+ConcurrentBitset2::operator&=(const ConcurrentBitset2& bitset) {
+	auto u8_1 = mutable_data();
+	auto u8_2 = bitset.data();
+	auto u64_1 = reinterpret_cast<uint64_t*>(u8_1);
+	auto u64_2 = reinterpret_cast<const uint64_t*>(u8_2);
+
+	size_t n64 = bitset_.size();
+//	size_t n64 = n8 / 8;
+
+	for (size_t i = 0; i < n64; i++) {
+	    u64_1[i] &= u64_2[i];
+	}
+
+	return *this;
+}
+
+ConcurrentBitset2&
+ConcurrentBitset2::operator&=(const BitsetView& view) {
+	auto u8_1 = mutable_data();
+	auto u8_2 = view.data();
+	auto u64_1 = reinterpret_cast<uint64_t*>(u8_1);
+	auto u64_2 = reinterpret_cast<const uint64_t*>(u8_2);
+
+	size_t n64 = bitset_.size();
+
+	for (size_t i = 0; i < n64; i++) {
+	    u64_1[i] &= u64_2[i];
+	}
+
+	return *this;
+}
+
+std::shared_ptr<ConcurrentBitset2>
+ConcurrentBitset2::operator&(const ConcurrentBitset2& bitset) const {
+auto result_bitset = std::make_shared<ConcurrentBitset2>(bitset.size());
+
+//auto result_8 = 
+auto result_64 = reinterpret_cast<uint64_t*>(result_bitset->mutable_data());
+
+auto u64_1 = reinterpret_cast<const uint64_t*>(data());
+auto u64_2 = reinterpret_cast<const uint64_t*>(bitset.data());
+
+size_t n64 = bitset_.size();
+
+for (size_t i = 0; i < n64; i++) {
+    result_64[i] = u64_1[i] & u64_2[i];
+}
+
+return result_bitset;
+}
+
+std::shared_ptr<ConcurrentBitset2>
+ConcurrentBitset2::operator&(const BitsetView& view) const {
+auto result_bitset = std::make_shared<ConcurrentBitset2>(view.size());
+
+auto result_64 = reinterpret_cast<uint64_t*>(result_bitset->mutable_data());
+
+auto u64_1 = reinterpret_cast<const uint64_t*>(data());
+auto u64_2 = reinterpret_cast<const uint64_t*>(view.data());
+
+size_t n64 = bitset_.size();
+
+for (size_t i = 0; i < n64; i++) {
+    result_64[i] = u64_1[i] & u64_2[i];
+}
+
+return result_bitset;
+}
+
+
+ConcurrentBitset2&
+ConcurrentBitset2::operator|=(const ConcurrentBitset2& bitset) {
+auto u64_1 = reinterpret_cast<uint64_t*>(mutable_data());
+auto u64_2 = reinterpret_cast<const uint64_t*>(bitset.data());
+
+size_t n64 = bitset_.size();
+
+for (size_t i = 0; i < n64; i++) {
+    u64_1[i] |= u64_2[i];
+}
+
+return *this;
+}
+
+ConcurrentBitset2&
+ConcurrentBitset2::operator|=(const BitsetView& view) {
+auto u64_1 = reinterpret_cast<uint64_t*>(mutable_data());
+auto u64_2 = reinterpret_cast<const uint64_t*>(view.data());
+
+size_t n64 = bitset_.size();
+
+for (size_t i = 0; i < n64; i++) {
+    u64_1[i] |= u64_2[i];
+}
+
+return *this;
+}
+
+
+std::shared_ptr<ConcurrentBitset2>
+ConcurrentBitset2::operator|(const ConcurrentBitset2& bitset) const {
+auto result_bitset = std::make_shared<ConcurrentBitset2>(bitset.size());
+
+auto result_64 = reinterpret_cast<uint64_t*>(result_bitset->mutable_data());
+
+auto u64_1 = reinterpret_cast<const uint64_t*>(data());
+auto u64_2 = reinterpret_cast<const uint64_t*>(bitset.data());
+
+size_t n64 = bitset_.size();
+
+for (size_t i = 0; i < n64; i++) {
+    result_64[i] = u64_1[i] | u64_2[i];
+}
+
+return result_bitset;
+}
+
+std::shared_ptr<ConcurrentBitset2>
+ConcurrentBitset2::operator|(const BitsetView& view) const {
+auto result_bitset = std::make_shared<ConcurrentBitset2>(view.size());
+
+auto result_64 = reinterpret_cast<uint64_t*>(result_bitset->mutable_data());
+
+auto u64_1 = reinterpret_cast<const uint64_t*>(data());
+auto u64_2 = reinterpret_cast<const uint64_t*>(view.data());
+
+size_t n64 = bitset_.size();
+
+for (size_t i = 0; i < n64; i++) {
+    result_64[i] = u64_1[i] | u64_2[i];
+}
+
+return result_bitset;
+}
+
+
+ConcurrentBitset2&
+ConcurrentBitset2::negate() {
+auto u64_1 = reinterpret_cast<uint64_t*>(mutable_data());
+
+size_t n64 = bitset_.size();
+
+for (size_t i = 0; i < n64; i++) {
+    u64_1[i] = ~u64_1[i];
+}
+
+return *this;
+}
+
+size_t
+ConcurrentBitset2::count() const {
+size_t ret = 0;
+auto p_data = reinterpret_cast<const uint64_t *>(data());
+
+auto len = size() >> 3;
+auto popcount8 = [&](uint8_t x) -> int{
+    x = (x & 0x55) + ((x >> 1) & 0x55);
+    x = (x & 0x33) + ((x >> 2) & 0x33);
+    x = (x & 0x0F) + ((x >> 4) & 0x0F);
+    return x;
+};
+
+for (size_t i = 0; i < len; ++i) {
+    ret += __builtin_popcountl(*p_data);
+    p_data++;
+}
+
+auto p_byte = data() + (len << 3);
+for (auto i = (len << 3); i < size(); ++i) {
+    ret += popcount8(*p_byte);
+    p_byte++;
+}
+return ret;
+}
+
+ConcurrentBitset2::operator std::string() const { 
+    const char one = '1';
+    const char zero = '0';
+    const size_t len = size();
+    std::string s;
+    s.assign (len, zero);
+
+    for (size_t i = 0; i < len; ++i) {
+        if (test(id_type_t(i)))
+            s.assign(len - 1 - i, one);
+    }
+    return s;
+}
+
+bool operator==(const ConcurrentBitset2& lhs, const ConcurrentBitset2& rhs) {
+    if (std::addressof(lhs) == std::addressof(rhs)){
+	return true;
+    }
+
+    if (lhs.size() != rhs.size()){
+	return false;
+    }
+
+    if (lhs.byte_size() != rhs.byte_size()){
+	return false;
+    }
+
+
+    auto ret = std::memcmp(lhs.data(), rhs.data(), lhs.byte_size());
+    return ret == 0;
+}
+
+bool operator!=(const ConcurrentBitset2& lhs, const ConcurrentBitset2& rhs){ 
+    return !(lhs == rhs); 
+}
+
+std::ostream& operator<<(std::ostream& os, const ConcurrentBitset2& bitset)
+{
+    os << std::string(bitset);
+    return os;
+}
+
+
+}  // namespace faiss