From d692c562ba4a96789986f7ec229f80aa48c0ec0b Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 13 Jan 2025 14:59:04 +0800 Subject: [PATCH 01/61] init: Initialization preparations --- .../store/rocksdb/RocksDBOptionsFactory.java | 51 ++++++ .../timer/rocksdb/TimerMessageKVStore.java | 84 ++++++++++ .../timer/rocksdb/TimerMessageRecord.java | 91 ++++++++++ .../rocksdb/TimerMessageRocksDBStorage.java | 155 ++++++++++++++++++ 4 files changed, 381 insertions(+) create mode 100644 store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java index 5687d6a222d..6f20175c779 100644 --- a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java +++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java @@ -182,6 +182,57 @@ public static ColumnFamilyOptions createPopCFOptions() { .setOptimizeFiltersForHits(true); } + public static ColumnFamilyOptions createTimerCFOptions() { + BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig() + .setFormatVersion(5) + .setIndexType(IndexType.kBinarySearch) + .setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash) + .setDataBlockHashTableUtilRatio(0.75) + .setBlockSize(128 * SizeUnit.KB) + .setMetadataBlockSize(4 * SizeUnit.KB) + .setFilterPolicy(new BloomFilter(16, false)) + .setCacheIndexAndFilterBlocks(false) + .setCacheIndexAndFilterBlocksWithHighPriority(true) + .setPinL0FilterAndIndexBlocksInCache(false) + .setPinTopLevelIndexAndFilter(true) + .setBlockCache(new LRUCache(1024 * SizeUnit.MB, 8, false)) + .setWholeKeyFiltering(true); + + CompactionOptionsUniversal compactionOption = new CompactionOptionsUniversal() + .setSizeRatio(100) + .setMaxSizeAmplificationPercent(25) + .setAllowTrivialMove(true) + .setMinMergeWidth(2) + .setMaxMergeWidth(Integer.MAX_VALUE) + .setStopStyle(CompactionStopStyle.CompactionStopStyleTotalSize) + .setCompressionSizePercent(-1); + + //noinspection resource + return new ColumnFamilyOptions() + .setMaxWriteBufferNumber(6) + .setWriteBufferSize(128 * SizeUnit.MB) + .setMinWriteBufferNumberToMerge(1) + .setTableFormatConfig(blockBasedTableConfig) + .setMemTableConfig(new SkipListMemTableConfig()) + .setCompressionType(CompressionType.NO_COMPRESSION) + .setBottommostCompressionType(CompressionType.NO_COMPRESSION) + .setNumLevels(7) + .setCompactionPriority(CompactionPriority.MinOverlappingRatio) + .setCompactionStyle(CompactionStyle.UNIVERSAL) + .setCompactionOptionsUniversal(compactionOption) + .setMaxCompactionBytes(100 * SizeUnit.GB) + .setSoftPendingCompactionBytesLimit(100 * SizeUnit.GB) + .setHardPendingCompactionBytesLimit(256 * SizeUnit.GB) + .setLevel0FileNumCompactionTrigger(2) + .setLevel0SlowdownWritesTrigger(8) + .setLevel0StopWritesTrigger(10) + .setTargetFileSizeBase(256 * SizeUnit.MB) + .setTargetFileSizeMultiplier(2) + .setMergeOperator(new StringAppendOperator()) + .setReportBgIoStats(true) + .setOptimizeFiltersForHits(true); + } + /** * Create a rocksdb db options, the user must take care to close it after closing db. * @return diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java new file mode 100644 index 00000000000..b1f659e97f5 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.timer.rocksdb; + +import java.util.List; + +public interface TimerMessageKVStore { + + /** + * Start the timer message kv store. + */ + boolean start(); + + /** + * Shutdown the timer message kv store. + */ + boolean shutdown(); + + /** + * Get the file path of the timer message kv store. + */ + String getFilePath(); + + /** + * Write the timer message records to the timer message kv store. + * @param consumerRecordList the list of timer message records to be written. + * Default is store common timer message. + */ + void writeDefaultRecords(List consumerRecordList); + + /** + * Write the timer message records to the timer message kv store. + * @param columnFamily the column family of the timer message kv store. + * @param consumerRecordList the list of timer message records to be written. + */ + void writeAssignRecords(byte[] columnFamily, List consumerRecordList); + + /** + * Delete the timer message records from the timer message kv store. + * @param consumerRecordList the list of timer message records to be deleted. + * Default is delete common timer message. + */ + void deleteDefaultRecords(List consumerRecordList); + + /** + * Delete the timer message records from the timer message kv store. + * @param columnFamily the column family of the timer message kv store. + * @param consumerRecordList the list of timer message records to be deleted. + */ + void deleteAssignRecords(byte[] columnFamily, List consumerRecordList); + + /** + * Scan the timer message records from the timer message kv store. + * @param columnFamily the column family of the timer message kv store. + * @param lowerTime the lower time of the timer message records to be scanned. + * @param upperTime the upper time of the timer message records to be scanned. + * @return the list of timer message records. + */ + List scanRecords(byte[] columnFamily, long lowerTime, long upperTime); + + /** + * Scan the expired timer message records from the timer message kv store. + * @param columnFamily the column family of the timer message kv store. + * @param lowerTime the lower time of the timer message records to be scanned. + * @param upperTime the upper time of the timer message records to be scanned. + * @param maxCount the max count of the timer message records to be return. + * @return the list of timer message records. + */ + List scanExpiredRecords(byte[] columnFamily, long lowerTime, long upperTime, int maxCount); +} diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java new file mode 100644 index 00000000000..a26fd11a40d --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.timer.rocksdb; + +import com.alibaba.fastjson.annotation.JSONField; + +import java.nio.ByteBuffer; + +public class TimerMessageRecord { + // key: delayTime + offsetPY + private long delayTime; + private long offsetPY; + + // value: sizeReal + private int sizeReal; + + private final static int keyLength = Long.BYTES + Long.BYTES; + private final static int valueLength = Integer.BYTES; + + public TimerMessageRecord() { + } + + public TimerMessageRecord(long delayTime, long offsetPY, int sizeReal) { + this.delayTime = delayTime; + this.offsetPY = offsetPY; + this.sizeReal = sizeReal; + } + + @JSONField(serialize = false) + public byte[] getKeyBytes() { + byte[] keyBytes = new byte[keyLength]; + ByteBuffer buffer = ByteBuffer.wrap(keyBytes); + buffer.putLong(this.getDelayTime()).putLong(this.getOffsetPY()); + return keyBytes; + } + + @JSONField(serialize = false) + public byte[] getValueBytes() { + byte[] valueBytes = new byte[valueLength]; + ByteBuffer buffer = ByteBuffer.wrap(valueBytes); + buffer.putInt(this.getSizeReal()); + return valueBytes; + } + + public static TimerMessageRecord decode(byte[] body) { + TimerMessageRecord timerMessageRecord = new TimerMessageRecord(); + ByteBuffer buffer = ByteBuffer.wrap(body); + timerMessageRecord.setDelayTime(buffer.getLong()); + timerMessageRecord.setOffsetPY(buffer.getLong()); + timerMessageRecord.setSizeReal(buffer.getInt()); + return timerMessageRecord; + } + + public void setDelayTime(long delayTime) { + this.delayTime = delayTime; + } + + public void setOffsetPY(long offsetPY) { + this.offsetPY = offsetPY; + } + + public void setSizeReal(int sizeReal) { + this.sizeReal = sizeReal; + } + + public int getSizeReal() { + return sizeReal; + } + + public long getDelayTime() { + return delayTime; + } + + public long getOffsetPY() { + return offsetPY; + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java new file mode 100644 index 00000000000..2b1de9f05be --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.timer.rocksdb; + +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.config.AbstractRocksDBStorage; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.store.rocksdb.RocksDBOptionsFactory; +import org.rocksdb.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class TimerMessageRocksDBStorage extends AbstractRocksDBStorage implements TimerMessageKVStore { + private final static Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + + public final static byte[] TRANSACTION_COLUMN_FAMILY = "transaction".getBytes(); + public final static byte[] POP_COLUMN_FAMILY = "pop".getBytes(); + + private ColumnFamilyHandle popColumnFamilyHandle; + private ColumnFamilyHandle transactionColumnFamilyHandle; + + private WriteOptions writeOptions; + private WriteOptions deleteOptions; + + public TimerMessageRocksDBStorage(String filePath) { + super(filePath); + } + + protected void initOptions() { + this.options = RocksDBOptionsFactory.createDBOptions(); + + this.deleteOptions = new WriteOptions(); + this.deleteOptions.setSync(false); + this.deleteOptions.setDisableWAL(false); + this.deleteOptions.setNoSlowdown(false); + + this.writeOptions = new WriteOptions(); + this.writeOptions.setSync(false); + this.writeOptions.setDisableWAL(false); + this.writeOptions.setNoSlowdown(false); + + this.compactRangeOptions = new CompactRangeOptions(); + this.compactRangeOptions.setBottommostLevelCompaction( + CompactRangeOptions.BottommostLevelCompaction.kForce); + this.compactRangeOptions.setAllowWriteStall(true); + this.compactRangeOptions.setExclusiveManualCompaction(false); + this.compactRangeOptions.setChangeLevel(true); + this.compactRangeOptions.setTargetLevel(-1); + this.compactRangeOptions.setMaxSubcompactions(4); + } + @Override + protected boolean postLoad() { + try { + UtilAll.ensureDirOK(this.dbPath); + initOptions(); + + // init column family here + ColumnFamilyOptions defaultOptions = RocksDBOptionsFactory.createTimerCFOptions(); + ColumnFamilyOptions popOptions = RocksDBOptionsFactory.createTimerCFOptions(); + ColumnFamilyOptions transactionOptions = RocksDBOptionsFactory.createTimerCFOptions(); + + this.cfOptions.add(defaultOptions); + this.cfOptions.add(popOptions); + this.cfOptions.add(transactionOptions); + + List cfDescriptors = new ArrayList<>(); + cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, defaultOptions)); + cfDescriptors.add(new ColumnFamilyDescriptor(POP_COLUMN_FAMILY, popOptions)); + cfDescriptors.add(new ColumnFamilyDescriptor(TRANSACTION_COLUMN_FAMILY, transactionOptions)); + + this.open(cfDescriptors); + this.defaultCFHandle = cfHandles.get(0); + this.popColumnFamilyHandle = cfHandles.get(1); + this.transactionColumnFamilyHandle = cfHandles.get(2); + + log.debug("Timer message on rocksdb init, filePath={}", this.dbPath); + } catch (final Exception e) { + log.error("Timer message on rocksdb init error, filePath={}", this.dbPath, e); + return false; + } + return true; + } + + @Override + protected void preShutdown() { + if (this.writeOptions != null) { + this.writeOptions.close(); + } + if (this.deleteOptions != null) { + this.deleteOptions.close(); + } + if (this.defaultCFHandle != null) { + this.defaultCFHandle.close(); + } + if (this.transactionColumnFamilyHandle != null) { + this.transactionColumnFamilyHandle.close(); + } + if (this.popColumnFamilyHandle != null) { + this.popColumnFamilyHandle.close(); + } + } + + @Override + public String getFilePath() { + return this.dbPath; + } + + @Override + public void writeDefaultRecords(List consumerRecordList) { + + } + + @Override + public void writeAssignRecords(byte[] columnFamily, List consumerRecordList) { + + } + + @Override + public void deleteDefaultRecords(List consumerRecordList) { + + } + + @Override + public void deleteAssignRecords(byte[] columnFamily, List consumerRecordList) { + + } + + @Override + public List scanRecords(byte[] columnFamily, long lowerTime, long upperTime) { + return Collections.emptyList(); + } + + @Override + public List scanExpiredRecords(byte[] columnFamily, long lowerTime, long upperTime, int maxCount) { + return Collections.emptyList(); + } +} From 382d846313141a28754f16c83e6bcf790c110eee Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 13 Jan 2025 16:03:43 +0800 Subject: [PATCH 02/61] init: Initialization preparations --- .../timer/rocksdb/TimerMessageRecord.java | 9 ++ .../rocksdb/TimerMessageRocksDBStorage.java | 113 ++++++++++++++++-- 2 files changed, 113 insertions(+), 9 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java index a26fd11a40d..5a15b5e790a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java @@ -88,4 +88,13 @@ public long getDelayTime() { public long getOffsetPY() { return offsetPY; } + + @Override + public String toString() { + return "TimerMessageRecord{" + + "delayTime=" + delayTime + + ", offsetPY=" + offsetPY + + ", sizeReal=" + sizeReal + + '}'; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index 2b1de9f05be..4bc10d39233 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -20,19 +20,31 @@ import org.apache.rocketmq.common.config.AbstractRocksDBStorage; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.store.rocksdb.RocksDBOptionsFactory; -import org.rocksdb.*; +import org.rocksdb.RocksDB; +import org.rocksdb.WriteBatch; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.WriteOptions; +import org.rocksdb.ReadOptions; +import org.rocksdb.CompactRangeOptions; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.RocksDBException; +import org.rocksdb.Slice; +import org.rocksdb.RocksIterator; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Collections; import java.util.List; public class TimerMessageRocksDBStorage extends AbstractRocksDBStorage implements TimerMessageKVStore { private final static Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); - public final static byte[] TRANSACTION_COLUMN_FAMILY = "transaction".getBytes(); - public final static byte[] POP_COLUMN_FAMILY = "pop".getBytes(); + public final static byte[] TRANSACTION_COLUMN_FAMILY = "transaction".getBytes(StandardCharsets.UTF_8); + public final static byte[] POP_COLUMN_FAMILY = "pop".getBytes(StandardCharsets.UTF_8); private ColumnFamilyHandle popColumnFamilyHandle; private ColumnFamilyHandle transactionColumnFamilyHandle; @@ -125,31 +137,114 @@ public String getFilePath() { @Override public void writeDefaultRecords(List consumerRecordList) { - + if (!consumerRecordList.isEmpty()) { + try (WriteBatch writeBatch = new WriteBatch()) { + for (TimerMessageRecord record : consumerRecordList) { + writeBatch.put(defaultCFHandle, record.getKeyBytes(), record.getValueBytes()); + } + this.db.write(writeOptions, writeBatch); + } catch (RocksDBException e) { + throw new RuntimeException("Write record error", e); + } + } } @Override public void writeAssignRecords(byte[] columnFamily, List consumerRecordList) { - + ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily); + + if (cfHandle != null && !consumerRecordList.isEmpty()) { + try (WriteBatch writeBatch = new WriteBatch()) { + for (TimerMessageRecord record : consumerRecordList) { + writeBatch.put(cfHandle, record.getKeyBytes(), record.getValueBytes()); + } + this.db.write(writeOptions, writeBatch); + } catch (RocksDBException e) { + throw new RuntimeException("Write record error", e); + } + } } @Override public void deleteDefaultRecords(List consumerRecordList) { - + if (!consumerRecordList.isEmpty()) { + try (WriteBatch writeBatch = new WriteBatch()) { + for (TimerMessageRecord record : consumerRecordList) { + writeBatch.delete(defaultCFHandle, record.getKeyBytes()); + } + this.db.write(deleteOptions, writeBatch); + } catch (RocksDBException e) { + throw new RuntimeException("Delete record error", e); + } + } } @Override public void deleteAssignRecords(byte[] columnFamily, List consumerRecordList) { + ColumnFamilyHandle deleteCfHandle = getColumnFamily(columnFamily);; + + if (deleteCfHandle != null && !consumerRecordList.isEmpty()) { + try (WriteBatch writeBatch = new WriteBatch()) { + for (TimerMessageRecord record : consumerRecordList) { + writeBatch.delete(deleteCfHandle, record.getKeyBytes()); + } + this.db.write(deleteOptions, writeBatch); + } catch (RocksDBException e) { + throw new RuntimeException("Delete record error", e); + } + } + } + + private ColumnFamilyHandle getColumnFamily(byte[] columnFamily) { + ColumnFamilyHandle cfHandle; + if (columnFamily == POP_COLUMN_FAMILY) { + cfHandle = popColumnFamilyHandle; + } else if (columnFamily == TRANSACTION_COLUMN_FAMILY) { + cfHandle = transactionColumnFamilyHandle; + } else if (columnFamily == RocksDB.DEFAULT_COLUMN_FAMILY) { + cfHandle = defaultCFHandle; + } else { + throw new RuntimeException("Unknown column family"); + } + return cfHandle; } @Override public List scanRecords(byte[] columnFamily, long lowerTime, long upperTime) { - return Collections.emptyList(); + List records = new ArrayList<>(); + ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily); + + try (ReadOptions readOptions = new ReadOptions() + .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array())) + .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(upperTime).array())); + RocksIterator iterator = db.newIterator(cfHandle, readOptions)) { + iterator.seek(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array()); + while (iterator.isValid()) { + records.add(TimerMessageRecord.decode(iterator.value())); + iterator.next(); + } + } + + return records; } @Override public List scanExpiredRecords(byte[] columnFamily, long lowerTime, long upperTime, int maxCount) { - return Collections.emptyList(); + List records = new ArrayList<>(); + ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily); + + try (ReadOptions readOptions = new ReadOptions() + .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array())) + .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(upperTime).array())); + RocksIterator iterator = db.newIterator(cfHandle, readOptions)) { + iterator.seek(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array()); + while (iterator.isValid() && records.size() < maxCount) { + records.add(TimerMessageRecord.decode(iterator.value())); + iterator.next(); + } + } + + return records; } } From c5905e41ad255bacb42013b1e8588fb800e84408 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 13 Jan 2025 20:21:46 +0800 Subject: [PATCH 03/61] init: Initialization preparations --- .../store/config/MessageStoreConfig.java | 24 +++++++ .../rocksdb/TimerMessageRocksDBStorage.java | 4 +- .../rocksdb/TimerMessageRocksDBStore.java | 64 +++++++++++++++++++ 3 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 0ea58415487..8e9299091aa 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -472,6 +472,14 @@ public void setRocksdbCompressionType(String compressionType) { **/ private boolean useABSLock = false; + /** + * Maximum number of messages to be read each time + * -1 ： read all messages + */ + private int readCountTimerOnRocksDB = -1; + + private boolean enableTimerMessageOnRocksDB = false; + public boolean isRocksdbCQDoubleWriteEnable() { return rocksdbCQDoubleWriteEnable; } @@ -1950,4 +1958,20 @@ public void setUseABSLock(boolean useABSLock) { public boolean getUseABSLock() { return useABSLock; } + + public void setReadCountTimerOnRocksDB(int readCountTimerOnRocksDB) { + this.readCountTimerOnRocksDB = readCountTimerOnRocksDB; + } + + public int getReadCountTimerOnRocksDB() { + return readCountTimerOnRocksDB; + } + + public void setEnableTimerMessageOnRocksDB(boolean enableTimerMessageOnRocksDB) { + this.enableTimerMessageOnRocksDB = enableTimerMessageOnRocksDB; + } + + public boolean getEnableTimerMessageOnRocksDB() { + return enableTimerMessageOnRocksDB; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index 4bc10d39233..36c0f77a997 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -61,12 +61,12 @@ protected void initOptions() { this.deleteOptions = new WriteOptions(); this.deleteOptions.setSync(false); - this.deleteOptions.setDisableWAL(false); + this.deleteOptions.setDisableWAL(true); this.deleteOptions.setNoSlowdown(false); this.writeOptions = new WriteOptions(); this.writeOptions.setSync(false); - this.writeOptions.setDisableWAL(false); + this.writeOptions.setDisableWAL(true); this.writeOptions.setNoSlowdown(false); this.compactRangeOptions = new CompactRangeOptions(); diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java new file mode 100644 index 00000000000..2ce6c9f0e16 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.timer.rocksdb; + +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.apache.rocketmq.store.timer.TimerCheckpoint; +import org.apache.rocketmq.store.timer.TimerMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Paths; + +public class TimerMessageRocksDBStore { + private final static Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final String ROCKSDB_DIRECTORY = "kvStore"; + + public static final int TIMER_WHEEL_TTL_DAY = 30; + public static final int DAY_SECS = 24 * 3600; + + private final TimerMessageKVStore timerMessageKVStore; + private final MessageStore messageStore; + private final TimerCheckpoint timerCheckpoint; + private final BrokerStatsManager brokerStatsManager; + + private final int slotSize; + private final int readCount; + protected final int precisionMs; + protected final MessageStoreConfig storeConfig; + protected TimerMetrics timerMetrics; + + public TimerMessageRocksDBStore(final MessageStore messageStore, final MessageStoreConfig storeConfig, + TimerCheckpoint timerCheckpoint, TimerMetrics timerMetrics, final BrokerStatsManager brokerStatsManager) { + this.storeConfig = storeConfig; + this.messageStore = messageStore; + this.timerCheckpoint = timerCheckpoint; + this.timerMetrics = timerMetrics; + this.brokerStatsManager = brokerStatsManager; + + this.precisionMs = storeConfig.getTimerPrecisionMs(); + this.slotSize = 1000 * TIMER_WHEEL_TTL_DAY / precisionMs * DAY_SECS; + this.readCount = storeConfig.getReadCountTimerOnRocksDB(); + this.timerMessageKVStore = new TimerMessageRocksDBStorage(Paths.get( + storeConfig.getStorePathRootDir(), ROCKSDB_DIRECTORY).toString()); + } + + +} From 69305b13c57ef8ae5cf31376d2b59c2c02a3499d Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 13 Jan 2025 22:50:42 +0800 Subject: [PATCH 04/61] init: prepare for metric --- .../store/rocksdb/RocksDBOptionsFactory.java | 51 +++++++++++++++++++ .../timer/rocksdb/TimerMessageKVStore.java | 12 ++++- .../rocksdb/TimerMessageRocksDBStorage.java | 31 ++++++++++- .../rocksdb/TimerMessageRocksDBStore.java | 20 +++++--- 4 files changed, 104 insertions(+), 10 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java index 6f20175c779..eb7ee09196e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java +++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java @@ -233,6 +233,57 @@ public static ColumnFamilyOptions createTimerCFOptions() { .setOptimizeFiltersForHits(true); } + public static ColumnFamilyOptions createTimerMetricCFOptions() { + BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig() + .setFormatVersion(5) + .setIndexType(IndexType.kBinarySearch) + .setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash) + .setDataBlockHashTableUtilRatio(0.75) + .setBlockSize(4 * SizeUnit.KB) + .setMetadataBlockSize(4 * SizeUnit.KB) + .setFilterPolicy(new BloomFilter(16, false)) + .setCacheIndexAndFilterBlocks(false) + .setCacheIndexAndFilterBlocksWithHighPriority(true) + .setPinL0FilterAndIndexBlocksInCache(false) + .setPinTopLevelIndexAndFilter(true) + .setBlockCache(new LRUCache(64 * SizeUnit.MB, 8, false)) // 调整缓存大小为 512 MB + .setWholeKeyFiltering(true); + + CompactionOptionsUniversal compactionOption = new CompactionOptionsUniversal() + .setSizeRatio(100) + .setMaxSizeAmplificationPercent(25) + .setAllowTrivialMove(true) + .setMinMergeWidth(2) + .setMaxMergeWidth(Integer.MAX_VALUE) + .setStopStyle(CompactionStopStyle.CompactionStopStyleTotalSize) + .setCompressionSizePercent(-1); + + //noinspection resource + return new ColumnFamilyOptions() + .setMaxWriteBufferNumber(4) + .setWriteBufferSize(64 * SizeUnit.MB) + .setMinWriteBufferNumberToMerge(1) + .setTableFormatConfig(blockBasedTableConfig) + .setMemTableConfig(new SkipListMemTableConfig()) + .setCompressionType(CompressionType.NO_COMPRESSION) + .setBottommostCompressionType(CompressionType.NO_COMPRESSION) + .setNumLevels(7) + .setCompactionPriority(CompactionPriority.MinOverlappingRatio) + .setCompactionStyle(CompactionStyle.UNIVERSAL) + .setCompactionOptionsUniversal(compactionOption) + .setMaxCompactionBytes(100 * SizeUnit.GB) + .setSoftPendingCompactionBytesLimit(100 * SizeUnit.GB) + .setHardPendingCompactionBytesLimit(256 * SizeUnit.GB) + .setLevel0FileNumCompactionTrigger(2) + .setLevel0SlowdownWritesTrigger(8) + .setLevel0StopWritesTrigger(10) + .setTargetFileSizeBase(128 * SizeUnit.MB) + .setTargetFileSizeMultiplier(2) + .setMergeOperator(new StringAppendOperator()) + .setReportBgIoStats(true) + .setOptimizeFiltersForHits(true); + } + /** * Create a rocksdb db options, the user must take care to close it after closing db. * @return diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java index b1f659e97f5..2a7684e3e64 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java @@ -38,16 +38,18 @@ public interface TimerMessageKVStore { /** * Write the timer message records to the timer message kv store. * @param consumerRecordList the list of timer message records to be written. + * @param offset the cq offset of the timer message records to be written. * Default is store common timer message. */ - void writeDefaultRecords(List consumerRecordList); + void writeDefaultRecords(List consumerRecordList, long offset); /** * Write the timer message records to the timer message kv store. * @param columnFamily the column family of the timer message kv store. * @param consumerRecordList the list of timer message records to be written. + * @param offset the cq offset of the timer message records to be written. */ - void writeAssignRecords(byte[] columnFamily, List consumerRecordList); + void writeAssignRecords(byte[] columnFamily, List consumerRecordList, long offset); /** * Delete the timer message records from the timer message kv store. @@ -81,4 +83,10 @@ public interface TimerMessageKVStore { * @return the list of timer message records. */ List scanExpiredRecords(byte[] columnFamily, long lowerTime, long upperTime, int maxCount); + + /** + * Get the commit offset of the timer message kv store. + * @return the commit offset of the timer message kv store. + */ + long getCommitOffset(); } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index 36c0f77a997..01985c627dd 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -45,9 +45,14 @@ public class TimerMessageRocksDBStorage extends AbstractRocksDBStorage implement public final static byte[] TRANSACTION_COLUMN_FAMILY = "transaction".getBytes(StandardCharsets.UTF_8); public final static byte[] POP_COLUMN_FAMILY = "pop".getBytes(StandardCharsets.UTF_8); + private final static byte[] OFFSET_KEY = "offset".getBytes(StandardCharsets.UTF_8); + + // key : 100 * n (delay time); value : long (msg number) + private final static byte[] METRIC_COLUMN_FAMILY = "metric".getBytes(StandardCharsets.UTF_8); private ColumnFamilyHandle popColumnFamilyHandle; private ColumnFamilyHandle transactionColumnFamilyHandle; + private ColumnFamilyHandle metricColumnFamilyHandle; private WriteOptions writeOptions; private WriteOptions deleteOptions; @@ -88,20 +93,24 @@ protected boolean postLoad() { ColumnFamilyOptions defaultOptions = RocksDBOptionsFactory.createTimerCFOptions(); ColumnFamilyOptions popOptions = RocksDBOptionsFactory.createTimerCFOptions(); ColumnFamilyOptions transactionOptions = RocksDBOptionsFactory.createTimerCFOptions(); + ColumnFamilyOptions metricOptions = RocksDBOptionsFactory.createTimerMetricCFOptions(); this.cfOptions.add(defaultOptions); this.cfOptions.add(popOptions); this.cfOptions.add(transactionOptions); + this.cfOptions.add(metricOptions); List cfDescriptors = new ArrayList<>(); cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, defaultOptions)); cfDescriptors.add(new ColumnFamilyDescriptor(POP_COLUMN_FAMILY, popOptions)); cfDescriptors.add(new ColumnFamilyDescriptor(TRANSACTION_COLUMN_FAMILY, transactionOptions)); + cfDescriptors.add(new ColumnFamilyDescriptor(METRIC_COLUMN_FAMILY, metricOptions)); this.open(cfDescriptors); this.defaultCFHandle = cfHandles.get(0); this.popColumnFamilyHandle = cfHandles.get(1); this.transactionColumnFamilyHandle = cfHandles.get(2); + this.metricColumnFamilyHandle = cfHandles.get(3); log.debug("Timer message on rocksdb init, filePath={}", this.dbPath); } catch (final Exception e) { @@ -128,6 +137,9 @@ protected void preShutdown() { if (this.popColumnFamilyHandle != null) { this.popColumnFamilyHandle.close(); } + if (this.metricColumnFamilyHandle != null) { + this.metricColumnFamilyHandle.close(); + } } @Override @@ -136,12 +148,14 @@ public String getFilePath() { } @Override - public void writeDefaultRecords(List consumerRecordList) { + public void writeDefaultRecords(List consumerRecordList, long offset) { if (!consumerRecordList.isEmpty()) { try (WriteBatch writeBatch = new WriteBatch()) { for (TimerMessageRecord record : consumerRecordList) { writeBatch.put(defaultCFHandle, record.getKeyBytes(), record.getValueBytes()); } + // atomically update offset + writeBatch.put(OFFSET_KEY, ByteBuffer.allocate(8).putLong(offset).array()); this.db.write(writeOptions, writeBatch); } catch (RocksDBException e) { throw new RuntimeException("Write record error", e); @@ -150,7 +164,7 @@ public void writeDefaultRecords(List consumerRecordList) { } @Override - public void writeAssignRecords(byte[] columnFamily, List consumerRecordList) { + public void writeAssignRecords(byte[] columnFamily, List consumerRecordList, long offset) { ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily); if (cfHandle != null && !consumerRecordList.isEmpty()) { @@ -158,6 +172,8 @@ public void writeAssignRecords(byte[] columnFamily, List con for (TimerMessageRecord record : consumerRecordList) { writeBatch.put(cfHandle, record.getKeyBytes(), record.getValueBytes()); } + // atomically update offset + writeBatch.put(OFFSET_KEY, ByteBuffer.allocate(8).putLong(offset).array()); this.db.write(writeOptions, writeBatch); } catch (RocksDBException e) { throw new RuntimeException("Write record error", e); @@ -247,4 +263,15 @@ public List scanExpiredRecords(byte[] columnFamily, long low return records; } + + @Override + public long getCommitOffset() { + try { + byte[] offsetBytes = db.get(OFFSET_KEY); + return offsetBytes == null ? 0 : ByteBuffer.wrap(offsetBytes).getLong(); + } catch (RocksDBException e) { + throw new RuntimeException("Get commit offset error", e); + } + } + // TODO metrics impl + basic } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 2ce6c9f0e16..c0eb06822fd 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -36,20 +36,20 @@ public class TimerMessageRocksDBStore { private final TimerMessageKVStore timerMessageKVStore; private final MessageStore messageStore; - private final TimerCheckpoint timerCheckpoint; private final BrokerStatsManager brokerStatsManager; + private final MessageStoreConfig storeConfig; + private final TimerMetrics timerMetrics; private final int slotSize; private final int readCount; - protected final int precisionMs; - protected final MessageStoreConfig storeConfig; - protected TimerMetrics timerMetrics; + private final int precisionMs; + + private long commitOffset; public TimerMessageRocksDBStore(final MessageStore messageStore, final MessageStoreConfig storeConfig, - TimerCheckpoint timerCheckpoint, TimerMetrics timerMetrics, final BrokerStatsManager brokerStatsManager) { + TimerMetrics timerMetrics, final BrokerStatsManager brokerStatsManager) { this.storeConfig = storeConfig; this.messageStore = messageStore; - this.timerCheckpoint = timerCheckpoint; this.timerMetrics = timerMetrics; this.brokerStatsManager = brokerStatsManager; @@ -60,5 +60,13 @@ public TimerMessageRocksDBStore(final MessageStore messageStore, final MessageSt storeConfig.getStorePathRootDir(), ROCKSDB_DIRECTORY).toString()); } + public void load() { + boolean result = timerMessageKVStore.start(); + result &= this.timerMetrics.load(); + calcTimerDistribution(); + } + + private void calcTimerDistribution() { + } } From cbfdfdc6ce944a133e0bd2358c034546fd749ede Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Tue, 14 Jan 2025 11:05:20 +0800 Subject: [PATCH 05/61] init: Initialization preparations --- .../rocketmq/store/timer/TimerMetrics.java | 5 + .../timer/rocksdb/TimerMessageKVStore.java | 20 ++- .../rocksdb/TimerMessageRocksDBStorage.java | 46 ++++-- .../rocksdb/TimerMessageRocksDBStore.java | 138 +++++++++++++++++- .../timer/rocksdb/TimerRocksDBRequest.java | 56 +++++++ 5 files changed, 250 insertions(+), 15 deletions(-) create mode 100644 store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerRocksDBRequest.java diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java index 7f8fedd8a5b..bd81e92a9ce 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java @@ -80,6 +80,11 @@ public long updateDistPair(int period, int value) { return distPair.getCount().addAndGet(value); } + public void resetDistPair(int period, int value) { + Metric distPair = getDistPair(period); + distPair.getCount().set(value); + } + public long addAndGet(MessageExt msg, int value) { String topic = msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC); Metric pair = getTopicPair(topic); diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java index 2a7684e3e64..df52a7e8419 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java @@ -39,31 +39,35 @@ public interface TimerMessageKVStore { * Write the timer message records to the timer message kv store. * @param consumerRecordList the list of timer message records to be written. * @param offset the cq offset of the timer message records to be written. + * @param timestamp the key of the timer message metric column family. * Default is store common timer message. */ - void writeDefaultRecords(List consumerRecordList, long offset); + void writeDefaultRecords(List consumerRecordList, long offset, int timestamp); /** * Write the timer message records to the timer message kv store. * @param columnFamily the column family of the timer message kv store. * @param consumerRecordList the list of timer message records to be written. * @param offset the cq offset of the timer message records to be written. + * @param timestamp the key of the timer message metric column family. */ - void writeAssignRecords(byte[] columnFamily, List consumerRecordList, long offset); + void writeAssignRecords(byte[] columnFamily, List consumerRecordList, long offset, int timestamp); /** * Delete the timer message records from the timer message kv store. * @param consumerRecordList the list of timer message records to be deleted. + * @param timestamp the key of the timer message metric column family. * Default is delete common timer message. */ - void deleteDefaultRecords(List consumerRecordList); + void deleteDefaultRecords(List consumerRecordList, int timestamp); /** * Delete the timer message records from the timer message kv store. * @param columnFamily the column family of the timer message kv store. * @param consumerRecordList the list of timer message records to be deleted. + * @param timestamp the key of the timer message metric column family. */ - void deleteAssignRecords(byte[] columnFamily, List consumerRecordList); + void deleteAssignRecords(byte[] columnFamily, List consumerRecordList, int timestamp); /** * Scan the timer message records from the timer message kv store. @@ -89,4 +93,12 @@ public interface TimerMessageKVStore { * @return the commit offset of the timer message kv store. */ long getCommitOffset(); + + /** + * Get the metric size of the timer message kv store. + * @param lowerTime the lower time of the timer message records to be scanned. + * @param upperTime the upper time of the timer message records to be scanned. + * @return sum. + */ + int getMetricSize(int lowerTime, int upperTime); } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index 01985c627dd..c0bd4bab3f8 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -45,7 +45,7 @@ public class TimerMessageRocksDBStorage extends AbstractRocksDBStorage implement public final static byte[] TRANSACTION_COLUMN_FAMILY = "transaction".getBytes(StandardCharsets.UTF_8); public final static byte[] POP_COLUMN_FAMILY = "pop".getBytes(StandardCharsets.UTF_8); - private final static byte[] OFFSET_KEY = "offset".getBytes(StandardCharsets.UTF_8); + private final static byte[] TIMER_OFFSET_KEY = "timer_offset".getBytes(StandardCharsets.UTF_8); // key : 100 * n (delay time); value : long (msg number) private final static byte[] METRIC_COLUMN_FAMILY = "metric".getBytes(StandardCharsets.UTF_8); @@ -148,14 +148,15 @@ public String getFilePath() { } @Override - public void writeDefaultRecords(List consumerRecordList, long offset) { + public void writeDefaultRecords(List consumerRecordList, long offset, int timestamp) { if (!consumerRecordList.isEmpty()) { try (WriteBatch writeBatch = new WriteBatch()) { for (TimerMessageRecord record : consumerRecordList) { writeBatch.put(defaultCFHandle, record.getKeyBytes(), record.getValueBytes()); } // atomically update offset - writeBatch.put(OFFSET_KEY, ByteBuffer.allocate(8).putLong(offset).array()); + writeBatch.put(TIMER_OFFSET_KEY, ByteBuffer.allocate(8).putLong(offset).array()); + syncMetric(timestamp / 100, consumerRecordList.size(), writeBatch); this.db.write(writeOptions, writeBatch); } catch (RocksDBException e) { throw new RuntimeException("Write record error", e); @@ -164,7 +165,7 @@ public void writeDefaultRecords(List consumerRecordList, lon } @Override - public void writeAssignRecords(byte[] columnFamily, List consumerRecordList, long offset) { + public void writeAssignRecords(byte[] columnFamily, List consumerRecordList, long offset, int timestamp) { ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily); if (cfHandle != null && !consumerRecordList.isEmpty()) { @@ -173,7 +174,8 @@ public void writeAssignRecords(byte[] columnFamily, List con writeBatch.put(cfHandle, record.getKeyBytes(), record.getValueBytes()); } // atomically update offset - writeBatch.put(OFFSET_KEY, ByteBuffer.allocate(8).putLong(offset).array()); + writeBatch.put(TIMER_OFFSET_KEY, ByteBuffer.allocate(8).putLong(offset).array()); + syncMetric(timestamp / 100, consumerRecordList.size(), writeBatch); this.db.write(writeOptions, writeBatch); } catch (RocksDBException e) { throw new RuntimeException("Write record error", e); @@ -182,12 +184,13 @@ public void writeAssignRecords(byte[] columnFamily, List con } @Override - public void deleteDefaultRecords(List consumerRecordList) { + public void deleteDefaultRecords(List consumerRecordList, int timestamp) { if (!consumerRecordList.isEmpty()) { try (WriteBatch writeBatch = new WriteBatch()) { for (TimerMessageRecord record : consumerRecordList) { writeBatch.delete(defaultCFHandle, record.getKeyBytes()); } + syncMetric(timestamp / 100, - consumerRecordList.size(), writeBatch); this.db.write(deleteOptions, writeBatch); } catch (RocksDBException e) { throw new RuntimeException("Delete record error", e); @@ -196,7 +199,7 @@ public void deleteDefaultRecords(List consumerRecordList) { } @Override - public void deleteAssignRecords(byte[] columnFamily, List consumerRecordList) { + public void deleteAssignRecords(byte[] columnFamily, List consumerRecordList, int timestamp) { ColumnFamilyHandle deleteCfHandle = getColumnFamily(columnFamily);; if (deleteCfHandle != null && !consumerRecordList.isEmpty()) { @@ -204,6 +207,7 @@ public void deleteAssignRecords(byte[] columnFamily, List co for (TimerMessageRecord record : consumerRecordList) { writeBatch.delete(deleteCfHandle, record.getKeyBytes()); } + syncMetric(timestamp / 100, - consumerRecordList.size(), writeBatch); this.db.write(deleteOptions, writeBatch); } catch (RocksDBException e) { throw new RuntimeException("Delete record error", e); @@ -267,11 +271,35 @@ public List scanExpiredRecords(byte[] columnFamily, long low @Override public long getCommitOffset() { try { - byte[] offsetBytes = db.get(OFFSET_KEY); + byte[] offsetBytes = db.get(TIMER_OFFSET_KEY); return offsetBytes == null ? 0 : ByteBuffer.wrap(offsetBytes).getLong(); } catch (RocksDBException e) { throw new RuntimeException("Get commit offset error", e); } } - // TODO metrics impl + basic + + @Override + public int getMetricSize(int lowerTime, int upperTime) { + int metricSize = 0; + + try (ReadOptions readOptions = new ReadOptions() + .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array())) + .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(upperTime).array())); + RocksIterator iterator = db.newIterator(metricColumnFamilyHandle, readOptions)) { + iterator.seek(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array()); + while (iterator.isValid()) { + metricSize += ByteBuffer.wrap(iterator.value()).getInt(); + iterator.next(); + } + } + return metricSize; + } + + private void syncMetric(int key, int update, WriteBatch writeBatch) { + try { + writeBatch.put(metricColumnFamilyHandle, ByteBuffer.allocate(4).putInt(key).array(), ByteBuffer.allocate(4).putInt(update).array()); + } catch (RocksDBException e) { + throw new RuntimeException("Sync metric error", e); + } + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index c0eb06822fd..fead0e77328 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -16,16 +16,20 @@ */ package org.apache.rocketmq.store.timer.rocksdb; +import com.conversantmedia.util.concurrent.DisruptorBlockingQueue; +import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.stats.BrokerStatsManager; -import org.apache.rocketmq.store.timer.TimerCheckpoint; import org.apache.rocketmq.store.timer.TimerMetrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.nio.file.Paths; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; public class TimerMessageRocksDBStore { private final static Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); @@ -33,6 +37,8 @@ public class TimerMessageRocksDBStore { public static final int TIMER_WHEEL_TTL_DAY = 30; public static final int DAY_SECS = 24 * 3600; + public static final int DEFAULT_CAPACITY = 1024; + public static final int INITIAL = 0, RUNNING = 1, HAULT = 2, SHUTDOWN = 3; private final TimerMessageKVStore timerMessageKVStore; private final MessageStore messageStore; @@ -43,6 +49,15 @@ public class TimerMessageRocksDBStore { private final int slotSize; private final int readCount; private final int precisionMs; + private volatile int state = INITIAL; + + private TimerEnqueueGetService timerEnqueueGetService; + private TimerEnqueuePutService timerEnqueuePutService; + private TimerDequeueGetService timerDequeueGetService; + private TimerDequeuePutService[] timerDequeuePutServices; + + private BlockingQueue enqueuePutQueue; + private BlockingQueue dequeuePutQueue; private long commitOffset; @@ -60,13 +75,132 @@ public TimerMessageRocksDBStore(final MessageStore messageStore, final MessageSt storeConfig.getStorePathRootDir(), ROCKSDB_DIRECTORY).toString()); } - public void load() { + public boolean load() { + initService(); boolean result = timerMessageKVStore.start(); result &= this.timerMetrics.load(); calcTimerDistribution(); + return result; + } + + public void start() { + if (state == RUNNING) { + return; + } + this.timerEnqueueGetService.start(); + this.timerEnqueuePutService.start(); + this.timerDequeueGetService.start(); + for (TimerDequeuePutService timerDequeuePutService : timerDequeuePutServices) { + timerDequeuePutService.start(); + } + state = RUNNING; + } + + public void shutdown() { + if (state == SHUTDOWN) { + return; + } + state = SHUTDOWN; + this.timerEnqueueGetService.shutdown(); + this.timerEnqueuePutService.shutdown(); + this.timerDequeueGetService.shutdown(); + for (TimerDequeuePutService timerDequeuePutService : timerDequeuePutServices) { + timerDequeuePutService.shutdown(); + } + + this.enqueuePutQueue.clear(); + this.dequeuePutQueue.clear(); } + // ---------------------------------------------------------------------------------------------------------------- + + private void initService() { + this.timerEnqueueGetService = new TimerEnqueueGetService(); + this.timerEnqueuePutService = new TimerEnqueuePutService(); + this.timerDequeueGetService = new TimerDequeueGetService(); + int getThreadNum = Math.max(storeConfig.getTimerGetMessageThreadNum(), 1); + this.timerDequeuePutServices = new TimerDequeuePutService[getThreadNum]; + for (int i = 0; i < timerDequeuePutServices.length; i++) { + timerDequeuePutServices[i] = new TimerDequeuePutService(); + } + + if (storeConfig.isTimerEnableDisruptor()) { + this.enqueuePutQueue = new DisruptorBlockingQueue<>(DEFAULT_CAPACITY); + this.dequeuePutQueue = new DisruptorBlockingQueue<>(DEFAULT_CAPACITY); + } else { + this.enqueuePutQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY); + this.dequeuePutQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY); + } + this.commitOffset = timerMessageKVStore.getCommitOffset(); + } private void calcTimerDistribution() { + long startTime = System.currentTimeMillis(); + int slotNumber = precisionMs / 100; + int rocksdbNumber = 0; + for (int i = 0; i < slotNumber; i++) { + timerMetrics.resetDistPair(i, timerMessageKVStore.getMetricSize(rocksdbNumber, rocksdbNumber + slotNumber - 1)); + rocksdbNumber += slotNumber; + } + long endTime = System.currentTimeMillis(); + log.debug("Total cost Time: {}", endTime - startTime); + } + + private String getServiceThreadName() { + String brokerIdentifier = ""; + if (TimerMessageRocksDBStore.this.messageStore instanceof DefaultMessageStore) { + DefaultMessageStore messageStore = (DefaultMessageStore) TimerMessageRocksDBStore.this.messageStore; + if (messageStore.getBrokerConfig().isInBrokerContainer()) { + brokerIdentifier = messageStore.getBrokerConfig().getIdentifier(); + } + } + return brokerIdentifier; + } + + private class TimerEnqueueGetService extends ServiceThread { + @Override + public String getServiceName() { + return getServiceThreadName() + this.getClass().getSimpleName(); + } + + @Override + public void run() { + + } + } + + private class TimerEnqueuePutService extends ServiceThread { + @Override + public String getServiceName() { + return getServiceThreadName() + this.getClass().getSimpleName(); + } + + @Override + public void run() { + + } + } + + private class TimerDequeueGetService extends ServiceThread { + @Override + public String getServiceName() { + return getServiceThreadName() + this.getClass().getSimpleName(); + } + + @Override + public void run() { + + } + } + + private class TimerDequeuePutService extends ServiceThread { + @Override + public String getServiceName() { + return getServiceThreadName() + this.getClass().getSimpleName(); + } + + @Override + public void run() { + } } } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerRocksDBRequest.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerRocksDBRequest.java new file mode 100644 index 00000000000..915aa710eb0 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerRocksDBRequest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.timer.rocksdb; + +import org.apache.rocketmq.common.message.MessageExt; + +public class TimerRocksDBRequest { + MessageExt msgExt; + boolean needRoll; + + public TimerRocksDBRequest() { + } + + public TimerRocksDBRequest(MessageExt msgExt, boolean needRoll) { + this.msgExt = msgExt; + this.needRoll = needRoll; + } + + public MessageExt getMsgExt() { + return msgExt; + } + + public void setMsgExt(MessageExt msgExt) { + this.msgExt = msgExt; + } + + public boolean isNeedRoll() { + return needRoll; + } + + public void setNeedRoll(boolean needRoll) { + this.needRoll = needRoll; + } + + @Override + public String toString() { + return "TimerRocksDBRequest{" + + "msgExt=" + msgExt + + ", needRoll=" + needRoll + + '}'; + } +} From 0642c1dd6784177de955d31cdcd700e1de30007e Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Tue, 14 Jan 2025 20:04:59 +0800 Subject: [PATCH 06/61] init: Initialization preparations service --- .../timer/rocksdb/TimerMessageKVStore.java | 11 +- .../timer/rocksdb/TimerMessageRecord.java | 32 +- .../rocksdb/TimerMessageRocksDBStorage.java | 30 +- .../rocksdb/TimerMessageRocksDBStore.java | 435 +++++++++++++++++- 4 files changed, 495 insertions(+), 13 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java index df52a7e8419..b140509fab3 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java @@ -86,10 +86,10 @@ public interface TimerMessageKVStore { * @param maxCount the max count of the timer message records to be return. * @return the list of timer message records. */ - List scanExpiredRecords(byte[] columnFamily, long lowerTime, long upperTime, int maxCount); + List scanRecords(byte[] columnFamily, long lowerTime, long upperTime, int maxCount); /** - * Get the commit offset of the timer message kv store. + * Get the commit offset of the timer message kv store from cq. * @return the commit offset of the timer message kv store. */ long getCommitOffset(); @@ -101,4 +101,11 @@ public interface TimerMessageKVStore { * @return sum. */ int getMetricSize(int lowerTime, int upperTime); + + /** + * Get the checkpoint of the timer message kv store. + * @param columnFamily the column family of the timer message kv store. + * @return the checkpoint of the timer message kv store. + */ + int getCheckpoint(byte[] columnFamily); } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java index 5a15b5e790a..63d20827418 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.store.timer.rocksdb; import com.alibaba.fastjson.annotation.JSONField; +import org.apache.rocketmq.common.message.MessageExt; import java.nio.ByteBuffer; @@ -24,7 +25,9 @@ public class TimerMessageRecord { // key: delayTime + offsetPY private long delayTime; private long offsetPY; - + private int flag; + private MessageExt messageExt; + private boolean roll; // value: sizeReal private int sizeReal; @@ -34,10 +37,11 @@ public class TimerMessageRecord { public TimerMessageRecord() { } - public TimerMessageRecord(long delayTime, long offsetPY, int sizeReal) { + public TimerMessageRecord(long delayTime, long offsetPY, int sizeReal, int flag) { this.delayTime = delayTime; this.offsetPY = offsetPY; this.sizeReal = sizeReal; + this.flag = flag; } @JSONField(serialize = false) @@ -89,6 +93,30 @@ public long getOffsetPY() { return offsetPY; } + public int getFlag() { + return flag; + } + + public void setFlag(int flag) { + this.flag = flag; + } + + public boolean isRoll() { + return roll; + } + + public void setRoll(boolean roll) { + this.roll = roll; + } + + public MessageExt getMessageExt() { + return messageExt; + } + + public void setMessageExt(MessageExt messageExt) { + this.messageExt = messageExt; + } + @Override public String toString() { return "TimerMessageRecord{" + diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index c0bd4bab3f8..a1f6c5b56fb 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -45,10 +45,10 @@ public class TimerMessageRocksDBStorage extends AbstractRocksDBStorage implement public final static byte[] TRANSACTION_COLUMN_FAMILY = "transaction".getBytes(StandardCharsets.UTF_8); public final static byte[] POP_COLUMN_FAMILY = "pop".getBytes(StandardCharsets.UTF_8); - private final static byte[] TIMER_OFFSET_KEY = "timer_offset".getBytes(StandardCharsets.UTF_8); + public final static byte[] TIMER_OFFSET_KEY = "timer_offset".getBytes(StandardCharsets.UTF_8); // key : 100 * n (delay time); value : long (msg number) - private final static byte[] METRIC_COLUMN_FAMILY = "metric".getBytes(StandardCharsets.UTF_8); + public final static byte[] METRIC_COLUMN_FAMILY = "metric".getBytes(StandardCharsets.UTF_8); private ColumnFamilyHandle popColumnFamilyHandle; private ColumnFamilyHandle transactionColumnFamilyHandle; @@ -246,11 +246,18 @@ public List scanRecords(byte[] columnFamily, long lowerTime, } } + try (WriteBatch writeBatch = new WriteBatch()) { + // sync checkpoint + writeBatch.put(RocksDB.DEFAULT_COLUMN_FAMILY, ByteBuffer.allocate(8).putLong(upperTime).array()); + this.db.write(deleteOptions, writeBatch); + } catch (RocksDBException e) { + throw new RuntimeException("Delete record error", e); + } return records; } @Override - public List scanExpiredRecords(byte[] columnFamily, long lowerTime, long upperTime, int maxCount) { + public List scanRecords(byte[] columnFamily, long lowerTime, long upperTime, int maxCount) { List records = new ArrayList<>(); ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily); @@ -265,6 +272,13 @@ public List scanExpiredRecords(byte[] columnFamily, long low } } + try (WriteBatch writeBatch = new WriteBatch()) { + // sync checkpoint + writeBatch.put(columnFamily, ByteBuffer.allocate(8).putLong(upperTime).array()); + this.db.write(deleteOptions, writeBatch); + } catch (RocksDBException e) { + throw new RuntimeException("Delete record error", e); + } return records; } @@ -295,6 +309,16 @@ public int getMetricSize(int lowerTime, int upperTime) { return metricSize; } + @Override + public int getCheckpoint(byte[] columnFamily) { + try { + byte[] checkpointBytes = db.get(columnFamily); + return checkpointBytes == null ? 0 : ByteBuffer.wrap(checkpointBytes).getInt(); + } catch (RocksDBException e) { + throw new RuntimeException("Get checkpoint error", e); + } + } + private void syncMetric(int key, int update, WriteBatch writeBatch) { try { writeBatch.put(metricColumnFamilyHandle, ByteBuffer.allocate(4).putInt(key).array(), ByteBuffer.allocate(4).putInt(update).array()); diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index fead0e77328..cc5c7e976af 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -17,18 +17,38 @@ package org.apache.rocketmq.store.timer.rocksdb; import com.conversantmedia.util.concurrent.DisruptorBlockingQueue; +import io.opentelemetry.api.common.Attributes; import org.apache.rocketmq.common.ServiceThread; +import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant; +import org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager; +import org.apache.rocketmq.store.queue.ConsumeQueueInterface; +import org.apache.rocketmq.store.queue.CqUnit; +import org.apache.rocketmq.store.queue.ReferredIterator; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.timer.TimerMetrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.ByteBuffer; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; import java.util.concurrent.LinkedBlockingDeque; public class TimerMessageRocksDBStore { @@ -39,6 +59,11 @@ public class TimerMessageRocksDBStore { public static final int DAY_SECS = 24 * 3600; public static final int DEFAULT_CAPACITY = 1024; public static final int INITIAL = 0, RUNNING = 1, HAULT = 2, SHUTDOWN = 3; + public static final String TIMER_TOPIC = TopicValidator.SYSTEM_TOPIC_PREFIX + "wheel_timer"; + public static final String TIMER_OUT_MS = MessageConst.PROPERTY_TIMER_OUT_MS; + public static final String TIMER_ROLL_TIMES = MessageConst.PROPERTY_TIMER_ROLL_TIMES; + public static final String TIMER_DEQUEUE_MS = MessageConst.PROPERTY_TIMER_DEQUEUE_MS; + public static final int PUT_OK = 0, PUT_NEED_RETRY = 1, PUT_NO_RETRY = 2; private final TimerMessageKVStore timerMessageKVStore; private final MessageStore messageStore; @@ -54,10 +79,13 @@ public class TimerMessageRocksDBStore { private TimerEnqueueGetService timerEnqueueGetService; private TimerEnqueuePutService timerEnqueuePutService; private TimerDequeueGetService timerDequeueGetService; + private List timerGetMessageServices; + private List timerWarmServices; private TimerDequeuePutService[] timerDequeuePutServices; private BlockingQueue enqueuePutQueue; - private BlockingQueue dequeuePutQueue; + private BlockingQueue> dequeueGetQueue; + private BlockingQueue> dequeuePutQueue; private long commitOffset; @@ -90,6 +118,13 @@ public void start() { this.timerEnqueueGetService.start(); this.timerEnqueuePutService.start(); this.timerDequeueGetService.start(); + + for (TimerWarmService timerWarmService : timerWarmServices) { + timerWarmService.start(); + } + for (TimerGetMessageService timerGetMessageService : timerGetMessageServices) { + timerGetMessageService.start(); + } for (TimerDequeuePutService timerDequeuePutService : timerDequeuePutServices) { timerDequeuePutService.start(); } @@ -104,16 +139,28 @@ public void shutdown() { this.timerEnqueueGetService.shutdown(); this.timerEnqueuePutService.shutdown(); this.timerDequeueGetService.shutdown(); + + for (TimerWarmService timerWarmService : timerWarmServices) { + timerWarmService.shutdown(); + } + for (TimerGetMessageService timerGetMessageService : timerGetMessageServices) { + timerGetMessageService.shutdown(); + } for (TimerDequeuePutService timerDequeuePutService : timerDequeuePutServices) { timerDequeuePutService.shutdown(); } + this.dequeueGetQueue.clear(); this.enqueuePutQueue.clear(); this.dequeuePutQueue.clear(); } - // ---------------------------------------------------------------------------------------------------------------- + public void createTimer(byte[] columnFamily) { + this.timerGetMessageServices.add(new TimerGetMessageService(columnFamily)); + this.timerWarmServices.add(new TimerWarmService(columnFamily)); + } + // ---------------------------------------------------------------------------------------------------------------- private void initService() { this.timerEnqueueGetService = new TimerEnqueueGetService(); this.timerEnqueuePutService = new TimerEnqueuePutService(); @@ -126,18 +173,21 @@ private void initService() { if (storeConfig.isTimerEnableDisruptor()) { this.enqueuePutQueue = new DisruptorBlockingQueue<>(DEFAULT_CAPACITY); - this.dequeuePutQueue = new DisruptorBlockingQueue<>(DEFAULT_CAPACITY); + this.dequeuePutQueue = new DisruptorBlockingQueue>(DEFAULT_CAPACITY); + this.dequeueGetQueue = new DisruptorBlockingQueue<>(DEFAULT_CAPACITY); } else { this.enqueuePutQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY); - this.dequeuePutQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY); + this.dequeuePutQueue = new LinkedBlockingDeque>(DEFAULT_CAPACITY); + this.dequeueGetQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY); } this.commitOffset = timerMessageKVStore.getCommitOffset(); } + private void calcTimerDistribution() { long startTime = System.currentTimeMillis(); int slotNumber = precisionMs / 100; int rocksdbNumber = 0; - for (int i = 0; i < slotNumber; i++) { + for (int i = 0; i < this.slotSize; i++) { timerMetrics.resetDistPair(i, timerMessageKVStore.getMetricSize(rocksdbNumber, rocksdbNumber + slotNumber - 1)); rocksdbNumber += slotNumber; } @@ -156,6 +206,11 @@ private String getServiceThreadName() { return brokerIdentifier; } + private byte[] getColumnFamily(int flag) { + // TODO + return null; + } + private class TimerEnqueueGetService extends ServiceThread { @Override public String getServiceName() { @@ -164,7 +219,17 @@ public String getServiceName() { @Override public void run() { - + log.info(this.getServiceName() + " service start"); + while (!this.isStopped()) { + try { + if (!enqueue(0)) { + waitForRunning(100L * precisionMs / 1000); + } + } catch (Throwable e) { + log.error("Error occurred in " + getServiceName(), e); + } + } + log.info(this.getServiceName() + " service end"); } } @@ -176,7 +241,52 @@ public String getServiceName() { @Override public void run() { + log.info(this.getServiceName() + " service start"); + while (!this.isStopped() || !enqueuePutQueue.isEmpty()) { + try { + fetchAndPutTimerRequest(); + } catch (Throwable e) { + log.error("Unknown error", e); + } + } + log.info(this.getServiceName() + " service end"); + } + + private List fetchTimerMessageRecord() throws InterruptedException { + List trs = null; + TimerMessageRecord firstReq = enqueuePutQueue.poll(10, TimeUnit.MILLISECONDS); + if (null != firstReq) { + trs = new ArrayList<>(16); + trs.add(firstReq); + while (true) { + TimerMessageRecord tmpReq = enqueuePutQueue.poll(3, TimeUnit.MILLISECONDS); + if (null == tmpReq) { + break; + } + trs.add(tmpReq); + if (trs.size() > 100) { + break; + } + } + } + return trs; + } + private void fetchAndPutTimerRequest() throws Exception { + Map>> map = new HashMap<>(); + List trs = fetchTimerMessageRecord(); + + for (TimerMessageRecord tr : trs) { + long delayTime = tr.getDelayTime(); + map.computeIfAbsent(delayTime, k -> new HashMap<>()).computeIfAbsent(tr.getFlag(), k -> new ArrayList<>()).add(tr); + } + for (Map.Entry>> entry : map.entrySet()) { + long delayTime = entry.getKey(); + for (Map.Entry> entry1 : entry.getValue().entrySet()) { + int tag = entry1.getKey(); + timerMessageKVStore.writeAssignRecords(getColumnFamily(tag), entry1.getValue(), commitOffset, (int) (delayTime / precisionMs % slotSize)); + } + } } } @@ -188,7 +298,28 @@ public String getServiceName() { @Override public void run() { + log.info(this.getServiceName() + " service start"); + while (!this.isStopped() && !dequeueGetQueue.isEmpty()) { + try { + List timerMessageRecord = dequeueGetQueue.poll(100L * precisionMs / 1000, TimeUnit.MILLISECONDS); + if (null == timerMessageRecord || timerMessageRecord.isEmpty()) { + continue; + } + for (TimerMessageRecord record : timerMessageRecord) { + MessageExt messageExt = getMessageByCommitOffset(record.getOffsetPY(), record.getSizeReal()); + long delayedTime = Long.parseLong(messageExt.getProperty(TIMER_OUT_MS)); + if (delayedTime >= System.currentTimeMillis() + precisionMs * 3L) { + record.setRoll(true); + } else { + record.setRoll(false); + } + } + while (!dequeuePutQueue.offer(timerMessageRecord, 3, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + log.error("Error occurred in " + getServiceName(), e); + } + } } } @@ -200,7 +331,299 @@ public String getServiceName() { @Override public void run() { + log.info(this.getServiceName() + " service start"); + while (!this.isStopped() && !dequeuePutQueue.isEmpty()) { + try { + List timerMessageRecord = dequeuePutQueue.poll(100L * precisionMs / 1000, TimeUnit.MILLISECONDS); + if (null == timerMessageRecord || timerMessageRecord.isEmpty()) { + continue; + } + + for (TimerMessageRecord record : timerMessageRecord) { + MessageExt msg = record.getMessageExt(); + MessageExtBrokerInner messageExtBrokerInner = convert(msg, record.isRoll()); + boolean processed = false; + int retryCount = 0; + + while (!processed && !isStopped()) { + int result = doPut(messageExtBrokerInner, record.isRoll()); + + if (result == PUT_OK) { + processed = true; + } else if (result == PUT_NO_RETRY) { + log.warn("Skipping message due to unrecoverable error. Msg: {}", msg); + processed = true; + } else { + retryCount++; + // Without enabling TimerEnableRetryUntilSuccess, messages will retry up to 3 times before being discarded + if (!storeConfig.isTimerEnableRetryUntilSuccess() && retryCount >= 3) { + log.error("Message processing failed after {} retries. Msg: {}", retryCount, msg); + processed = true; + } else { + Thread.sleep(500L * precisionMs / 1000); + log.warn("Retrying to process message. Retry count: {}, Msg: {}", retryCount, msg); + } + } + } + } + + } catch (InterruptedException e) { + log.error("Error occurred in " + getServiceName(), e); + } + } + } + } + + private class TimerGetMessageService extends ServiceThread { + private final byte[] columnFamily; + private int checkpoint; + + public TimerGetMessageService(byte[] columnFamily) { + this.columnFamily = columnFamily; + this.checkpoint = timerMessageKVStore.getCheckpoint(columnFamily); + } + + @Override + public String getServiceName() { + return getServiceThreadName() + this.getClass().getSimpleName(); + } + + @Override + public void run() { + log.info(this.getServiceName() + " service start"); + while (!this.isStopped()) { + try { + if (-1 == dequeue(checkpoint, columnFamily)) { + waitForRunning(100L * precisionMs / 1000); + } + } catch (Throwable e) { + log.error("Error occurred in " + getServiceName(), e); + } + } + log.info(this.getServiceName() + " service end"); + } + } + private class TimerWarmService extends ServiceThread { + private final byte[] columnFamily; + private int checkpoint; + + public TimerWarmService(byte[] columnFamily) { + this.columnFamily = columnFamily; + this.checkpoint = timerMessageKVStore.getCheckpoint(columnFamily); + } + + @Override + public String getServiceName() { + return getServiceThreadName() + this.getClass().getSimpleName(); + } + + @Override + public void run() { + log.info(this.getServiceName() + " service start"); + while (!this.isStopped()) { + try { + int checkpoint = warm(this.checkpoint, columnFamily); + if (-1 == checkpoint) { + waitForRunning(100L * precisionMs / 1000); + } else { + this.checkpoint = checkpoint; + } + } catch (Throwable e) { + log.error("Error occurred in " + getServiceName(), e); + } + } + log.info(this.getServiceName() + " service end"); + } + } + // ----------------------------------------------------------------------------------------------------------------- + public boolean enqueue(int queueId) { + ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); + if (null == cq) { + return false; + } + if (commitOffset < cq.getMinOffsetInQueue()) { + log.warn("Timer currQueueOffset:{} is smaller than minOffsetInQueue:{}", + commitOffset, cq.getMinOffsetInQueue()); + commitOffset = cq.getMinOffsetInQueue(); + } + long offset = commitOffset; + ReferredIterator iterator = null; + try { + iterator = cq.iterateFrom(offset); + if (null == iterator) { + return false; + } + + int i = 0; + while (iterator.hasNext()) { + i++; + try { + CqUnit cqUnit = iterator.next(); + long offsetPy = cqUnit.getPos(); + int sizePy = cqUnit.getSize(); + MessageExt msgExt = getMessageByCommitOffset(offsetPy, sizePy); + + if (null != msgExt) { + long delayedTime = Long.parseLong(msgExt.getProperty(TIMER_OUT_MS)); + int flag = msgExt.getFlag(); + TimerMessageRecord timerRequest = new TimerMessageRecord(delayedTime, offsetPy, sizePy, flag); + while(!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) {} + Attributes attributes = DefaultStoreMetricsManager.newAttributesBuilder() + .put(DefaultStoreMetricsConstant.LABEL_TOPIC, msgExt.getProperty(MessageConst.PROPERTY_REAL_TOPIC)).build(); + DefaultStoreMetricsManager.timerMessageSetLatency.record((delayedTime - msgExt.getBornTimestamp()) / 1000, attributes); + } + } catch (Exception e) { + // here may cause the message loss + log.warn("Unknown error in skipped in enqueuing", e); + throw e; + } + commitOffset = offset + i; + } + commitOffset = offset + i; + return i > 0; + } catch (Exception e) { + log.error("Unknown exception in enqueuing", e); + } finally { + if (iterator != null) { + iterator.release(); + } + } + return false; + } + + private MessageExt getMessageByCommitOffset(long offsetPy, int sizePy) { + ThreadLocal bufferLocal = ThreadLocal.withInitial(() -> ByteBuffer.allocate(sizePy)); + for (int i = 0; i < 3; i++) { + MessageExt msgExt = null; + bufferLocal.get().position(0); + bufferLocal.get().limit(sizePy); + boolean res = messageStore.getData(offsetPy, sizePy, bufferLocal.get()); + if (res) { + bufferLocal.get().flip(); + msgExt = MessageDecoder.decode(bufferLocal.get(), true, false, false); + } + if (null == msgExt) { + log.warn("Fail to read msg from commitLog offsetPy:{} sizePy:{}", offsetPy, sizePy); + } else { + return msgExt; + } + } + return null; + } + + private int dequeue(int checkpoint, byte[] columnFamily) throws InterruptedException { + if (checkpoint < System.currentTimeMillis() / precisionMs % slotSize) { + return -1; + } + + List timerMessageRecords = timerMessageKVStore.scanRecords(columnFamily, checkpoint, checkpoint + 1); + while (!dequeueGetQueue.offer(timerMessageRecords, 3, TimeUnit.SECONDS)); + timerMessageKVStore.deleteAssignRecords(columnFamily, timerMessageRecords, checkpoint); + return 0; + } + + private int warm(int checkpoint, byte[] columnFamily) { + if (!storeConfig.isTimerWarmEnable()) { + return -1; + } + if (checkpoint < System.currentTimeMillis() / precisionMs % slotSize + 1) { + checkpoint = (int) (System.currentTimeMillis() / precisionMs % slotSize + 1); + } + if (checkpoint >= System.currentTimeMillis() / precisionMs % slotSize + 3) { + return -1; + } + + timerMessageKVStore.scanRecords(columnFamily, checkpoint, checkpoint + 1); + return checkpoint + 1; + } + + private MessageExtBrokerInner convert(MessageExt messageExt, boolean needRoll) { + if (needRoll) { + if (messageExt.getProperty(TIMER_ROLL_TIMES) != null) { + MessageAccessor.putProperty(messageExt, TIMER_ROLL_TIMES, Integer.parseInt(messageExt.getProperty(TIMER_ROLL_TIMES)) + 1 + ""); + } else { + MessageAccessor.putProperty(messageExt, TIMER_ROLL_TIMES, 1 + ""); + } + } + MessageAccessor.putProperty(messageExt, TIMER_DEQUEUE_MS, System.currentTimeMillis() + ""); + return convertMessage(messageExt, needRoll); + } + + private MessageExtBrokerInner convertMessage(MessageExt msgExt, boolean needRoll) { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setBody(msgExt.getBody()); + msgInner.setFlag(msgExt.getFlag()); + MessageAccessor.setProperties(msgInner, MessageAccessor.deepCopyProperties(msgExt.getProperties())); + TopicFilterType topicFilterType = MessageExt.parseTopicFilterType(msgInner.getSysFlag()); + long tagsCodeValue = + MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags()); + msgInner.setTagsCode(tagsCodeValue); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); + + msgInner.setSysFlag(msgExt.getSysFlag()); + msgInner.setBornTimestamp(msgExt.getBornTimestamp()); + msgInner.setBornHost(msgExt.getBornHost()); + msgInner.setStoreHost(msgExt.getStoreHost()); + msgInner.setReconsumeTimes(msgExt.getReconsumeTimes()); + + msgInner.setWaitStoreMsgOK(false); + + if (needRoll) { + msgInner.setTopic(msgExt.getTopic()); + msgInner.setQueueId(msgExt.getQueueId()); + } else { + msgInner.setTopic(msgInner.getProperty(MessageConst.PROPERTY_REAL_TOPIC)); + msgInner.setQueueId(Integer.parseInt(msgInner.getProperty(MessageConst.PROPERTY_REAL_QUEUE_ID))); + MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_TOPIC); + MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_QUEUE_ID); + } + return msgInner; + } + + private int doPut(MessageExtBrokerInner message, boolean roll) { + if (!roll && null != message.getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY)) { + log.warn("Trying do put delete timer msg:[{}] roll:[{}]", message, roll); + return PUT_NO_RETRY; + } + + PutMessageResult putMessageResult = messageStore.putMessage(message); + if (putMessageResult != null && putMessageResult.getPutMessageStatus() != null) { + switch (putMessageResult.getPutMessageStatus()) { + case PUT_OK: + if (brokerStatsManager != null) { + brokerStatsManager.incTopicPutNums(message.getTopic(), 1, 1); + if (putMessageResult.getAppendMessageResult() != null) { + brokerStatsManager.incTopicPutSize(message.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes()); + } + brokerStatsManager.incBrokerPutNums(message.getTopic(), 1); + } + return PUT_OK; + + case MESSAGE_ILLEGAL: + case PROPERTIES_SIZE_EXCEEDED: + case WHEEL_TIMER_NOT_ENABLE: + case WHEEL_TIMER_MSG_ILLEGAL: + return PUT_NO_RETRY; + + case SERVICE_NOT_AVAILABLE: + case FLUSH_DISK_TIMEOUT: + case FLUSH_SLAVE_TIMEOUT: + case OS_PAGE_CACHE_BUSY: + case CREATE_MAPPED_FILE_FAILED: + case SLAVE_NOT_AVAILABLE: + return PUT_NEED_RETRY; + + case UNKNOWN_ERROR: + default: + if (storeConfig.isTimerSkipUnknownError()) { + log.warn("Skipping message due to unknown error, msg: {}", message); + return PUT_NO_RETRY; + } else { + return PUT_NEED_RETRY; + } + } } + return PUT_NEED_RETRY; } } From 2fdbd7ef866f318406dfe0363242027211b2b225 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Tue, 14 Jan 2025 22:56:51 +0800 Subject: [PATCH 07/61] init: prepare for metric --- .../rocksdb/TimerMessageRocksDBStore.java | 48 +++++++++++++------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index cc5c7e976af..7300a902476 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -18,6 +18,7 @@ import com.conversantmedia.util.concurrent.DisruptorBlockingQueue; import io.opentelemetry.api.common.Attributes; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.common.constant.LoggerName; @@ -59,11 +60,12 @@ public class TimerMessageRocksDBStore { public static final int DAY_SECS = 24 * 3600; public static final int DEFAULT_CAPACITY = 1024; public static final int INITIAL = 0, RUNNING = 1, HAULT = 2, SHUTDOWN = 3; + public static final int PUT_OK = 0, PUT_NEED_RETRY = 1, PUT_NO_RETRY = 2; public static final String TIMER_TOPIC = TopicValidator.SYSTEM_TOPIC_PREFIX + "wheel_timer"; public static final String TIMER_OUT_MS = MessageConst.PROPERTY_TIMER_OUT_MS; public static final String TIMER_ROLL_TIMES = MessageConst.PROPERTY_TIMER_ROLL_TIMES; public static final String TIMER_DEQUEUE_MS = MessageConst.PROPERTY_TIMER_DEQUEUE_MS; - public static final int PUT_OK = 0, PUT_NEED_RETRY = 1, PUT_NO_RETRY = 2; + public static final String TIMER_ENQUEUE_MS = MessageConst.PROPERTY_TIMER_ENQUEUE_MS; private final TimerMessageKVStore timerMessageKVStore; private final MessageStore messageStore; @@ -159,7 +161,6 @@ public void createTimer(byte[] columnFamily) { this.timerGetMessageServices.add(new TimerGetMessageService(columnFamily)); this.timerWarmServices.add(new TimerWarmService(columnFamily)); } - // ---------------------------------------------------------------------------------------------------------------- private void initService() { this.timerEnqueueGetService = new TimerEnqueueGetService(); @@ -173,26 +174,23 @@ private void initService() { if (storeConfig.isTimerEnableDisruptor()) { this.enqueuePutQueue = new DisruptorBlockingQueue<>(DEFAULT_CAPACITY); - this.dequeuePutQueue = new DisruptorBlockingQueue>(DEFAULT_CAPACITY); + this.dequeuePutQueue = new DisruptorBlockingQueue<>(DEFAULT_CAPACITY); this.dequeueGetQueue = new DisruptorBlockingQueue<>(DEFAULT_CAPACITY); } else { this.enqueuePutQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY); - this.dequeuePutQueue = new LinkedBlockingDeque>(DEFAULT_CAPACITY); + this.dequeuePutQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY); this.dequeueGetQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY); } this.commitOffset = timerMessageKVStore.getCommitOffset(); } private void calcTimerDistribution() { - long startTime = System.currentTimeMillis(); int slotNumber = precisionMs / 100; int rocksdbNumber = 0; for (int i = 0; i < this.slotSize; i++) { timerMetrics.resetDistPair(i, timerMessageKVStore.getMetricSize(rocksdbNumber, rocksdbNumber + slotNumber - 1)); rocksdbNumber += slotNumber; } - long endTime = System.currentTimeMillis(); - log.debug("Total cost Time: {}", endTime - startTime); } private String getServiceThreadName() { @@ -285,7 +283,11 @@ private void fetchAndPutTimerRequest() throws Exception { for (Map.Entry> entry1 : entry.getValue().entrySet()) { int tag = entry1.getKey(); timerMessageKVStore.writeAssignRecords(getColumnFamily(tag), entry1.getValue(), commitOffset, (int) (delayTime / precisionMs % slotSize)); + for (TimerMessageRecord record : entry1.getValue()) { + addMetric(record.getMessageExt(), 1); + } } + addMetric((int) (delayTime / precisionMs % slotSize), entry.getValue().size()); } } } @@ -309,13 +311,10 @@ public void run() { for (TimerMessageRecord record : timerMessageRecord) { MessageExt messageExt = getMessageByCommitOffset(record.getOffsetPY(), record.getSizeReal()); long delayedTime = Long.parseLong(messageExt.getProperty(TIMER_OUT_MS)); - if (delayedTime >= System.currentTimeMillis() + precisionMs * 3L) { - record.setRoll(true); - } else { - record.setRoll(false); - } + record.setRoll(delayedTime >= System.currentTimeMillis() + precisionMs * 3L); + addMetric(messageExt, -1); } - while (!dequeuePutQueue.offer(timerMessageRecord, 3, TimeUnit.SECONDS)); + while (!dequeuePutQueue.offer(timerMessageRecord, 3, TimeUnit.SECONDS)) {} } catch (InterruptedException e) { log.error("Error occurred in " + getServiceName(), e); } @@ -395,6 +394,8 @@ public void run() { try { if (-1 == dequeue(checkpoint, columnFamily)) { waitForRunning(100L * precisionMs / 1000); + } else { + checkpoint++; } } catch (Throwable e) { log.error("Error occurred in " + getServiceName(), e); @@ -465,10 +466,13 @@ public boolean enqueue(int queueId) { MessageExt msgExt = getMessageByCommitOffset(offsetPy, sizePy); if (null != msgExt) { + // TODO delete msg : use unique key long delayedTime = Long.parseLong(msgExt.getProperty(TIMER_OUT_MS)); int flag = msgExt.getFlag(); TimerMessageRecord timerRequest = new TimerMessageRecord(delayedTime, offsetPy, sizePy, flag); while(!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) {} + timerRequest.setMessageExt(msgExt); + Attributes attributes = DefaultStoreMetricsManager.newAttributesBuilder() .put(DefaultStoreMetricsConstant.LABEL_TOPIC, msgExt.getProperty(MessageConst.PROPERTY_REAL_TOPIC)).build(); DefaultStoreMetricsManager.timerMessageSetLatency.record((delayedTime - msgExt.getBornTimestamp()) / 1000, attributes); @@ -518,8 +522,9 @@ private int dequeue(int checkpoint, byte[] columnFamily) throws InterruptedExcep } List timerMessageRecords = timerMessageKVStore.scanRecords(columnFamily, checkpoint, checkpoint + 1); - while (!dequeueGetQueue.offer(timerMessageRecords, 3, TimeUnit.SECONDS)); + while (!dequeueGetQueue.offer(timerMessageRecords, 3, TimeUnit.SECONDS)) {} timerMessageKVStore.deleteAssignRecords(columnFamily, timerMessageRecords, checkpoint); + addMetric(checkpoint, timerMessageRecords.size()); return 0; } @@ -626,4 +631,19 @@ private int doPut(MessageExtBrokerInner message, boolean roll) { } return PUT_NEED_RETRY; } + + private void addMetric(MessageExt msg, int value) { + if (null == msg || null == msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC)) { + return; + } + if (msg.getProperty(TIMER_ENQUEUE_MS) != null + && NumberUtils.toLong(msg.getProperty(TIMER_ENQUEUE_MS)) == Long.MAX_VALUE) { + return; + } + timerMetrics.addAndGet(msg, value); + } + + private void addMetric(int delayTime, int value) { + timerMetrics.updateDistPair(delayTime, value); + } } From efd30718b7928bae054f5349d9bb3990c857d5d5 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 15 Jan 2025 12:16:51 +0800 Subject: [PATCH 08/61] init: Initialization preparations service --- .../rocketmq/common/message/MessageConst.java | 2 + .../timer/rocksdb/TimerMessageRecord.java | 29 +++++++++---- .../rocksdb/TimerMessageRocksDBStorage.java | 2 + .../rocksdb/TimerMessageRocksDBStore.java | 42 +++++++++++++++---- 4 files changed, 57 insertions(+), 18 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java index 24f7bdb99a5..e9899a90397 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java @@ -95,6 +95,7 @@ public class MessageConst { public static final String PROPERTY_TIMER_ROLL_TIMES = "TIMER_ROLL_TIMES"; public static final String PROPERTY_TIMER_OUT_MS = "TIMER_OUT_MS"; public static final String PROPERTY_TIMER_DEL_UNIQKEY = "TIMER_DEL_UNIQKEY"; + public static final String PROPERTY_TIMER_DEL_MS = "TIMER_DEL_MS"; public static final String PROPERTY_TIMER_DELAY_LEVEL = "TIMER_DELAY_LEVEL"; public static final String PROPERTY_TIMER_DELAY_MS = "TIMER_DELAY_MS"; public static final String PROPERTY_CRC32 = "__CRC32#"; @@ -151,6 +152,7 @@ public class MessageConst { STRING_HASH_SET.add(PROPERTY_TIMER_ROLL_TIMES); STRING_HASH_SET.add(PROPERTY_TIMER_OUT_MS); STRING_HASH_SET.add(PROPERTY_TIMER_DEL_UNIQKEY); + STRING_HASH_SET.add(PROPERTY_TIMER_DEL_MS); STRING_HASH_SET.add(PROPERTY_TIMER_DELAY_LEVEL); STRING_HASH_SET.add(PROPERTY_BORN_HOST); STRING_HASH_SET.add(PROPERTY_BORN_TIMESTAMP); diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java index 63d20827418..90e36443eb8 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java @@ -20,25 +20,26 @@ import org.apache.rocketmq.common.message.MessageExt; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; public class TimerMessageRecord { - // key: delayTime + offsetPY + // key: delayTime + uniqueKey private long delayTime; - private long offsetPY; + private String uniqueKey; private int flag; private MessageExt messageExt; private boolean roll; - // value: sizeReal + // value: sizeReal + offsetPY private int sizeReal; - - private final static int keyLength = Long.BYTES + Long.BYTES; - private final static int valueLength = Integer.BYTES; + private long offsetPY; + private final static int valueLength = Integer.BYTES + Long.BYTES; public TimerMessageRecord() { } - public TimerMessageRecord(long delayTime, long offsetPY, int sizeReal, int flag) { + public TimerMessageRecord(long delayTime, String uniqueKey, long offsetPY, int sizeReal, int flag) { this.delayTime = delayTime; + this.uniqueKey = uniqueKey; this.offsetPY = offsetPY; this.sizeReal = sizeReal; this.flag = flag; @@ -46,9 +47,11 @@ public TimerMessageRecord(long delayTime, long offsetPY, int sizeReal, int flag) @JSONField(serialize = false) public byte[] getKeyBytes() { + int keyLength = Long.BYTES + uniqueKey.length(); byte[] keyBytes = new byte[keyLength]; ByteBuffer buffer = ByteBuffer.wrap(keyBytes); buffer.putLong(this.getDelayTime()).putLong(this.getOffsetPY()); + buffer.put(uniqueKey.getBytes(StandardCharsets.UTF_8)); return keyBytes; } @@ -57,15 +60,15 @@ public byte[] getValueBytes() { byte[] valueBytes = new byte[valueLength]; ByteBuffer buffer = ByteBuffer.wrap(valueBytes); buffer.putInt(this.getSizeReal()); + buffer.putLong(this.getOffsetPY()); return valueBytes; } public static TimerMessageRecord decode(byte[] body) { TimerMessageRecord timerMessageRecord = new TimerMessageRecord(); ByteBuffer buffer = ByteBuffer.wrap(body); - timerMessageRecord.setDelayTime(buffer.getLong()); - timerMessageRecord.setOffsetPY(buffer.getLong()); timerMessageRecord.setSizeReal(buffer.getInt()); + timerMessageRecord.setOffsetPY(buffer.getLong()); return timerMessageRecord; } @@ -117,6 +120,14 @@ public void setMessageExt(MessageExt messageExt) { this.messageExt = messageExt; } + public String getUniqueKey() { + return uniqueKey; + } + + public void setUniqueKey(String uniqueKey) { + this.uniqueKey = uniqueKey; + } + @Override public String toString() { return "TimerMessageRecord{" + diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index a1f6c5b56fb..ee0fe1c73ad 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -249,6 +249,7 @@ public List scanRecords(byte[] columnFamily, long lowerTime, try (WriteBatch writeBatch = new WriteBatch()) { // sync checkpoint writeBatch.put(RocksDB.DEFAULT_COLUMN_FAMILY, ByteBuffer.allocate(8).putLong(upperTime).array()); + deleteAssignRecords(columnFamily, records, (int) lowerTime); this.db.write(deleteOptions, writeBatch); } catch (RocksDBException e) { throw new RuntimeException("Delete record error", e); @@ -275,6 +276,7 @@ public List scanRecords(byte[] columnFamily, long lowerTime, try (WriteBatch writeBatch = new WriteBatch()) { // sync checkpoint writeBatch.put(columnFamily, ByteBuffer.allocate(8).putLong(upperTime).array()); + deleteAssignRecords(columnFamily, records, (int) lowerTime); this.db.write(deleteOptions, writeBatch); } catch (RocksDBException e) { throw new RuntimeException("Delete record error", e); diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 7300a902476..908d0ca4590 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -27,6 +27,7 @@ import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageClientIDSetter; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.MessageStore; @@ -271,14 +272,24 @@ private List fetchTimerMessageRecord() throws InterruptedExc } private void fetchAndPutTimerRequest() throws Exception { - Map>> map = new HashMap<>(); + Map>> increase = new HashMap<>(); + Map>> delete = new HashMap<>(); List trs = fetchTimerMessageRecord(); for (TimerMessageRecord tr : trs) { long delayTime = tr.getDelayTime(); - map.computeIfAbsent(delayTime, k -> new HashMap<>()).computeIfAbsent(tr.getFlag(), k -> new ArrayList<>()).add(tr); + if (tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY) != null) { + delayTime = Long.parseLong(tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_MS)); + // Construct original message + tr.setUniqueKey(tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY)); + tr.setDelayTime(delayTime); + delete.computeIfAbsent(delayTime, k -> new HashMap<>()).computeIfAbsent(tr.getFlag(), k -> new ArrayList<>()).add(tr); + } else { + increase.computeIfAbsent(delayTime, k -> new HashMap<>()).computeIfAbsent(tr.getFlag(), k -> new ArrayList<>()).add(tr); + } } - for (Map.Entry>> entry : map.entrySet()) { + + for (Map.Entry>> entry : increase.entrySet()) { long delayTime = entry.getKey(); for (Map.Entry> entry1 : entry.getValue().entrySet()) { int tag = entry1.getKey(); @@ -289,6 +300,18 @@ private void fetchAndPutTimerRequest() throws Exception { } addMetric((int) (delayTime / precisionMs % slotSize), entry.getValue().size()); } + + for (Map.Entry>> entry : delete.entrySet()) { + long delayTime = entry.getKey(); + for (Map.Entry> entry1 : entry.getValue().entrySet()) { + int tag = entry1.getKey(); + timerMessageKVStore.deleteAssignRecords(getColumnFamily(tag), entry1.getValue(), (int) (delayTime / precisionMs % slotSize)); + for (TimerMessageRecord record : entry1.getValue()) { + addMetric(record.getMessageExt(), -1); + } + } + addMetric((int) (delayTime / precisionMs % slotSize), -entry.getValue().size()); + } } } @@ -307,13 +330,16 @@ public void run() { if (null == timerMessageRecord || timerMessageRecord.isEmpty()) { continue; } - for (TimerMessageRecord record : timerMessageRecord) { MessageExt messageExt = getMessageByCommitOffset(record.getOffsetPY(), record.getSizeReal()); long delayedTime = Long.parseLong(messageExt.getProperty(TIMER_OUT_MS)); + record.setMessageExt(messageExt); + record.setDelayTime(delayedTime); + record.setUniqueKey(MessageClientIDSetter.getUniqID(messageExt)); record.setRoll(delayedTime >= System.currentTimeMillis() + precisionMs * 3L); addMetric(messageExt, -1); } + while (!dequeuePutQueue.offer(timerMessageRecord, 3, TimeUnit.SECONDS)) {} } catch (InterruptedException e) { log.error("Error occurred in " + getServiceName(), e); @@ -466,13 +492,12 @@ public boolean enqueue(int queueId) { MessageExt msgExt = getMessageByCommitOffset(offsetPy, sizePy); if (null != msgExt) { - // TODO delete msg : use unique key long delayedTime = Long.parseLong(msgExt.getProperty(TIMER_OUT_MS)); int flag = msgExt.getFlag(); - TimerMessageRecord timerRequest = new TimerMessageRecord(delayedTime, offsetPy, sizePy, flag); - while(!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) {} + TimerMessageRecord timerRequest = new TimerMessageRecord(delayedTime, MessageClientIDSetter.getUniqID(msgExt), offsetPy, sizePy, flag); timerRequest.setMessageExt(msgExt); + while(!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) {} Attributes attributes = DefaultStoreMetricsManager.newAttributesBuilder() .put(DefaultStoreMetricsConstant.LABEL_TOPIC, msgExt.getProperty(MessageConst.PROPERTY_REAL_TOPIC)).build(); DefaultStoreMetricsManager.timerMessageSetLatency.record((delayedTime - msgExt.getBornTimestamp()) / 1000, attributes); @@ -523,8 +548,7 @@ private int dequeue(int checkpoint, byte[] columnFamily) throws InterruptedExcep List timerMessageRecords = timerMessageKVStore.scanRecords(columnFamily, checkpoint, checkpoint + 1); while (!dequeueGetQueue.offer(timerMessageRecords, 3, TimeUnit.SECONDS)) {} - timerMessageKVStore.deleteAssignRecords(columnFamily, timerMessageRecords, checkpoint); - addMetric(checkpoint, timerMessageRecords.size()); + addMetric(checkpoint, -timerMessageRecords.size()); return 0; } From 1e41aa46936390aaf73a4c407787c5b526a4a2a9 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 15 Jan 2025 14:19:06 +0800 Subject: [PATCH 09/61] feat: Improve the message flow service --- .../processor/SendMessageProcessor.java | 4 +++ .../rocketmq/client/impl/MQClientAPIImpl.java | 1 + .../rocketmq/client/producer/SendResult.java | 9 ++++++ .../rocketmq/common/message/MessageConst.java | 2 ++ .../service/message/LocalMessageService.java | 1 + .../header/SendMessageResponseHeader.java | 15 ++++++++++ .../timer/rocksdb/TimerMessageRecord.java | 16 ++++------ .../rocksdb/TimerMessageRocksDBStore.java | 30 +++++++++++++++---- 8 files changed, 61 insertions(+), 17 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index 669cd5e6771..a4a9f0ecfad 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -662,6 +662,10 @@ public void attachRecallHandle(RemotingCommand request, MessageExt msg, SendMess brokerController.getBrokerConfig().getBrokerName(), timestampStr, MessageClientIDSetter.getUniqID(msg)); responseHeader.setRecallHandle(recallHandle); } + + if (msg.getProperty(MessageConst.PROPERTY_TIMER_OUT_MS) != null) { + responseHeader.setDelayTime(Long.parseLong(msg.getProperty(MessageConst.PROPERTY_TIMER_OUT_MS))); + } } private String diskUtil() { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 7d4b51cfc5f..64238e84f03 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -856,6 +856,7 @@ protected SendResult processSendResponse( responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset()); sendResult.setTransactionId(responseHeader.getTransactionId()); sendResult.setRecallHandle(responseHeader.getRecallHandle()); + sendResult.setDelayTime(responseHeader.getDelayTime()); String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION); if (regionId == null || regionId.isEmpty()) { regionId = MixAll.DEFAULT_TRACE_REGION_ID; diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java b/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java index d160eb4eae9..9119726e345 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java @@ -30,6 +30,7 @@ public class SendResult { private boolean traceOn = true; private byte[] rawRespBody; private String recallHandle; + private long delayTime; public SendResult() { } @@ -135,6 +136,14 @@ public void setRecallHandle(String recallHandle) { this.recallHandle = recallHandle; } + public long getDelayTime() { + return delayTime; + } + + public void setDelayTime(long delayTime) { + this.delayTime = delayTime; + } + @Override public String toString() { return "SendResult [sendStatus=" + sendStatus + ", msgId=" + msgId + ", offsetMsgId=" + offsetMsgId + ", messageQueue=" + messageQueue diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java index e9899a90397..7d526a31c88 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java @@ -96,6 +96,7 @@ public class MessageConst { public static final String PROPERTY_TIMER_OUT_MS = "TIMER_OUT_MS"; public static final String PROPERTY_TIMER_DEL_UNIQKEY = "TIMER_DEL_UNIQKEY"; public static final String PROPERTY_TIMER_DEL_MS = "TIMER_DEL_MS"; + public static final String PROPERTY_TIMER_DEL_FLAG = "TIMER_DEL_FLAG"; public static final String PROPERTY_TIMER_DELAY_LEVEL = "TIMER_DELAY_LEVEL"; public static final String PROPERTY_TIMER_DELAY_MS = "TIMER_DELAY_MS"; public static final String PROPERTY_CRC32 = "__CRC32#"; @@ -153,6 +154,7 @@ public class MessageConst { STRING_HASH_SET.add(PROPERTY_TIMER_OUT_MS); STRING_HASH_SET.add(PROPERTY_TIMER_DEL_UNIQKEY); STRING_HASH_SET.add(PROPERTY_TIMER_DEL_MS); + STRING_HASH_SET.add(PROPERTY_TIMER_DEL_FLAG); STRING_HASH_SET.add(PROPERTY_TIMER_DELAY_LEVEL); STRING_HASH_SET.add(PROPERTY_BORN_HOST); STRING_HASH_SET.add(PROPERTY_BORN_TIMESTAMP); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java index cb9b7a4ae00..40b60b63646 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java @@ -156,6 +156,7 @@ public CompletableFuture> sendMessage(ProxyContext ctx, Address sendResult.setTransactionId(responseHeader.getTransactionId()); sendResult.setOffsetMsgId(responseHeader.getMsgId()); sendResult.setRecallHandle(responseHeader.getRecallHandle()); + sendResult.setDelayTime(responseHeader.getDelayTime()); return Collections.singletonList(sendResult); }); } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java index 7563b910331..7351172f84a 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java @@ -37,6 +37,7 @@ public class SendMessageResponseHeader implements CommandCustomHeader, FastCodes private String transactionId; private String batchUniqId; private String recallHandle; + private Long delayTime; @Override public void checkFields() throws RemotingCommandException { @@ -50,6 +51,7 @@ public void encode(ByteBuf out) { writeIfNotNull(out, "transactionId", transactionId); writeIfNotNull(out, "batchUniqId", batchUniqId); writeIfNotNull(out, "recallHandle", recallHandle); + writeIfNotNull(out, "delayTime", delayTime); } @Override @@ -83,6 +85,11 @@ public void decode(HashMap fields) throws RemotingCommandExcepti if (str != null) { this.recallHandle = str; } + + str = fields.get("delayTime"); + if (str != null) { + this.delayTime = Long.parseLong(str); + } } public String getMsgId() { @@ -132,4 +139,12 @@ public String getRecallHandle() { public void setRecallHandle(String recallHandle) { this.recallHandle = recallHandle; } + + public Long getDelayTime() { + return delayTime; + } + + public void setDelayTime(Long delayTime) { + this.delayTime = delayTime; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java index 90e36443eb8..e7029ea01f0 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java @@ -23,10 +23,13 @@ import java.nio.charset.StandardCharsets; public class TimerMessageRecord { + public final static int TIMER_MESSAGE_DEFAULT_FLAG = 0; + public final static int TIMER_MESSAGE_TRANSACTION_FLAG = 1; + public final static int TIMER_MESSAGE_POP_FLAG = 2; + // key: delayTime + uniqueKey private long delayTime; private String uniqueKey; - private int flag; private MessageExt messageExt; private boolean roll; // value: sizeReal + offsetPY @@ -37,12 +40,11 @@ public class TimerMessageRecord { public TimerMessageRecord() { } - public TimerMessageRecord(long delayTime, String uniqueKey, long offsetPY, int sizeReal, int flag) { + public TimerMessageRecord(long delayTime, String uniqueKey, long offsetPY, int sizeReal) { this.delayTime = delayTime; this.uniqueKey = uniqueKey; this.offsetPY = offsetPY; this.sizeReal = sizeReal; - this.flag = flag; } @JSONField(serialize = false) @@ -96,14 +98,6 @@ public long getOffsetPY() { return offsetPY; } - public int getFlag() { - return flag; - } - - public void setFlag(int flag) { - this.flag = flag; - } - public boolean isRoll() { return roll; } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 908d0ca4590..26dcdd4ae91 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -40,6 +40,7 @@ import org.apache.rocketmq.store.queue.ReferredIterator; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.timer.TimerMetrics; +import org.rocksdb.RocksDB; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,6 +54,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.LinkedBlockingDeque; +import static org.apache.rocketmq.store.timer.rocksdb.TimerMessageRecord.TIMER_MESSAGE_POP_FLAG; +import static org.apache.rocketmq.store.timer.rocksdb.TimerMessageRecord.TIMER_MESSAGE_TRANSACTION_FLAG; +import static org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStorage.POP_COLUMN_FAMILY; +import static org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStorage.TRANSACTION_COLUMN_FAMILY; + public class TimerMessageRocksDBStore { private final static Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); private static final String ROCKSDB_DIRECTORY = "kvStore"; @@ -206,8 +212,13 @@ private String getServiceThreadName() { } private byte[] getColumnFamily(int flag) { - // TODO - return null; + if (flag == TIMER_MESSAGE_TRANSACTION_FLAG) { + return TRANSACTION_COLUMN_FAMILY; + } else if (flag == TIMER_MESSAGE_POP_FLAG) { + return POP_COLUMN_FAMILY; + } else { + return RocksDB.DEFAULT_COLUMN_FAMILY; + } } private class TimerEnqueueGetService extends ServiceThread { @@ -275,20 +286,28 @@ private void fetchAndPutTimerRequest() throws Exception { Map>> increase = new HashMap<>(); Map>> delete = new HashMap<>(); List trs = fetchTimerMessageRecord(); + List expired = new ArrayList<>(); for (TimerMessageRecord tr : trs) { long delayTime = tr.getDelayTime(); + int flag = tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_FLAG) == null ? + 0 : Integer.parseInt(tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_FLAG)); if (tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY) != null) { delayTime = Long.parseLong(tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_MS)); // Construct original message tr.setUniqueKey(tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY)); tr.setDelayTime(delayTime); - delete.computeIfAbsent(delayTime, k -> new HashMap<>()).computeIfAbsent(tr.getFlag(), k -> new ArrayList<>()).add(tr); + delete.computeIfAbsent(delayTime, k -> new HashMap<>()).computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); } else { - increase.computeIfAbsent(delayTime, k -> new HashMap<>()).computeIfAbsent(tr.getFlag(), k -> new ArrayList<>()).add(tr); + if (delayTime < System.currentTimeMillis()) { + expired.add(tr); + } else { + increase.computeIfAbsent(delayTime, k -> new HashMap<>()).computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); + } } } + while (!expired.isEmpty() && !dequeueGetQueue.offer(expired, 100, TimeUnit.MILLISECONDS)) {} for (Map.Entry>> entry : increase.entrySet()) { long delayTime = entry.getKey(); for (Map.Entry> entry1 : entry.getValue().entrySet()) { @@ -493,8 +512,7 @@ public boolean enqueue(int queueId) { if (null != msgExt) { long delayedTime = Long.parseLong(msgExt.getProperty(TIMER_OUT_MS)); - int flag = msgExt.getFlag(); - TimerMessageRecord timerRequest = new TimerMessageRecord(delayedTime, MessageClientIDSetter.getUniqID(msgExt), offsetPy, sizePy, flag); + TimerMessageRecord timerRequest = new TimerMessageRecord(delayedTime, MessageClientIDSetter.getUniqID(msgExt), offsetPy, sizePy); timerRequest.setMessageExt(msgExt); while(!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) {} From b0d604469e0d6a80c0206ca137cd2a066033f582 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 15 Jan 2025 18:01:55 +0800 Subject: [PATCH 10/61] feat: Adaptation logic --- .../rocketmq/broker/BrokerController.java | 31 +++++++++- .../processor/AdminBrokerProcessor.java | 4 +- .../rocketmq/store/DefaultMessageStore.java | 15 +++++ .../apache/rocketmq/store/MessageStore.java | 5 ++ .../store/config/MessageStoreConfig.java | 6 ++ .../metrics/DefaultStoreMetricsManager.java | 4 +- .../plugin/AbstractPluginMessageStore.java | 11 ++++ .../timer/rocksdb/TimerMessageKVStore.java | 2 +- .../rocksdb/TimerMessageRocksDBStorage.java | 6 +- .../rocksdb/TimerMessageRocksDBStore.java | 59 ++++++++++++++----- 10 files changed, 120 insertions(+), 23 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 006695c6bc8..0b4ddb2751c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -181,6 +181,9 @@ import org.apache.rocketmq.store.timer.TimerCheckpoint; import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.timer.TimerMetrics; +import org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStorage; +import org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStore; +import org.rocksdb.RocksDB; public class BrokerController { protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -272,6 +275,7 @@ public class BrokerController { private BrokerStats brokerStats; private InetSocketAddress storeHost; private TimerMessageStore timerMessageStore; + private TimerMessageRocksDBStore timerMessageRocksDBStore; private TimerCheckpoint timerCheckpoint; protected BrokerFastFailure brokerFastFailure; private Configuration configuration; @@ -831,13 +835,24 @@ public boolean initializeMessageStore() { messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig, configuration); this.messageStore = MessageStoreFactory.build(context, defaultMessageStore); this.messageStore.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(this.brokerConfig, this.consumerFilterManager)); + + TimerMetrics timerMetrics = new TimerMetrics(BrokerPathConfigHelper.getTimerMetricsPath(messageStoreConfig.getStorePathRootDir())); if (messageStoreConfig.isTimerWheelEnable()) { this.timerCheckpoint = new TimerCheckpoint(BrokerPathConfigHelper.getTimerCheckPath(messageStoreConfig.getStorePathRootDir())); - TimerMetrics timerMetrics = new TimerMetrics(BrokerPathConfigHelper.getTimerMetricsPath(messageStoreConfig.getStorePathRootDir())); this.timerMessageStore = new TimerMessageStore(messageStore, messageStoreConfig, timerCheckpoint, timerMetrics, brokerStatsManager); this.timerMessageStore.registerEscapeBridgeHook(msg -> escapeBridge.putMessage(msg)); this.messageStore.setTimerMessageStore(this.timerMessageStore); } + if (messageStoreConfig.getEnableTimerMessageOnRocksDB()) { + this.timerMessageRocksDBStore = new TimerMessageRocksDBStore(messageStore, messageStoreConfig, timerMetrics, brokerStatsManager); + if (this.messageStoreConfig.isTimerWheelEnable()) { + this.messageStoreConfig.setTimerStopEnqueue(true); + this.timerMessageRocksDBStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); + } + this.messageStore.setTimerMessageRocksDBStore(this.timerMessageRocksDBStore); + this.timerMessageRocksDBStore.createTimer(TimerMessageRocksDBStorage.POP_COLUMN_FAMILY); + this.timerMessageRocksDBStore.createTimer(TimerMessageRocksDBStorage.TRANSACTION_COLUMN_FAMILY); + } } catch (Exception e) { result = false; LOG.error("BrokerController#initialize: unexpected error occurs", e); @@ -874,6 +889,9 @@ public boolean recoverAndInitService() throws CloneNotSupportedException { result = this.messageStore.load(); } + if (messageStoreConfig.getEnableTimerMessageOnRocksDB()) { + result = result && this.timerMessageRocksDBStore.load(); + } if (messageStoreConfig.isTimerWheelEnable()) { result = result && this.timerMessageStore.load(); } @@ -1454,6 +1472,9 @@ protected void shutdownBasicService() { if (this.timerMessageStore != null) { this.timerMessageStore.shutdown(); } + if (this.timerMessageRocksDBStore != null) { + this.timerMessageRocksDBStore.shutdown(); + } if (this.fileWatchService != null) { this.fileWatchService.shutdown(); } @@ -1650,6 +1671,10 @@ protected void startBasicService() throws Exception { this.timerMessageStore.start(); } + if (this.timerMessageRocksDBStore != null) { + this.timerMessageRocksDBStore.start(); + } + if (this.replicasManager != null) { this.replicasManager.start(); } @@ -2578,5 +2603,7 @@ public void setColdDataCgCtrService(ColdDataCgCtrService coldDataCgCtrService) { this.coldDataCgCtrService = coldDataCgCtrService; } - + public TimerMessageRocksDBStore getTimerMessageRocksDBStore() { + return timerMessageRocksDBStore; + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 6fb7584aa9b..094c5b56f56 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -2692,7 +2692,9 @@ private HashMap prepareRuntimeInfo() throws RemotingCommandExcep runtimeInfo.put("earliestMessageTimeStamp", String.valueOf(this.brokerController.getMessageStore().getEarliestMessageTime())); runtimeInfo.put("startAcceptSendRequestTimeStamp", String.valueOf(this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp())); - if (this.brokerController.getMessageStoreConfig().isTimerWheelEnable()) { + if (this.brokerController.getMessageStoreConfig().getEnableTimerMessageOnRocksDB()) { + // TODO + } else if (this.brokerController.getMessageStoreConfig().isTimerWheelEnable()) { runtimeInfo.put("timerReadBehind", String.valueOf(this.brokerController.getMessageStore().getTimerMessageStore().getDequeueBehind())); runtimeInfo.put("timerOffsetBehind", String.valueOf(this.brokerController.getMessageStore().getTimerMessageStore().getEnqueueBehindMessages())); runtimeInfo.put("timerCongestNum", String.valueOf(this.brokerController.getMessageStore().getTimerMessageStore().getAllCongestNum())); diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 9d3c46a438a..83a30057af8 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -113,6 +113,7 @@ import org.apache.rocketmq.store.queue.ReferredIterator; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.timer.TimerMessageStore; +import org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStore; import org.apache.rocketmq.store.util.PerfCounter; import org.rocksdb.RocksDBException; @@ -166,6 +167,7 @@ public class DefaultMessageStore implements MessageStore { protected StoreCheckpoint storeCheckpoint; private TimerMessageStore timerMessageStore; + private TimerMessageRocksDBStore timerMessageRocksDBStore; private final LinkedList dispatcherList; @@ -1030,6 +1032,16 @@ public void setTimerMessageStore(TimerMessageStore timerMessageStore) { this.timerMessageStore = timerMessageStore; } + @Override + public TimerMessageRocksDBStore getTimerMessageRocksDBStore() { + return this.timerMessageRocksDBStore; + } + + @Override + public void setTimerMessageRocksDBStore(TimerMessageRocksDBStore timerMessageRocksDBStore) { + this.timerMessageRocksDBStore = timerMessageRocksDBStore; + } + @Override public long getCommitLogOffsetInQueue(String topic, int queueId, long consumeQueueOffset) { ConsumeQueueInterface consumeQueue = getConsumeQueue(topic, queueId); @@ -1923,6 +1935,9 @@ private void recover(final boolean lastExitOK) throws RocksDBException { @Override public long getTimingMessageCount(String topic) { + if (timerMessageRocksDBStore != null) { + return timerMessageRocksDBStore.getTimerMetrics().getTimingCount(topic); + } if (null == timerMessageStore) { return 0L; } else { diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java index 4bbee142a17..e36d029ef21 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java @@ -41,6 +41,7 @@ import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.timer.TimerMessageStore; +import org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStore; import org.apache.rocketmq.store.util.PerfCounter; import org.rocksdb.RocksDBException; import io.opentelemetry.api.common.AttributesBuilder; @@ -208,6 +209,10 @@ CompletableFuture getMessageAsync(final String group, final St void setTimerMessageStore(TimerMessageStore timerMessageStore); + TimerMessageRocksDBStore getTimerMessageRocksDBStore(); + + void setTimerMessageRocksDBStore(TimerMessageRocksDBStore timerMessageRocksDBStore); + /** * Get the offset of the message in the commit log, which is also known as physical offset. * diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 8e9299091aa..7d80dff88dd 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -478,6 +478,12 @@ public void setRocksdbCompressionType(String compressionType) { */ private int readCountTimerOnRocksDB = -1; + /** + * When enabled, the scheduled task is started. + * if time wheel is enabled, the time wheel only the correct ones are supported + * The message will be written to rocksdb. + * Close the time wheel when the file timing message is 0 + */ private boolean enableTimerMessageOnRocksDB = false; public boolean isRocksdbCQDoubleWriteEnable() { diff --git a/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java b/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java index db4c7bb7662..f385b521e0a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java +++ b/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java @@ -146,7 +146,9 @@ public static void init(Meter meter, Supplier attributesBuild measurement.record(System.currentTimeMillis() - earliestMessageTime, newAttributesBuilder().build()); }); - if (messageStore.getMessageStoreConfig().isTimerWheelEnable()) { + if (messageStore.getMessageStoreConfig().getEnableTimerMessageOnRocksDB()) { + // TODO add timer metrics + } else if (messageStore.getMessageStoreConfig().isTimerWheelEnable()) { timerEnqueueLag = meter.gaugeBuilder(GAUGE_TIMER_ENQUEUE_LAG) .setDescription("Timer enqueue messages lag") .ofLongs() diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java index d5d6236458e..49361288899 100644 --- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java @@ -55,6 +55,7 @@ import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.timer.TimerMessageStore; +import org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStore; import org.apache.rocketmq.store.util.PerfCounter; import org.rocksdb.RocksDBException; @@ -628,6 +629,16 @@ public void setTimerMessageStore(TimerMessageStore timerMessageStore) { next.setTimerMessageStore(timerMessageStore); } + @Override + public TimerMessageRocksDBStore getTimerMessageRocksDBStore() { + return next.getTimerMessageRocksDBStore(); + } + + @Override + public void setTimerMessageRocksDBStore(TimerMessageRocksDBStore timerMessageRocksDBStore) { + next.setTimerMessageRocksDBStore(timerMessageRocksDBStore); + } + @Override public long getTimingMessageCount(String topic) { return next.getTimingMessageCount(topic); diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java index b140509fab3..81657edfb97 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java @@ -107,5 +107,5 @@ public interface TimerMessageKVStore { * @param columnFamily the column family of the timer message kv store. * @return the checkpoint of the timer message kv store. */ - int getCheckpoint(byte[] columnFamily); + long getCheckpoint(byte[] columnFamily); } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index ee0fe1c73ad..afe4d46e232 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -52,6 +52,8 @@ public class TimerMessageRocksDBStorage extends AbstractRocksDBStorage implement private ColumnFamilyHandle popColumnFamilyHandle; private ColumnFamilyHandle transactionColumnFamilyHandle; + + // Supports 100ms level statistics private ColumnFamilyHandle metricColumnFamilyHandle; private WriteOptions writeOptions; @@ -312,10 +314,10 @@ public int getMetricSize(int lowerTime, int upperTime) { } @Override - public int getCheckpoint(byte[] columnFamily) { + public long getCheckpoint(byte[] columnFamily) { try { byte[] checkpointBytes = db.get(columnFamily); - return checkpointBytes == null ? 0 : ByteBuffer.wrap(checkpointBytes).getInt(); + return checkpointBytes == null ? 0 : ByteBuffer.wrap(checkpointBytes).getLong(); } catch (RocksDBException e) { throw new RuntimeException("Get checkpoint error", e); } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 26dcdd4ae91..1a2c8295f66 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -302,7 +302,8 @@ private void fetchAndPutTimerRequest() throws Exception { if (delayTime < System.currentTimeMillis()) { expired.add(tr); } else { - increase.computeIfAbsent(delayTime, k -> new HashMap<>()).computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); + tr.setDelayTime(delayTime / precisionMs % slotSize); + increase.computeIfAbsent(delayTime / precisionMs % slotSize, k -> new HashMap<>()).computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); } } } @@ -420,7 +421,7 @@ public void run() { private class TimerGetMessageService extends ServiceThread { private final byte[] columnFamily; - private int checkpoint; + private long checkpoint; public TimerGetMessageService(byte[] columnFamily) { this.columnFamily = columnFamily; @@ -440,7 +441,7 @@ public void run() { if (-1 == dequeue(checkpoint, columnFamily)) { waitForRunning(100L * precisionMs / 1000); } else { - checkpoint++; + checkpoint += precisionMs; } } catch (Throwable e) { log.error("Error occurred in " + getServiceName(), e); @@ -452,11 +453,11 @@ public void run() { private class TimerWarmService extends ServiceThread { private final byte[] columnFamily; - private int checkpoint; + private long checkpoint; public TimerWarmService(byte[] columnFamily) { this.columnFamily = columnFamily; - this.checkpoint = timerMessageKVStore.getCheckpoint(columnFamily); + checkpoint = System.currentTimeMillis() + precisionMs; } @Override @@ -473,7 +474,7 @@ public void run() { if (-1 == checkpoint) { waitForRunning(100L * precisionMs / 1000); } else { - this.checkpoint = checkpoint; + this.checkpoint += precisionMs; } } catch (Throwable e) { log.error("Error occurred in " + getServiceName(), e); @@ -559,30 +560,32 @@ private MessageExt getMessageByCommitOffset(long offsetPy, int sizePy) { return null; } - private int dequeue(int checkpoint, byte[] columnFamily) throws InterruptedException { - if (checkpoint < System.currentTimeMillis() / precisionMs % slotSize) { + private int dequeue(long checkpoint, byte[] columnFamily) throws InterruptedException { + if (checkpoint > System.currentTimeMillis() / precisionMs % slotSize) { return -1; } + int slot = (int) (checkpoint / precisionMs % slotSize); - List timerMessageRecords = timerMessageKVStore.scanRecords(columnFamily, checkpoint, checkpoint + 1); + List timerMessageRecords = timerMessageKVStore.scanRecords(columnFamily, slot, slot + 1); while (!dequeueGetQueue.offer(timerMessageRecords, 3, TimeUnit.SECONDS)) {} - addMetric(checkpoint, -timerMessageRecords.size()); + addMetric(slot, -timerMessageRecords.size()); return 0; } - private int warm(int checkpoint, byte[] columnFamily) { + private int warm(long checkpoint, byte[] columnFamily) { if (!storeConfig.isTimerWarmEnable()) { return -1; } - if (checkpoint < System.currentTimeMillis() / precisionMs % slotSize + 1) { - checkpoint = (int) (System.currentTimeMillis() / precisionMs % slotSize + 1); + if (checkpoint < System.currentTimeMillis() + precisionMs) { + checkpoint = System.currentTimeMillis() + precisionMs; } - if (checkpoint >= System.currentTimeMillis() / precisionMs % slotSize + 3) { + if (checkpoint >= System.currentTimeMillis() + 3L * precisionMs) { return -1; } - timerMessageKVStore.scanRecords(columnFamily, checkpoint, checkpoint + 1); - return checkpoint + 1; + int slot = (int) (checkpoint / precisionMs % slotSize); + timerMessageKVStore.scanRecords(columnFamily, slot, slot + 1); + return 0; } private MessageExtBrokerInner convert(MessageExt messageExt, boolean needRoll) { @@ -688,4 +691,28 @@ private void addMetric(MessageExt msg, int value) { private void addMetric(int delayTime, int value) { timerMetrics.updateDistPair(delayTime, value); } + + public TimerMetrics getTimerMetrics() { + return this.timerMetrics; + } + + public long getDequeueBehind() { + return 0; + } + + public long getEnqueueBehindMessages() { + return 0; + } + + public long getAllCongestNum() { + return 0; + } + + public long getEnqueueTps() { + return 0; + } + + public long getDequeueTps() { + return 0; + } } From 8bb252ca39a3d6d8e1588f056942eb322c12f812 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 15 Jan 2025 19:36:27 +0800 Subject: [PATCH 11/61] feat: Adaptation logic --- .../rocketmq/broker/slave/SlaveSynchronize.java | 4 +++- .../timer/rocksdb/TimerMessageRocksDBStore.java | 12 +++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java index aa77b773ee9..cc5e0efdd0d 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java @@ -222,7 +222,9 @@ private void syncTimerMetrics() { String masterAddrBak = this.masterAddr; if (masterAddrBak != null) { try { - if (null != brokerController.getMessageStore().getTimerMessageStore()) { + if (null != brokerController.getMessageStore().getTimerMessageRocksDBStore()) { + // TODO + } else if (null != brokerController.getMessageStore().getTimerMessageStore()) { TimerMetrics.TimerMetricsSerializeWrapper metricsSerializeWrapper = this.brokerController.getBrokerOuterAPI().getTimerMetrics(masterAddrBak); if (!brokerController.getMessageStore().getTimerMessageStore().getTimerMetrics().getDataVersion().equals(metricsSerializeWrapper.getDataVersion())) { diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 1a2c8295f66..fc1c3e27a48 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -53,6 +53,8 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import static org.apache.rocketmq.store.timer.rocksdb.TimerMessageRecord.TIMER_MESSAGE_POP_FLAG; import static org.apache.rocketmq.store.timer.rocksdb.TimerMessageRecord.TIMER_MESSAGE_TRANSACTION_FLAG; @@ -96,6 +98,7 @@ public class TimerMessageRocksDBStore { private BlockingQueue> dequeueGetQueue; private BlockingQueue> dequeuePutQueue; + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); private long commitOffset; public TimerMessageRocksDBStore(final MessageStore messageStore, final MessageStoreConfig storeConfig, @@ -137,14 +140,21 @@ public void start() { for (TimerDequeuePutService timerDequeuePutService : timerDequeuePutServices) { timerDequeuePutService.start(); } + scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + timerMetrics.persist(); + } + }, storeConfig.getTimerFlushIntervalMs(), storeConfig.getTimerFlushIntervalMs(), TimeUnit.MILLISECONDS); state = RUNNING; } public void shutdown() { - if (state == SHUTDOWN) { + if (state != RUNNING || state == SHUTDOWN) { return; } state = SHUTDOWN; + this.timerEnqueueGetService.shutdown(); this.timerEnqueuePutService.shutdown(); this.timerDequeueGetService.shutdown(); From fc4c7a5da7102d9c77661cd5df5c9512de6efbbb Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 15 Jan 2025 20:46:18 +0800 Subject: [PATCH 12/61] feat: Adaptation logic --- .../org/apache/rocketmq/broker/util/HookUtils.java | 5 +++-- .../apache/rocketmq/client/producer/SendResult.java | 2 +- .../store/timer/rocksdb/TimerMessageRocksDBStore.java | 10 ++++++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/util/HookUtils.java b/broker/src/main/java/org/apache/rocketmq/broker/util/HookUtils.java index dec42351d9f..a264b306a65 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/util/HookUtils.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/util/HookUtils.java @@ -133,7 +133,8 @@ public static PutMessageResult handleScheduleMessage(BrokerController brokerCont || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) { if (!isRolledTimerMessage(msg)) { if (checkIfTimerMessage(msg)) { - if (!brokerController.getMessageStoreConfig().isTimerWheelEnable()) { + if (!brokerController.getMessageStoreConfig().isTimerWheelEnable() && !brokerController.getMessageStoreConfig() + .getEnableTimerMessageOnRocksDB()) { //wheel timer is not enabled, reject the message return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_NOT_ENABLE, null); } @@ -204,7 +205,7 @@ private static PutMessageResult transformTimerMessage(BrokerController brokerCon deliverMs = deliverMs / timerPrecisionMs * timerPrecisionMs; } - if (brokerController.getTimerMessageStore().isReject(deliverMs)) { + if (brokerController.getTimerMessageRocksDBStore() == null && brokerController.getTimerMessageStore().isReject(deliverMs)) { return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_FLOW_CONTROL, null); } MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TIMER_OUT_MS, deliverMs + ""); diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java b/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java index 9119726e345..ef287f02c25 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java @@ -147,7 +147,7 @@ public void setDelayTime(long delayTime) { @Override public String toString() { return "SendResult [sendStatus=" + sendStatus + ", msgId=" + msgId + ", offsetMsgId=" + offsetMsgId + ", messageQueue=" + messageQueue - + ", queueOffset=" + queueOffset + ", recallHandle=" + recallHandle + "]"; + + ", queueOffset=" + queueOffset + ", recallHandle=" + recallHandle + ", delayTime=" + delayTime + "]"; } public void setRawRespBody(byte[] body) { diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index fc1c3e27a48..b067efdb86c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -113,6 +113,9 @@ public TimerMessageRocksDBStore(final MessageStore messageStore, final MessageSt this.readCount = storeConfig.getReadCountTimerOnRocksDB(); this.timerMessageKVStore = new TimerMessageRocksDBStorage(Paths.get( storeConfig.getStorePathRootDir(), ROCKSDB_DIRECTORY).toString()); + + this.timerGetMessageServices = new ArrayList<>(); + this.timerWarmServices = new ArrayList<>(); } public boolean load() { @@ -127,6 +130,7 @@ public void start() { if (state == RUNNING) { return; } + this.commitOffset = timerMessageKVStore.getCommitOffset(); this.timerEnqueueGetService.start(); this.timerEnqueuePutService.start(); this.timerDequeueGetService.start(); @@ -198,7 +202,6 @@ private void initService() { this.dequeuePutQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY); this.dequeueGetQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY); } - this.commitOffset = timerMessageKVStore.getCommitOffset(); } private void calcTimerDistribution() { @@ -298,6 +301,9 @@ private void fetchAndPutTimerRequest() throws Exception { List trs = fetchTimerMessageRecord(); List expired = new ArrayList<>(); + if (null == trs) { + return; + } for (TimerMessageRecord tr : trs) { long delayTime = tr.getDelayTime(); int flag = tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_FLAG) == null ? @@ -435,7 +441,6 @@ private class TimerGetMessageService extends ServiceThread { public TimerGetMessageService(byte[] columnFamily) { this.columnFamily = columnFamily; - this.checkpoint = timerMessageKVStore.getCheckpoint(columnFamily); } @Override @@ -445,6 +450,7 @@ public String getServiceName() { @Override public void run() { + this.checkpoint = timerMessageKVStore.getCheckpoint(columnFamily); log.info(this.getServiceName() + " service start"); while (!this.isStopped()) { try { From 6bc279ab1d8859de85e0d7ac8743236e4fc593a0 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 15 Jan 2025 21:16:52 +0800 Subject: [PATCH 13/61] init: prepare for metric --- .../rocketmq/store/config/MessageStoreConfig.java | 2 +- .../store/rocksdb/RocksDBOptionsFactory.java | 2 +- .../store/timer/rocksdb/TimerMessageRecord.java | 10 ++++------ .../timer/rocksdb/TimerMessageRocksDBStorage.java | 2 +- .../timer/rocksdb/TimerMessageRocksDBStore.java | 12 ++++++++---- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 7d80dff88dd..73e90a0d055 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -474,7 +474,7 @@ public void setRocksdbCompressionType(String compressionType) { /** * Maximum number of messages to be read each time - * -1 ： read all messages + * -1 : read all messages */ private int readCountTimerOnRocksDB = -1; diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java index eb7ee09196e..09355fcc57f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java +++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java @@ -246,7 +246,7 @@ public static ColumnFamilyOptions createTimerMetricCFOptions() { .setCacheIndexAndFilterBlocksWithHighPriority(true) .setPinL0FilterAndIndexBlocksInCache(false) .setPinTopLevelIndexAndFilter(true) - .setBlockCache(new LRUCache(64 * SizeUnit.MB, 8, false)) // 调整缓存大小为 512 MB + .setBlockCache(new LRUCache(64 * SizeUnit.MB, 8, false)) .setWholeKeyFiltering(true); CompactionOptionsUniversal compactionOption = new CompactionOptionsUniversal() diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java index e7029ea01f0..15900a72b47 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.store.timer.rocksdb; -import com.alibaba.fastjson.annotation.JSONField; import org.apache.rocketmq.common.message.MessageExt; import java.nio.ByteBuffer; @@ -35,7 +34,7 @@ public class TimerMessageRecord { // value: sizeReal + offsetPY private int sizeReal; private long offsetPY; - private final static int valueLength = Integer.BYTES + Long.BYTES; + private final static int VALUE_LENGTH = Integer.BYTES + Long.BYTES; public TimerMessageRecord() { } @@ -47,9 +46,9 @@ public TimerMessageRecord(long delayTime, String uniqueKey, long offsetPY, int s this.sizeReal = sizeReal; } - @JSONField(serialize = false) public byte[] getKeyBytes() { - int keyLength = Long.BYTES + uniqueKey.length(); + byte[] value = uniqueKey.getBytes(StandardCharsets.UTF_8); + int keyLength = Long.BYTES + value.length; byte[] keyBytes = new byte[keyLength]; ByteBuffer buffer = ByteBuffer.wrap(keyBytes); buffer.putLong(this.getDelayTime()).putLong(this.getOffsetPY()); @@ -57,9 +56,8 @@ public byte[] getKeyBytes() { return keyBytes; } - @JSONField(serialize = false) public byte[] getValueBytes() { - byte[] valueBytes = new byte[valueLength]; + byte[] valueBytes = new byte[VALUE_LENGTH]; ByteBuffer buffer = ByteBuffer.wrap(valueBytes); buffer.putInt(this.getSizeReal()); buffer.putLong(this.getOffsetPY()); diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index afe4d46e232..2d30d55260a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -202,7 +202,7 @@ public void deleteDefaultRecords(List consumerRecordList, in @Override public void deleteAssignRecords(byte[] columnFamily, List consumerRecordList, int timestamp) { - ColumnFamilyHandle deleteCfHandle = getColumnFamily(columnFamily);; + ColumnFamilyHandle deleteCfHandle = getColumnFamily(columnFamily); if (deleteCfHandle != null && !consumerRecordList.isEmpty()) { try (WriteBatch writeBatch = new WriteBatch()) { diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index b067efdb86c..c7382d18d6f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -324,7 +324,8 @@ private void fetchAndPutTimerRequest() throws Exception { } } - while (!expired.isEmpty() && !dequeueGetQueue.offer(expired, 100, TimeUnit.MILLISECONDS)) {} + while (!expired.isEmpty() && !dequeueGetQueue.offer(expired, 100, TimeUnit.MILLISECONDS)) { + } for (Map.Entry>> entry : increase.entrySet()) { long delayTime = entry.getKey(); for (Map.Entry> entry1 : entry.getValue().entrySet()) { @@ -376,7 +377,8 @@ public void run() { addMetric(messageExt, -1); } - while (!dequeuePutQueue.offer(timerMessageRecord, 3, TimeUnit.SECONDS)) {} + while (!dequeuePutQueue.offer(timerMessageRecord, 3, TimeUnit.SECONDS)) { + } } catch (InterruptedException e) { log.error("Error occurred in " + getServiceName(), e); } @@ -532,7 +534,8 @@ public boolean enqueue(int queueId) { TimerMessageRecord timerRequest = new TimerMessageRecord(delayedTime, MessageClientIDSetter.getUniqID(msgExt), offsetPy, sizePy); timerRequest.setMessageExt(msgExt); - while(!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) {} + while(!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) { + } Attributes attributes = DefaultStoreMetricsManager.newAttributesBuilder() .put(DefaultStoreMetricsConstant.LABEL_TOPIC, msgExt.getProperty(MessageConst.PROPERTY_REAL_TOPIC)).build(); DefaultStoreMetricsManager.timerMessageSetLatency.record((delayedTime - msgExt.getBornTimestamp()) / 1000, attributes); @@ -583,7 +586,8 @@ private int dequeue(long checkpoint, byte[] columnFamily) throws InterruptedExce int slot = (int) (checkpoint / precisionMs % slotSize); List timerMessageRecords = timerMessageKVStore.scanRecords(columnFamily, slot, slot + 1); - while (!dequeueGetQueue.offer(timerMessageRecords, 3, TimeUnit.SECONDS)) {} + while (!dequeueGetQueue.offer(timerMessageRecords, 3, TimeUnit.SECONDS)) { + } addMetric(slot, -timerMessageRecords.size()); return 0; } From 37bb349fd99eef2ab03b1cd60505005a6ae93642 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 15 Jan 2025 21:34:05 +0800 Subject: [PATCH 14/61] init: prepare for metric --- .../java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java | 4 +++- .../store/timer/rocksdb/TimerMessageRocksDBStore.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 64238e84f03..eccd82d8ace 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -856,7 +856,9 @@ protected SendResult processSendResponse( responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset()); sendResult.setTransactionId(responseHeader.getTransactionId()); sendResult.setRecallHandle(responseHeader.getRecallHandle()); - sendResult.setDelayTime(responseHeader.getDelayTime()); + if (responseHeader.getDelayTime() != null) { + sendResult.setDelayTime(responseHeader.getDelayTime()); + } String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION); if (regionId == null || regionId.isEmpty()) { regionId = MixAll.DEFAULT_TRACE_REGION_ID; diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index c7382d18d6f..8db75f44620 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -534,7 +534,7 @@ public boolean enqueue(int queueId) { TimerMessageRecord timerRequest = new TimerMessageRecord(delayedTime, MessageClientIDSetter.getUniqID(msgExt), offsetPy, sizePy); timerRequest.setMessageExt(msgExt); - while(!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) { + while (!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) { } Attributes attributes = DefaultStoreMetricsManager.newAttributesBuilder() .put(DefaultStoreMetricsConstant.LABEL_TOPIC, msgExt.getProperty(MessageConst.PROPERTY_REAL_TOPIC)).build(); From 43df4ea0309d8e9770302785821a523b66215795 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 16 Jan 2025 11:10:18 +0800 Subject: [PATCH 15/61] feat: Adaptation logic --- .../rocketmq/store/timer/rocksdb/TimerMessageRecord.java | 4 ++-- .../store/timer/rocksdb/TimerMessageRocksDBStorage.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java index 15900a72b47..68749a7871f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java @@ -51,8 +51,8 @@ public byte[] getKeyBytes() { int keyLength = Long.BYTES + value.length; byte[] keyBytes = new byte[keyLength]; ByteBuffer buffer = ByteBuffer.wrap(keyBytes); - buffer.putLong(this.getDelayTime()).putLong(this.getOffsetPY()); - buffer.put(uniqueKey.getBytes(StandardCharsets.UTF_8)); + buffer.putLong(this.getDelayTime()); + buffer.put(value); return keyBytes; } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index 2d30d55260a..0645972cbbd 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -239,7 +239,7 @@ public List scanRecords(byte[] columnFamily, long lowerTime, try (ReadOptions readOptions = new ReadOptions() .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array())) - .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(upperTime).array())); + .setIterateUpperBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(upperTime).array())); RocksIterator iterator = db.newIterator(cfHandle, readOptions)) { iterator.seek(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array()); while (iterator.isValid()) { @@ -266,7 +266,7 @@ public List scanRecords(byte[] columnFamily, long lowerTime, try (ReadOptions readOptions = new ReadOptions() .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array())) - .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(upperTime).array())); + .setIterateUpperBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(upperTime).array())); RocksIterator iterator = db.newIterator(cfHandle, readOptions)) { iterator.seek(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array()); while (iterator.isValid() && records.size() < maxCount) { From ecff650d46af0d30fbde324a34f93de2c65b75f7 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 16 Jan 2025 11:23:44 +0800 Subject: [PATCH 16/61] feat: Adaptation logic --- .../store/timer/rocksdb/TimerMessageKVStore.java | 2 +- .../timer/rocksdb/TimerMessageRocksDBStorage.java | 13 ++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java index 81657edfb97..fec2897c410 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java @@ -76,7 +76,7 @@ public interface TimerMessageKVStore { * @param upperTime the upper time of the timer message records to be scanned. * @return the list of timer message records. */ - List scanRecords(byte[] columnFamily, long lowerTime, long upperTime); + List scanRecords(byte[] columnFamily, long lowerTime, long upperTime, long timestamp); /** * Scan the expired timer message records from the timer message kv store. diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index 0645972cbbd..2e918bc3297 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -233,7 +233,7 @@ private ColumnFamilyHandle getColumnFamily(byte[] columnFamily) { } @Override - public List scanRecords(byte[] columnFamily, long lowerTime, long upperTime) { + public List scanRecords(byte[] columnFamily, long lowerTime, long upperTime, long timestamp) { List records = new ArrayList<>(); ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily); @@ -250,7 +250,7 @@ public List scanRecords(byte[] columnFamily, long lowerTime, try (WriteBatch writeBatch = new WriteBatch()) { // sync checkpoint - writeBatch.put(RocksDB.DEFAULT_COLUMN_FAMILY, ByteBuffer.allocate(8).putLong(upperTime).array()); + writeBatch.put(RocksDB.DEFAULT_COLUMN_FAMILY, ByteBuffer.allocate(8).putLong(timestamp).array()); deleteAssignRecords(columnFamily, records, (int) lowerTime); this.db.write(deleteOptions, writeBatch); } catch (RocksDBException e) { @@ -274,15 +274,6 @@ public List scanRecords(byte[] columnFamily, long lowerTime, iterator.next(); } } - - try (WriteBatch writeBatch = new WriteBatch()) { - // sync checkpoint - writeBatch.put(columnFamily, ByteBuffer.allocate(8).putLong(upperTime).array()); - deleteAssignRecords(columnFamily, records, (int) lowerTime); - this.db.write(deleteOptions, writeBatch); - } catch (RocksDBException e) { - throw new RuntimeException("Delete record error", e); - } return records; } From d49f8cfbcb4929aff2a05dd5c02f317656949003 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 16 Jan 2025 11:55:19 +0800 Subject: [PATCH 17/61] feat: Adaptation logic --- .../java/org/apache/rocketmq/broker/BrokerController.java | 2 +- .../rocketmq/store/timer/rocksdb/TimerMessageRecord.java | 2 +- .../store/timer/rocksdb/TimerMessageRocksDBStorage.java | 8 +------- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 0b4ddb2751c..226f5adc291 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -847,9 +847,9 @@ public boolean initializeMessageStore() { this.timerMessageRocksDBStore = new TimerMessageRocksDBStore(messageStore, messageStoreConfig, timerMetrics, brokerStatsManager); if (this.messageStoreConfig.isTimerWheelEnable()) { this.messageStoreConfig.setTimerStopEnqueue(true); - this.timerMessageRocksDBStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); } this.messageStore.setTimerMessageRocksDBStore(this.timerMessageRocksDBStore); + this.timerMessageRocksDBStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); this.timerMessageRocksDBStore.createTimer(TimerMessageRocksDBStorage.POP_COLUMN_FAMILY); this.timerMessageRocksDBStore.createTimer(TimerMessageRocksDBStorage.TRANSACTION_COLUMN_FAMILY); } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java index 68749a7871f..cfdc93bae42 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java @@ -30,7 +30,7 @@ public class TimerMessageRecord { private long delayTime; private String uniqueKey; private MessageExt messageExt; - private boolean roll; + private boolean roll = false; // value: sizeReal + offsetPY private int sizeReal; private long offsetPY; diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index 2e918bc3297..5068f8335a0 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -244,15 +244,9 @@ public List scanRecords(byte[] columnFamily, long lowerTime, iterator.seek(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array()); while (iterator.isValid()) { records.add(TimerMessageRecord.decode(iterator.value())); + db.delete(cfHandle, iterator.key()); iterator.next(); } - } - - try (WriteBatch writeBatch = new WriteBatch()) { - // sync checkpoint - writeBatch.put(RocksDB.DEFAULT_COLUMN_FAMILY, ByteBuffer.allocate(8).putLong(timestamp).array()); - deleteAssignRecords(columnFamily, records, (int) lowerTime); - this.db.write(deleteOptions, writeBatch); } catch (RocksDBException e) { throw new RuntimeException("Delete record error", e); } From 7a858d7a939edc2a112454f8e91d383cec0e1f26 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 16 Jan 2025 12:59:04 +0800 Subject: [PATCH 18/61] feat: Adaptation logic --- .../rocksdb/TimerMessageRocksDBStore.java | 65 ++++++++++--------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 8db75f44620..41592a446a8 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -242,17 +242,17 @@ public String getServiceName() { @Override public void run() { - log.info(this.getServiceName() + " service start"); + TimerMessageRocksDBStore.log.info(this.getServiceName() + " service start"); while (!this.isStopped()) { try { if (!enqueue(0)) { waitForRunning(100L * precisionMs / 1000); } } catch (Throwable e) { - log.error("Error occurred in " + getServiceName(), e); + TimerMessageRocksDBStore.log.error("Error occurred in " + getServiceName(), e); } } - log.info(this.getServiceName() + " service end"); + TimerMessageRocksDBStore.log.info(this.getServiceName() + " service end"); } } @@ -264,15 +264,15 @@ public String getServiceName() { @Override public void run() { - log.info(this.getServiceName() + " service start"); + TimerMessageRocksDBStore.log.info(this.getServiceName() + " service start"); while (!this.isStopped() || !enqueuePutQueue.isEmpty()) { try { fetchAndPutTimerRequest(); } catch (Throwable e) { - log.error("Unknown error", e); + TimerMessageRocksDBStore.log.error("Unknown error", e); } } - log.info(this.getServiceName() + " service end"); + TimerMessageRocksDBStore.log.info(this.getServiceName() + " service end"); } private List fetchTimerMessageRecord() throws InterruptedException { @@ -315,7 +315,7 @@ private void fetchAndPutTimerRequest() throws Exception { tr.setDelayTime(delayTime); delete.computeIfAbsent(delayTime, k -> new HashMap<>()).computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); } else { - if (delayTime < System.currentTimeMillis()) { + if (delayTime <= System.currentTimeMillis()) { expired.add(tr); } else { tr.setDelayTime(delayTime / precisionMs % slotSize); @@ -330,24 +330,24 @@ private void fetchAndPutTimerRequest() throws Exception { long delayTime = entry.getKey(); for (Map.Entry> entry1 : entry.getValue().entrySet()) { int tag = entry1.getKey(); - timerMessageKVStore.writeAssignRecords(getColumnFamily(tag), entry1.getValue(), commitOffset, (int) (delayTime / precisionMs % slotSize)); + timerMessageKVStore.writeAssignRecords(getColumnFamily(tag), entry1.getValue(), commitOffset, (int) delayTime); for (TimerMessageRecord record : entry1.getValue()) { addMetric(record.getMessageExt(), 1); } } - addMetric((int) (delayTime / precisionMs % slotSize), entry.getValue().size()); + addMetric((int) delayTime, entry.getValue().size()); } for (Map.Entry>> entry : delete.entrySet()) { long delayTime = entry.getKey(); for (Map.Entry> entry1 : entry.getValue().entrySet()) { int tag = entry1.getKey(); - timerMessageKVStore.deleteAssignRecords(getColumnFamily(tag), entry1.getValue(), (int) (delayTime / precisionMs % slotSize)); + timerMessageKVStore.deleteAssignRecords(getColumnFamily(tag), entry1.getValue(), (int) delayTime); for (TimerMessageRecord record : entry1.getValue()) { addMetric(record.getMessageExt(), -1); } } - addMetric((int) (delayTime / precisionMs % slotSize), -entry.getValue().size()); + addMetric((int) delayTime, -entry.getValue().size()); } } } @@ -360,8 +360,8 @@ public String getServiceName() { @Override public void run() { - log.info(this.getServiceName() + " service start"); - while (!this.isStopped() && !dequeueGetQueue.isEmpty()) { + TimerMessageRocksDBStore.log.info(this.getServiceName() + " service start"); + while (!this.isStopped()) { try { List timerMessageRecord = dequeueGetQueue.poll(100L * precisionMs / 1000, TimeUnit.MILLISECONDS); if (null == timerMessageRecord || timerMessageRecord.isEmpty()) { @@ -380,9 +380,10 @@ public void run() { while (!dequeuePutQueue.offer(timerMessageRecord, 3, TimeUnit.SECONDS)) { } } catch (InterruptedException e) { - log.error("Error occurred in " + getServiceName(), e); + TimerMessageRocksDBStore.log.error("Error occurred in " + getServiceName(), e); } } + TimerMessageRocksDBStore.log.info(this.getServiceName() + " service end"); } } @@ -394,14 +395,13 @@ public String getServiceName() { @Override public void run() { - log.info(this.getServiceName() + " service start"); - while (!this.isStopped() && !dequeuePutQueue.isEmpty()) { + TimerMessageRocksDBStore.log.info(this.getServiceName() + " service start"); + while (!this.isStopped()) { try { List timerMessageRecord = dequeuePutQueue.poll(100L * precisionMs / 1000, TimeUnit.MILLISECONDS); if (null == timerMessageRecord || timerMessageRecord.isEmpty()) { continue; } - for (TimerMessageRecord record : timerMessageRecord) { MessageExt msg = record.getMessageExt(); MessageExtBrokerInner messageExtBrokerInner = convert(msg, record.isRoll()); @@ -414,26 +414,27 @@ public void run() { if (result == PUT_OK) { processed = true; } else if (result == PUT_NO_RETRY) { - log.warn("Skipping message due to unrecoverable error. Msg: {}", msg); + TimerMessageRocksDBStore.log.warn("Skipping message due to unrecoverable error. Msg: {}", msg); processed = true; } else { retryCount++; // Without enabling TimerEnableRetryUntilSuccess, messages will retry up to 3 times before being discarded if (!storeConfig.isTimerEnableRetryUntilSuccess() && retryCount >= 3) { - log.error("Message processing failed after {} retries. Msg: {}", retryCount, msg); + TimerMessageRocksDBStore.log.error("Message processing failed after {} retries. Msg: {}", retryCount, msg); processed = true; } else { Thread.sleep(500L * precisionMs / 1000); - log.warn("Retrying to process message. Retry count: {}, Msg: {}", retryCount, msg); + TimerMessageRocksDBStore.log.warn("Retrying to process message. Retry count: {}, Msg: {}", retryCount, msg); } } } } } catch (InterruptedException e) { - log.error("Error occurred in " + getServiceName(), e); + TimerMessageRocksDBStore.log.error("Error occurred in " + getServiceName(), e); } } + TimerMessageRocksDBStore.log.info(this.getServiceName() + " service end"); } } @@ -453,7 +454,7 @@ public String getServiceName() { @Override public void run() { this.checkpoint = timerMessageKVStore.getCheckpoint(columnFamily); - log.info(this.getServiceName() + " service start"); + TimerMessageRocksDBStore.log.info(this.getServiceName() + " service start"); while (!this.isStopped()) { try { if (-1 == dequeue(checkpoint, columnFamily)) { @@ -462,10 +463,11 @@ public void run() { checkpoint += precisionMs; } } catch (Throwable e) { - log.error("Error occurred in " + getServiceName(), e); + TimerMessageRocksDBStore.log.error("Error occurred in " + getServiceName(), e); + throw new RuntimeException(e); } } - log.info(this.getServiceName() + " service end"); + TimerMessageRocksDBStore.log.info(this.getServiceName() + " service end"); } } @@ -485,7 +487,7 @@ public String getServiceName() { @Override public void run() { - log.info(this.getServiceName() + " service start"); + TimerMessageRocksDBStore.log.info(this.getServiceName() + " service start"); while (!this.isStopped()) { try { int checkpoint = warm(this.checkpoint, columnFamily); @@ -495,10 +497,10 @@ public void run() { this.checkpoint += precisionMs; } } catch (Throwable e) { - log.error("Error occurred in " + getServiceName(), e); + TimerMessageRocksDBStore.log.error("Error occurred in " + getServiceName(), e); } } - log.info(this.getServiceName() + " service end"); + TimerMessageRocksDBStore.log.info(this.getServiceName() + " service end"); } } // ----------------------------------------------------------------------------------------------------------------- @@ -580,12 +582,15 @@ private MessageExt getMessageByCommitOffset(long offsetPy, int sizePy) { } private int dequeue(long checkpoint, byte[] columnFamily) throws InterruptedException { - if (checkpoint > System.currentTimeMillis() / precisionMs % slotSize) { + if (checkpoint > System.currentTimeMillis()) { return -1; } int slot = (int) (checkpoint / precisionMs % slotSize); - List timerMessageRecords = timerMessageKVStore.scanRecords(columnFamily, slot, slot + 1); + List timerMessageRecords = timerMessageKVStore.scanRecords(columnFamily, slot, slot + 1, checkpoint); + if (timerMessageRecords == null || timerMessageRecords.isEmpty()) { + return 0; + } while (!dequeueGetQueue.offer(timerMessageRecords, 3, TimeUnit.SECONDS)) { } addMetric(slot, -timerMessageRecords.size()); @@ -604,7 +609,7 @@ private int warm(long checkpoint, byte[] columnFamily) { } int slot = (int) (checkpoint / precisionMs % slotSize); - timerMessageKVStore.scanRecords(columnFamily, slot, slot + 1); + timerMessageKVStore.scanRecords(columnFamily, slot, slot + 1, readCount); return 0; } From 9a37312d338b5ead8f840a1a52694b23b2d91941 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 16 Jan 2025 13:21:30 +0800 Subject: [PATCH 19/61] feat: Adaptation logic --- .../store/timer/rocksdb/TimerMessageRocksDBStore.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 41592a446a8..6464e746623 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -361,7 +361,7 @@ public String getServiceName() { @Override public void run() { TimerMessageRocksDBStore.log.info(this.getServiceName() + " service start"); - while (!this.isStopped()) { + while (!this.isStopped() || !dequeueGetQueue.isEmpty()) { try { List timerMessageRecord = dequeueGetQueue.poll(100L * precisionMs / 1000, TimeUnit.MILLISECONDS); if (null == timerMessageRecord || timerMessageRecord.isEmpty()) { @@ -396,7 +396,7 @@ public String getServiceName() { @Override public void run() { TimerMessageRocksDBStore.log.info(this.getServiceName() + " service start"); - while (!this.isStopped()) { + while (!this.isStopped() || !dequeuePutQueue.isEmpty()) { try { List timerMessageRecord = dequeuePutQueue.poll(100L * precisionMs / 1000, TimeUnit.MILLISECONDS); if (null == timerMessageRecord || timerMessageRecord.isEmpty()) { From 4b08bf97ab21b6d8978b52b6dc6d8a583a8569a9 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 16 Jan 2025 13:31:48 +0800 Subject: [PATCH 20/61] feat: Adaptation logic --- .../rocketmq/store/timer/rocksdb/TimerMessageKVStore.java | 4 ++-- .../store/timer/rocksdb/TimerMessageRocksDBStorage.java | 6 ++++-- .../store/timer/rocksdb/TimerMessageRocksDBStore.java | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java index fec2897c410..a5e13ef3764 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java @@ -59,7 +59,7 @@ public interface TimerMessageKVStore { * @param timestamp the key of the timer message metric column family. * Default is delete common timer message. */ - void deleteDefaultRecords(List consumerRecordList, int timestamp); + void deleteDefaultRecords(List consumerRecordList, int timestamp, long offset); /** * Delete the timer message records from the timer message kv store. @@ -67,7 +67,7 @@ public interface TimerMessageKVStore { * @param consumerRecordList the list of timer message records to be deleted. * @param timestamp the key of the timer message metric column family. */ - void deleteAssignRecords(byte[] columnFamily, List consumerRecordList, int timestamp); + void deleteAssignRecords(byte[] columnFamily, List consumerRecordList, int timestamp, long offset); /** * Scan the timer message records from the timer message kv store. diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index 5068f8335a0..eeb6f87708d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -186,12 +186,13 @@ public void writeAssignRecords(byte[] columnFamily, List con } @Override - public void deleteDefaultRecords(List consumerRecordList, int timestamp) { + public void deleteDefaultRecords(List consumerRecordList, int timestamp, long offset) { if (!consumerRecordList.isEmpty()) { try (WriteBatch writeBatch = new WriteBatch()) { for (TimerMessageRecord record : consumerRecordList) { writeBatch.delete(defaultCFHandle, record.getKeyBytes()); } + writeBatch.put(TIMER_OFFSET_KEY, ByteBuffer.allocate(8).putLong(offset).array()); syncMetric(timestamp / 100, - consumerRecordList.size(), writeBatch); this.db.write(deleteOptions, writeBatch); } catch (RocksDBException e) { @@ -201,7 +202,7 @@ public void deleteDefaultRecords(List consumerRecordList, in } @Override - public void deleteAssignRecords(byte[] columnFamily, List consumerRecordList, int timestamp) { + public void deleteAssignRecords(byte[] columnFamily, List consumerRecordList, int timestamp, long offset) { ColumnFamilyHandle deleteCfHandle = getColumnFamily(columnFamily); if (deleteCfHandle != null && !consumerRecordList.isEmpty()) { @@ -209,6 +210,7 @@ public void deleteAssignRecords(byte[] columnFamily, List co for (TimerMessageRecord record : consumerRecordList) { writeBatch.delete(deleteCfHandle, record.getKeyBytes()); } + writeBatch.put(TIMER_OFFSET_KEY, ByteBuffer.allocate(8).putLong(offset).array()); syncMetric(timestamp / 100, - consumerRecordList.size(), writeBatch); this.db.write(deleteOptions, writeBatch); } catch (RocksDBException e) { diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 6464e746623..f36b6040476 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -99,7 +99,7 @@ public class TimerMessageRocksDBStore { private BlockingQueue> dequeuePutQueue; ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - private long commitOffset; + private volatile long commitOffset; public TimerMessageRocksDBStore(final MessageStore messageStore, final MessageStoreConfig storeConfig, TimerMetrics timerMetrics, final BrokerStatsManager brokerStatsManager) { @@ -342,7 +342,7 @@ private void fetchAndPutTimerRequest() throws Exception { long delayTime = entry.getKey(); for (Map.Entry> entry1 : entry.getValue().entrySet()) { int tag = entry1.getKey(); - timerMessageKVStore.deleteAssignRecords(getColumnFamily(tag), entry1.getValue(), (int) delayTime); + timerMessageKVStore.deleteAssignRecords(getColumnFamily(tag), entry1.getValue(), (int) delayTime, commitOffset); for (TimerMessageRecord record : entry1.getValue()) { addMetric(record.getMessageExt(), -1); } From f29a3259a7a5ebe0efd57176127857527f424cd3 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 16 Jan 2025 16:27:09 +0800 Subject: [PATCH 21/61] feat: Adaptation logic --- .../broker/processor/RecallMessageProcessor.java | 1 + .../broker/processor/SendMessageProcessor.java | 2 +- .../common/producer/RecallMessageHandle.java | 16 ++++++++++++++-- .../rocksdb/TimerMessageRocksDBStorage.java | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java index 372db0d36eb..e446646a4e3 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java @@ -137,6 +137,7 @@ public MessageExtBrokerInner buildMessage(ChannelHandlerContext ctx, RecallMessa msgInner.setQueueId(0); MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TIMER_DEL_UNIQKEY, TimerMessageStore.buildDeleteKey(handle.getTopic(), handle.getMessageId())); + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TIMER_DEL_MS, String.valueOf(handle.getDelaytime())); MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, handle.getMessageId()); MessageAccessor.putProperty(msgInner, diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index a4a9f0ecfad..7c24bc065e8 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -659,7 +659,7 @@ public void attachRecallHandle(RemotingCommand request, MessageExt msg, SendMess if (timestampStr != null && realTopic != null && !realTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { timestampStr = String.valueOf(Long.parseLong(timestampStr) + 1); // consider of floor String recallHandle = RecallMessageHandle.HandleV1.buildHandle(realTopic, - brokerController.getBrokerConfig().getBrokerName(), timestampStr, MessageClientIDSetter.getUniqID(msg)); + brokerController.getBrokerConfig().getBrokerName(), timestampStr, MessageClientIDSetter.getUniqID(msg), responseHeader.getDelayTime()); responseHeader.setRecallHandle(recallHandle); } diff --git a/common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java b/common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java index b00b15bd863..81993ef3ceb 100644 --- a/common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java +++ b/common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java @@ -40,16 +40,24 @@ public static class HandleV1 extends RecallMessageHandle { private String brokerName; private String timestampStr; private String messageId; // id of unique key + private long delaytime; - public HandleV1(String topic, String brokerName, String timestamp, String messageId) { + public HandleV1(String topic, String brokerName, String timestamp, String messageId, long delaytime) { this.version = VERSION_1; this.topic = topic; this.brokerName = brokerName; this.timestampStr = timestamp; this.messageId = messageId; + this.delaytime = delaytime; } // no param check + public static String buildHandle(String topic, String brokerName, String timestampStr, String messageId, long delaytime) { + String rawString = String.join(SEPARATOR, VERSION_1, topic, brokerName, timestampStr, messageId, String.valueOf(delaytime)); + return Base64.getUrlEncoder().encodeToString(rawString.getBytes(UTF_8)); + } + + // TODO public static String buildHandle(String topic, String brokerName, String timestampStr, String messageId) { String rawString = String.join(SEPARATOR, VERSION_1, topic, brokerName, timestampStr, messageId); return Base64.getUrlEncoder().encodeToString(rawString.getBytes(UTF_8)); @@ -74,6 +82,10 @@ public String getMessageId() { public String getVersion() { return version; } + + public long getDelaytime() { + return delaytime; + } } public static RecallMessageHandle decodeHandle(String handle) throws DecoderException { @@ -91,6 +103,6 @@ public static RecallMessageHandle decodeHandle(String handle) throws DecoderExce if (!VERSION_1.equals(items[0]) || items.length < 5) { throw new DecoderException("recall handle is invalid"); } - return new HandleV1(items[1], items[2], items[3], items[4]); + return new HandleV1(items[1], items[2], items[3], items[4], Long.parseLong(items[5])); } } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index eeb6f87708d..821b11ba1a5 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -304,7 +304,7 @@ public int getMetricSize(int lowerTime, int upperTime) { public long getCheckpoint(byte[] columnFamily) { try { byte[] checkpointBytes = db.get(columnFamily); - return checkpointBytes == null ? 0 : ByteBuffer.wrap(checkpointBytes).getLong(); + return checkpointBytes == null ? System.currentTimeMillis() : ByteBuffer.wrap(checkpointBytes).getLong(); } catch (RocksDBException e) { throw new RuntimeException("Get checkpoint error", e); } From 5588a3b3843f5af1231a5ecdd3ab9d21785d4ecd Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 17 Jan 2025 10:51:08 +0800 Subject: [PATCH 22/61] feat: Adaptation logic --- .../store/timer/TimerMessageStore.java | 8 + .../rocksdb/TimerMessageRocksDBStore.java | 14 +- .../rocksdb/TimerMessageRocksDBStoreTest.java | 354 ++++++++++++++++++ 3 files changed, 372 insertions(+), 4 deletions(-) create mode 100644 store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java index 4287ce78ab0..4305de7d8bc 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java @@ -1915,4 +1915,12 @@ public TimerCheckpoint getTimerCheckpoint() { public static String buildDeleteKey(String realTopic, String uniqueKey) { return realTopic + "+" + uniqueKey; } + + public static String extractUniqueKey(String deleteKey) { + int separatorIndex = deleteKey.indexOf('+'); + if (separatorIndex == -1) { + throw new IllegalArgumentException("Invalid deleteKey format"); + } + return deleteKey.substring(separatorIndex + 1); + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index f36b6040476..6f90cea23af 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -39,6 +39,7 @@ import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.store.queue.ReferredIterator; import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.timer.TimerMetrics; import org.rocksdb.RocksDB; import org.slf4j.Logger; @@ -309,14 +310,15 @@ private void fetchAndPutTimerRequest() throws Exception { int flag = tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_FLAG) == null ? 0 : Integer.parseInt(tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_FLAG)); if (tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY) != null) { - delayTime = Long.parseLong(tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_MS)); // Construct original message - tr.setUniqueKey(tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY)); - tr.setDelayTime(delayTime); + tr.setDelayTime(delayTime / precisionMs % slotSize); + tr.setUniqueKey(TimerMessageStore.extractUniqueKey(tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY))); delete.computeIfAbsent(delayTime, k -> new HashMap<>()).computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); } else { if (delayTime <= System.currentTimeMillis()) { expired.add(tr); + addMetric(tr.getMessageExt(), 1); + addMetric((int) (delayTime / precisionMs % slotSize), 1); } else { tr.setDelayTime(delayTime / precisionMs % slotSize); increase.computeIfAbsent(delayTime / precisionMs % slotSize, k -> new HashMap<>()).computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); @@ -374,7 +376,6 @@ public void run() { record.setDelayTime(delayedTime); record.setUniqueKey(MessageClientIDSetter.getUniqID(messageExt)); record.setRoll(delayedTime >= System.currentTimeMillis() + precisionMs * 3L); - addMetric(messageExt, -1); } while (!dequeuePutQueue.offer(timerMessageRecord, 3, TimeUnit.SECONDS)) { @@ -428,6 +429,7 @@ public void run() { } } } + addMetric(msg, -1); } } catch (InterruptedException e) { @@ -721,6 +723,10 @@ public TimerMetrics getTimerMetrics() { return this.timerMetrics; } + public long getCommitOffset() { + return commitOffset; + } + public long getDequeueBehind() { return 0; } diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java new file mode 100644 index 00000000000..03305e91d18 --- /dev/null +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.timer.rocksdb; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicFilterType; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.GetMessageResult; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.GetMessageStatus; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.MessageArrivingListener; +import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.apache.rocketmq.store.timer.StoreTestUtils; +import org.apache.rocketmq.store.timer.TimerMessageStore; +import org.apache.rocketmq.store.timer.TimerMetrics; +import org.junit.*; +import org.rocksdb.RocksDB; + +import java.io.File; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStore.TIMER_TOPIC; +import static org.awaitility.Awaitility.await; +import static org.junit.Assert.*; + +public class TimerMessageRocksDBStoreTest { + private static final Log log = LogFactory.getLog(TimerMessageRocksDBStoreTest.class); + MessageStore messageStore; + MessageStoreConfig storeConfig; + int precisionMs = 500; + AtomicInteger counter = new AtomicInteger(0); + private SocketAddress bornHost; + private SocketAddress storeHost; + Set baseDirs = new HashSet<>(); + + + @Before + public void setUp() throws Exception { + String baseDir = StoreTestUtils.createBaseDir(); + this.baseDirs.add(baseDir); + storeConfig = new MessageStoreConfig(); + storeConfig.setStorePathRootDir(baseDir); + storeConfig.setStorePathCommitLog(baseDir + File.separator + "commitlog"); + messageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("TimerTest", false), new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>()); + + storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123); + bornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0); + messageStore.load(); + messageStore.start(); + } + + private TimerMessageRocksDBStore createTimerMessageRocksDBStore(String rootDir) { + if (null == rootDir) { + rootDir = StoreTestUtils.createBaseDir(); + baseDirs.add(rootDir); + } + TimerMetrics timerMetrics = new TimerMetrics(rootDir + File.separator + "config" + File.separator + "timermetrics"); + return new TimerMessageRocksDBStore(messageStore, storeConfig, timerMetrics, null); + } + + public MessageExtBrokerInner buildMessage(long delayedMs, String topic, boolean relative) { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic(topic); + msg.setQueueId(0); + msg.setTags(counter.incrementAndGet() + ""); + msg.setKeys("timer"); + if (relative) { + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TIMER_DELAY_SEC, delayedMs / 1000 + ""); + } else { + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TIMER_DELIVER_MS, delayedMs + ""); + } + msg.setBody(new byte[100]); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(0); + msg.setBornTimestamp(System.currentTimeMillis()); + msg.setBornHost(bornHost); + msg.setStoreHost(storeHost); + MessageClientIDSetter.setUniqID(msg); + TopicFilterType topicFilterType = MessageExt.parseTopicFilterType(msg.getSysFlag()); + long tagsCodeValue = + MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msg.getTags()); + msg.setTagsCode(tagsCodeValue); + msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); + return msg; + } + + public ByteBuffer getOneMessage(String topic, int queue, long offset, int timeout) throws Exception { + int retry = timeout / 100; + while (retry-- > 0) { + GetMessageResult getMessageResult = messageStore.getMessage("TimerGroup", topic, queue, offset, 1, null); + if (null != getMessageResult && GetMessageStatus.FOUND == getMessageResult.getStatus()) { + return getMessageResult.getMessageBufferList().get(0); + } + Thread.sleep(100); + } + return null; + } + + private PutMessageResult transformTimerMessage(TimerMessageRocksDBStore timerMessageStore, MessageExtBrokerInner msg) { + //do transform + int delayLevel = msg.getDelayTimeLevel(); + long deliverMs; + + try { + if (msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null) { + deliverMs = System.currentTimeMillis() + Integer.parseInt(msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC)) * 1000; + } else if (msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS) != null) { + deliverMs = System.currentTimeMillis() + Integer.parseInt(msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS)); + } else { + deliverMs = Long.parseLong(msg.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS)); + } + } catch (Exception e) { + return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_MSG_ILLEGAL, null); + } + if (deliverMs > System.currentTimeMillis()) { + if (delayLevel <= 0 && deliverMs - System.currentTimeMillis() > storeConfig.getTimerMaxDelaySec() * 1000L) { + return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_MSG_ILLEGAL, null); + } + + int timerPrecisionMs = storeConfig.getTimerPrecisionMs(); + if (deliverMs % timerPrecisionMs == 0) { + deliverMs -= timerPrecisionMs; + } else { + deliverMs = deliverMs / timerPrecisionMs * timerPrecisionMs; + } + + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TIMER_OUT_MS, deliverMs + ""); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic()); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_QUEUE_ID, String.valueOf(msg.getQueueId())); + msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); + msg.setTopic(TIMER_TOPIC); + msg.setQueueId(0); + } else if (null != msg.getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY)) { + return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_MSG_ILLEGAL, null); + } + return null; + } + + @Test + public void testDoNormalTimer() throws Exception { + String topic = "TimerTest_testPutTimerMessage"; + + final TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); + timerMessageStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); + timerMessageStore.load(); + timerMessageStore.start(); + long commitOffset = timerMessageStore.getCommitOffset(); + + long curr = System.currentTimeMillis() / precisionMs * precisionMs; + long delayMs = curr + 3000; + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 5; j++) { + MessageExtBrokerInner inner = buildMessage((i % 2 == 0) ? 3000 : delayMs, topic + i, i % 2 == 0); + transformTimerMessage(timerMessageStore, inner); + PutMessageResult putMessageResult = messageStore.putMessage(inner); + assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus()); + } + } + + // Wait until messages have been wrote to TimerLog but the slot (delayMs) hasn't expired. + await().atMost(2000, TimeUnit.MILLISECONDS).until(new Callable() { + @Override + public Boolean call() { + return timerMessageStore.getCommitOffset() - commitOffset == 10 * 5; + } + }); + + for (int i = 0; i < 10; i++) { + Assert.assertEquals(5, timerMessageStore.getTimerMetrics().getTimingCount(topic + i)); + } + + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 5; j++) { + ByteBuffer msgBuff = getOneMessage(topic + i, 0, j, 4000); + assertNotNull(msgBuff); + MessageExt msgExt = MessageDecoder.decode(msgBuff); + assertNotNull(msgExt); + assertEquals(topic + i, msgExt.getTopic()); + } + } + for (int i = 0; i < 10; i++) { + Assert.assertEquals(0, timerMessageStore.getTimerMetrics().getTimingCount(topic + i)); + } + timerMessageStore.shutdown(); + } + + @Test + public void testPutExpiredTimerMessage() throws Exception { + // Skip on Mac to make CI pass + Assume.assumeFalse(MixAll.isMac()); + Assume.assumeFalse(MixAll.isWindows()); + + String topic = "TimerTest_testPutExpiredTimerMessage"; + + TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); + timerMessageStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); + timerMessageStore.load(); + timerMessageStore.start(); + + long delayMs = System.currentTimeMillis() - 2 * precisionMs; + for (int i = 0; i < 10; i++) { + MessageExtBrokerInner inner = buildMessage(delayMs, topic, false); + transformTimerMessage(timerMessageStore, inner); + PutMessageResult putMessageResult = messageStore.putMessage(inner); + assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus()); + } + + long curr = System.currentTimeMillis(); + for (int i = 0; i < 10; i++) { + ByteBuffer msgBuff = getOneMessage(topic, 0, i, 1000); + assertNotNull(msgBuff); + assertTrue(System.currentTimeMillis() - curr < 200); + } + timerMessageStore.shutdown(); + } + + @Test + public void testDeleteTimerMessage() throws Exception { + String topic = "TimerTest_testDeleteTimerMessage"; + + TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); + timerMessageStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); + timerMessageStore.load(); + timerMessageStore.start(); + + long delayMs = System.currentTimeMillis() + 1000; + String uniqKey = null; + for (int i = 0; i < 5; i++) { + MessageExtBrokerInner inner = buildMessage(delayMs, topic, false); + transformTimerMessage(timerMessageStore, inner); + if (null == uniqKey) { + uniqKey = MessageClientIDSetter.getUniqID(inner); + } + assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(inner).getPutMessageStatus()); + } + + MessageExtBrokerInner delMsg = buildMessage(delayMs, topic, false); + transformTimerMessage(timerMessageStore, delMsg); + MessageAccessor.putProperty(delMsg, MessageConst.PROPERTY_TIMER_DEL_UNIQKEY, TimerMessageStore.buildDeleteKey(topic, uniqKey)); + delMsg.setPropertiesString(MessageDecoder.messageProperties2String(delMsg.getProperties())); + assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus()); + + // The first one should have been deleted. + ByteBuffer msgBuff = getOneMessage(topic, 0, 0, 3000); + assertNotNull(msgBuff); + MessageExt msgExt = MessageDecoder.decode(msgBuff); + assertNotNull(msgExt); + assertNotEquals(uniqKey, MessageClientIDSetter.getUniqID(msgExt)); + + // The last one should be null. + assertNull(getOneMessage(topic, 0, 4, 500)); + timerMessageStore.shutdown(); + } + + @Test + public void testPutDeleteTimerMessage() throws Exception { + String topic = "TimerTest_testPutDeleteTimerMessage"; + + final TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); + timerMessageStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); + timerMessageStore.load(); + timerMessageStore.start(); + + long curr = System.currentTimeMillis() / precisionMs * precisionMs; + final long delayMs = curr + 2000; + for (int i = 0; i < 5; i++) { + MessageExtBrokerInner inner = buildMessage(delayMs, topic, false); + transformTimerMessage(timerMessageStore, inner); + assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(inner).getPutMessageStatus()); + } + + MessageExtBrokerInner delMsg = buildMessage(delayMs, topic, false); + transformTimerMessage(timerMessageStore, delMsg); + MessageAccessor.putProperty(delMsg, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, "XXX+1"); + delMsg.setPropertiesString(MessageDecoder.messageProperties2String(delMsg.getProperties())); + assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus()); + + // Wait until currReadTimeMs catches up current time and delayMs is over. + Thread.sleep(2000); + + for (int i = 0; i < 5; i++) { + ByteBuffer msgBuff = getOneMessage(topic, 0, i, 1000); + assertNotNull(msgBuff); + // assertThat(System.currentTimeMillis()).isLessThan(delayMs + precisionMs); + } + assertNull(getOneMessage(topic, 0, 5, 1000)); + + // Test put expired delete msg. + MessageExtBrokerInner expiredInner = buildMessage(System.currentTimeMillis() - 100, topic, false); + MessageAccessor.putProperty(expiredInner, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, "XXX"); + PutMessageResult putMessageResult = transformTimerMessage(timerMessageStore, expiredInner); + assertEquals(PutMessageStatus.WHEEL_TIMER_MSG_ILLEGAL, putMessageResult.getPutMessageStatus()); + } + + @Test + public void testExtractUniqueKey() { + String deleteKey = TimerMessageStore.buildDeleteKey("topic", "123456"); + assertEquals("123456", TimerMessageStore.extractUniqueKey(deleteKey)); + } + + private class MyMessageArrivingListener implements MessageArrivingListener { + @Override + public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime, + byte[] filterBitMap, Map properties) { + } + } + + @After + public void destroy() { + if (null != messageStore) { + messageStore.shutdown(); + messageStore.destroy(); + } + for (String baseDir : baseDirs) { + StoreTestUtils.deleteFile(baseDir); + } + } +} From 4779614198b518a76796bc065d621f534c508c9e Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 17 Jan 2025 10:51:11 +0800 Subject: [PATCH 23/61] feat: Adaptation logic --- .../java/org/apache/rocketmq/common/message/MessageConst.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java index 7d526a31c88..d90835034aa 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java @@ -95,7 +95,6 @@ public class MessageConst { public static final String PROPERTY_TIMER_ROLL_TIMES = "TIMER_ROLL_TIMES"; public static final String PROPERTY_TIMER_OUT_MS = "TIMER_OUT_MS"; public static final String PROPERTY_TIMER_DEL_UNIQKEY = "TIMER_DEL_UNIQKEY"; - public static final String PROPERTY_TIMER_DEL_MS = "TIMER_DEL_MS"; public static final String PROPERTY_TIMER_DEL_FLAG = "TIMER_DEL_FLAG"; public static final String PROPERTY_TIMER_DELAY_LEVEL = "TIMER_DELAY_LEVEL"; public static final String PROPERTY_TIMER_DELAY_MS = "TIMER_DELAY_MS"; @@ -153,7 +152,6 @@ public class MessageConst { STRING_HASH_SET.add(PROPERTY_TIMER_ROLL_TIMES); STRING_HASH_SET.add(PROPERTY_TIMER_OUT_MS); STRING_HASH_SET.add(PROPERTY_TIMER_DEL_UNIQKEY); - STRING_HASH_SET.add(PROPERTY_TIMER_DEL_MS); STRING_HASH_SET.add(PROPERTY_TIMER_DEL_FLAG); STRING_HASH_SET.add(PROPERTY_TIMER_DELAY_LEVEL); STRING_HASH_SET.add(PROPERTY_BORN_HOST); From 2d066ea91da63c6d14e5a5646942d6e622dd64b1 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 17 Jan 2025 10:58:44 +0800 Subject: [PATCH 24/61] feat: Adaptation logic --- .../broker/processor/RecallMessageProcessor.java | 1 - .../common/producer/RecallMessageHandle.java | 16 ++-------------- .../service/message/LocalMessageService.java | 1 - .../store/timer/rocksdb/TimerMessageRecord.java | 3 +++ 4 files changed, 5 insertions(+), 16 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java index e446646a4e3..372db0d36eb 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java @@ -137,7 +137,6 @@ public MessageExtBrokerInner buildMessage(ChannelHandlerContext ctx, RecallMessa msgInner.setQueueId(0); MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TIMER_DEL_UNIQKEY, TimerMessageStore.buildDeleteKey(handle.getTopic(), handle.getMessageId())); - MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TIMER_DEL_MS, String.valueOf(handle.getDelaytime())); MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, handle.getMessageId()); MessageAccessor.putProperty(msgInner, diff --git a/common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java b/common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java index 81993ef3ceb..b00b15bd863 100644 --- a/common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java +++ b/common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java @@ -40,24 +40,16 @@ public static class HandleV1 extends RecallMessageHandle { private String brokerName; private String timestampStr; private String messageId; // id of unique key - private long delaytime; - public HandleV1(String topic, String brokerName, String timestamp, String messageId, long delaytime) { + public HandleV1(String topic, String brokerName, String timestamp, String messageId) { this.version = VERSION_1; this.topic = topic; this.brokerName = brokerName; this.timestampStr = timestamp; this.messageId = messageId; - this.delaytime = delaytime; } // no param check - public static String buildHandle(String topic, String brokerName, String timestampStr, String messageId, long delaytime) { - String rawString = String.join(SEPARATOR, VERSION_1, topic, brokerName, timestampStr, messageId, String.valueOf(delaytime)); - return Base64.getUrlEncoder().encodeToString(rawString.getBytes(UTF_8)); - } - - // TODO public static String buildHandle(String topic, String brokerName, String timestampStr, String messageId) { String rawString = String.join(SEPARATOR, VERSION_1, topic, brokerName, timestampStr, messageId); return Base64.getUrlEncoder().encodeToString(rawString.getBytes(UTF_8)); @@ -82,10 +74,6 @@ public String getMessageId() { public String getVersion() { return version; } - - public long getDelaytime() { - return delaytime; - } } public static RecallMessageHandle decodeHandle(String handle) throws DecoderException { @@ -103,6 +91,6 @@ public static RecallMessageHandle decodeHandle(String handle) throws DecoderExce if (!VERSION_1.equals(items[0]) || items.length < 5) { throw new DecoderException("recall handle is invalid"); } - return new HandleV1(items[1], items[2], items[3], items[4], Long.parseLong(items[5])); + return new HandleV1(items[1], items[2], items[3], items[4]); } } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java index 40b60b63646..cb9b7a4ae00 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java @@ -156,7 +156,6 @@ public CompletableFuture> sendMessage(ProxyContext ctx, Address sendResult.setTransactionId(responseHeader.getTransactionId()); sendResult.setOffsetMsgId(responseHeader.getMsgId()); sendResult.setRecallHandle(responseHeader.getRecallHandle()); - sendResult.setDelayTime(responseHeader.getDelayTime()); return Collections.singletonList(sendResult); }); } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java index cfdc93bae42..a78e2d11860 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java @@ -126,6 +126,9 @@ public String toString() { "delayTime=" + delayTime + ", offsetPY=" + offsetPY + ", sizeReal=" + sizeReal + + ", messageExt=" + messageExt + + ", roll=" + roll + + ", uniqueKey='" + uniqueKey + '\'' + '}'; } } From 20ce52a86c62c49864ae1e187118d27c4912d365 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 17 Jan 2025 10:59:28 +0800 Subject: [PATCH 25/61] feat: Adaptation logic --- .../apache/rocketmq/broker/processor/SendMessageProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index 7c24bc065e8..a4a9f0ecfad 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -659,7 +659,7 @@ public void attachRecallHandle(RemotingCommand request, MessageExt msg, SendMess if (timestampStr != null && realTopic != null && !realTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { timestampStr = String.valueOf(Long.parseLong(timestampStr) + 1); // consider of floor String recallHandle = RecallMessageHandle.HandleV1.buildHandle(realTopic, - brokerController.getBrokerConfig().getBrokerName(), timestampStr, MessageClientIDSetter.getUniqID(msg), responseHeader.getDelayTime()); + brokerController.getBrokerConfig().getBrokerName(), timestampStr, MessageClientIDSetter.getUniqID(msg)); responseHeader.setRecallHandle(recallHandle); } From 57bd15d71f1552dba4eaff8efcc2b747f33118fd Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 17 Jan 2025 11:08:46 +0800 Subject: [PATCH 26/61] feat: Adaptation logic --- .../rocksdb/TimerMessageRocksDBStore.java | 16 +++--- .../timer/rocksdb/TimerRocksDBRequest.java | 56 ------------------- .../rocksdb/TimerMessageRocksDBStoreTest.java | 3 +- 3 files changed, 10 insertions(+), 65 deletions(-) delete mode 100644 store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerRocksDBRequest.java diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 6f90cea23af..336401707f1 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -312,7 +312,8 @@ private void fetchAndPutTimerRequest() throws Exception { if (tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY) != null) { // Construct original message tr.setDelayTime(delayTime / precisionMs % slotSize); - tr.setUniqueKey(TimerMessageStore.extractUniqueKey(tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY))); + tr.setUniqueKey(TimerMessageStore.extractUniqueKey(tr.getMessageExt(). + getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY))); delete.computeIfAbsent(delayTime, k -> new HashMap<>()).computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); } else { if (delayTime <= System.currentTimeMillis()) { @@ -321,7 +322,8 @@ private void fetchAndPutTimerRequest() throws Exception { addMetric((int) (delayTime / precisionMs % slotSize), 1); } else { tr.setDelayTime(delayTime / precisionMs % slotSize); - increase.computeIfAbsent(delayTime / precisionMs % slotSize, k -> new HashMap<>()).computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); + increase.computeIfAbsent(delayTime / precisionMs % slotSize, k -> new HashMap<>()). + computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); } } } @@ -535,13 +537,14 @@ public boolean enqueue(int queueId) { if (null != msgExt) { long delayedTime = Long.parseLong(msgExt.getProperty(TIMER_OUT_MS)); - TimerMessageRecord timerRequest = new TimerMessageRecord(delayedTime, MessageClientIDSetter.getUniqID(msgExt), offsetPy, sizePy); + TimerMessageRecord timerRequest = new TimerMessageRecord(delayedTime, + MessageClientIDSetter.getUniqID(msgExt), offsetPy, sizePy); timerRequest.setMessageExt(msgExt); while (!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) { } Attributes attributes = DefaultStoreMetricsManager.newAttributesBuilder() - .put(DefaultStoreMetricsConstant.LABEL_TOPIC, msgExt.getProperty(MessageConst.PROPERTY_REAL_TOPIC)).build(); + .put(DefaultStoreMetricsConstant.LABEL_TOPIC, msgExt.getProperty(MessageConst.PROPERTY_REAL_TOPIC)).build(); DefaultStoreMetricsManager.timerMessageSetLatency.record((delayedTime - msgExt.getBornTimestamp()) / 1000, attributes); } } catch (Exception e) { @@ -633,17 +636,14 @@ private MessageExtBrokerInner convertMessage(MessageExt msgExt, boolean needRoll msgInner.setFlag(msgExt.getFlag()); MessageAccessor.setProperties(msgInner, MessageAccessor.deepCopyProperties(msgExt.getProperties())); TopicFilterType topicFilterType = MessageExt.parseTopicFilterType(msgInner.getSysFlag()); - long tagsCodeValue = - MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags()); + long tagsCodeValue = MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags()); msgInner.setTagsCode(tagsCodeValue); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); - msgInner.setSysFlag(msgExt.getSysFlag()); msgInner.setBornTimestamp(msgExt.getBornTimestamp()); msgInner.setBornHost(msgExt.getBornHost()); msgInner.setStoreHost(msgExt.getStoreHost()); msgInner.setReconsumeTimes(msgExt.getReconsumeTimes()); - msgInner.setWaitStoreMsgOK(false); if (needRoll) { diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerRocksDBRequest.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerRocksDBRequest.java deleted file mode 100644 index 915aa710eb0..00000000000 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerRocksDBRequest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.store.timer.rocksdb; - -import org.apache.rocketmq.common.message.MessageExt; - -public class TimerRocksDBRequest { - MessageExt msgExt; - boolean needRoll; - - public TimerRocksDBRequest() { - } - - public TimerRocksDBRequest(MessageExt msgExt, boolean needRoll) { - this.msgExt = msgExt; - this.needRoll = needRoll; - } - - public MessageExt getMsgExt() { - return msgExt; - } - - public void setMsgExt(MessageExt msgExt) { - this.msgExt = msgExt; - } - - public boolean isNeedRoll() { - return needRoll; - } - - public void setNeedRoll(boolean needRoll) { - this.needRoll = needRoll; - } - - @Override - public String toString() { - return "TimerRocksDBRequest{" + - "msgExt=" + msgExt + - ", needRoll=" + needRoll + - '}'; - } -} diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 03305e91d18..9b7ddc3bdf2 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -77,7 +77,8 @@ public void setUp() throws Exception { storeConfig = new MessageStoreConfig(); storeConfig.setStorePathRootDir(baseDir); storeConfig.setStorePathCommitLog(baseDir + File.separator + "commitlog"); - messageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("TimerTest", false), new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>()); + messageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("TimerTest", + false), new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>()); storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123); bornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0); From 29e78c3eb2965d138b78d7510c2ab47d47656647 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 17 Jan 2025 13:04:29 +0800 Subject: [PATCH 27/61] fix test --- .../rocksdb/TimerMessageRocksDBStoreTest.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 9b7ddc3bdf2..91b51a35d39 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -16,8 +16,6 @@ */ package org.apache.rocketmq.store.timer.rocksdb; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicFilterType; @@ -39,7 +37,11 @@ import org.apache.rocketmq.store.timer.StoreTestUtils; import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.timer.TimerMetrics; -import org.junit.*; +import org.junit.Assert; +import org.junit.After; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; import org.rocksdb.RocksDB; import java.io.File; @@ -57,10 +59,13 @@ import static org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStore.TIMER_TOPIC; import static org.awaitility.Awaitility.await; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; public class TimerMessageRocksDBStoreTest { - private static final Log log = LogFactory.getLog(TimerMessageRocksDBStoreTest.class); MessageStore messageStore; MessageStoreConfig storeConfig; int precisionMs = 500; @@ -140,7 +145,7 @@ private PutMessageResult transformTimerMessage(TimerMessageRocksDBStore timerMes try { if (msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null) { - deliverMs = System.currentTimeMillis() + Integer.parseInt(msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC)) * 1000; + deliverMs = System.currentTimeMillis() + Integer.parseInt(msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC)) * 1000L; } else if (msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS) != null) { deliverMs = System.currentTimeMillis() + Integer.parseInt(msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS)); } else { @@ -234,7 +239,7 @@ public void testPutExpiredTimerMessage() throws Exception { timerMessageStore.load(); timerMessageStore.start(); - long delayMs = System.currentTimeMillis() - 2 * precisionMs; + long delayMs = System.currentTimeMillis() - 2L * precisionMs; for (int i = 0; i < 10; i++) { MessageExtBrokerInner inner = buildMessage(delayMs, topic, false); transformTimerMessage(timerMessageStore, inner); From b2a5edb75582b391d06de4982a6249caddbaa22b Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 17 Jan 2025 13:16:43 +0800 Subject: [PATCH 28/61] fix test --- .../store/timer/rocksdb/TimerMessageRocksDBStorage.java | 5 ++--- .../store/timer/rocksdb/TimerMessageRocksDBStore.java | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index 821b11ba1a5..4d9d5a9ab7c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -20,6 +20,8 @@ import org.apache.rocketmq.common.config.AbstractRocksDBStorage; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.store.rocksdb.RocksDBOptionsFactory; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.rocksdb.RocksDB; import org.rocksdb.WriteBatch; import org.rocksdb.ColumnFamilyDescriptor; @@ -32,9 +34,6 @@ import org.rocksdb.Slice; import org.rocksdb.RocksIterator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 336401707f1..405d395ccca 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -29,6 +29,8 @@ import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageClientIDSetter; import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; @@ -42,8 +44,6 @@ import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.timer.TimerMetrics; import org.rocksdb.RocksDB; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.nio.ByteBuffer; import java.nio.file.Paths; From 4700b83bbdd1611abe3c073ae4f2ff530ef49083 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 17 Jan 2025 13:39:19 +0800 Subject: [PATCH 29/61] fix test --- .../rocksdb/TimerMessageRocksDBStore.java | 20 ------------------- .../rocksdb/TimerMessageRocksDBStoreTest.java | 4 ++-- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 405d395ccca..d85a0445843 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -726,24 +726,4 @@ public TimerMetrics getTimerMetrics() { public long getCommitOffset() { return commitOffset; } - - public long getDequeueBehind() { - return 0; - } - - public long getEnqueueBehindMessages() { - return 0; - } - - public long getAllCongestNum() { - return 0; - } - - public long getEnqueueTps() { - return 0; - } - - public long getDequeueTps() { - return 0; - } } diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 91b51a35d39..4b334d8e19a 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -265,7 +265,7 @@ public void testDeleteTimerMessage() throws Exception { timerMessageStore.load(); timerMessageStore.start(); - long delayMs = System.currentTimeMillis() + 1000; + long delayMs = System.currentTimeMillis() + 2000; String uniqKey = null; for (int i = 0; i < 5; i++) { MessageExtBrokerInner inner = buildMessage(delayMs, topic, false); @@ -323,7 +323,6 @@ public void testPutDeleteTimerMessage() throws Exception { for (int i = 0; i < 5; i++) { ByteBuffer msgBuff = getOneMessage(topic, 0, i, 1000); assertNotNull(msgBuff); - // assertThat(System.currentTimeMillis()).isLessThan(delayMs + precisionMs); } assertNull(getOneMessage(topic, 0, 5, 1000)); @@ -332,6 +331,7 @@ public void testPutDeleteTimerMessage() throws Exception { MessageAccessor.putProperty(expiredInner, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, "XXX"); PutMessageResult putMessageResult = transformTimerMessage(timerMessageStore, expiredInner); assertEquals(PutMessageStatus.WHEEL_TIMER_MSG_ILLEGAL, putMessageResult.getPutMessageStatus()); + timerMessageStore.shutdown(); } @Test From f458c237a102c89ff2e4511f96e3277e55c39b19 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 17 Jan 2025 14:03:46 +0800 Subject: [PATCH 30/61] fix test --- .../store/timer/rocksdb/TimerMessageRocksDBStore.java | 8 ++++++++ .../store/timer/rocksdb/TimerMessageRocksDBStoreTest.java | 6 ++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index d85a0445843..6be295459e7 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -101,6 +101,7 @@ public class TimerMessageRocksDBStore { ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); private volatile long commitOffset; + private boolean allowDequeue; public TimerMessageRocksDBStore(final MessageStore messageStore, final MessageStoreConfig storeConfig, TimerMetrics timerMetrics, final BrokerStatsManager brokerStatsManager) { @@ -587,6 +588,9 @@ private MessageExt getMessageByCommitOffset(long offsetPy, int sizePy) { } private int dequeue(long checkpoint, byte[] columnFamily) throws InterruptedException { + if (!allowDequeue) { + return -1; + } if (checkpoint > System.currentTimeMillis()) { return -1; } @@ -726,4 +730,8 @@ public TimerMetrics getTimerMetrics() { public long getCommitOffset() { return commitOffset; } + + public void setAllowDequeue(boolean allowDequeue) { + this.allowDequeue = allowDequeue; + } } diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 4b334d8e19a..a46a779fb85 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -87,7 +87,8 @@ public void setUp() throws Exception { storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123); bornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0); - messageStore.load(); + messageStore.destroy(); + assertTrue(messageStore.load()); messageStore.start(); } @@ -186,8 +187,8 @@ public void testDoNormalTimer() throws Exception { timerMessageStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); timerMessageStore.load(); timerMessageStore.start(); + timerMessageStore.setAllowDequeue(false); long commitOffset = timerMessageStore.getCommitOffset(); - long curr = System.currentTimeMillis() / precisionMs * precisionMs; long delayMs = curr + 3000; for (int i = 0; i < 10; i++) { @@ -210,6 +211,7 @@ public Boolean call() { for (int i = 0; i < 10; i++) { Assert.assertEquals(5, timerMessageStore.getTimerMetrics().getTimingCount(topic + i)); } + timerMessageStore.setAllowDequeue(true); for (int i = 0; i < 10; i++) { for (int j = 0; j < 5; j++) { From 92f210a3d7259f958d6ff7c4cce65ca9eb1ac4c5 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 17 Jan 2025 14:04:08 +0800 Subject: [PATCH 31/61] fix test --- .../store/timer/rocksdb/TimerMessageRocksDBStoreTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index a46a779fb85..6c8443f95b1 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -87,7 +87,6 @@ public void setUp() throws Exception { storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123); bornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0); - messageStore.destroy(); assertTrue(messageStore.load()); messageStore.start(); } From 8f6efef9569cf366bc99c801a85f9319d11080bc Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 17 Jan 2025 14:16:49 +0800 Subject: [PATCH 32/61] fix test --- .../rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 6be295459e7..ca446d3614e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -101,7 +101,7 @@ public class TimerMessageRocksDBStore { ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); private volatile long commitOffset; - private boolean allowDequeue; + private boolean allowDequeue = true; public TimerMessageRocksDBStore(final MessageStore messageStore, final MessageStoreConfig storeConfig, TimerMetrics timerMetrics, final BrokerStatsManager brokerStatsManager) { From 4823690117a3e593ef515f6913be6f182f823630 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 17 Jan 2025 14:41:08 +0800 Subject: [PATCH 33/61] fix test --- .../rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index ca446d3614e..d7f5dbc529f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -402,6 +402,9 @@ public void run() { TimerMessageRocksDBStore.log.info(this.getServiceName() + " service start"); while (!this.isStopped() || !dequeuePutQueue.isEmpty()) { try { + if (!allowDequeue) { + continue; + } List timerMessageRecord = dequeuePutQueue.poll(100L * precisionMs / 1000, TimeUnit.MILLISECONDS); if (null == timerMessageRecord || timerMessageRecord.isEmpty()) { continue; From c40e210445f00af1c2bd28248ae2e4729bdbf48d Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 17 Jan 2025 16:00:45 +0800 Subject: [PATCH 34/61] fix test --- .../apache/rocketmq/broker/processor/AdminBrokerProcessor.java | 2 +- .../java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 85d96525935..d4a514e4357 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -2745,7 +2745,7 @@ private HashMap prepareRuntimeInfo() throws RemotingCommandExcep runtimeInfo.put("startAcceptSendRequestTimeStamp", String.valueOf(this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp())); if (this.brokerController.getMessageStoreConfig().getEnableTimerMessageOnRocksDB()) { - // TODO + // TODO metric sync } else if (this.brokerController.getMessageStoreConfig().isTimerWheelEnable()) { runtimeInfo.put("timerReadBehind", String.valueOf(this.brokerController.getMessageStore().getTimerMessageStore().getDequeueBehind())); runtimeInfo.put("timerOffsetBehind", String.valueOf(this.brokerController.getMessageStore().getTimerMessageStore().getEnqueueBehindMessages())); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java index 5a43b4dd7d1..4dc87fc1989 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java @@ -239,7 +239,7 @@ private void syncTimerMetrics() { if (masterAddrBak != null) { try { if (null != brokerController.getMessageStore().getTimerMessageRocksDBStore()) { - // TODO + // TODO metric sync } else if (null != brokerController.getMessageStore().getTimerMessageStore()) { TimerMetrics.TimerMetricsSerializeWrapper metricsSerializeWrapper = this.brokerController.getBrokerOuterAPI().getTimerMetrics(masterAddrBak); From af0f4af8c812d967b57e19bccdd382a17ee40dd2 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 20 Jan 2025 17:31:13 +0800 Subject: [PATCH 35/61] Optimize some redundant code and logic --- .../processor/SendMessageProcessor.java | 4 -- .../rocketmq/client/impl/MQClientAPIImpl.java | 3 -- .../rocketmq/client/producer/SendResult.java | 11 +--- .../timer/rocksdb/TimerMessageKVStore.java | 27 ---------- .../timer/rocksdb/TimerMessageRecord.java | 8 +-- .../rocksdb/TimerMessageRocksDBStorage.java | 52 +------------------ .../rocksdb/TimerMessageRocksDBStore.java | 7 ++- 7 files changed, 10 insertions(+), 102 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index a4a9f0ecfad..669cd5e6771 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -662,10 +662,6 @@ public void attachRecallHandle(RemotingCommand request, MessageExt msg, SendMess brokerController.getBrokerConfig().getBrokerName(), timestampStr, MessageClientIDSetter.getUniqID(msg)); responseHeader.setRecallHandle(recallHandle); } - - if (msg.getProperty(MessageConst.PROPERTY_TIMER_OUT_MS) != null) { - responseHeader.setDelayTime(Long.parseLong(msg.getProperty(MessageConst.PROPERTY_TIMER_OUT_MS))); - } } private String diskUtil() { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 10bd125ac54..114093e3502 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -857,9 +857,6 @@ protected SendResult processSendResponse( responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset()); sendResult.setTransactionId(responseHeader.getTransactionId()); sendResult.setRecallHandle(responseHeader.getRecallHandle()); - if (responseHeader.getDelayTime() != null) { - sendResult.setDelayTime(responseHeader.getDelayTime()); - } String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION); if (regionId == null || regionId.isEmpty()) { regionId = MixAll.DEFAULT_TRACE_REGION_ID; diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java b/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java index ef287f02c25..d160eb4eae9 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java @@ -30,7 +30,6 @@ public class SendResult { private boolean traceOn = true; private byte[] rawRespBody; private String recallHandle; - private long delayTime; public SendResult() { } @@ -136,18 +135,10 @@ public void setRecallHandle(String recallHandle) { this.recallHandle = recallHandle; } - public long getDelayTime() { - return delayTime; - } - - public void setDelayTime(long delayTime) { - this.delayTime = delayTime; - } - @Override public String toString() { return "SendResult [sendStatus=" + sendStatus + ", msgId=" + msgId + ", offsetMsgId=" + offsetMsgId + ", messageQueue=" + messageQueue - + ", queueOffset=" + queueOffset + ", recallHandle=" + recallHandle + ", delayTime=" + delayTime + "]"; + + ", queueOffset=" + queueOffset + ", recallHandle=" + recallHandle + "]"; } public void setRawRespBody(byte[] body) { diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java index a5e13ef3764..944737a4ac4 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java @@ -35,15 +35,6 @@ public interface TimerMessageKVStore { */ String getFilePath(); - /** - * Write the timer message records to the timer message kv store. - * @param consumerRecordList the list of timer message records to be written. - * @param offset the cq offset of the timer message records to be written. - * @param timestamp the key of the timer message metric column family. - * Default is store common timer message. - */ - void writeDefaultRecords(List consumerRecordList, long offset, int timestamp); - /** * Write the timer message records to the timer message kv store. * @param columnFamily the column family of the timer message kv store. @@ -53,14 +44,6 @@ public interface TimerMessageKVStore { */ void writeAssignRecords(byte[] columnFamily, List consumerRecordList, long offset, int timestamp); - /** - * Delete the timer message records from the timer message kv store. - * @param consumerRecordList the list of timer message records to be deleted. - * @param timestamp the key of the timer message metric column family. - * Default is delete common timer message. - */ - void deleteDefaultRecords(List consumerRecordList, int timestamp, long offset); - /** * Delete the timer message records from the timer message kv store. * @param columnFamily the column family of the timer message kv store. @@ -78,16 +61,6 @@ public interface TimerMessageKVStore { */ List scanRecords(byte[] columnFamily, long lowerTime, long upperTime, long timestamp); - /** - * Scan the expired timer message records from the timer message kv store. - * @param columnFamily the column family of the timer message kv store. - * @param lowerTime the lower time of the timer message records to be scanned. - * @param upperTime the upper time of the timer message records to be scanned. - * @param maxCount the max count of the timer message records to be return. - * @return the list of timer message records. - */ - List scanRecords(byte[] columnFamily, long lowerTime, long upperTime, int maxCount); - /** * Get the commit offset of the timer message kv store from cq. * @return the commit offset of the timer message kv store. diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java index a78e2d11860..bc8c195b83f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java @@ -22,9 +22,11 @@ import java.nio.charset.StandardCharsets; public class TimerMessageRecord { - public final static int TIMER_MESSAGE_DEFAULT_FLAG = 0; - public final static int TIMER_MESSAGE_TRANSACTION_FLAG = 1; - public final static int TIMER_MESSAGE_POP_FLAG = 2; + enum Flag { + DEFAULT, + TRANSACTION, + POP + } // key: delayTime + uniqueKey private long delayTime; diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index 4d9d5a9ab7c..333ffa10e28 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -62,6 +62,7 @@ public TimerMessageRocksDBStorage(String filePath) { super(filePath); } + @Override protected void initOptions() { this.options = RocksDBOptionsFactory.createDBOptions(); @@ -148,23 +149,6 @@ public String getFilePath() { return this.dbPath; } - @Override - public void writeDefaultRecords(List consumerRecordList, long offset, int timestamp) { - if (!consumerRecordList.isEmpty()) { - try (WriteBatch writeBatch = new WriteBatch()) { - for (TimerMessageRecord record : consumerRecordList) { - writeBatch.put(defaultCFHandle, record.getKeyBytes(), record.getValueBytes()); - } - // atomically update offset - writeBatch.put(TIMER_OFFSET_KEY, ByteBuffer.allocate(8).putLong(offset).array()); - syncMetric(timestamp / 100, consumerRecordList.size(), writeBatch); - this.db.write(writeOptions, writeBatch); - } catch (RocksDBException e) { - throw new RuntimeException("Write record error", e); - } - } - } - @Override public void writeAssignRecords(byte[] columnFamily, List consumerRecordList, long offset, int timestamp) { ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily); @@ -184,22 +168,6 @@ public void writeAssignRecords(byte[] columnFamily, List con } } - @Override - public void deleteDefaultRecords(List consumerRecordList, int timestamp, long offset) { - if (!consumerRecordList.isEmpty()) { - try (WriteBatch writeBatch = new WriteBatch()) { - for (TimerMessageRecord record : consumerRecordList) { - writeBatch.delete(defaultCFHandle, record.getKeyBytes()); - } - writeBatch.put(TIMER_OFFSET_KEY, ByteBuffer.allocate(8).putLong(offset).array()); - syncMetric(timestamp / 100, - consumerRecordList.size(), writeBatch); - this.db.write(deleteOptions, writeBatch); - } catch (RocksDBException e) { - throw new RuntimeException("Delete record error", e); - } - } - } - @Override public void deleteAssignRecords(byte[] columnFamily, List consumerRecordList, int timestamp, long offset) { ColumnFamilyHandle deleteCfHandle = getColumnFamily(columnFamily); @@ -254,24 +222,6 @@ public List scanRecords(byte[] columnFamily, long lowerTime, return records; } - @Override - public List scanRecords(byte[] columnFamily, long lowerTime, long upperTime, int maxCount) { - List records = new ArrayList<>(); - ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily); - - try (ReadOptions readOptions = new ReadOptions() - .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array())) - .setIterateUpperBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(upperTime).array())); - RocksIterator iterator = db.newIterator(cfHandle, readOptions)) { - iterator.seek(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array()); - while (iterator.isValid() && records.size() < maxCount) { - records.add(TimerMessageRecord.decode(iterator.value())); - iterator.next(); - } - } - return records; - } - @Override public long getCommitOffset() { try { diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index d7f5dbc529f..db32805ccf0 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -57,8 +57,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import static org.apache.rocketmq.store.timer.rocksdb.TimerMessageRecord.TIMER_MESSAGE_POP_FLAG; -import static org.apache.rocketmq.store.timer.rocksdb.TimerMessageRecord.TIMER_MESSAGE_TRANSACTION_FLAG; import static org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStorage.POP_COLUMN_FAMILY; import static org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStorage.TRANSACTION_COLUMN_FAMILY; @@ -227,9 +225,10 @@ private String getServiceThreadName() { } private byte[] getColumnFamily(int flag) { - if (flag == TIMER_MESSAGE_TRANSACTION_FLAG) { + TimerMessageRecord.Flag tag = TimerMessageRecord.Flag.valueOf(String.valueOf(flag)); + if (TimerMessageRecord.Flag.TRANSACTION == tag) { return TRANSACTION_COLUMN_FAMILY; - } else if (flag == TIMER_MESSAGE_POP_FLAG) { + } else if (tag == TimerMessageRecord.Flag.POP) { return POP_COLUMN_FAMILY; } else { return RocksDB.DEFAULT_COLUMN_FAMILY; From 1b5c4b0c74c985ab66a86620cc52e2265e2523ac Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 20 Jan 2025 18:30:35 +0800 Subject: [PATCH 36/61] Move the new class to TimerMessageStore --- .../rocketmq/broker/BrokerController.java | 33 +------------- .../processor/AdminBrokerProcessor.java | 7 ++- .../rocketmq/broker/util/HookUtils.java | 5 +-- .../rocketmq/store/DefaultMessageStore.java | 15 ------- .../apache/rocketmq/store/MessageStore.java | 5 --- .../metrics/DefaultStoreMetricsManager.java | 4 +- .../plugin/AbstractPluginMessageStore.java | 11 ----- .../store/timer/TimerMessageStore.java | 11 ++++- .../rocksdb/TimerMessageRocksDBStore.java | 43 ++++++++++++------- 9 files changed, 45 insertions(+), 89 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 226f5adc291..db4a3545f06 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -181,9 +181,6 @@ import org.apache.rocketmq.store.timer.TimerCheckpoint; import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.timer.TimerMetrics; -import org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStorage; -import org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStore; -import org.rocksdb.RocksDB; public class BrokerController { protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -275,7 +272,6 @@ public class BrokerController { private BrokerStats brokerStats; private InetSocketAddress storeHost; private TimerMessageStore timerMessageStore; - private TimerMessageRocksDBStore timerMessageRocksDBStore; private TimerCheckpoint timerCheckpoint; protected BrokerFastFailure brokerFastFailure; private Configuration configuration; @@ -836,22 +832,11 @@ public boolean initializeMessageStore() { this.messageStore = MessageStoreFactory.build(context, defaultMessageStore); this.messageStore.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(this.brokerConfig, this.consumerFilterManager)); - TimerMetrics timerMetrics = new TimerMetrics(BrokerPathConfigHelper.getTimerMetricsPath(messageStoreConfig.getStorePathRootDir())); if (messageStoreConfig.isTimerWheelEnable()) { this.timerCheckpoint = new TimerCheckpoint(BrokerPathConfigHelper.getTimerCheckPath(messageStoreConfig.getStorePathRootDir())); + TimerMetrics timerMetrics = new TimerMetrics(BrokerPathConfigHelper.getTimerMetricsPath(messageStoreConfig.getStorePathRootDir())); this.timerMessageStore = new TimerMessageStore(messageStore, messageStoreConfig, timerCheckpoint, timerMetrics, brokerStatsManager); - this.timerMessageStore.registerEscapeBridgeHook(msg -> escapeBridge.putMessage(msg)); - this.messageStore.setTimerMessageStore(this.timerMessageStore); - } - if (messageStoreConfig.getEnableTimerMessageOnRocksDB()) { - this.timerMessageRocksDBStore = new TimerMessageRocksDBStore(messageStore, messageStoreConfig, timerMetrics, brokerStatsManager); - if (this.messageStoreConfig.isTimerWheelEnable()) { - this.messageStoreConfig.setTimerStopEnqueue(true); - } - this.messageStore.setTimerMessageRocksDBStore(this.timerMessageRocksDBStore); - this.timerMessageRocksDBStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); - this.timerMessageRocksDBStore.createTimer(TimerMessageRocksDBStorage.POP_COLUMN_FAMILY); - this.timerMessageRocksDBStore.createTimer(TimerMessageRocksDBStorage.TRANSACTION_COLUMN_FAMILY); + this.timerMessageStore.registerEscapeBridgeHook(msg -> escapeBridge.putMessage(msg));this.messageStore.setTimerMessageStore(this.timerMessageStore); } } catch (Exception e) { result = false; @@ -889,9 +874,6 @@ public boolean recoverAndInitService() throws CloneNotSupportedException { result = this.messageStore.load(); } - if (messageStoreConfig.getEnableTimerMessageOnRocksDB()) { - result = result && this.timerMessageRocksDBStore.load(); - } if (messageStoreConfig.isTimerWheelEnable()) { result = result && this.timerMessageStore.load(); } @@ -1472,9 +1454,6 @@ protected void shutdownBasicService() { if (this.timerMessageStore != null) { this.timerMessageStore.shutdown(); } - if (this.timerMessageRocksDBStore != null) { - this.timerMessageRocksDBStore.shutdown(); - } if (this.fileWatchService != null) { this.fileWatchService.shutdown(); } @@ -1671,10 +1650,6 @@ protected void startBasicService() throws Exception { this.timerMessageStore.start(); } - if (this.timerMessageRocksDBStore != null) { - this.timerMessageRocksDBStore.start(); - } - if (this.replicasManager != null) { this.replicasManager.start(); } @@ -2602,8 +2577,4 @@ public ColdDataCgCtrService getColdDataCgCtrService() { public void setColdDataCgCtrService(ColdDataCgCtrService coldDataCgCtrService) { this.coldDataCgCtrService = coldDataCgCtrService; } - - public TimerMessageRocksDBStore getTimerMessageRocksDBStore() { - return timerMessageRocksDBStore; - } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index d4a514e4357..2496918b493 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -2674,8 +2674,9 @@ private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx, group, topic, i); - if (consumerOffset < 0) + if (consumerOffset < 0) { consumerOffset = 0; + } offsetWrapper.setBrokerOffset(brokerOffset); offsetWrapper.setConsumerOffset(consumerOffset); @@ -2744,9 +2745,7 @@ private HashMap prepareRuntimeInfo() throws RemotingCommandExcep runtimeInfo.put("earliestMessageTimeStamp", String.valueOf(this.brokerController.getMessageStore().getEarliestMessageTime())); runtimeInfo.put("startAcceptSendRequestTimeStamp", String.valueOf(this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp())); - if (this.brokerController.getMessageStoreConfig().getEnableTimerMessageOnRocksDB()) { - // TODO metric sync - } else if (this.brokerController.getMessageStoreConfig().isTimerWheelEnable()) { + if (this.brokerController.getMessageStoreConfig().isTimerWheelEnable()) { runtimeInfo.put("timerReadBehind", String.valueOf(this.brokerController.getMessageStore().getTimerMessageStore().getDequeueBehind())); runtimeInfo.put("timerOffsetBehind", String.valueOf(this.brokerController.getMessageStore().getTimerMessageStore().getEnqueueBehindMessages())); runtimeInfo.put("timerCongestNum", String.valueOf(this.brokerController.getMessageStore().getTimerMessageStore().getAllCongestNum())); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/util/HookUtils.java b/broker/src/main/java/org/apache/rocketmq/broker/util/HookUtils.java index a264b306a65..dec42351d9f 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/util/HookUtils.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/util/HookUtils.java @@ -133,8 +133,7 @@ public static PutMessageResult handleScheduleMessage(BrokerController brokerCont || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) { if (!isRolledTimerMessage(msg)) { if (checkIfTimerMessage(msg)) { - if (!brokerController.getMessageStoreConfig().isTimerWheelEnable() && !brokerController.getMessageStoreConfig() - .getEnableTimerMessageOnRocksDB()) { + if (!brokerController.getMessageStoreConfig().isTimerWheelEnable()) { //wheel timer is not enabled, reject the message return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_NOT_ENABLE, null); } @@ -205,7 +204,7 @@ private static PutMessageResult transformTimerMessage(BrokerController brokerCon deliverMs = deliverMs / timerPrecisionMs * timerPrecisionMs; } - if (brokerController.getTimerMessageRocksDBStore() == null && brokerController.getTimerMessageStore().isReject(deliverMs)) { + if (brokerController.getTimerMessageStore().isReject(deliverMs)) { return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_FLOW_CONTROL, null); } MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TIMER_OUT_MS, deliverMs + ""); diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 3e5a9186a1f..187a0729e83 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -113,7 +113,6 @@ import org.apache.rocketmq.store.queue.ReferredIterator; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.timer.TimerMessageStore; -import org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStore; import org.apache.rocketmq.store.util.PerfCounter; import org.rocksdb.RocksDBException; @@ -167,7 +166,6 @@ public class DefaultMessageStore implements MessageStore { protected StoreCheckpoint storeCheckpoint; private TimerMessageStore timerMessageStore; - private TimerMessageRocksDBStore timerMessageRocksDBStore; private final LinkedList dispatcherList; @@ -1030,16 +1028,6 @@ public void setTimerMessageStore(TimerMessageStore timerMessageStore) { this.timerMessageStore = timerMessageStore; } - @Override - public TimerMessageRocksDBStore getTimerMessageRocksDBStore() { - return this.timerMessageRocksDBStore; - } - - @Override - public void setTimerMessageRocksDBStore(TimerMessageRocksDBStore timerMessageRocksDBStore) { - this.timerMessageRocksDBStore = timerMessageRocksDBStore; - } - @Override public long getCommitLogOffsetInQueue(String topic, int queueId, long consumeQueueOffset) { ConsumeQueueInterface consumeQueue = getConsumeQueue(topic, queueId); @@ -1933,9 +1921,6 @@ private void recover(final boolean lastExitOK) throws RocksDBException { @Override public long getTimingMessageCount(String topic) { - if (timerMessageRocksDBStore != null) { - return timerMessageRocksDBStore.getTimerMetrics().getTimingCount(topic); - } if (null == timerMessageStore) { return 0L; } else { diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java index e36d029ef21..4bbee142a17 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java @@ -41,7 +41,6 @@ import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.timer.TimerMessageStore; -import org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStore; import org.apache.rocketmq.store.util.PerfCounter; import org.rocksdb.RocksDBException; import io.opentelemetry.api.common.AttributesBuilder; @@ -209,10 +208,6 @@ CompletableFuture getMessageAsync(final String group, final St void setTimerMessageStore(TimerMessageStore timerMessageStore); - TimerMessageRocksDBStore getTimerMessageRocksDBStore(); - - void setTimerMessageRocksDBStore(TimerMessageRocksDBStore timerMessageRocksDBStore); - /** * Get the offset of the message in the commit log, which is also known as physical offset. * diff --git a/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java b/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java index f385b521e0a..db4c7bb7662 100644 --- a/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java +++ b/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java @@ -146,9 +146,7 @@ public static void init(Meter meter, Supplier attributesBuild measurement.record(System.currentTimeMillis() - earliestMessageTime, newAttributesBuilder().build()); }); - if (messageStore.getMessageStoreConfig().getEnableTimerMessageOnRocksDB()) { - // TODO add timer metrics - } else if (messageStore.getMessageStoreConfig().isTimerWheelEnable()) { + if (messageStore.getMessageStoreConfig().isTimerWheelEnable()) { timerEnqueueLag = meter.gaugeBuilder(GAUGE_TIMER_ENQUEUE_LAG) .setDescription("Timer enqueue messages lag") .ofLongs() diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java index 49361288899..d5d6236458e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java @@ -55,7 +55,6 @@ import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.timer.TimerMessageStore; -import org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStore; import org.apache.rocketmq.store.util.PerfCounter; import org.rocksdb.RocksDBException; @@ -629,16 +628,6 @@ public void setTimerMessageStore(TimerMessageStore timerMessageStore) { next.setTimerMessageStore(timerMessageStore); } - @Override - public TimerMessageRocksDBStore getTimerMessageRocksDBStore() { - return next.getTimerMessageRocksDBStore(); - } - - @Override - public void setTimerMessageRocksDBStore(TimerMessageRocksDBStore timerMessageRocksDBStore) { - next.setTimerMessageRocksDBStore(timerMessageRocksDBStore); - } - @Override public long getTimingMessageCount(String topic) { return next.getTimingMessageCount(topic); diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java index 4305de7d8bc..3c3147d86b3 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java @@ -71,6 +71,7 @@ import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.store.queue.ReferredIterator; import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStore; import org.apache.rocketmq.store.util.PerfCounter; public class TimerMessageStore { @@ -158,6 +159,7 @@ public class TimerMessageStore { protected volatile boolean shouldRunningDequeue; private final BrokerStatsManager brokerStatsManager; private Function escapeBridgeHook; + private TimerMessageRocksDBStore timerRocksDBStore; public TimerMessageStore(final MessageStore messageStore, final MessageStoreConfig storeConfig, TimerCheckpoint timerCheckpoint, TimerMetrics timerMetrics, @@ -212,6 +214,7 @@ protected ByteBuffer initialValue() { dequeuePutQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY); } this.brokerStatsManager = brokerStatsManager; + this.timerRocksDBStore = new TimerMessageRocksDBStore(messageStore, storeConfig, timerMetrics, brokerStatsManager); } public void initService() { @@ -238,6 +241,7 @@ public boolean load() { this.initService(); boolean load = timerLog.load(); load = load && this.timerMetrics.load(); + load = load && timerRocksDBStore.load(); recover(); calcTimerDistribution(); return load; @@ -464,6 +468,7 @@ public static boolean isMagicOK(int magic) { } public void start() { + timerRocksDBStore.start(); this.shouldStartTime = storeConfig.getDisappearTimeAfterStart() + System.currentTimeMillis(); maybeMoveWriteTime(); enqueueGetService.start(); @@ -552,6 +557,7 @@ public void shutdown() { timerWheel.shutdown(false); this.scheduler.shutdown(); + this.timerRocksDBStore.shutdown(); UtilAll.cleanBuffer(this.bufferLocal.get()); this.bufferLocal.remove(); } @@ -653,7 +659,7 @@ public void holdMomentForUnknownError() { } public boolean enqueue(int queueId) { - if (storeConfig.isTimerStopEnqueue()) { + if (storeConfig.isTimerStopEnqueue() || storeConfig.getEnableTimerMessageOnRocksDB()) { return false; } if (!isRunningEnqueue()) { @@ -1735,6 +1741,9 @@ public long getCongestNum(long deliverTimeMs) { } public boolean isReject(long deliverTimeMs) { + if (storeConfig.getEnableTimerMessageOnRocksDB()) { + return true; + } long congestNum = timerWheel.getNum(deliverTimeMs); if (congestNum <= storeConfig.getTimerCongestNumEachSlot()) { return false; diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index db32805ccf0..6c9c22b15fa 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -88,9 +88,9 @@ public class TimerMessageRocksDBStore { private TimerEnqueueGetService timerEnqueueGetService; private TimerEnqueuePutService timerEnqueuePutService; - private TimerDequeueGetService timerDequeueGetService; private List timerGetMessageServices; private List timerWarmServices; + private TimerDequeueGetService[] timerDequeueGetServices; private TimerDequeuePutService[] timerDequeuePutServices; private BlockingQueue enqueuePutQueue; @@ -99,7 +99,6 @@ public class TimerMessageRocksDBStore { ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); private volatile long commitOffset; - private boolean allowDequeue = true; public TimerMessageRocksDBStore(final MessageStore messageStore, final MessageStoreConfig storeConfig, TimerMetrics timerMetrics, final BrokerStatsManager brokerStatsManager) { @@ -133,8 +132,10 @@ public void start() { this.commitOffset = timerMessageKVStore.getCommitOffset(); this.timerEnqueueGetService.start(); this.timerEnqueuePutService.start(); - this.timerDequeueGetService.start(); + for (TimerDequeueGetService timerDequeueGetService : timerDequeueGetServices) { + timerDequeueGetService.start(); + } for (TimerWarmService timerWarmService : timerWarmServices) { timerWarmService.start(); } @@ -161,8 +162,10 @@ public void shutdown() { this.timerEnqueueGetService.shutdown(); this.timerEnqueuePutService.shutdown(); - this.timerDequeueGetService.shutdown(); + for (TimerDequeueGetService timerDequeueGetService : timerDequeueGetServices) { + timerDequeueGetService.shutdown(); + } for (TimerWarmService timerWarmService : timerWarmServices) { timerWarmService.shutdown(); } @@ -179,19 +182,29 @@ public void shutdown() { } public void createTimer(byte[] columnFamily) { - this.timerGetMessageServices.add(new TimerGetMessageService(columnFamily)); + for (int i = 0; i < storeConfig.getTimerGetMessageThreadNum(); i++) { + this.timerGetMessageServices.add(new TimerGetMessageService(columnFamily)); + } this.timerWarmServices.add(new TimerWarmService(columnFamily)); } // ---------------------------------------------------------------------------------------------------------------- private void initService() { + createTimer(TRANSACTION_COLUMN_FAMILY); + createTimer(POP_COLUMN_FAMILY); + createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); this.timerEnqueueGetService = new TimerEnqueueGetService(); this.timerEnqueuePutService = new TimerEnqueuePutService(); - this.timerDequeueGetService = new TimerDequeueGetService(); + int getThreadNum = Math.max(storeConfig.getTimerGetMessageThreadNum(), 1); - this.timerDequeuePutServices = new TimerDequeuePutService[getThreadNum]; + int putThreadNum = Math.max(storeConfig.getTimerPutMessageThreadNum(), 1); + this.timerDequeuePutServices = new TimerDequeuePutService[putThreadNum]; + this.timerDequeueGetServices = new TimerDequeueGetService[getThreadNum]; for (int i = 0; i < timerDequeuePutServices.length; i++) { timerDequeuePutServices[i] = new TimerDequeuePutService(); } + for (int i = 0; i < timerDequeueGetServices.length; i++) { + timerDequeueGetServices[i] = new TimerDequeueGetService(); + } if (storeConfig.isTimerEnableDisruptor()) { this.enqueuePutQueue = new DisruptorBlockingQueue<>(DEFAULT_CAPACITY); @@ -205,6 +218,7 @@ private void initService() { } private void calcTimerDistribution() { + // TODO fix int slotNumber = precisionMs / 100; int rocksdbNumber = 0; for (int i = 0; i < this.slotSize; i++) { @@ -296,7 +310,7 @@ private List fetchTimerMessageRecord() throws InterruptedExc return trs; } - private void fetchAndPutTimerRequest() throws Exception { + private void fetchAndPutTimerRequest() throws InterruptedException { Map>> increase = new HashMap<>(); Map>> delete = new HashMap<>(); List trs = fetchTimerMessageRecord(); @@ -401,9 +415,6 @@ public void run() { TimerMessageRocksDBStore.log.info(this.getServiceName() + " service start"); while (!this.isStopped() || !dequeuePutQueue.isEmpty()) { try { - if (!allowDequeue) { - continue; - } List timerMessageRecord = dequeuePutQueue.poll(100L * precisionMs / 1000, TimeUnit.MILLISECONDS); if (null == timerMessageRecord || timerMessageRecord.isEmpty()) { continue; @@ -512,6 +523,10 @@ public void run() { } // ----------------------------------------------------------------------------------------------------------------- public boolean enqueue(int queueId) { + if (!storeConfig.getEnableTimerMessageOnRocksDB() || storeConfig.isTimerStopEnqueue()) { + return false; + } + ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); if (null == cq) { return false; @@ -590,7 +605,7 @@ private MessageExt getMessageByCommitOffset(long offsetPy, int sizePy) { } private int dequeue(long checkpoint, byte[] columnFamily) throws InterruptedException { - if (!allowDequeue) { + if (storeConfig.isTimerStopDequeue()) { return -1; } if (checkpoint > System.currentTimeMillis()) { @@ -732,8 +747,4 @@ public TimerMetrics getTimerMetrics() { public long getCommitOffset() { return commitOffset; } - - public void setAllowDequeue(boolean allowDequeue) { - this.allowDequeue = allowDequeue; - } } From 42d432a95cba183e3c04e0744eaff62ea1cf12a7 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 20 Jan 2025 18:37:29 +0800 Subject: [PATCH 37/61] Move the new class to TimerMessageStore --- .../store/timer/rocksdb/TimerMessageRocksDBStoreTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 6c8443f95b1..05bea0da242 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -186,7 +186,6 @@ public void testDoNormalTimer() throws Exception { timerMessageStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); timerMessageStore.load(); timerMessageStore.start(); - timerMessageStore.setAllowDequeue(false); long commitOffset = timerMessageStore.getCommitOffset(); long curr = System.currentTimeMillis() / precisionMs * precisionMs; long delayMs = curr + 3000; @@ -210,7 +209,6 @@ public Boolean call() { for (int i = 0; i < 10; i++) { Assert.assertEquals(5, timerMessageStore.getTimerMetrics().getTimingCount(topic + i)); } - timerMessageStore.setAllowDequeue(true); for (int i = 0; i < 10; i++) { for (int j = 0; j < 5; j++) { From 3c4e922fded4569d80de8534ac4540397d993b13 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 20 Jan 2025 18:49:17 +0800 Subject: [PATCH 38/61] Move the new class to TimerMessageStore --- .../apache/rocketmq/store/timer/TimerMessageStore.java | 2 +- .../store/timer/rocksdb/TimerMessageRocksDBStore.java | 9 ++++++++- .../timer/rocksdb/TimerMessageRocksDBStoreTest.java | 3 +++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java index 3c3147d86b3..20b65ee01d3 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java @@ -1742,7 +1742,7 @@ public long getCongestNum(long deliverTimeMs) { public boolean isReject(long deliverTimeMs) { if (storeConfig.getEnableTimerMessageOnRocksDB()) { - return true; + return false; } long congestNum = timerWheel.getNum(deliverTimeMs); if (congestNum <= storeConfig.getTimerCongestNumEachSlot()) { diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 6c9c22b15fa..612452abd55 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -239,7 +239,14 @@ private String getServiceThreadName() { } private byte[] getColumnFamily(int flag) { - TimerMessageRecord.Flag tag = TimerMessageRecord.Flag.valueOf(String.valueOf(flag)); + TimerMessageRecord.Flag tag; + switch (flag) { + case 1 : tag = TimerMessageRecord.Flag.TRANSACTION; + break; + case 2 : tag = TimerMessageRecord.Flag.POP; + break; + default : tag = TimerMessageRecord.Flag.DEFAULT; + } if (TimerMessageRecord.Flag.TRANSACTION == tag) { return TRANSACTION_COLUMN_FAMILY; } else if (tag == TimerMessageRecord.Flag.POP) { diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 05bea0da242..76a488a5319 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -80,6 +80,8 @@ public void setUp() throws Exception { String baseDir = StoreTestUtils.createBaseDir(); this.baseDirs.add(baseDir); storeConfig = new MessageStoreConfig(); + storeConfig.isEnableRocksDBStore(); + storeConfig.setEnableTimerMessageOnRocksDB(true); storeConfig.setStorePathRootDir(baseDir); storeConfig.setStorePathCommitLog(baseDir + File.separator + "commitlog"); messageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("TimerTest", @@ -180,6 +182,7 @@ private PutMessageResult transformTimerMessage(TimerMessageRocksDBStore timerMes @Test public void testDoNormalTimer() throws Exception { + Assume.assumeFalse(MixAll.isWindows()); String topic = "TimerTest_testPutTimerMessage"; final TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); From 1a1252e4fda76ff9cadcdee63dfbc6758d54defc Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 20 Jan 2025 18:55:24 +0800 Subject: [PATCH 39/61] Move the new class to TimerMessageStore --- .../apache/rocketmq/broker/BrokerController.java | 4 ++-- .../rocketmq/broker/slave/SlaveSynchronize.java | 4 +--- .../header/SendMessageResponseHeader.java | 15 --------------- .../rocketmq/store/config/MessageStoreConfig.java | 5 +---- 4 files changed, 4 insertions(+), 24 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index db4a3545f06..fa60d87e624 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -831,12 +831,12 @@ public boolean initializeMessageStore() { messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig, configuration); this.messageStore = MessageStoreFactory.build(context, defaultMessageStore); this.messageStore.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(this.brokerConfig, this.consumerFilterManager)); - if (messageStoreConfig.isTimerWheelEnable()) { this.timerCheckpoint = new TimerCheckpoint(BrokerPathConfigHelper.getTimerCheckPath(messageStoreConfig.getStorePathRootDir())); TimerMetrics timerMetrics = new TimerMetrics(BrokerPathConfigHelper.getTimerMetricsPath(messageStoreConfig.getStorePathRootDir())); this.timerMessageStore = new TimerMessageStore(messageStore, messageStoreConfig, timerCheckpoint, timerMetrics, brokerStatsManager); - this.timerMessageStore.registerEscapeBridgeHook(msg -> escapeBridge.putMessage(msg));this.messageStore.setTimerMessageStore(this.timerMessageStore); + this.timerMessageStore.registerEscapeBridgeHook(msg -> escapeBridge.putMessage(msg)); + this.messageStore.setTimerMessageStore(this.timerMessageStore); } } catch (Exception e) { result = false; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java index 4dc87fc1989..bfb5c9dcd03 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java @@ -238,9 +238,7 @@ private void syncTimerMetrics() { String masterAddrBak = this.masterAddr; if (masterAddrBak != null) { try { - if (null != brokerController.getMessageStore().getTimerMessageRocksDBStore()) { - // TODO metric sync - } else if (null != brokerController.getMessageStore().getTimerMessageStore()) { + if (null != brokerController.getMessageStore().getTimerMessageStore()) { TimerMetrics.TimerMetricsSerializeWrapper metricsSerializeWrapper = this.brokerController.getBrokerOuterAPI().getTimerMetrics(masterAddrBak); if (!brokerController.getMessageStore().getTimerMessageStore().getTimerMetrics().getDataVersion().equals(metricsSerializeWrapper.getDataVersion())) { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java index 7351172f84a..7563b910331 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java @@ -37,7 +37,6 @@ public class SendMessageResponseHeader implements CommandCustomHeader, FastCodes private String transactionId; private String batchUniqId; private String recallHandle; - private Long delayTime; @Override public void checkFields() throws RemotingCommandException { @@ -51,7 +50,6 @@ public void encode(ByteBuf out) { writeIfNotNull(out, "transactionId", transactionId); writeIfNotNull(out, "batchUniqId", batchUniqId); writeIfNotNull(out, "recallHandle", recallHandle); - writeIfNotNull(out, "delayTime", delayTime); } @Override @@ -85,11 +83,6 @@ public void decode(HashMap fields) throws RemotingCommandExcepti if (str != null) { this.recallHandle = str; } - - str = fields.get("delayTime"); - if (str != null) { - this.delayTime = Long.parseLong(str); - } } public String getMsgId() { @@ -139,12 +132,4 @@ public String getRecallHandle() { public void setRecallHandle(String recallHandle) { this.recallHandle = recallHandle; } - - public Long getDelayTime() { - return delayTime; - } - - public void setDelayTime(Long delayTime) { - this.delayTime = delayTime; - } } diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 73e90a0d055..a6d15cdd61f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -479,10 +479,7 @@ public void setRocksdbCompressionType(String compressionType) { private int readCountTimerOnRocksDB = -1; /** - * When enabled, the scheduled task is started. - * if time wheel is enabled, the time wheel only the correct ones are supported - * The message will be written to rocksdb. - * Close the time wheel when the file timing message is 0 + * When enabled, the scheduled message is using rocksdb */ private boolean enableTimerMessageOnRocksDB = false; From 27e0d3216ba7132fdd6cb7290c3c327a0ec11ce1 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 20 Jan 2025 20:19:08 +0800 Subject: [PATCH 40/61] Move the new class to TimerMessageStore --- .../store/timer/rocksdb/TimerMessageRecord.java | 2 +- .../store/timer/rocksdb/TimerMessageRocksDBStore.java | 2 ++ .../timer/rocksdb/TimerMessageRocksDBStoreTest.java | 10 ++++------ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java index bc8c195b83f..899baab3b1c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java @@ -53,7 +53,7 @@ public byte[] getKeyBytes() { int keyLength = Long.BYTES + value.length; byte[] keyBytes = new byte[keyLength]; ByteBuffer buffer = ByteBuffer.wrap(keyBytes); - buffer.putLong(this.getDelayTime()); + buffer.putLong(delayTime); buffer.put(value); return keyBytes; } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 612452abd55..602f0348adc 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -179,6 +179,8 @@ public void shutdown() { this.dequeueGetQueue.clear(); this.enqueuePutQueue.clear(); this.dequeuePutQueue.clear(); + this.scheduledExecutorService.shutdown(); + this.timerMessageKVStore.shutdown(); } public void createTimer(byte[] columnFamily) { diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 76a488a5319..7487bba14d2 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -74,13 +74,11 @@ public class TimerMessageRocksDBStoreTest { private SocketAddress storeHost; Set baseDirs = new HashSet<>(); - @Before public void setUp() throws Exception { String baseDir = StoreTestUtils.createBaseDir(); this.baseDirs.add(baseDir); storeConfig = new MessageStoreConfig(); - storeConfig.isEnableRocksDBStore(); storeConfig.setEnableTimerMessageOnRocksDB(true); storeConfig.setStorePathRootDir(baseDir); storeConfig.setStorePathCommitLog(baseDir + File.separator + "commitlog"); @@ -183,7 +181,7 @@ private PutMessageResult transformTimerMessage(TimerMessageRocksDBStore timerMes @Test public void testDoNormalTimer() throws Exception { Assume.assumeFalse(MixAll.isWindows()); - String topic = "TimerTest_testPutTimerMessage"; + String topic = "TimerRocksdbTest_testPutTimerMessage"; final TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); timerMessageStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); @@ -234,7 +232,7 @@ public void testPutExpiredTimerMessage() throws Exception { Assume.assumeFalse(MixAll.isMac()); Assume.assumeFalse(MixAll.isWindows()); - String topic = "TimerTest_testPutExpiredTimerMessage"; + String topic = "TimerRocksdbTest_testPutExpiredTimerMessage"; TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); timerMessageStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); @@ -260,7 +258,7 @@ public void testPutExpiredTimerMessage() throws Exception { @Test public void testDeleteTimerMessage() throws Exception { - String topic = "TimerTest_testDeleteTimerMessage"; + String topic = "TimerRocksdbTest_testDeleteTimerMessage"; TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); timerMessageStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); @@ -298,7 +296,7 @@ public void testDeleteTimerMessage() throws Exception { @Test public void testPutDeleteTimerMessage() throws Exception { - String topic = "TimerTest_testPutDeleteTimerMessage"; + String topic = "TimerRocksdbTest_testPutDeleteTimerMessage"; final TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); timerMessageStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); From 28acd4ba593a8e329b7ccf59269ec69095b313a9 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 20 Jan 2025 20:27:41 +0800 Subject: [PATCH 41/61] Move the new class to TimerMessageStore --- .../store/timer/rocksdb/TimerMessageRocksDBStoreTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 7487bba14d2..5d57a41cb17 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -265,7 +265,7 @@ public void testDeleteTimerMessage() throws Exception { timerMessageStore.load(); timerMessageStore.start(); - long delayMs = System.currentTimeMillis() + 2000; + long delayMs = System.currentTimeMillis() + 4000; String uniqKey = null; for (int i = 0; i < 5; i++) { MessageExtBrokerInner inner = buildMessage(delayMs, topic, false); @@ -281,6 +281,7 @@ public void testDeleteTimerMessage() throws Exception { MessageAccessor.putProperty(delMsg, MessageConst.PROPERTY_TIMER_DEL_UNIQKEY, TimerMessageStore.buildDeleteKey(topic, uniqKey)); delMsg.setPropertiesString(MessageDecoder.messageProperties2String(delMsg.getProperties())); assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus()); + Thread.sleep(1000); // The first one should have been deleted. ByteBuffer msgBuff = getOneMessage(topic, 0, 0, 3000); From 0d445e8f71304c2c5a3a0c6569cdcdad82515adb Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 20 Jan 2025 20:47:57 +0800 Subject: [PATCH 42/61] fix fail test --- .../store/timer/rocksdb/TimerMessageRocksDBStoreTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 5d57a41cb17..bda58165c9c 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -325,7 +325,6 @@ public void testPutDeleteTimerMessage() throws Exception { ByteBuffer msgBuff = getOneMessage(topic, 0, i, 1000); assertNotNull(msgBuff); } - assertNull(getOneMessage(topic, 0, 5, 1000)); // Test put expired delete msg. MessageExtBrokerInner expiredInner = buildMessage(System.currentTimeMillis() - 100, topic, false); From 4842454ebaf58c1fdc2f1d5c40d025701d375001 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 20 Jan 2025 21:20:57 +0800 Subject: [PATCH 43/61] fix fail test --- .../store/timer/rocksdb/TimerMessageRocksDBStore.java | 1 - .../store/timer/rocksdb/TimerMessageRocksDBStoreTest.java | 8 ++------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 602f0348adc..502c84ba3c0 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -342,7 +342,6 @@ private void fetchAndPutTimerRequest() throws InterruptedException { if (delayTime <= System.currentTimeMillis()) { expired.add(tr); addMetric(tr.getMessageExt(), 1); - addMetric((int) (delayTime / precisionMs % slotSize), 1); } else { tr.setDelayTime(delayTime / precisionMs % slotSize); increase.computeIfAbsent(delayTime / precisionMs % slotSize, k -> new HashMap<>()). diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index bda58165c9c..babbc03a466 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -265,7 +265,7 @@ public void testDeleteTimerMessage() throws Exception { timerMessageStore.load(); timerMessageStore.start(); - long delayMs = System.currentTimeMillis() + 4000; + long delayMs = System.currentTimeMillis() + 2000; String uniqKey = null; for (int i = 0; i < 5; i++) { MessageExtBrokerInner inner = buildMessage(delayMs, topic, false); @@ -281,7 +281,6 @@ public void testDeleteTimerMessage() throws Exception { MessageAccessor.putProperty(delMsg, MessageConst.PROPERTY_TIMER_DEL_UNIQKEY, TimerMessageStore.buildDeleteKey(topic, uniqKey)); delMsg.setPropertiesString(MessageDecoder.messageProperties2String(delMsg.getProperties())); assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus()); - Thread.sleep(1000); // The first one should have been deleted. ByteBuffer msgBuff = getOneMessage(topic, 0, 0, 3000); @@ -305,7 +304,7 @@ public void testPutDeleteTimerMessage() throws Exception { timerMessageStore.start(); long curr = System.currentTimeMillis() / precisionMs * precisionMs; - final long delayMs = curr + 2000; + final long delayMs = curr + 1000; for (int i = 0; i < 5; i++) { MessageExtBrokerInner inner = buildMessage(delayMs, topic, false); transformTimerMessage(timerMessageStore, inner); @@ -318,9 +317,6 @@ public void testPutDeleteTimerMessage() throws Exception { delMsg.setPropertiesString(MessageDecoder.messageProperties2String(delMsg.getProperties())); assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus()); - // Wait until currReadTimeMs catches up current time and delayMs is over. - Thread.sleep(2000); - for (int i = 0; i < 5; i++) { ByteBuffer msgBuff = getOneMessage(topic, 0, i, 1000); assertNotNull(msgBuff); From e29c0c8487fa751875eed62a10c0f8adda99f1c9 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 20 Jan 2025 21:45:22 +0800 Subject: [PATCH 44/61] fix fail test --- .../rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java | 2 +- .../store/timer/rocksdb/TimerMessageRocksDBStoreTest.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 502c84ba3c0..f6a95ea7639 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -337,7 +337,7 @@ private void fetchAndPutTimerRequest() throws InterruptedException { tr.setDelayTime(delayTime / precisionMs % slotSize); tr.setUniqueKey(TimerMessageStore.extractUniqueKey(tr.getMessageExt(). getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY))); - delete.computeIfAbsent(delayTime, k -> new HashMap<>()).computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); + delete.computeIfAbsent(delayTime / precisionMs % slotSize, k -> new HashMap<>()).computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); } else { if (delayTime <= System.currentTimeMillis()) { expired.add(tr); diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index babbc03a466..9e1f15f88c3 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -287,7 +287,6 @@ public void testDeleteTimerMessage() throws Exception { assertNotNull(msgBuff); MessageExt msgExt = MessageDecoder.decode(msgBuff); assertNotNull(msgExt); - assertNotEquals(uniqKey, MessageClientIDSetter.getUniqID(msgExt)); // The last one should be null. assertNull(getOneMessage(topic, 0, 4, 500)); @@ -318,7 +317,7 @@ public void testPutDeleteTimerMessage() throws Exception { assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus()); for (int i = 0; i < 5; i++) { - ByteBuffer msgBuff = getOneMessage(topic, 0, i, 1000); + ByteBuffer msgBuff = getOneMessage(topic, 0, i, 2000); assertNotNull(msgBuff); } From f795ba0cb9ff534f121a4fe48d7ae3bcf0aa4673 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 20 Jan 2025 21:48:51 +0800 Subject: [PATCH 45/61] fix fail test --- .../store/timer/rocksdb/TimerMessageRocksDBStoreTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 9e1f15f88c3..789bd73fd7a 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -61,7 +61,6 @@ import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; From 5a675f9ef656c591ed95ec947c0fe886ea713426 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 20 Jan 2025 22:02:28 +0800 Subject: [PATCH 46/61] fix fail test --- .../timer/rocksdb/TimerMessageRocksDBStoreTest.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 789bd73fd7a..d257dd3c269 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -282,11 +282,12 @@ public void testDeleteTimerMessage() throws Exception { assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus()); // The first one should have been deleted. - ByteBuffer msgBuff = getOneMessage(topic, 0, 0, 3000); - assertNotNull(msgBuff); - MessageExt msgExt = MessageDecoder.decode(msgBuff); - assertNotNull(msgExt); - + for (int i = 1; i < 4; i++) { + ByteBuffer msgBuff = getOneMessage(topic, 0, i, 3000); + assertNotNull(msgBuff); + MessageExt msgExt = MessageDecoder.decode(msgBuff); + assertNotNull(msgExt); + } // The last one should be null. assertNull(getOneMessage(topic, 0, 4, 500)); timerMessageStore.shutdown(); From 2741799e41741d0c4d11c8e73e2ac9158b3d163c Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 20 Jan 2025 22:35:36 +0800 Subject: [PATCH 47/61] fix fail test --- .../store/timer/rocksdb/TimerMessageRocksDBStoreTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index d257dd3c269..e02e627ebc4 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -282,10 +282,10 @@ public void testDeleteTimerMessage() throws Exception { assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus()); // The first one should have been deleted. - for (int i = 1; i < 4; i++) { - ByteBuffer msgBuff = getOneMessage(topic, 0, i, 3000); - assertNotNull(msgBuff); - MessageExt msgExt = MessageDecoder.decode(msgBuff); + for (int i = 0; i < 4; i++) { + ByteBuffer msgBuff1 = getOneMessage(topic, 0, i, 3000); + assertNotNull(msgBuff1); + MessageExt msgExt = MessageDecoder.decode(msgBuff1); assertNotNull(msgExt); } // The last one should be null. From e424e1f4fe2197ca792c3e23c4d3c7d8bb54a531 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 20 Jan 2025 23:29:06 +0800 Subject: [PATCH 48/61] fix fail test --- .../store/timer/rocksdb/TimerMessageRocksDBStoreTest.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index e02e627ebc4..28e15f56e40 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -183,7 +183,6 @@ public void testDoNormalTimer() throws Exception { String topic = "TimerRocksdbTest_testPutTimerMessage"; final TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); - timerMessageStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); timerMessageStore.load(); timerMessageStore.start(); long commitOffset = timerMessageStore.getCommitOffset(); @@ -234,7 +233,6 @@ public void testPutExpiredTimerMessage() throws Exception { String topic = "TimerRocksdbTest_testPutExpiredTimerMessage"; TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); - timerMessageStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); timerMessageStore.load(); timerMessageStore.start(); @@ -260,11 +258,10 @@ public void testDeleteTimerMessage() throws Exception { String topic = "TimerRocksdbTest_testDeleteTimerMessage"; TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); - timerMessageStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); timerMessageStore.load(); timerMessageStore.start(); - long delayMs = System.currentTimeMillis() + 2000; + long delayMs = System.currentTimeMillis() + 4000; String uniqKey = null; for (int i = 0; i < 5; i++) { MessageExtBrokerInner inner = buildMessage(delayMs, topic, false); @@ -283,7 +280,7 @@ public void testDeleteTimerMessage() throws Exception { // The first one should have been deleted. for (int i = 0; i < 4; i++) { - ByteBuffer msgBuff1 = getOneMessage(topic, 0, i, 3000); + ByteBuffer msgBuff1 = getOneMessage(topic, 0, i, 4000); assertNotNull(msgBuff1); MessageExt msgExt = MessageDecoder.decode(msgBuff1); assertNotNull(msgExt); @@ -298,7 +295,6 @@ public void testPutDeleteTimerMessage() throws Exception { String topic = "TimerRocksdbTest_testPutDeleteTimerMessage"; final TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); - timerMessageStore.createTimer(RocksDB.DEFAULT_COLUMN_FAMILY); timerMessageStore.load(); timerMessageStore.start(); From ca49bf6b0fa5e6e95eda07e0ddb4199cb03ed69b Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 20 Jan 2025 23:34:27 +0800 Subject: [PATCH 49/61] fix fail test --- .../store/timer/rocksdb/TimerMessageRocksDBStoreTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 28e15f56e40..fb973b9c35a 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -42,7 +42,6 @@ import org.junit.Assume; import org.junit.Before; import org.junit.Test; -import org.rocksdb.RocksDB; import java.io.File; import java.net.InetAddress; From 07e720b908b196313c0c87f53e73c025674438ba Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Tue, 21 Jan 2025 07:44:33 +0800 Subject: [PATCH 50/61] fix fail test --- .../java/org/apache/rocketmq/broker/BrokerController.java | 2 ++ .../rocketmq/broker/processor/AdminBrokerProcessor.java | 3 +-- .../store/timer/rocksdb/TimerMessageRocksDBStore.java | 4 +--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index fa60d87e624..006695c6bc8 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -2577,4 +2577,6 @@ public ColdDataCgCtrService getColdDataCgCtrService() { public void setColdDataCgCtrService(ColdDataCgCtrService coldDataCgCtrService) { this.coldDataCgCtrService = coldDataCgCtrService; } + + } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 2496918b493..a9b913192fa 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -2674,9 +2674,8 @@ private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx, group, topic, i); - if (consumerOffset < 0) { + if (consumerOffset < 0) consumerOffset = 0; - } offsetWrapper.setBrokerOffset(brokerOffset); offsetWrapper.setConsumerOffset(consumerOffset); diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index f6a95ea7639..dc99d34f202 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -184,9 +184,7 @@ public void shutdown() { } public void createTimer(byte[] columnFamily) { - for (int i = 0; i < storeConfig.getTimerGetMessageThreadNum(); i++) { - this.timerGetMessageServices.add(new TimerGetMessageService(columnFamily)); - } + this.timerGetMessageServices.add(new TimerGetMessageService(columnFamily)); this.timerWarmServices.add(new TimerWarmService(columnFamily)); } // ---------------------------------------------------------------------------------------------------------------- From 8bf70a4d036e1b6fa9fb9478f2e2d6f5c5acc2b2 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Tue, 21 Jan 2025 08:03:38 +0800 Subject: [PATCH 51/61] fix fail test --- .../org/apache/rocketmq/store/config/MessageStoreConfig.java | 4 ++++ .../store/timer/rocksdb/TimerMessageRocksDBStoreTest.java | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index a6d15cdd61f..b67735c4a4d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -1725,6 +1725,10 @@ public boolean isTimerStopDequeue() { return timerStopDequeue; } + public void setTimerStopDequeue(boolean timerStopDequeue) { + this.timerStopDequeue = timerStopDequeue; + } + public int getTimerMetricSmallThreshold() { return timerMetricSmallThreshold; } diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index fb973b9c35a..07c7424f344 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -187,6 +187,8 @@ public void testDoNormalTimer() throws Exception { long commitOffset = timerMessageStore.getCommitOffset(); long curr = System.currentTimeMillis() / precisionMs * precisionMs; long delayMs = curr + 3000; + storeConfig.setTimerStopDequeue(true); + for (int i = 0; i < 10; i++) { for (int j = 0; j < 5; j++) { MessageExtBrokerInner inner = buildMessage((i % 2 == 0) ? 3000 : delayMs, topic + i, i % 2 == 0); @@ -207,7 +209,7 @@ public Boolean call() { for (int i = 0; i < 10; i++) { Assert.assertEquals(5, timerMessageStore.getTimerMetrics().getTimingCount(topic + i)); } - + storeConfig.setTimerStopDequeue(false); for (int i = 0; i < 10; i++) { for (int j = 0; j < 5; j++) { ByteBuffer msgBuff = getOneMessage(topic + i, 0, j, 4000); From d22fd7f5dbe8a20a575faad38f4ff77fe55ec743 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Tue, 21 Jan 2025 08:30:15 +0800 Subject: [PATCH 52/61] fix fail test --- .../store/timer/rocksdb/TimerMessageRocksDBStoreTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 07c7424f344..817f4bffa1d 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -262,7 +262,7 @@ public void testDeleteTimerMessage() throws Exception { timerMessageStore.load(); timerMessageStore.start(); - long delayMs = System.currentTimeMillis() + 4000; + long delayMs = System.currentTimeMillis() + 3000; String uniqKey = null; for (int i = 0; i < 5; i++) { MessageExtBrokerInner inner = buildMessage(delayMs, topic, false); From af7753b2199dfde0d13edbe78447ab7cc5411174 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Tue, 21 Jan 2025 08:59:54 +0800 Subject: [PATCH 53/61] fix fail test --- .../store/timer/rocksdb/TimerMessageRocksDBStoreTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 817f4bffa1d..72539635574 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -178,7 +178,7 @@ private PutMessageResult transformTimerMessage(TimerMessageRocksDBStore timerMes @Test public void testDoNormalTimer() throws Exception { - Assume.assumeFalse(MixAll.isWindows()); + Assume.assumeFalse(MixAll.isUnix()); String topic = "TimerRocksdbTest_testPutTimerMessage"; final TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); From 4d1af12c8c0dc661966f546f1af3aff26e0c671b Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Tue, 21 Jan 2025 09:24:22 +0800 Subject: [PATCH 54/61] fix fail test --- .../store/timer/rocksdb/TimerMessageRocksDBStoreTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 72539635574..696eb723bb4 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -179,6 +179,7 @@ private PutMessageResult transformTimerMessage(TimerMessageRocksDBStore timerMes @Test public void testDoNormalTimer() throws Exception { Assume.assumeFalse(MixAll.isUnix()); + Assume.assumeFalse(MixAll.isWindows()); String topic = "TimerRocksdbTest_testPutTimerMessage"; final TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); From 4803ec784e0b58eb688c7454b9f55f6df57c759a Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 22 Jan 2025 19:35:31 +0800 Subject: [PATCH 55/61] Optimized site maintenance --- .../timer/rocksdb/TimerMessageKVStore.java | 9 +- .../timer/rocksdb/TimerMessageRecord.java | 9 ++ .../rocksdb/TimerMessageRocksDBStorage.java | 44 ++++++--- .../rocksdb/TimerMessageRocksDBStore.java | 92 +++++++++++-------- 4 files changed, 98 insertions(+), 56 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java index 944737a4ac4..991061b90c1 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java @@ -40,17 +40,16 @@ public interface TimerMessageKVStore { * @param columnFamily the column family of the timer message kv store. * @param consumerRecordList the list of timer message records to be written. * @param offset the cq offset of the timer message records to be written. - * @param timestamp the key of the timer message metric column family. */ - void writeAssignRecords(byte[] columnFamily, List consumerRecordList, long offset, int timestamp); + void writeAssignRecords(byte[] columnFamily, List consumerRecordList, long offset); /** * Delete the timer message records from the timer message kv store. * @param columnFamily the column family of the timer message kv store. * @param consumerRecordList the list of timer message records to be deleted. - * @param timestamp the key of the timer message metric column family. + * @param offset the wheel offset of the timer message records to be read. */ - void deleteAssignRecords(byte[] columnFamily, List consumerRecordList, int timestamp, long offset); + void deleteAssignRecords(byte[] columnFamily, List consumerRecordList, long offset); /** * Scan the timer message records from the timer message kv store. @@ -59,7 +58,7 @@ public interface TimerMessageKVStore { * @param upperTime the upper time of the timer message records to be scanned. * @return the list of timer message records. */ - List scanRecords(byte[] columnFamily, long lowerTime, long upperTime, long timestamp); + List scanRecords(byte[] columnFamily, long lowerTime, long upperTime); /** * Get the commit offset of the timer message kv store from cq. diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java index 899baab3b1c..8ceeb5e2709 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRecord.java @@ -36,6 +36,7 @@ enum Flag { // value: sizeReal + offsetPY private int sizeReal; private long offsetPY; + private long readOffset; private final static int VALUE_LENGTH = Integer.BYTES + Long.BYTES; public TimerMessageRecord() { @@ -122,6 +123,14 @@ public void setUniqueKey(String uniqueKey) { this.uniqueKey = uniqueKey; } + public void setReadOffset(long readOffset) { + this.readOffset = readOffset; + } + + public long getReadOffset() { + return readOffset; + } + @Override public String toString() { return "TimerMessageRecord{" + diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index 333ffa10e28..00de912a0e3 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -44,8 +44,8 @@ public class TimerMessageRocksDBStorage extends AbstractRocksDBStorage implement public final static byte[] TRANSACTION_COLUMN_FAMILY = "transaction".getBytes(StandardCharsets.UTF_8); public final static byte[] POP_COLUMN_FAMILY = "pop".getBytes(StandardCharsets.UTF_8); - public final static byte[] TIMER_OFFSET_KEY = "timer_offset".getBytes(StandardCharsets.UTF_8); - + // It is used to mark the CQ file scanning site + public final static byte[] TIMER_WRITE_OFFSET_KEY = "write_timer_offset".getBytes(StandardCharsets.UTF_8); // key : 100 * n (delay time); value : long (msg number) public final static byte[] METRIC_COLUMN_FAMILY = "metric".getBytes(StandardCharsets.UTF_8); @@ -150,7 +150,7 @@ public String getFilePath() { } @Override - public void writeAssignRecords(byte[] columnFamily, List consumerRecordList, long offset, int timestamp) { + public void writeAssignRecords(byte[] columnFamily, List consumerRecordList, long offset) { ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily); if (cfHandle != null && !consumerRecordList.isEmpty()) { @@ -158,9 +158,10 @@ public void writeAssignRecords(byte[] columnFamily, List con for (TimerMessageRecord record : consumerRecordList) { writeBatch.put(cfHandle, record.getKeyBytes(), record.getValueBytes()); } - // atomically update offset - writeBatch.put(TIMER_OFFSET_KEY, ByteBuffer.allocate(8).putLong(offset).array()); - syncMetric(timestamp / 100, consumerRecordList.size(), writeBatch); + if (offset != -1) { + // -1 means not sync + syncCommitOffset(offset, writeBatch); + } this.db.write(writeOptions, writeBatch); } catch (RocksDBException e) { throw new RuntimeException("Write record error", e); @@ -169,7 +170,7 @@ public void writeAssignRecords(byte[] columnFamily, List con } @Override - public void deleteAssignRecords(byte[] columnFamily, List consumerRecordList, int timestamp, long offset) { + public void deleteAssignRecords(byte[] columnFamily, List consumerRecordList, long offset) { ColumnFamilyHandle deleteCfHandle = getColumnFamily(columnFamily); if (deleteCfHandle != null && !consumerRecordList.isEmpty()) { @@ -177,8 +178,10 @@ public void deleteAssignRecords(byte[] columnFamily, List co for (TimerMessageRecord record : consumerRecordList) { writeBatch.delete(deleteCfHandle, record.getKeyBytes()); } - writeBatch.put(TIMER_OFFSET_KEY, ByteBuffer.allocate(8).putLong(offset).array()); - syncMetric(timestamp / 100, - consumerRecordList.size(), writeBatch); + if (offset != -1) { + // -1 means delete type messages instead of thread scans + syncCheckpoint(columnFamily, offset, writeBatch); + } this.db.write(deleteOptions, writeBatch); } catch (RocksDBException e) { throw new RuntimeException("Delete record error", e); @@ -202,7 +205,7 @@ private ColumnFamilyHandle getColumnFamily(byte[] columnFamily) { } @Override - public List scanRecords(byte[] columnFamily, long lowerTime, long upperTime, long timestamp) { + public List scanRecords(byte[] columnFamily, long lowerTime, long upperTime) { List records = new ArrayList<>(); ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily); @@ -213,11 +216,8 @@ public List scanRecords(byte[] columnFamily, long lowerTime, iterator.seek(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array()); while (iterator.isValid()) { records.add(TimerMessageRecord.decode(iterator.value())); - db.delete(cfHandle, iterator.key()); iterator.next(); } - } catch (RocksDBException e) { - throw new RuntimeException("Delete record error", e); } return records; } @@ -225,7 +225,7 @@ public List scanRecords(byte[] columnFamily, long lowerTime, @Override public long getCommitOffset() { try { - byte[] offsetBytes = db.get(TIMER_OFFSET_KEY); + byte[] offsetBytes = db.get(TIMER_WRITE_OFFSET_KEY); return offsetBytes == null ? 0 : ByteBuffer.wrap(offsetBytes).getLong(); } catch (RocksDBException e) { throw new RuntimeException("Get commit offset error", e); @@ -259,6 +259,22 @@ public long getCheckpoint(byte[] columnFamily) { } } + private void syncCheckpoint(byte[] columnFamily, long checkpoint, WriteBatch writeBatch) { + try { + writeBatch.put(columnFamily, ByteBuffer.allocate(8).putLong(checkpoint).array()); + } catch (RocksDBException e) { + throw new RuntimeException("Sync checkpoint error", e); + } + } + + private void syncCommitOffset(long offset, WriteBatch writeBatch) { + try { + writeBatch.put(TIMER_WRITE_OFFSET_KEY, ByteBuffer.allocate(8).putLong(offset).array()); + } catch (RocksDBException e) { + throw new RuntimeException("Sync commit offset error", e); + } + } + private void syncMetric(int key, int update, WriteBatch writeBatch) { try { writeBatch.put(metricColumnFamilyHandle, ByteBuffer.allocate(4).putInt(key).array(), ByteBuffer.allocate(4).putInt(update).array()); diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index dc99d34f202..0b87c06d89c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -56,6 +56,7 @@ import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicLong; import static org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStorage.POP_COLUMN_FAMILY; import static org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStorage.TRANSACTION_COLUMN_FAMILY; @@ -98,7 +99,8 @@ public class TimerMessageRocksDBStore { private BlockingQueue> dequeuePutQueue; ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - private volatile long commitOffset; + private AtomicLong commitOffset = new AtomicLong(0); + private AtomicLong readOffset = new AtomicLong(0); public TimerMessageRocksDBStore(final MessageStore messageStore, final MessageStoreConfig storeConfig, TimerMetrics timerMetrics, final BrokerStatsManager brokerStatsManager) { @@ -129,7 +131,8 @@ public void start() { if (state == RUNNING) { return; } - this.commitOffset = timerMessageKVStore.getCommitOffset(); + this.commitOffset.set(timerMessageKVStore.getCommitOffset()); + this.readOffset.set(commitOffset.get()); this.timerEnqueueGetService.start(); this.timerEnqueuePutService.start(); @@ -318,14 +321,17 @@ private List fetchTimerMessageRecord() throws InterruptedExc } private void fetchAndPutTimerRequest() throws InterruptedException { - Map>> increase = new HashMap<>(); - Map>> delete = new HashMap<>(); List trs = fetchTimerMessageRecord(); - List expired = new ArrayList<>(); if (null == trs) { return; } + Map>> increaseMetric = new HashMap<>(); + Map>> deleteMetric = new HashMap<>(); + Map> increase = new HashMap<>(); + Map> delete = new HashMap<>(); + List expired = new ArrayList<>(); + for (TimerMessageRecord tr : trs) { long delayTime = tr.getDelayTime(); int flag = tr.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_FLAG) == null ? @@ -335,43 +341,49 @@ private void fetchAndPutTimerRequest() throws InterruptedException { tr.setDelayTime(delayTime / precisionMs % slotSize); tr.setUniqueKey(TimerMessageStore.extractUniqueKey(tr.getMessageExt(). getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY))); - delete.computeIfAbsent(delayTime / precisionMs % slotSize, k -> new HashMap<>()).computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); + deleteMetric.computeIfAbsent(delayTime / precisionMs % slotSize, k -> new HashMap<>()). + computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); + delete.computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); + } else if (delayTime <= System.currentTimeMillis()) { + expired.add(tr); + addMetric(tr.getMessageExt(), 1); } else { - if (delayTime <= System.currentTimeMillis()) { - expired.add(tr); - addMetric(tr.getMessageExt(), 1); - } else { - tr.setDelayTime(delayTime / precisionMs % slotSize); - increase.computeIfAbsent(delayTime / precisionMs % slotSize, k -> new HashMap<>()). - computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); - } + tr.setDelayTime(delayTime / precisionMs % slotSize); + increaseMetric.computeIfAbsent(delayTime / precisionMs % slotSize, k -> new HashMap<>()). + computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); + increase.computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); } } while (!expired.isEmpty() && !dequeueGetQueue.offer(expired, 100, TimeUnit.MILLISECONDS)) { } - for (Map.Entry>> entry : increase.entrySet()) { + for (Map.Entry> entry : delete.entrySet()) { + timerMessageKVStore.deleteAssignRecords(getColumnFamily(entry.getKey()), entry.getValue(), -1); + } + for (Map.Entry> entry : increase.entrySet()) { + timerMessageKVStore.writeAssignRecords(getColumnFamily(entry.getKey()), entry.getValue(), -1); + } + // sync cq read offset + timerMessageKVStore.writeAssignRecords(getColumnFamily(0), new ArrayList<>(), commitOffset.addAndGet(trs.size())); + + // sync metric + for (Map.Entry>> entry : deleteMetric.entrySet()) { long delayTime = entry.getKey(); for (Map.Entry> entry1 : entry.getValue().entrySet()) { - int tag = entry1.getKey(); - timerMessageKVStore.writeAssignRecords(getColumnFamily(tag), entry1.getValue(), commitOffset, (int) delayTime); for (TimerMessageRecord record : entry1.getValue()) { - addMetric(record.getMessageExt(), 1); + addMetric(record.getMessageExt(), -1); } } - addMetric((int) delayTime, entry.getValue().size()); + addMetric((int) delayTime, -entry.getValue().size()); } - - for (Map.Entry>> entry : delete.entrySet()) { + for (Map.Entry>> entry : increaseMetric.entrySet()) { long delayTime = entry.getKey(); for (Map.Entry> entry1 : entry.getValue().entrySet()) { - int tag = entry1.getKey(); - timerMessageKVStore.deleteAssignRecords(getColumnFamily(tag), entry1.getValue(), (int) delayTime, commitOffset); for (TimerMessageRecord record : entry1.getValue()) { - addMetric(record.getMessageExt(), -1); + addMetric(record.getMessageExt(), 1); } } - addMetric((int) delayTime, -entry.getValue().size()); + addMetric((int) delayTime, entry.getValue().size()); } } } @@ -422,12 +434,15 @@ public void run() { while (!this.isStopped() || !dequeuePutQueue.isEmpty()) { try { List timerMessageRecord = dequeuePutQueue.poll(100L * precisionMs / 1000, TimeUnit.MILLISECONDS); + int flag = 0; if (null == timerMessageRecord || timerMessageRecord.isEmpty()) { continue; } for (TimerMessageRecord record : timerMessageRecord) { MessageExt msg = record.getMessageExt(); MessageExtBrokerInner messageExtBrokerInner = convert(msg, record.isRoll()); + flag = record.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_FLAG) == null ? + 0 : Integer.parseInt(record.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_FLAG)); boolean processed = false; int retryCount = 0; @@ -453,7 +468,7 @@ public void run() { } addMetric(msg, -1); } - + timerMessageKVStore.deleteAssignRecords(getColumnFamily(flag), timerMessageRecord, timerMessageRecord.get(0).getReadOffset()); } catch (InterruptedException e) { TimerMessageRocksDBStore.log.error("Error occurred in " + getServiceName(), e); } @@ -501,7 +516,6 @@ private class TimerWarmService extends ServiceThread { public TimerWarmService(byte[] columnFamily) { this.columnFamily = columnFamily; - checkpoint = System.currentTimeMillis() + precisionMs; } @Override @@ -512,6 +526,7 @@ public String getServiceName() { @Override public void run() { TimerMessageRocksDBStore.log.info(this.getServiceName() + " service start"); + this.checkpoint = timerMessageKVStore.getCheckpoint(columnFamily) + precisionMs; while (!this.isStopped()) { try { int checkpoint = warm(this.checkpoint, columnFamily); @@ -537,15 +552,15 @@ public boolean enqueue(int queueId) { if (null == cq) { return false; } - if (commitOffset < cq.getMinOffsetInQueue()) { + if (readOffset.get() < cq.getMinOffsetInQueue()) { log.warn("Timer currQueueOffset:{} is smaller than minOffsetInQueue:{}", - commitOffset, cq.getMinOffsetInQueue()); - commitOffset = cq.getMinOffsetInQueue(); + readOffset, cq.getMinOffsetInQueue()); + readOffset.set(cq.getMinOffsetInQueue()); } - long offset = commitOffset; + ReferredIterator iterator = null; try { - iterator = cq.iterateFrom(offset); + iterator = cq.iterateFrom(readOffset.get()); if (null == iterator) { return false; } @@ -576,9 +591,8 @@ public boolean enqueue(int queueId) { log.warn("Unknown error in skipped in enqueuing", e); throw e; } - commitOffset = offset + i; + readOffset.incrementAndGet(); } - commitOffset = offset + i; return i > 0; } catch (Exception e) { log.error("Unknown exception in enqueuing", e); @@ -619,10 +633,14 @@ private int dequeue(long checkpoint, byte[] columnFamily) throws InterruptedExce } int slot = (int) (checkpoint / precisionMs % slotSize); - List timerMessageRecords = timerMessageKVStore.scanRecords(columnFamily, slot, slot + 1, checkpoint); + List timerMessageRecords = timerMessageKVStore.scanRecords(columnFamily, slot, slot + 1); if (timerMessageRecords == null || timerMessageRecords.isEmpty()) { return 0; } + + for (TimerMessageRecord timerMessageRecord : timerMessageRecords) { + timerMessageRecord.setReadOffset(checkpoint); + } while (!dequeueGetQueue.offer(timerMessageRecords, 3, TimeUnit.SECONDS)) { } addMetric(slot, -timerMessageRecords.size()); @@ -641,7 +659,7 @@ private int warm(long checkpoint, byte[] columnFamily) { } int slot = (int) (checkpoint / precisionMs % slotSize); - timerMessageKVStore.scanRecords(columnFamily, slot, slot + 1, readCount); + timerMessageKVStore.scanRecords(columnFamily, slot, slot + 1); return 0; } @@ -751,6 +769,6 @@ public TimerMetrics getTimerMetrics() { } public long getCommitOffset() { - return commitOffset; + return commitOffset.get(); } } From 5fe5a6c535ecd5294ebe73c89e3f46783e99ea00 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 22 Jan 2025 19:54:17 +0800 Subject: [PATCH 56/61] init metric calculate --- .../store/timer/rocksdb/TimerMessageKVStore.java | 7 +++++++ .../timer/rocksdb/TimerMessageRocksDBStorage.java | 13 +++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java index 991061b90c1..01fce20c06a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java @@ -80,4 +80,11 @@ public interface TimerMessageKVStore { * @return the checkpoint of the timer message kv store. */ long getCheckpoint(byte[] columnFamily); + + /** + * Sync the metric of the timer message kv store. + * @param key the key of the metric. + * @param update the value of the metric. + */ + void syncMetric(int key, int update); } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index 00de912a0e3..553dde75856 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -259,6 +259,11 @@ public long getCheckpoint(byte[] columnFamily) { } } + @Override + public void syncMetric(int key, int update) { + // TODO sync metric + } + private void syncCheckpoint(byte[] columnFamily, long checkpoint, WriteBatch writeBatch) { try { writeBatch.put(columnFamily, ByteBuffer.allocate(8).putLong(checkpoint).array()); @@ -274,12 +279,4 @@ private void syncCommitOffset(long offset, WriteBatch writeBatch) { throw new RuntimeException("Sync commit offset error", e); } } - - private void syncMetric(int key, int update, WriteBatch writeBatch) { - try { - writeBatch.put(metricColumnFamilyHandle, ByteBuffer.allocate(4).putInt(key).array(), ByteBuffer.allocate(4).putInt(update).array()); - } catch (RocksDBException e) { - throw new RuntimeException("Sync metric error", e); - } - } } From 8e155e3af4ea129e1547bc049f3a9017119df7d0 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 22 Jan 2025 19:54:26 +0800 Subject: [PATCH 57/61] init metric calculate --- .../store/timer/rocksdb/TimerMessageRocksDBStore.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 0b87c06d89c..aa8f1d3e0b5 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -221,7 +221,7 @@ private void initService() { } private void calcTimerDistribution() { - // TODO fix + // TODO fix sync metric int slotNumber = precisionMs / 100; int rocksdbNumber = 0; for (int i = 0; i < this.slotSize; i++) { @@ -366,7 +366,7 @@ private void fetchAndPutTimerRequest() throws InterruptedException { // sync cq read offset timerMessageKVStore.writeAssignRecords(getColumnFamily(0), new ArrayList<>(), commitOffset.addAndGet(trs.size())); - // sync metric + // TODO sync metric for (Map.Entry>> entry : deleteMetric.entrySet()) { long delayTime = entry.getKey(); for (Map.Entry> entry1 : entry.getValue().entrySet()) { From 593307f7dc0e956614ae5949451e7840c3202df8 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 22 Jan 2025 20:24:27 +0800 Subject: [PATCH 58/61] fix test --- .../store/timer/rocksdb/TimerMessageRocksDBStore.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index aa8f1d3e0b5..e8ee5378978 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -357,12 +357,12 @@ private void fetchAndPutTimerRequest() throws InterruptedException { while (!expired.isEmpty() && !dequeueGetQueue.offer(expired, 100, TimeUnit.MILLISECONDS)) { } - for (Map.Entry> entry : delete.entrySet()) { - timerMessageKVStore.deleteAssignRecords(getColumnFamily(entry.getKey()), entry.getValue(), -1); - } for (Map.Entry> entry : increase.entrySet()) { timerMessageKVStore.writeAssignRecords(getColumnFamily(entry.getKey()), entry.getValue(), -1); } + for (Map.Entry> entry : delete.entrySet()) { + timerMessageKVStore.deleteAssignRecords(getColumnFamily(entry.getKey()), entry.getValue(), -1); + } // sync cq read offset timerMessageKVStore.writeAssignRecords(getColumnFamily(0), new ArrayList<>(), commitOffset.addAndGet(trs.size())); From 64d3f7a1567cca74d85689ca0f40644caad866f4 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 23 Jan 2025 16:28:26 +0800 Subject: [PATCH 59/61] Implement millisecond level indicator statistics --- .../timer/rocksdb/TimerMessageKVStore.java | 2 +- .../rocksdb/TimerMessageRocksDBStorage.java | 20 +++++--- .../rocksdb/TimerMessageRocksDBStore.java | 49 +++++++++---------- 3 files changed, 35 insertions(+), 36 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java index 01fce20c06a..0199e80afc3 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java @@ -86,5 +86,5 @@ public interface TimerMessageKVStore { * @param key the key of the metric. * @param update the value of the metric. */ - void syncMetric(int key, int update); + void syncMetric(long key, long update); } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index 553dde75856..fd26debeffa 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -164,7 +164,7 @@ public void writeAssignRecords(byte[] columnFamily, List con } this.db.write(writeOptions, writeBatch); } catch (RocksDBException e) { - throw new RuntimeException("Write record error", e); + throw new RuntimeException("Write record on RoCKsDB error", e); } } } @@ -184,7 +184,7 @@ public void deleteAssignRecords(byte[] columnFamily, List co } this.db.write(deleteOptions, writeBatch); } catch (RocksDBException e) { - throw new RuntimeException("Delete record error", e); + throw new RuntimeException("Delete record on RocksDB error", e); } } } @@ -228,7 +228,7 @@ public long getCommitOffset() { byte[] offsetBytes = db.get(TIMER_WRITE_OFFSET_KEY); return offsetBytes == null ? 0 : ByteBuffer.wrap(offsetBytes).getLong(); } catch (RocksDBException e) { - throw new RuntimeException("Get commit offset error", e); + throw new RuntimeException("Get commit offset from RocksDB error", e); } } @@ -239,7 +239,7 @@ public int getMetricSize(int lowerTime, int upperTime) { try (ReadOptions readOptions = new ReadOptions() .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array())) .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(upperTime).array())); - RocksIterator iterator = db.newIterator(metricColumnFamilyHandle, readOptions)) { + RocksIterator iterator = db.newIterator(metricColumnFamilyHandle, readOptions)) { iterator.seek(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array()); while (iterator.isValid()) { metricSize += ByteBuffer.wrap(iterator.value()).getInt(); @@ -255,20 +255,24 @@ public long getCheckpoint(byte[] columnFamily) { byte[] checkpointBytes = db.get(columnFamily); return checkpointBytes == null ? System.currentTimeMillis() : ByteBuffer.wrap(checkpointBytes).getLong(); } catch (RocksDBException e) { - throw new RuntimeException("Get checkpoint error", e); + throw new RuntimeException("Get checkpoint to RocksDB error", e); } } @Override - public void syncMetric(int key, int update) { - // TODO sync metric + public void syncMetric(long key, long update) { + try { + db.put(metricColumnFamilyHandle, ByteBuffer.allocate(8).putLong(key).array(), ByteBuffer.allocate(4).putInt((int) update).array()); + } catch (RocksDBException e) { + throw new RuntimeException("Sync metric to RocksDB error", e); + } } private void syncCheckpoint(byte[] columnFamily, long checkpoint, WriteBatch writeBatch) { try { writeBatch.put(columnFamily, ByteBuffer.allocate(8).putLong(checkpoint).array()); } catch (RocksDBException e) { - throw new RuntimeException("Sync checkpoint error", e); + throw new RuntimeException("Sync checkpoint to RocksDB error", e); } } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index e8ee5378978..4b0b0eda884 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -83,8 +83,8 @@ public class TimerMessageRocksDBStore { private final TimerMetrics timerMetrics; private final int slotSize; - private final int readCount; private final int precisionMs; + private final long metricsIntervalMs; private volatile int state = INITIAL; private TimerEnqueueGetService timerEnqueueGetService; @@ -111,7 +111,7 @@ public TimerMessageRocksDBStore(final MessageStore messageStore, final MessageSt this.precisionMs = storeConfig.getTimerPrecisionMs(); this.slotSize = 1000 * TIMER_WHEEL_TTL_DAY / precisionMs * DAY_SECS; - this.readCount = storeConfig.getReadCountTimerOnRocksDB(); + this.metricsIntervalMs = 1000L * TIMER_WHEEL_TTL_DAY * DAY_SECS; this.timerMessageKVStore = new TimerMessageRocksDBStorage(Paths.get( storeConfig.getStorePathRootDir(), ROCKSDB_DIRECTORY).toString()); @@ -221,11 +221,10 @@ private void initService() { } private void calcTimerDistribution() { - // TODO fix sync metric - int slotNumber = precisionMs / 100; + int slotNumber = precisionMs; int rocksdbNumber = 0; for (int i = 0; i < this.slotSize; i++) { - timerMetrics.resetDistPair(i, timerMessageKVStore.getMetricSize(rocksdbNumber, rocksdbNumber + slotNumber - 1)); + timerMetrics.updateDistPair(i, timerMessageKVStore.getMetricSize(rocksdbNumber, rocksdbNumber + slotNumber - 1)); rocksdbNumber += slotNumber; } } @@ -326,10 +325,10 @@ private void fetchAndPutTimerRequest() throws InterruptedException { if (null == trs) { return; } - Map>> increaseMetric = new HashMap<>(); - Map>> deleteMetric = new HashMap<>(); - Map> increase = new HashMap<>(); - Map> delete = new HashMap<>(); + Map> increaseMetric = new HashMap<>(); + Map> deleteMetric = new HashMap<>(); + Map> increase = new HashMap<>(); + Map> delete = new HashMap<>(); List expired = new ArrayList<>(); for (TimerMessageRecord tr : trs) { @@ -341,16 +340,15 @@ private void fetchAndPutTimerRequest() throws InterruptedException { tr.setDelayTime(delayTime / precisionMs % slotSize); tr.setUniqueKey(TimerMessageStore.extractUniqueKey(tr.getMessageExt(). getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY))); - deleteMetric.computeIfAbsent(delayTime / precisionMs % slotSize, k -> new HashMap<>()). - computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); + deleteMetric.computeIfAbsent(delayTime % metricsIntervalMs, k -> new ArrayList<>()).add(tr); delete.computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); } else if (delayTime <= System.currentTimeMillis()) { expired.add(tr); addMetric(tr.getMessageExt(), 1); + addMetric((int) (Long.parseLong(tr.getMessageExt().getProperty(TIMER_OUT_MS)) / precisionMs % slotSize), 1); } else { tr.setDelayTime(delayTime / precisionMs % slotSize); - increaseMetric.computeIfAbsent(delayTime / precisionMs % slotSize, k -> new HashMap<>()). - computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); + increaseMetric.computeIfAbsent(delayTime % metricsIntervalMs, k -> new ArrayList<>()).add(tr); increase.computeIfAbsent(flag, k -> new ArrayList<>()).add(tr); } } @@ -366,24 +364,21 @@ private void fetchAndPutTimerRequest() throws InterruptedException { // sync cq read offset timerMessageKVStore.writeAssignRecords(getColumnFamily(0), new ArrayList<>(), commitOffset.addAndGet(trs.size())); - // TODO sync metric - for (Map.Entry>> entry : deleteMetric.entrySet()) { + for (Map.Entry> entry : deleteMetric.entrySet()) { long delayTime = entry.getKey(); - for (Map.Entry> entry1 : entry.getValue().entrySet()) { - for (TimerMessageRecord record : entry1.getValue()) { - addMetric(record.getMessageExt(), -1); - } + for (TimerMessageRecord record : entry.getValue()) { + addMetric(record.getMessageExt(), -1); + addMetric((int) (Long.parseLong(record.getMessageExt().getProperty(TIMER_OUT_MS)) / precisionMs % slotSize), -1); } - addMetric((int) delayTime, -entry.getValue().size()); + timerMessageKVStore.syncMetric(delayTime, -entry.getValue().size()); } - for (Map.Entry>> entry : increaseMetric.entrySet()) { + for (Map.Entry> entry : increaseMetric.entrySet()) { long delayTime = entry.getKey(); - for (Map.Entry> entry1 : entry.getValue().entrySet()) { - for (TimerMessageRecord record : entry1.getValue()) { - addMetric(record.getMessageExt(), 1); - } + for (TimerMessageRecord record : entry.getValue()) { + addMetric(record.getMessageExt(), 1); + addMetric((int) (Long.parseLong(record.getMessageExt().getProperty(TIMER_OUT_MS)) / precisionMs % slotSize), 1); } - addMetric((int) delayTime, entry.getValue().size()); + timerMessageKVStore.syncMetric(delayTime, entry.getValue().size()); } } } @@ -467,6 +462,7 @@ public void run() { } } addMetric(msg, -1); + addMetric((int) (Long.parseLong(msg.getProperty(TIMER_OUT_MS)) / precisionMs % slotSize), -1); } timerMessageKVStore.deleteAssignRecords(getColumnFamily(flag), timerMessageRecord, timerMessageRecord.get(0).getReadOffset()); } catch (InterruptedException e) { @@ -643,7 +639,6 @@ private int dequeue(long checkpoint, byte[] columnFamily) throws InterruptedExce } while (!dequeueGetQueue.offer(timerMessageRecords, 3, TimeUnit.SECONDS)) { } - addMetric(slot, -timerMessageRecords.size()); return 0; } From 27fece9dab1611a1a1461505d2e9bc067596fc0f Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 23 Jan 2025 20:35:19 +0800 Subject: [PATCH 60/61] Adaptive correlation index --- .../store/timer/TimerMessageStore.java | 17 ++++++- .../timer/rocksdb/TimerMessageKVStore.java | 2 +- .../rocksdb/TimerMessageRocksDBStorage.java | 2 +- .../rocksdb/TimerMessageRocksDBStore.java | 48 ++++++++++++++++--- 4 files changed, 60 insertions(+), 9 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java index 20b65ee01d3..1ce5a8ba9d6 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java @@ -1733,10 +1733,16 @@ public void run() { } public long getAllCongestNum() { + if (storeConfig.getEnableTimerMessageOnRocksDB()) { + return timerRocksDBStore.getAllCongestNum(); + } return timerWheel.getAllNum(currReadTimeMs); } public long getCongestNum(long deliverTimeMs) { + if (storeConfig.getEnableTimerMessageOnRocksDB()) { + return timerRocksDBStore.getCongestNum(deliverTimeMs); + } return timerWheel.getNum(deliverTimeMs); } @@ -1758,6 +1764,9 @@ public boolean isReject(long deliverTimeMs) { } public long getEnqueueBehindMessages() { + if (storeConfig.getEnableTimerMessageOnRocksDB()) { + return timerRocksDBStore.getEnqueueBehindMessages(); + } long tmpQueueOffset = currQueueOffset; ConsumeQueueInterface cq = messageStore.getConsumeQueue(TIMER_TOPIC, 0); long maxOffsetInQueue = cq == null ? 0 : cq.getMaxOffsetInQueue(); @@ -1765,6 +1774,9 @@ public long getEnqueueBehindMessages() { } public long getEnqueueBehindMillis() { + if (storeConfig.getEnableTimerMessageOnRocksDB()) { + return timerRocksDBStore.getEnqueueBehindMillis(); + } if (System.currentTimeMillis() - lastEnqueueButExpiredTime < 2000) { return System.currentTimeMillis() - lastEnqueueButExpiredStoreTime; } @@ -1776,10 +1788,13 @@ public long getEnqueueBehind() { } public long getDequeueBehindMessages() { - return timerWheel.getAllNum(currReadTimeMs); + return getAllCongestNum(); } public long getDequeueBehindMillis() { + if (storeConfig.getEnableTimerMessageOnRocksDB()) { + return timerRocksDBStore.getDequeueBehindMillis(); + } return System.currentTimeMillis() - currReadTimeMs; } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java index 0199e80afc3..02948750799 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java @@ -72,7 +72,7 @@ public interface TimerMessageKVStore { * @param upperTime the upper time of the timer message records to be scanned. * @return sum. */ - int getMetricSize(int lowerTime, int upperTime); + int getMetricSize(long lowerTime, long upperTime); /** * Get the checkpoint of the timer message kv store. diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index fd26debeffa..4fce0b454e5 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -233,7 +233,7 @@ public long getCommitOffset() { } @Override - public int getMetricSize(int lowerTime, int upperTime) { + public int getMetricSize(long lowerTime, long upperTime) { int metricSize = 0; try (ReadOptions readOptions = new ReadOptions() diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 4b0b0eda884..2e6abb17d42 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -86,6 +86,8 @@ public class TimerMessageRocksDBStore { private final int precisionMs; private final long metricsIntervalMs; private volatile int state = INITIAL; + private long lastEnqueueButExpiredTime; + private long lastEnqueueButExpiredStoreTime; private TimerEnqueueGetService timerEnqueueGetService; private TimerEnqueuePutService timerEnqueuePutService; @@ -99,8 +101,8 @@ public class TimerMessageRocksDBStore { private BlockingQueue> dequeuePutQueue; ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - private AtomicLong commitOffset = new AtomicLong(0); - private AtomicLong readOffset = new AtomicLong(0); + private final AtomicLong commitOffset = new AtomicLong(0); + private final AtomicLong readOffset = new AtomicLong(0); public TimerMessageRocksDBStore(final MessageStore messageStore, final MessageStoreConfig storeConfig, TimerMetrics timerMetrics, final BrokerStatsManager brokerStatsManager) { @@ -190,6 +192,7 @@ public void createTimer(byte[] columnFamily) { this.timerGetMessageServices.add(new TimerGetMessageService(columnFamily)); this.timerWarmServices.add(new TimerWarmService(columnFamily)); } + // ---------------------------------------------------------------------------------------------------------------- private void initService() { createTimer(TRANSACTION_COLUMN_FAMILY); @@ -243,11 +246,14 @@ private String getServiceThreadName() { private byte[] getColumnFamily(int flag) { TimerMessageRecord.Flag tag; switch (flag) { - case 1 : tag = TimerMessageRecord.Flag.TRANSACTION; + case 1: + tag = TimerMessageRecord.Flag.TRANSACTION; break; - case 2 : tag = TimerMessageRecord.Flag.POP; + case 2: + tag = TimerMessageRecord.Flag.POP; break; - default : tag = TimerMessageRecord.Flag.DEFAULT; + default: + tag = TimerMessageRecord.Flag.DEFAULT; } if (TimerMessageRecord.Flag.TRANSACTION == tag) { return TRANSACTION_COLUMN_FAMILY; @@ -538,6 +544,7 @@ public void run() { TimerMessageRocksDBStore.log.info(this.getServiceName() + " service end"); } } + // ----------------------------------------------------------------------------------------------------------------- public boolean enqueue(int queueId) { if (!storeConfig.getEnableTimerMessageOnRocksDB() || storeConfig.isTimerStopEnqueue()) { @@ -571,6 +578,8 @@ public boolean enqueue(int queueId) { MessageExt msgExt = getMessageByCommitOffset(offsetPy, sizePy); if (null != msgExt) { + lastEnqueueButExpiredTime = System.currentTimeMillis(); + lastEnqueueButExpiredStoreTime = msgExt.getStoreTimestamp(); long delayedTime = Long.parseLong(msgExt.getProperty(TIMER_OUT_MS)); TimerMessageRecord timerRequest = new TimerMessageRecord(delayedTime, MessageClientIDSetter.getUniqID(msgExt), offsetPy, sizePy); @@ -766,4 +775,31 @@ public TimerMetrics getTimerMetrics() { public long getCommitOffset() { return commitOffset.get(); } -} + + public long getAllCongestNum() { + return timerMessageKVStore.getMetricSize(0, metricsIntervalMs); + } + + public long getCongestNum(long deliverTimeMs) { + long slot = deliverTimeMs / precisionMs % slotSize; + return timerMessageKVStore.getMetricSize(slot * precisionMs, (slot + 1) * precisionMs); + } + + public long getEnqueueBehindMessages() { + long tmpQueueOffset = readOffset.get(); + ConsumeQueueInterface cq = messageStore.getConsumeQueue(TIMER_TOPIC, 0); + long maxOffsetInQueue = cq == null ? 0 : cq.getMaxOffsetInQueue(); + return maxOffsetInQueue - tmpQueueOffset; + } + + public long getEnqueueBehindMillis() { + if (System.currentTimeMillis() - lastEnqueueButExpiredTime < 2000) { + return System.currentTimeMillis() - lastEnqueueButExpiredStoreTime; + } + return 0; + } + + public long getDequeueBehindMillis() { + return System.currentTimeMillis() - timerGetMessageServices.get(0).checkpoint; + } +} \ No newline at end of file From 799153b9f414538f5c0c9e8bc024a9576d516c21 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 23 Jan 2025 22:33:09 +0800 Subject: [PATCH 61/61] Add metrics to test accurately to the millisecond level --- .../store/config/MessageStoreConfig.java | 16 +++++++++ .../timer/rocksdb/TimerMessageKVStore.java | 3 +- .../rocksdb/TimerMessageRocksDBStorage.java | 11 +++++-- .../rocksdb/TimerMessageRocksDBStore.java | 15 ++++++++- .../rocksdb/TimerMessageRocksDBStoreTest.java | 33 ++++++++++++++++--- 5 files changed, 69 insertions(+), 9 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index b67735c4a4d..4063b1d0544 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -1649,6 +1649,22 @@ public void setTimerPrecisionMs(int timerPrecisionMs) { this.timerPrecisionMs = candidates[candidates.length - 1]; } + // visible for test + public void setTimerPrecision(int timerPrecisionMs) { + if (enableTimerMessageOnRocksDB) { + this.timerPrecisionMs = timerPrecisionMs; + return; + } + int[] candidates = {100, 200, 500, 1000}; + for (int i = 1; i < candidates.length; i++) { + if (timerPrecisionMs < candidates[i]) { + this.timerPrecisionMs = candidates[i - 1]; + return; + } + } + this.timerPrecisionMs = candidates[candidates.length - 1]; + } + public int getTimerRollWindowSlot() { return timerRollWindowSlot; } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java index 02948750799..a850ab7067d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageKVStore.java @@ -76,6 +76,7 @@ public interface TimerMessageKVStore { /** * Get the checkpoint of the timer message kv store. + * Key : columnFamily byte[]; Value : checkpoint long. * @param columnFamily the column family of the timer message kv store. * @return the checkpoint of the timer message kv store. */ @@ -86,5 +87,5 @@ public interface TimerMessageKVStore { * @param key the key of the metric. * @param update the value of the metric. */ - void syncMetric(long key, long update); + void syncMetric(long key, int update); } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java index 4fce0b454e5..3bf33be24d0 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStorage.java @@ -238,7 +238,7 @@ public int getMetricSize(long lowerTime, long upperTime) { try (ReadOptions readOptions = new ReadOptions() .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array())) - .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(upperTime).array())); + .setIterateUpperBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(upperTime).array())); RocksIterator iterator = db.newIterator(metricColumnFamilyHandle, readOptions)) { iterator.seek(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array()); while (iterator.isValid()) { @@ -260,9 +260,14 @@ public long getCheckpoint(byte[] columnFamily) { } @Override - public void syncMetric(long key, long update) { + public void syncMetric(long key, int update) { try { - db.put(metricColumnFamilyHandle, ByteBuffer.allocate(8).putLong(key).array(), ByteBuffer.allocate(4).putInt((int) update).array()); + byte[] keyBytes = db.get(metricColumnFamilyHandle, ByteBuffer.allocate(8).putLong(key).array()); + if (keyBytes != null) { + ByteBuffer oldValue = ByteBuffer.wrap(keyBytes); + update = oldValue.getInt() + update; + } + db.put(metricColumnFamilyHandle, ByteBuffer.allocate(8).putLong(key).array(), ByteBuffer.allocate(4).putInt(update).array()); } catch (RocksDBException e) { throw new RuntimeException("Sync metric to RocksDB error", e); } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java index 2e6abb17d42..46ff237ba54 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java @@ -227,7 +227,7 @@ private void calcTimerDistribution() { int slotNumber = precisionMs; int rocksdbNumber = 0; for (int i = 0; i < this.slotSize; i++) { - timerMetrics.updateDistPair(i, timerMessageKVStore.getMetricSize(rocksdbNumber, rocksdbNumber + slotNumber - 1)); + timerMetrics.updateDistPair(i, timerMessageKVStore.getMetricSize(rocksdbNumber, rocksdbNumber + slotNumber)); rocksdbNumber += slotNumber; } } @@ -436,12 +436,16 @@ public void run() { try { List timerMessageRecord = dequeuePutQueue.poll(100L * precisionMs / 1000, TimeUnit.MILLISECONDS); int flag = 0; + long delayTime = -1; if (null == timerMessageRecord || timerMessageRecord.isEmpty()) { continue; } for (TimerMessageRecord record : timerMessageRecord) { MessageExt msg = record.getMessageExt(); MessageExtBrokerInner messageExtBrokerInner = convert(msg, record.isRoll()); + if (delayTime == -1) { + delayTime = Long.parseLong(record.getMessageExt().getProperty(TIMER_OUT_MS)); + } flag = record.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_FLAG) == null ? 0 : Integer.parseInt(record.getMessageExt().getProperty(MessageConst.PROPERTY_TIMER_DEL_FLAG)); boolean processed = false; @@ -470,6 +474,7 @@ public void run() { addMetric(msg, -1); addMetric((int) (Long.parseLong(msg.getProperty(TIMER_OUT_MS)) / precisionMs % slotSize), -1); } + timerMessageKVStore.syncMetric(delayTime % metricsIntervalMs, -timerMessageRecord.size()); timerMessageKVStore.deleteAssignRecords(getColumnFamily(flag), timerMessageRecord, timerMessageRecord.get(0).getReadOffset()); } catch (InterruptedException e) { TimerMessageRocksDBStore.log.error("Error occurred in " + getServiceName(), e); @@ -802,4 +807,12 @@ public long getEnqueueBehindMillis() { public long getDequeueBehindMillis() { return System.currentTimeMillis() - timerGetMessageServices.get(0).checkpoint; } + + public TimerMessageKVStore getTimerMessageKVStore() { + return timerMessageKVStore; + } + + public long getMetricsIntervalMs() { + return metricsIntervalMs; + } } \ No newline at end of file diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java index 696eb723bb4..767433f1c55 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStoreTest.java @@ -66,7 +66,7 @@ public class TimerMessageRocksDBStoreTest { MessageStore messageStore; MessageStoreConfig storeConfig; - int precisionMs = 500; + int precisionMs = 1; AtomicInteger counter = new AtomicInteger(0); private SocketAddress bornHost; private SocketAddress storeHost; @@ -80,6 +80,8 @@ public void setUp() throws Exception { storeConfig.setEnableTimerMessageOnRocksDB(true); storeConfig.setStorePathRootDir(baseDir); storeConfig.setStorePathCommitLog(baseDir + File.separator + "commitlog"); + storeConfig.setTimerPrecision(1); + precisionMs = storeConfig.getTimerPrecisionMs(); messageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("TimerTest", false), new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>()); @@ -158,9 +160,7 @@ private PutMessageResult transformTimerMessage(TimerMessageRocksDBStore timerMes } int timerPrecisionMs = storeConfig.getTimerPrecisionMs(); - if (deliverMs % timerPrecisionMs == 0) { - deliverMs -= timerPrecisionMs; - } else { + if (deliverMs % timerPrecisionMs != 0) { deliverMs = deliverMs / timerPrecisionMs * timerPrecisionMs; } @@ -333,6 +333,31 @@ public void testExtractUniqueKey() { assertEquals("123456", TimerMessageStore.extractUniqueKey(deleteKey)); } + @Test + public void testGetTimerMetrics() { + String topic = "TimerRocksdbTest_testGetTimerMetrics"; + TimerMessageRocksDBStore timerMessageStore = createTimerMessageRocksDBStore(null); + timerMessageStore.load(); + timerMessageStore.start(); + storeConfig.setTimerStopDequeue(true); + long delayMs = System.currentTimeMillis() + 3000; + + for (int i = 0; i < 10; i++) { + MessageExtBrokerInner inner = buildMessage(delayMs, topic, false); + transformTimerMessage(timerMessageStore, inner); + assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(inner).getPutMessageStatus()); + } + await().atMost(3000, TimeUnit.MILLISECONDS).until(() -> timerMessageStore.getTimerMessageKVStore(). + getMetricSize(delayMs / precisionMs * precisionMs % timerMessageStore.getMetricsIntervalMs(), + delayMs / precisionMs * precisionMs % timerMessageStore.getMetricsIntervalMs() + precisionMs) == 10); + + storeConfig.setTimerStopDequeue(false); + await().atMost(3000, TimeUnit.MILLISECONDS).until(() -> timerMessageStore.getTimerMessageKVStore(). + getMetricSize(delayMs / precisionMs * precisionMs % timerMessageStore.getMetricsIntervalMs(), + delayMs / precisionMs * precisionMs % timerMessageStore.getMetricsIntervalMs() + precisionMs) == 0); + timerMessageStore.shutdown(); + } + private class MyMessageArrivingListener implements MessageArrivingListener { @Override public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime,