feat: add compress and structured_array options.

nkaz001 · Mar 22, 2024 · 3627a57 · 3627a57
1 parent eaf181a
commit 3627a57
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 10 deletions.
diff --git a/hftbacktest/data/utils/binancefutures.py b/hftbacktest/data/utils/binancefutures.py
@@ -5,10 +5,17 @@
 import numpy as np
 from numpy.typing import NDArray
 
-from .. import correct, validate_data
+from .. import validate_data
 from ..validation import correct_event_order, convert_to_struct_arr
-from ... import DEPTH_EVENT, DEPTH_CLEAR_EVENT, DEPTH_SNAPSHOT_EVENT, TRADE_EVENT, correct_local_timestamp, \
-    COL_EXCH_TIMESTAMP, COL_LOCAL_TIMESTAMP
+from ... import (
+    DEPTH_EVENT,
+    DEPTH_CLEAR_EVENT,
+    DEPTH_SNAPSHOT_EVENT,
+    TRADE_EVENT,
+    COL_EXCH_TIMESTAMP,
+    COL_LOCAL_TIMESTAMP,
+    correct_local_timestamp
+)
 
 
 def convert(
@@ -57,7 +64,7 @@ def convert(
         base_latency: The value to be added to the feed latency.
                       See :func:`.correct_local_timestamp`.
         compress: If this is set to True, the output file will be compressed.
-        structured_array: If this is set to True, the output is converted into the new format.
+        structured_array: If this is set to True, the output is converted into the new format(currently only Rust impl).
 
     Returns:
         Converted data compatible with HftBacktest.

diff --git a/hftbacktest/data/utils/snapshot.py b/hftbacktest/data/utils/snapshot.py
@@ -3,6 +3,7 @@
 import numpy as np
 from numpy.typing import NDArray
 
+from ..validation import convert_to_struct_arr
 from ... import HftBacktest, Linear, ConstantLatency
 from ...typing import DataCollection, Data
 from ...reader import UNTIL_END_OF_DATA, DEPTH_SNAPSHOT_EVENT
@@ -13,7 +14,9 @@ def create_last_snapshot(
         tick_size: float,
         lot_size: float,
         initial_snapshot: Optional[Data] = None,
-        output_snapshot_filename: Optional[str] = None
+        output_snapshot_filename: Optional[str] = None,
+        compress: bool = False,
+        structured_array: bool = False
 ) -> NDArray:
     r"""
     Creates a snapshot of the last market depth for the specified data, which can be used as the initial snapshot data
@@ -26,6 +29,9 @@ def create_last_snapshot(
          initial_snapshot: The initial market depth snapshot.
          output_snapshot_filename: If provided, the snapshot data will be saved to the specified filename in ``npz``
                                    format.
+         compress: If this is set to True, the output file will be compressed.
+         structured_array: If this is set to True, the output is converted into the new
+                           format(currently only Rust impl).
 
     Returns:
         Snapshot of the last market depth compatible with HftBacktest.
@@ -65,7 +71,13 @@ def create_last_snapshot(
 
     snapshot = np.asarray(snapshot, np.float64)
 
+    if structured_array:
+        snapshot = convert_to_struct_arr(snapshot)
+
     if output_snapshot_filename is not None:
-        np.savez(output_snapshot_filename, data=snapshot)
+        if compress:
+            np.savez_compressed(output_snapshot_filename, data=snapshot)
+        else:
+            np.savez(output_snapshot_filename, data=snapshot)
 
     return snapshot
diff --git a/hftbacktest/data/utils/tardis.py b/hftbacktest/data/utils/tardis.py
@@ -6,15 +6,22 @@
 
 from .. import validate_data
 from ..validation import correct_event_order, convert_to_struct_arr
-from ... import DEPTH_CLEAR_EVENT, DEPTH_SNAPSHOT_EVENT, TRADE_EVENT, DEPTH_EVENT, COL_LOCAL_TIMESTAMP, \
-    correct_local_timestamp, COL_EXCH_TIMESTAMP
+from ... import (
+    DEPTH_CLEAR_EVENT,
+    DEPTH_SNAPSHOT_EVENT,
+    TRADE_EVENT,
+    DEPTH_EVENT,
+    COL_LOCAL_TIMESTAMP,
+    COL_EXCH_TIMESTAMP,
+    correct_local_timestamp
+)
 
 
 def convert(
         input_files: List[str],
         output_filename: Optional[str] = None,
         buffer_size: int = 100_000_000,
-        ss_buffer_size: int = 10_000,
+        ss_buffer_size: int = 1_000_000,
         base_latency: float = 0,
         snapshot_mode: Literal['process', 'ignore_sod', 'ignore'] = 'process',
         compress: bool = False,
@@ -28,6 +35,7 @@ def convert(
                      e.g. ['incremental_book.csv', 'trades.csv'].
         output_filename: If provided, the converted data will be saved to the specified filename in ``npz`` format.
         buffer_size: Sets a preallocated row size for the buffer.
+        ss_buffer_size: Sets a preallocated row size for the snapshot.
         base_latency: The value to be added to the feed latency.
                       See :func:`.correct_local_timestamp`.
         snapshot_mode: - If this is set to 'ignore', all snapshots are ignored. The order book will converge to a
@@ -39,7 +47,7 @@ def convert(
                          for more details.
                        - Otherwise, all snapshot events will be processed.
         compress: If this is set to True, the output file will be compressed.
-        structured_array: If this is set to True, the output is converted into the new format.
+        structured_array: If this is set to True, the output is converted into the new format(currently only Rust impl).
 
     Returns:
         Converted data compatible with HftBacktest.