Merge pull request #81 from fslaborg/speed

Reader speedboost

RELEASE_NOTES.md

@@ -1,3 +1,10 @@
+### 5.1.0+4732d22 (Released 2024-1-24)
+* Additions:
+    * Increase Parser speed
+    * [[#4732d22](https://github.com/CSBiology/FsSpreadsheet/commit/4732d22e6acbf70b284355149fd4d06932023783)] include parse speed test in js
+    * [[#9aa368f](https://github.com/CSBiology/FsSpreadsheet/commit/9aa368f3cefeaa50c9eb4c13865bb952a57a273c)] add speedtest for excel file reader
+    * [[#f401c96](https://github.com/CSBiology/FsSpreadsheet/commit/f401c9618bfd66648688cfbb8b99e51f46b14bdd)] make RescanRows speed linear
+
 ### 5.0.2+41eca2f (Released 2023-11-3)
 * Bugfixes:
     * [[#b45db41](https://github.com/CSBiology/FsSpreadsheet/commit/b45db41a89e78576999afd28a2d7b1e087caf335)] Fxi critical bug in fable compatibility

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fslab/fsspreadsheet",
-  "version": "5.0.2",
+  "version": "5.1.0",
   "description": "Minimal spreadsheet creation and manipulation using exceljs io.",
   "type": "module",
   "main": "Xlsx.js",

src/FsSpreadsheet.ExcelIO/FsExtensions.fs

@@ -225,7 +225,6 @@ module FsExtensions =
                 xlsxSheets
                 |> Seq.map (
                     fun xlsxSheet ->
-                        let sheetIndex = Sheet.getSheetIndex xlsxSheet //unused?
                         let sheetId = Sheet.getID xlsxSheet
                         let xlsxCells = 
                             Spreadsheet.getCellsBySheetID sheetId doc

src/FsSpreadsheet/FsRow.fs

@@ -14,7 +14,7 @@ open Fable.Core
 /// <remarks>The FsCellsCollection must only cover 1 row!</remarks>
 /// <exception cref="System.Exception">if given FsCellsCollection has more than 1 row.</exception>
 [<AttachMembers>]
-type FsRow (rangeAddress : FsRangeAddress, cells : FsCellsCollection)= 
+type FsRow (rangeAddress : FsRangeAddress, cells : FsCellsCollection) = 
 
     inherit FsRangeBase(rangeAddress)
 

src/FsSpreadsheet/FsWorksheet.fs

@@ -144,10 +144,20 @@ type FsWorksheet (name, ?fsRows, ?fsTables, ?fsCellsCollection) =
     /// <summary>
     /// Returns the FsRow at the given FsRangeAddress. If it does not exist, it is created and appended first.
     /// </summary>
-    member self.RowWithRange(rangeAddress : FsRangeAddress) = 
+    /// <param name="SkipSearch">If true, the FsRow is created and appended without checking if it already exists.</param>
+    member self.RowWithRange(rangeAddress : FsRangeAddress, ?SkipSearch) = 
+        let skipSearch = defaultArg SkipSearch false
         if rangeAddress.FirstAddress.RowNumber <> rangeAddress.LastAddress.RowNumber then
             failwithf "Row may not have a range address spanning over different row indices"
-        self.Row(rangeAddress.FirstAddress.RowNumber).RangeAddress <- rangeAddress
+        if skipSearch then
+            let row = FsRow.createAt(rangeAddress.FirstAddress.RowNumber,self.CellCollection) 
+            row.RangeAddress <- rangeAddress
+            _rows.Add row
+            row
+        else
+            let row = self.Row(rangeAddress.FirstAddress.RowNumber) 
+            row.RangeAddress <- rangeAddress
+            row
 
     /// <summary>
     /// Appends an FsRow to an FsWorksheet if the rowIndex is not already taken.
@@ -361,7 +371,8 @@ type FsWorksheet (name, ?fsRows, ?fsTables, ?fsCellsCollection) =
             | Some row -> 
                 row.RangeAddress <- newRange
             | None ->
-                self.RowWithRange(newRange)
+                // Use SkipSearch flag to avoid checking if row already exists with linear speed, as this is already done in the tryFind above
+                self.RowWithRange(newRange,true) |> ignore
         )
 
 

tests/FsSpreadsheet.ExcelIO.Tests/FsWorkbook.fs

@@ -65,10 +65,23 @@ let writeAndReadBytes =
         )
     ]
 
+let performance =
+    testList "Performace" [
+        testCase "ReadBigFile" (fun () -> 
+            let sw = Stopwatch()        
+            let p = "./TestFiles/BigFile.xlsx"
+            sw.Start()
+            let wb = FsWorkbook.fromXlsxFile(p)
+            sw.Stop()
+            Expect.isLessThan sw.Elapsed.Milliseconds 2000  "Elapsed time should be less than 2000ms"
+            Expect.equal (wb.GetWorksheetAt(1).Rows.Count) 153991 "Row count should be 153991"
 
+        )
+    ]
 
 [<Tests>]
 let tests =
     testList "FsWorkbook" [
         writeAndReadBytes
+        performance
     ]

tests/FsSpreadsheet.Exceljs.Tests/Workbook.Tests.fs

@@ -288,9 +288,24 @@ let tests_xlsx = testList "xlsx" [
     ]
 ]
 
+let performance =
+    testList "Performace" [
+        testCaseAsync "ReadBigFile" <| async {
+            let sw = Stopwatch()        
+            let p = DefaultTestObject.BigFile.asRelativePathNode
+            sw.Start()
+            let! wb = FsWorkbook.fromXlsxFile(p) |> Async.AwaitPromise
+            sw.Stop()
+            let ms = sw.Elapsed.Milliseconds
+            Expect.isTrue (ms < 2000)  $"Elapsed time should be less than 2000ms but was {ms}ms"
+            Expect.equal (wb.GetWorksheetAt(1).Rows.Count) 153991 "Row count should be 153991"
+        }
+    ]
+
 let main = testList "JsWorkbook<->FsWorkbook" [
     tests_toFsWorkbook
     tests_toJsWorkbook
     tests_xlsx
+    performance
 ]
 

tests/TestUtils/DefaultTestObjects.fs

@@ -18,6 +18,7 @@ type TestFiles =
 | ClosedXML
 | FsSpreadsheetNET
 | FsSpreadsheetJS
+| BigFile
 
     member this.asFileName =
         match this with
@@ -27,6 +28,7 @@ type TestFiles =
         | ClosedXML         -> "TestWorkbook_ClosedXML.xlsx"
         | FsSpreadsheetNET  -> "TestWorkbook_FsSpreadsheet.net.xlsx"
         | FsSpreadsheetJS   -> "TestWorkbook_FsSpreadsheet.js.xlsx"
+        | BigFile           -> "BigFile.xlsx"
 
     member this.asRelativePath = $"../TestUtils/{testFolder}/{this.asFileName}"
     member this.asRelativePathNode = $"./tests/TestUtils/{testFolder}/{this.asFileName}"

tests/TestUtils/TestUtils.fsproj

@@ -6,24 +6,27 @@
   </PropertyGroup>
 
   <ItemGroup>
-	<EmbeddedResource Include="TestFiles\TestWorkbook_ClosedXML.xlsx">
-		<CopyToOutputDirectory>Always</CopyToOutputDirectory>
-	</EmbeddedResource>
-	<EmbeddedResource Include="TestFiles\TestWorkbook_Excel.xlsx">
-		<CopyToOutputDirectory>Always</CopyToOutputDirectory>
-	</EmbeddedResource>
-	<EmbeddedResource Include="TestFiles\TestWorkbook_FableExceljs.xlsx">
-		<CopyToOutputDirectory>Always</CopyToOutputDirectory>
-	</EmbeddedResource>
-	<EmbeddedResource Include="TestFiles\TestWorkbook_Libre.xlsx">
-		<CopyToOutputDirectory>Always</CopyToOutputDirectory>
-	</EmbeddedResource>
-	<EmbeddedResource Include="TestFiles\TestWorkbook_FsSpreadsheet.net.xlsx">
-		<CopyToOutputDirectory>Always</CopyToOutputDirectory>
-	</EmbeddedResource>
-	<EmbeddedResource Include="TestFiles\TestWorkbook_FsSpreadsheet.js.xlsx">
-		<CopyToOutputDirectory>Always</CopyToOutputDirectory>
-	</EmbeddedResource>
+		<EmbeddedResource Include="TestFiles\TestWorkbook_ClosedXML.xlsx">
+			<CopyToOutputDirectory>Always</CopyToOutputDirectory>
+		</EmbeddedResource>
+		<EmbeddedResource Include="TestFiles\TestWorkbook_Excel.xlsx">
+			<CopyToOutputDirectory>Always</CopyToOutputDirectory>
+		</EmbeddedResource>
+		<EmbeddedResource Include="TestFiles\TestWorkbook_FableExceljs.xlsx">
+			<CopyToOutputDirectory>Always</CopyToOutputDirectory>
+		</EmbeddedResource>
+		<EmbeddedResource Include="TestFiles\TestWorkbook_Libre.xlsx">
+			<CopyToOutputDirectory>Always</CopyToOutputDirectory>
+		</EmbeddedResource>
+		<EmbeddedResource Include="TestFiles\TestWorkbook_FsSpreadsheet.net.xlsx">
+			<CopyToOutputDirectory>Always</CopyToOutputDirectory>
+		</EmbeddedResource>
+		<EmbeddedResource Include="TestFiles\TestWorkbook_FsSpreadsheet.js.xlsx">
+			<CopyToOutputDirectory>Always</CopyToOutputDirectory>
+		</EmbeddedResource>
+    <EmbeddedResource Include="TestFiles\BigFile.xlsx" >
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </EmbeddedResource>
     <Compile Include="DefaultTestObjects.fs" />
     <Compile Include="TestingUtils.fs" />
   </ItemGroup>

tests/TestUtils/TestingUtils.fs

@@ -1,6 +1,8 @@
 module TestingUtils
 
 open FsSpreadsheet
+open Fable.Core
+
 #if FABLE_COMPILER
 open Fable.Mocha
 #else
@@ -139,4 +141,20 @@ module Test =
     let ftestCaseAsync = ftestCaseAsync
 
 
-    let testList = testList
+    let testList = testList
+
+open System
+
+[<AttachMembers>]
+type Stopwatch() =
+    member val StartTime: DateTime option = None with get, set
+    member val StopTime: DateTime option = None with get, set
+    member this.Start() = this.StartTime <- Some DateTime.Now
+    member this.Stop() = 
+        match this.StartTime with
+        | Some _ -> this.StopTime <- Some DateTime.Now
+        | None -> failwith "Error. Unable to call `Stop` before `Start`."
+    member this.Elapsed : TimeSpan = 
+        match this.StartTime, this.StopTime with
+        | Some start, Some stop -> stop - start
+        | _, _ -> failwith "Error. Unable to call `Elapsed` without calling `Start` and `Stop` before."