Stream database file line by line to avoid string length limit in Node.js

browser-version/lib/byline.js

@@ -0,0 +1 @@
+module.exports = {}

lib/persistence.js

@@ -6,6 +6,7 @@
  */
 const path = require('path')
 const async = require('async')
+const byline = require('byline')
 const customUtils = require('./customUtils.js')
 const Index = require('./indexes.js')
 const model = require('./model.js')
@@ -183,6 +184,55 @@ class Persistence {
     return { data: tdata, indexes: indexes }
   }
 
+  /**
+   * From a database's raw stream, return the corresponding
+   * machine understandable collection
+   */
+  treatRawStream (rawStream, cb) {
+    const dataById = {}
+    const tdata = []
+    const indexes = {}
+    let corruptItems = 0
+
+    const lineStream = byline(rawStream)
+    const that = this
+    let length = 0
+
+    lineStream.on('data', function(line) {
+
+      try {
+        const doc = model.deserialize(that.beforeDeserialization(line))
+        if (doc._id) {
+          if (doc.$$deleted === true) delete dataById[doc._id]
+          else dataById[doc._id] = doc
+        } else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != null) indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated
+        else if (typeof doc.$$indexRemoved === 'string') delete indexes[doc.$$indexRemoved]
+      } catch (e) {
+        corruptItems += 1
+      }
+
+      length++
+    })
+
+    lineStream.on('end', function() {
+      // A bit lenient on corruption
+      let err
+      if (length > 0 && corruptItems / length > that.corruptAlertThreshold) {
+        err = new Error("More than " + Math.floor(100 * that.corruptAlertThreshold) + "% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss")
+      }
+
+      Object.keys(dataById).forEach(function (k) {
+        tdata.push(dataById[k])
+      })
+
+      cb(err, { data: tdata, indexes: indexes })
+    })
+
+    lineStream.on('error', function(err) {
+      cb(err)
+    })
+  }
+
   /**
    * Load the database
    * 1) Create all indexes
@@ -207,14 +257,9 @@ class Persistence {
           // eslint-disable-next-line node/handle-callback-err
           storage.ensureDatafileIntegrity(this.filename, err => {
             // TODO: handle error
-            storage.readFile(this.filename, 'utf8', (err, rawData) => {
+            const treatedDataCallback = (err, treatedData) => {
+
               if (err) return cb(err)
-              let treatedData
-              try {
-                treatedData = this.treatRawData(rawData)
-              } catch (e) {
-                return cb(e)
-              }
 
               // Recreate all indexes in the datafile
               Object.keys(treatedData.indexes).forEach(key => {
@@ -230,6 +275,25 @@ class Persistence {
               }
 
               this.db.persistence.persistCachedDatabase(cb)
+            }
+
+            if (storage.readFileStream) {
+              // Server side
+              const fileStream = storage.readFileStream(this.filename, { encoding : 'utf8' })
+              this.treatRawStream(fileStream, treatedDataCallback)
+              return
+            }
+
+            // Browser
+            storage.readFile(this.filename, 'utf8', (err, rawData) => {
+              if (err) return cb(err)
+
+              try {
+                const treatedData = this.treatRawData(rawData)
+                treatedDataCallback(null, treatedData)
+              } catch (e) {
+                return cb(e)
+              }
             })
           })
         })

lib/storage.js

@@ -18,6 +18,7 @@ storage.writeFile = fs.writeFile
 storage.unlink = fs.unlink
 storage.appendFile = fs.appendFile
 storage.readFile = fs.readFile
+storage.readFileStream = fs.createReadStream
 storage.mkdir = fs.mkdir
 
 /**

package.json

@@ -36,6 +36,7 @@
   "dependencies": {
     "@seald-io/binary-search-tree": "^1.0.2",
     "async": "0.2.10",
+    "byline": "^5.0.0",
     "localforage": "^1.9.0"
   },
   "devDependencies": {