Merge branch 'release/0.10.0'

qcif · Feb 14, 2018 · 354f2c0 · 354f2c0
2 parents a5b4b8a + 9f9ac27
commit 354f2c0
Show file tree

Hide file tree

Showing 35 changed files with 659 additions and 227 deletions.
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
@@ -23,4 +23,4 @@
 > Include details about the environment you experienced the problem - this will help us fix the bug quicker.
 
 * Data Curator version: 0.Y.Z
-* Operating System and version: e.g. macOS High Sierra 10.13.2
+* Operating System and version: e.g. macOS High Sierra 10.13.3
diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md
@@ -16,4 +16,4 @@
 ### Your Environment
 
 * Data Curator version: 0.Y.Z
-* Operating System and version: macOS High Sierra 10.13.2, Windows 7 64bit
+* Operating System and version: macOS High Sierra 10.13.3, Windows 7 64bit
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "DataCurator",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "author": " <[email protected]>",
   "description": "Data Curator is a simple desktop CSV editor to help describe, validate and share usable open data",
   "license": "MIT",
@@ -82,6 +82,7 @@
     "escape-regexp": "^0.0.1",
     "etl": "^0.5.8",
     "exports-loader": "^0.6.4",
+    "fs-extra": "^5.0.0",
     "handsontable": "^0.34.5",
     "imports-loader": "^0.7.1",
     "jquery": "^3.2.1",
@@ -93,15 +94,14 @@
     "lodash": "^4.17.4",
     "markdown-it": "^8.4.0",
     "moment": "^2.19.1",
-    "node-fs-extra": "^0.8.2",
     "pikaday": "^1.6.1",
     "pouchdb-adapter-idb": "^6.3.4",
     "request": "^2.81.0",
     "rxjs": "^5.5.2",
     "slug": "^0.9.1",
     "sortablejs": "^1.6.0",
     "svgo": "^1.0.0",
-    "tableschema": "^1.5.1",
+    "tableschema": "^1.7.0",
     "temp": "^0.8.3",
     "unzipper": "^0.8.11",
     "vee-validate": "^2.0.3",
@@ -138,7 +138,7 @@
     "devtron": "^1.1.0",
     "electron": "^1.7.11",
     "electron-builder": "^19.27.2",
-    "electron-debug": "^1.5.0",
+    "electron-debug": "~1.4.0",
     "electron-devtools-installer": "^2.0.1",
     "eslint": "^4.10.0",
     "eslint-config-standard": "^10.2.1",

diff --git a/src/main/file.js b/src/main/file.js
@@ -8,12 +8,12 @@ function makeCustomFormat(separator, delimiter) {
   return {
     label: 'Custom',
     filters: [],
-    options: {
-      separator: separator,
-      delimiter: delimiter
+    dialect: {
+      delimiter: delimiter,
+      quoteChar: quoteChar
     },
-    mime_type: 'text/plain',
-    default_extension: 'txt'
+    mediatype: 'text/plain',
+    format: 'txt'
   }
 }
 
@@ -27,7 +27,7 @@ function saveAsCustom() {
   })
   ipc.once('formatSelected', function(event, data) {
     dialog.close()
-    let format = makeCustomFormat(data.separator, data.delimiter)
+    let format = makeCustomFormat(data.delimiter, data.quoteChar)
     saveFileAs(format, currentWindow)
   })
   dialog.loadURL(`http://localhost:9080/#/customformat`)
@@ -120,8 +120,8 @@ function openFile(format) {
   })
 }
 
-ipc.on('openFileIntoTab', (event, arg) => {
-  readFile(arg)
+ipc.on('openFileIntoTab', (event, arg1, arg2) => {
+  readFile(arg1, arg2)
 })
 
 function readFile(filename, format) {
@@ -131,7 +131,7 @@ function readFile(filename, format) {
   }
   Fs.readFile(filename, 'utf-8', function(err, data) {
     if (err) {
-      console.log(err.stack)
+      console.log(err)
     } else {
       createWindowTabWithFormattedDataFile(data, format, filename)
       // enableSave()

diff --git a/src/main/menu.js b/src/main/menu.js
@@ -99,8 +99,13 @@ const template = [
         //            enabled: false
         //          }
         //        ]
-      }, {
         // Placeholder for non-macOS Settings for future feature
+        //      }, {
+        //        type: 'separator'
+        //      }, {
+        //        label: 'Settings',
+        //        enabled: false
+      }, {
         type: 'separator'
       }, {
         label: 'Save',
@@ -243,9 +248,9 @@ const template = [
     submenu: [
       {
         label: 'Header Row',
+        accelerator: 'Shift+CmdOrCtrl+H',
         type: 'checkbox',
         checked: false,
-        accelerator: 'Shift+CmdOrCtrl+H',
         click(menuItem) {
           // revert 'checked' toggle so only controlled by header row event
           menuItem.checked = !menuItem.checked
@@ -270,28 +275,33 @@ const template = [
         //  type: 'separator'
         // }, {
         label: 'Guess Column Properties',
+        accelerator: 'Shift+CmdOrCtrl+G',
         click: function() {
           guessColumnProperties()
         }
       }, {
         type: 'separator'
       }, {
         label: 'Set Column Properties',
+        accelerator: 'Shift+CmdOrCtrl+C',
         click() {
           triggerMenuButton('Column')
         }
       }, {
         label: 'Set Table Properties',
+        accelerator: 'Shift+CmdOrCtrl+T',
         click() {
           triggerMenuButton('Table')
         }
       }, {
         label: 'Set Provenance Information',
+        accelerator: 'Shift+CmdOrCtrl+P',
         click() {
           triggerMenuButton('Provenance')
         }
       }, {
         label: 'Set Data Package Properties',
+        accelerator: 'Shift+CmdOrCtrl+D',
         click() {
           triggerMenuButton('Package')
         }
@@ -307,7 +317,7 @@ const template = [
         type: 'separator'
       }, {
         label: 'Export Data Package...',
-        accelerator: 'CmdOrCtrl+D',
+        accelerator: 'Shift+CmdOrCtrl+X',
         click() {
           triggerMenuButton('Export')
         }

diff --git a/src/renderer/components/CustomFormat.vue b/src/renderer/components/CustomFormat.vue
@@ -1,50 +1,64 @@
 <template>
   <div id="container" class="container-fluid">
-  <form id="custom-format">
+  <form id="custom-format" v-on:submit.prevent="submit()">
     <div class="form-wrapper">
 
       <div class="form-group">
-        <label for="separator">Separator</label>
-        <input type="text" class="form-control" id="separator" name="separator" value=",">
+        <label for="separator">Delimiter</label>
+        <input type="text" class="form-control" id="delimiter" name="delimiter" :value="getDelimiter" @input="setDelimiter($event.target.value)" >
       </div>
 
       <div class="form-group">
-        <label for="delimiter">Text Delimiter</label>
-        <input type="text" class="form-control" id="delimiter" name="delimiter" value='"'>
+        <label for="delimiter">Quote Character</label>
+        <input type="text" class="form-control" id="quoteChar" name="quoteChar" :value="getQuoteChar" @input="setQuoteChar($event.target.value)">
       </div>
 
     </div>
 
     <div class="well">
       <button id="submit" class="btn btn-default">Go</button>
-      <button id="cancel" class="btn btn-default">Cancel</button>
+      <button id="cancel" class="btn btn-default" @click="cancel()">Cancel</button>
     </div>
   </form>
   </div>
 </template>
 <script>
-window.$ = window.jQuery = require('jquery/dist/jquery.js')
 const {
   shell
 } = require('electron')
 var ipc = require('electron').ipcRenderer
 require('bootstrap/dist/js/bootstrap.min.js')
 export default {
   name: 'customformat',
-  methods: {},
-  mounted: function() {
-    $('#cancel').click(function(e) {
-      window.close()
-    })
-
-    $('#submit').click(function(e) {
-      e.preventDefault()
+  computed: {
+    getDelimiter() {
+      return ','
+      // TODO : couple to vuex
+    },
+    getQuoteChar() {
+      return ','
+      // TODO : couple to vuex
+    }
+  },
+  methods: {
+    setDelimiter(delimiter) {
+      // TODO: complete
+    },
+    setQuoteChar(quoteChar) {
+      // TODO: complete
+    },
+    submit() {
       let format = {
-        separator: $('#separator').val(),
-        delimiter: $('#delimiter').val()
+        delimiter: document.querySelector('#delimiter').val(),
+        quoteChar: document.querySelector('#quoteChar').val()
       }
       ipc.send('formatSelected', format)
-    })
+    },
+    cancel() {
+      window.close()
+    }
+  },
+  mounted: function() {
   }
 }
 </script>

diff --git a/src/renderer/components/KeyboardHelp.vue b/src/renderer/components/KeyboardHelp.vue
@@ -404,15 +404,45 @@
       </tr>
     </thead>
     <tbody>
+      <tr>
+        <td><b>Header Row</b> - Freeze first row and use values for Column Name</td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>H</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Command &#8984;</kbd> <kbd>H</kbd></td>
+      </tr>
+      <tr>
+        <td><b>Guess Column Properties</b> - set Type and Format based on column values</td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>G</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Command &#8984;</kbd> <kbd>G</kbd></td>
+      </tr>
+      <tr>
+        <td><b>Set Column Properties</b></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>C</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Command &#8984;</kbd> <kbd>C</kbd></td>
+      </tr>
+      <tr>
+        <td><b>Set Table Properties</b></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>T</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Command &#8984;</kbd> <kbd>T</kbd></td>
+      </tr>
+      <tr>
+        <td><b>Set Provenance Information</b></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>P</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Command &#8984;</kbd> <kbd>P</kbd></td>
+      </tr>
+      <tr>
+        <td><b>Set Data Package Properties</b></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>D</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Command &#8984;</kbd> <kbd>D</kbd></td>
+      </tr>
       <tr>
         <td><b>Validate Table</b> (uses Column Properties if set)</td>
         <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>V</kbd></td>
         <td><kbd>Shift &#8679;</kbd> <kbd>Command &#8984;</kbd> <kbd>V</kbd></td>
       </tr>
       <tr>
         <td><b>Export Data Package</b> - save all data and properties to a .zip file</td>
-        <td><kbd>Ctrl</kbd> <kbd>D</kbd></td>
-        <td><kbd>Command &#8984;</kbd> <kbd>D</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>X</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Command &#8984;</kbd> <kbd>X</kbd></td>
       </tr>
     </tbody>
   </table>

diff --git a/src/renderer/data-actions.js b/src/renderer/data-actions.js
@@ -1,21 +1,31 @@
-import tabStore from '../renderer/store/modules/tabs.js'
+import tabStore from '@/store/modules/tabs.js'
 import fs from 'fs'
 import {fixRaggedRows} from '@/ragged-rows.js'
 import {includeHeadersInData} from '@/frictionlessUtilities.js'
 import {toggleHeaderNoFeedback} from '@/headerRow.js'
+import {pushCsvDialect} from '@/dialect.js'
 // import parse from 'csv-parse/lib/sync'
 // import stringify from 'csv-stringify'
+// TODO : replace jQuery with node 'csv' library's stringify and transform
 const $ = global.jQuery = require('jquery/dist/jquery.js')
 require('jquery-csv/src/jquery.csv.js')
+var parse = require('csv-parse/lib/sync')
+var stringify = require('csv-stringify/lib/sync')
 
+// { delimiter: ',', lineTerminator, quoteChar, doubleQuote, escapeChar, nullSequence, skipInitialSpace, header, caseSensitiveHeader, csvddfVersion }
+const frictionlessToCsvmapper = {delimiter: 'delimiter', lineTerminator: 'rowDelimiter', quoteChar: 'quote', escapeChar: 'escape', skipInitialSpace: 'ltrim'}
 export function loadDataIntoHot(hot, data, format) {
   let arrays
   // if no format specified, default to csv
   if (typeof format === 'undefined' || !format) {
-    arrays = $.csv.toArrays(data)
+    arrays = parse(data)
   } else {
-    arrays = $.csv.toArrays(data, format.options)
+    let csvOptions = dialectToCsvOptions(format.dialect)
+    // TODO: update to stream
+    arrays = parse(data, csvOptions)
+    pushCsvDialect(hot.guid, format)
   }
+
   fixRaggedRows(arrays)
   hot.loadData(arrays)
   hot.render()
@@ -48,9 +58,24 @@ export function saveDataToFile(hot, format, filename, callback) {
   let data
   // if no format specified, default to csv
   if (typeof format === 'undefined' || !format) {
-    data = $.csv.fromArrays(arrays)
+    // TODO: update to stream
+    data = stringify(arrays)
   } else {
-    data = $.csv.fromArrays(arrays, format.options)
+    let csvOptions = dialectToCsvOptions(format.dialect)
+    data = stringify(arrays, csvOptions)
+    pushCsvDialect(hot.guid, format)
   }
   fs.writeFile(filename, data, callback)
 }
+
+function dialectToCsvOptions(dialect) {
+  let csvOptions = {}
+  if (dialect) {
+    _.forEach(frictionlessToCsvmapper, function(csvKey, frictionlessKey) {
+      if (_.has(dialect, frictionlessKey)) {
+        csvOptions[csvKey] = dialect[frictionlessKey]
+      }
+    })
+  }
+  return csvOptions
+}
diff --git a/src/renderer/dialect.js b/src/renderer/dialect.js
@@ -0,0 +1,20 @@
+import store from '@/store'
+
+export function pushCsvDialect(guid, formatOriginal = {}) {
+  let format = {}
+  _.assign(format, formatOriginal)
+  // TODO : neaten up to merge existing values and then push all up
+  if (format.dialect) {
+    _.unset(format.dialect, 'objectMode')
+    _.merge(format.dialect, {header: true})
+    _.forEach(format.dialect, function(value, key) {
+      store.commit('pushTableProperty', {hotId: guid, key: `dialect.${key}`, value: value})
+    })
+  }
+  if (format.mediatype) {
+    store.commit('pushTableProperty', {hotId: guid, key: 'mediatype', value: format.mediatype})
+  }
+  if (format.format) {
+    store.commit('pushTableProperty', {hotId: guid, key: 'format', value: format.format})
+  }
+}
diff --git a/src/renderer/exportPackage.js b/src/renderer/exportPackage.js
@@ -7,7 +7,8 @@ import hotStore from '@/store/modules/hots.js'
 import {extractNameFromFile} from '@/store/tabStoreUtilities.js'
 const Dialog = remote.dialog
 
-export function createZipFile(json) {
+export function createZipFile(text) {
+  let json = JSON.stringify(text, null, 4)
   Dialog.showSaveDialog({
     filters: [
       {