From 468d98e034f8f109857ff8fe25c9116cf53b5858 Mon Sep 17 00:00:00 2001
From: Stephen-Gates <Stephen.Gates@me.com>
Date: Sat, 23 Dec 2017 07:43:22 +1000
Subject: [PATCH 01/26] Add tool shortcuts

Fixes #327 #268
---
 src/main/menu.js                         | 35 ++++++++++++++++--------
 src/renderer/components/KeyboardHelp.vue | 34 +++++++++++++++++++++--
 2 files changed, 56 insertions(+), 13 deletions(-)

diff --git a/src/main/menu.js b/src/main/menu.js
index 06cdebf33..77b3570f0 100644
--- a/src/main/menu.js
+++ b/src/main/menu.js
@@ -99,8 +99,13 @@ const template = [
         //            enabled: false
         //          }
         //        ]
-      }, {
         // Placeholder for non-macOS Settings for future feature
+        //      }, {
+        //        type: 'separator'
+        //      }, {
+        //        label: 'Settings',
+        //        enabled: false
+      }, {
         type: 'separator'
       }, {
         label: 'Save',
@@ -247,6 +252,7 @@ const template = [
     submenu: [
       {
         label: 'Header Row',
+        accelerator: 'Shift+CmdOrCtrl+H',
         type: 'checkbox',
         checked: false,
         click(menuItem) {
@@ -273,6 +279,7 @@ const template = [
         //  type: 'separator'
         // }, {
         label: 'Guess Column Properties',
+        accelerator: 'Shift+CmdOrCtrl+G',
         click: function() {
           guessColumnProperties()
         }
@@ -280,21 +287,25 @@ const template = [
         type: 'separator'
       }, {
         label: 'Set Column Properties',
+        accelerator: 'Shift+CmdOrCtrl+C',
         click() {
           triggerMenuButton('Column')
         }
       }, {
         label: 'Set Table Properties',
+        accelerator: 'Shift+CmdOrCtrl+T',
         click() {
           triggerMenuButton('Table')
         }
       }, {
         label: 'Set Provenance Information',
+        accelerator: 'Shift+CmdOrCtrl+P',
         click() {
           triggerMenuButton('Provenance')
         }
       }, {
         label: 'Set Data Package Properties',
+        accelerator: 'Shift+CmdOrCtrl+D',
         click() {
           triggerMenuButton('Package')
         }
@@ -310,7 +321,7 @@ const template = [
         type: 'separator'
       }, {
         label: 'Export Data Package...',
-        accelerator: 'CmdOrCtrl+D',
+        accelerator: 'Shift+CmdOrCtrl+X',
         click() {
           triggerMenuButton('Export')
         }
@@ -402,15 +413,17 @@ const template = [
   }
 ]
 
-// Tailor menu for Windows - add About to Help menu if (process.platform !== 'darwin') {
-template[4].submenu.push({
-  type: 'separator'
-}, {
-  label: 'About Data Curator',
-  click: function() {
-    showSidePanel('about')
-  }
-})
+// Tailor menu for Windows - add About to Help menu
+if (process.platform !== 'darwin') {
+  template[4].submenu.push({
+    type: 'separator'
+  }, {
+    label: 'About Data Curator',
+    click: function() {
+      showSidePanel('about')
+    }
+  })
+}
 
 // Tailor menu for macOS
 if (process.platform === 'darwin') {
diff --git a/src/renderer/components/KeyboardHelp.vue b/src/renderer/components/KeyboardHelp.vue
index a992a0bdf..064bcb84d 100644
--- a/src/renderer/components/KeyboardHelp.vue
+++ b/src/renderer/components/KeyboardHelp.vue
@@ -404,6 +404,36 @@
       </tr>
     </thead>
     <tbody>
+      <tr>
+        <td><b>Header Row</b> - Freeze first row and use values for Column Name</td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>H</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Command &#8984;</kbd> <kbd>H</kbd></td>
+      </tr>
+      <tr>
+        <td><b>Guess Column Properties</b> - set Type and Format based on column values</td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>G</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Command &#8984;</kbd> <kbd>G</kbd></td>
+      </tr>
+      <tr>
+        <td><b>Set Column Properties</b></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>C</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Command &#8984;</kbd> <kbd>C</kbd></td>
+      </tr>
+      <tr>
+        <td><b>Set Table Properties</b></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>T</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Command &#8984;</kbd> <kbd>T</kbd></td>
+      </tr>
+      <tr>
+        <td><b>Set Provenance Information</b></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>P</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Command &#8984;</kbd> <kbd>P</kbd></td>
+      </tr>
+      <tr>
+        <td><b>Set Data Package Properties</b></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>D</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Command &#8984;</kbd> <kbd>D</kbd></td>
+      </tr>
       <tr>
         <td><b>Validate Table</b> (uses Column Properties if set)</td>
         <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>V</kbd></td>
@@ -411,8 +441,8 @@
       </tr>
       <tr>
         <td><b>Export Data Package</b> - save all data and properties to a .zip file</td>
-        <td><kbd>Ctrl</kbd> <kbd>D</kbd></td>
-        <td><kbd>Command &#8984;</kbd> <kbd>D</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Ctrl</kbd> <kbd>X</kbd></td>
+        <td><kbd>Shift &#8679;</kbd> <kbd>Command &#8984;</kbd> <kbd>X</kbd></td>
       </tr>
     </tbody>
   </table>

From b6ec55b4ea093c179e1aa63d61888b3992f8ccb2 Mon Sep 17 00:00:00 2001
From: Stephen-Gates <Stephen.Gates@me.com>
Date: Sat, 23 Dec 2017 07:56:29 +1000
Subject: [PATCH 02/26] update provenance placeholder text

---
 src/renderer/partials/ProvenanceProperties.vue | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/renderer/partials/ProvenanceProperties.vue b/src/renderer/partials/ProvenanceProperties.vue
index ba9bf8ec7..bf36fb4d8 100644
--- a/src/renderer/partials/ProvenanceProperties.vue
+++ b/src/renderer/partials/ProvenanceProperties.vue
@@ -63,19 +63,22 @@ export default {
     return {
       isPreview: false,
       provenance: '',
-      placeholder: `### Introduction
+      placeholder: `Short description of the dataset (the first sentence and first paragraph should be extractable to provide short standalone descriptions)
 
-### Why was the dataset created? (reference legislation if relevant)
+### Why was the dataset created?
+reference legislation if relevant
 
-### How was it collected - what events lead up to its collection?
+### How was it collected
+what events lead up to its collection?
 
-### When was it collected? (Temporal extent)
+### When was it collected?
 
-### Where was it collected? (Spatial extent name, coordinate reference system, minimum bounding rectangle)
+### Where was it collected?
 
 ### Which instruments were used to collect it?
 
-### What does “null” mean? Unknown, missing or not applicable?
+### What does “null” mean?
+are null values unknown, missing or not applicable?
 
 ### Other comments
 

From cee908570bc981567c44cc571d9109e21b1b35cd Mon Sep 17 00:00:00 2001
From: Stephen-Gates <Stephen.Gates@me.com>
Date: Sat, 23 Dec 2017 16:32:06 +1000
Subject: [PATCH 03/26] fix #323

---
 src/renderer/partials/ColumnProperties.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/renderer/partials/ColumnProperties.vue b/src/renderer/partials/ColumnProperties.vue
index 6fd893a59..b37d630e0 100644
--- a/src/renderer/partials/ColumnProperties.vue
+++ b/src/renderer/partials/ColumnProperties.vue
@@ -18,7 +18,7 @@
             </option>
           </select>
         </template>
-        <div v-else-if="formprop.key === 'constraints'" id="constraints" class="col-sm-9">
+        <div v-else-if="formprop.key === 'constraints'" id="constraints" class="col-sm-9"><br>
           <div class="input-group row" v-for="option in constraintValues" :key="option">
             <input type="checkbox" :id="option" :checked="getConstraintCheck(option)" @click="setConstraintCheck(option, $event.target)"></input>
             <label :for="option" class="form-control-static">{{option}}</label>

From 51115e1c9b7303866e8ec5f5e497455c7e8ef417 Mon Sep 17 00:00:00 2001
From: Stephen-Gates <Stephen.Gates@me.com>
Date: Wed, 27 Dec 2017 13:07:21 +1000
Subject: [PATCH 04/26] remove name from PDM

name must be in http://licenses.opendefinition.org/licenses/groups/od.json
otherwise leave blank as per specification https://frictionlessdata.io/specs/data-package/#licenses
---
 src/renderer/partials/Licenses.vue | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/renderer/partials/Licenses.vue b/src/renderer/partials/Licenses.vue
index 65f6a3760..ccea786b9 100644
--- a/src/renderer/partials/Licenses.vue
+++ b/src/renderer/partials/Licenses.vue
@@ -69,7 +69,6 @@ export default {
         'path': 'https://data.gov.tw/license/'
       },
       {
-        'name': 'pdm',
         'title': 'Public Domain Mark',
         'path': 'http://creativecommons.org/publicdomain/mark/1.0/'
       }],

From 4e25cb6f37a92492302c2905bdbdc99cee7e451a Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Thu, 1 Feb 2018 16:18:17 +1000
Subject: [PATCH 05/26] resolved double merge.

---
 src/main/menu.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/menu.js b/src/main/menu.js
index 4e5d48a9e..0fb3b34bb 100644
--- a/src/main/menu.js
+++ b/src/main/menu.js
@@ -251,7 +251,6 @@ const template = [
         accelerator: 'Shift+CmdOrCtrl+H',
         type: 'checkbox',
         checked: false,
-        accelerator: 'Shift+CmdOrCtrl+H',
         click(menuItem) {
           // revert 'checked' toggle so only controlled by header row event
           menuItem.checked = !menuItem.checked

From a41a785953f37e027e95a626950d2ef473d10f59 Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Thu, 1 Feb 2018 16:23:37 +1000
Subject: [PATCH 06/26] replace tag with css.

---
 src/renderer/partials/ColumnProperties.vue | 2 +-
 static/css/columnprops.styl                | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/renderer/partials/ColumnProperties.vue b/src/renderer/partials/ColumnProperties.vue
index 1d918ddc3..9f5c8f50f 100644
--- a/src/renderer/partials/ColumnProperties.vue
+++ b/src/renderer/partials/ColumnProperties.vue
@@ -24,7 +24,7 @@
             </div>
           </div>
         </template>
-        <div v-else-if="formprop.key === 'constraints'" id="constraints" class="col-sm-9"><br>
+        <div v-else-if="formprop.key === 'constraints'" id="constraints" class="col-sm-9">
           <div class="input-group row" v-for="option in constraintValues" :key="option">
             <input type="checkbox" :id="option" :checked="getConstraintCheck(option)" @click="setConstraintCheck(option, $event.target)"></input>
             <label :for="option" class="form-control-static">{{option}}</label>
diff --git a/static/css/columnprops.styl b/static/css/columnprops.styl
index 0caeae89c..54390fa1a 100644
--- a/static/css/columnprops.styl
+++ b/static/css/columnprops.styl
@@ -17,6 +17,7 @@ dangerColor = #ff3860
         width 70%
   div#constraints
     display block
+    margin-top 15px
     padding-right 0
     padding-top 5px
     div

From f03c0b14354bb99be1fb7808f417fca4076137f4 Mon Sep 17 00:00:00 2001
From: Stephen-Gates <Stephen.Gates@me.com>
Date: Sat, 10 Feb 2018 14:12:53 +1000
Subject: [PATCH 07/26] update macOS version in issue template

---
 .github/ISSUE_TEMPLATE.md     | 2 +-
 .github/ISSUE_TEMPLATE/bug.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index c64f8cfb1..4050e4c0c 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -23,4 +23,4 @@
 > Include details about the environment you experienced the problem - this will help us fix the bug quicker.
 
 * Data Curator version: 0.Y.Z
-* Operating System and version: e.g. macOS High Sierra 10.13.2
+* Operating System and version: e.g. macOS High Sierra 10.13.3
diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md
index 52fe0bf67..af83a0d47 100644
--- a/.github/ISSUE_TEMPLATE/bug.md
+++ b/.github/ISSUE_TEMPLATE/bug.md
@@ -16,4 +16,4 @@
 ### Your Environment
 
 * Data Curator version: 0.Y.Z
-* Operating System and version: macOS High Sierra 10.13.2, Windows 7 64bit
+* Operating System and version: macOS High Sierra 10.13.3, Windows 7 64bit

From 4a1ef6bcb9dc0f06a5a200a2669996841f5de923 Mon Sep 17 00:00:00 2001
From: Stephen-Gates <Stephen.Gates@me.com>
Date: Sun, 11 Feb 2018 09:40:52 +1000
Subject: [PATCH 08/26] Add frictionless data to About panel

---
 src/renderer/partials/About.vue  |  12 +++++++++++-
 static/img/frictionless-data.png | Bin 0 -> 8021 bytes
 2 files changed, 11 insertions(+), 1 deletion(-)
 create mode 100644 static/img/frictionless-data.png

diff --git a/src/renderer/partials/About.vue b/src/renderer/partials/About.vue
index 5485d226c..055ad2610 100644
--- a/src/renderer/partials/About.vue
+++ b/src/renderer/partials/About.vue
@@ -46,7 +46,7 @@ export default {
       {
         items: [
           {
-            label: `Beta version ${this.getApplicationVersion()} - access the support forum or report issues via the help menu`
+            label: `Beta version ${this.getApplicationVersion()} - Access the support forum or report issues via the Help menu`
           }
         ]
       },
@@ -82,6 +82,16 @@ export default {
           label: 'Includes software developed by the Queensland Cyber Infrastructure Foundation on behalf of the Queensland Government and the ODI Australian Network'
         }
         ]
+      },
+      {
+        items: [{
+          image: 'static/img/frictionless-data.png',
+          link: 'https://frictionlessdata.io'
+        },
+        {
+          label: 'Using Frictionless Data specifications and software'
+        }
+        ]
       }
       ]
     }
diff --git a/static/img/frictionless-data.png b/static/img/frictionless-data.png
new file mode 100644
index 0000000000000000000000000000000000000000..4add354906a32a68ed414c093cccf011abeaf891
GIT binary patch
literal 8021
zcmV-bAFAMqP)<h;3K|Lk000e1NJLTq007$n001Wl1^@s7@zf2p00004XF*Lt006O%
z3;baP0018hNkl<ZcmeI534B!5^~c|vOhOWN1R;nEn~I9nx*{PUTa^mc{sk?l5H`gH
zH?WFJLH=r8s7keH686Pb+p5@tvJ=26V5_1~1-Jf%fFS!4lFZEi`@1uD@-mqTi=f2j
zem-yRz2}^J?!D)nyWjW5-7+vW<ak{Dyh7I#J>o&XIJ$1qWv<^TIO5#nF!ULi&j|*L
z10U#7yj{C?8J0FPvu%;W_@Q>~GPg&<0mUs%R#sL=aI3?(%+3}bp$$R1rf1KdDT@0Y
z*ZDLQ2+Z+{a><e<rna3rb!s?&{(M8|y1voS7k22-p~<2}i;|RBj>0`03Wb~{OO~Dy
zhB&{%pO=}LdB5wrd*S6mbK9m(n;HV^X=q~gDB$?<xeps8KU1dUHnrh+zGx>-%xxEo
zd->&;O<}!o*PEHQmR_PG6*dOrOHTU9kz!a8A7d<DLqV|{Z31LI+=k`bc-U~-WpwD!
z_F}<hELmA?`vQ*`R^}YRi3jCdXEKxctO#A#`K?9bzum4~+i^CI(w~HBFQ3mHg%NEE
z1bp3u!%#W|gKpN+rOS+-d_HHI;KY5lK#DtC2c`Yq1mgUl<Qf({abnInHau(AtZE96
zA3wea2J)1J=jG)MlfmdA+#7rwi<4i*K)eWrZEa)Wl_hQrQiVkoC)dl__n&wLN7n(`
z>L%K7W5R2oIUSX#-40JqPM%Rzv|q3gG<#5C2EO`-bm;3qZ#o{)D;!Cep*}<^U9@P~
zJIa#|X@h*9FEk(!aGr)oI^D)>%%}sb7c5z_YzX}Wwijv!?i6_YvCT{J^Ybw+lePiL
zH%H27@^RF(6FtgI&So5rg?r90yxnkHcrw07FdHY$_8vWY)GSStIkObkcDdkX?9~!R
zd`qjS4Tme|IPKfF?IJjNe8BT@ghm3kz4`~F`M2WH3Yy4|@u#AOzHs5fsQIe#s{ZQr
zCrp@Nw57O4iir~^*1;@I4<*>aI68udL}&Rb3=dZ$$oTO&y};jTan=XcbLGmF$&qxA
z$&cp}b|{s-Gf`y^Ny_o0b1cK6=JtIJ;oqk)qLr&RZqJADGFEpR3I+cxoVs~DdLj2i
zx@aHWZcp2jg}eB0cMiU1Qc}{Ml9Cbz-_R`Li<d54`b&DdZ2XClG@D4%NPHF7@WFKQ
z+F&rSSTH+u*sy$h#&0M?1K|``I-4)UMk=6pd_LcFhKK3GN6j*JI1sYbx#uM%`OQ2P
zX00f2FX5<oOEI^ZX^aU!OO^@mx}hh)9q0G^`zxND?EF)HJV(W@qfq(af|ef@-eSW_
zLj@jC+$&JQl_#_kTsMR>&f=q$6Hy|k0hfunSrLdPBcrjI4pEl1iP8v8VY1|-!5sMu
z7cOP|$^1;=SO>2Kf>HSB0)Rp{U^S5zORKml5E<v9<r17qet}i3Q&?tsKhGn8r`6Bf
zcfEjKmy#0v+QuWR5NFe-f)Rv`sWMB^%-}i%@U*3i_!}i<@K1v?nid@CHV5`F!QByG
z;L!?WQnN#$k`oaM7O?CYU_&-;+_($;G!fx45Kd2cxL8eV1hOhCE7Mp7&EOMo_ahK9
z{b!|gx(4tOVUtXo8czS?`SmfLJz7z9CQX{uTzC!Id3ia8matiO*g;iz!e%hl033-w
zJay{S49(Ovxaa2PUL!aI@MZGl2;av*eUrlA&z2wS;8(8cknUvBk>~IU6LPOsc+#Yt
zGv#}oDTn}D2BuAoXO=KJiwrjV{m!Y`*@KqY#F`;|@WH<x957&znPI2`*!ywm>FJwx
z?b>lSbV(^GspIzV-**%66xzW%nA^qF{aremQ&LmYC->>oXB(9oQoDBTNjrAzXo+bZ
z-@kwVc~hoLIlic<sH9e{T0466>J_kg$|s*j!C>)~HEY(H0Io=tqkMIflAO2u^&4#F
znU+4`48;<}^M4Gpmw5H6Ot5n1T$u|0UtQMNdCAtnX(#;<0vS1S<Y5~TBbp{&N7Z%i
z+_`qV47|(nc=)3Ri>J8Vw^whtePH@|iJsWPXlBhk$ZUXkadv`dVKxr!D@;9x3ocmK
zu3e9}H0-lY^(xV(DaB`QM^R>5Ot=`@_1c%Se1(U$zT)J^)F!-{gTFD}{)WZdyeyxN
z9Xp<2VV7TixtVvyqqQ{QbeVIRE)BLY!cS4}&86|Sdnv0{5_uLDPS<t~W&K4AjCp3;
zJD}B6(zxhy#{z=E&{u_f%8Z#7<@KWjzXw=X3JJ&EeCCp7Z?bF!-H{%8ttd2u-#TZ`
zoNp|g`Q-IHHk#0<5_d5v_%n(NS|TR%ClwTIC?Wo`VBvXf+b1a3%|u%Yb|A+*b)==q
z&tEz+GB<8ZJ>~&Feb=sCI&L^eaLUk=$;db+CdV<)W=^}rJJX9EHHaT`;bHL5#0F!{
z9FI2u{(lAbpbU{k$@{{^Iutga$w?dS5K8eVHHUmQms6Zc=sNpd$JIpqP~0{q<2_>&
zsDgsEZ=ZQpFzC!6ILgv6p`S%19tkVgy<0Ofgm7Eu&Rvqc3bFKFA3^+1hQd=wy4y+D
zW8=nxpV|B@Ek<w;;d{b)Sg7M_mmDRXY9R=5io(nczjWQ85juv|3d33rbUuo?55F6;
zyg_Q^^XAQ!b(6-|7kXMk=r-bB#fe>e1kDf-yj<n;`;#-;w{L#|JbNRn0V1Hh=SIp_
zt}&37S!M7vyREC$pQSa#TJXSw)tq(lS9KO!<^S*!FfGpK__hR{XpeJPrD8$Wk2TlE
zfddD=$pUKxmRmyji3t;Ot{}{Y3w^_YK~J)Sdr2@D{3*ESSmbpbFkrwO=*@U1{PgM5
z>-J#{#2TwNYoy)aXDY4u^zT3TZ=$8;R`>awJy`n-;CImqT<9NY)Tq&z?%liZ)j~5n
zd(b;dC#QuUG~EUb8Z<5p3IDR=TbuRL4lljRl#-INs<5!ICtCum;J?tOflqhB^(gyf
z!C0r!OSrESUk7d&EuX0K1+A;8Eh9(B%UQoIhVS{5AsyO$)@!Y^v$I!Co0i+;y6Xmg
zt#~k4av9ZF1^!KV-3PCdq@?7(_UqU0Gn>EHH#50<d3$rTHuJgeb6(`3g_Gt;I7)+m
z5|59<*5t76V!uE1Iu)3t%2A28DO@C<V5aD)<Xd_a4g~gUwSGS|Gpy23*0X_tTQGTY
z&Z$x^O7qD_z~&qeX@yYAXemuR5L`p;7$__Uudnb5z1#leF_MLG3*i~p&146Bdt#X=
ztpWVTN@>vDD;hZ#&GX2rIdyY^pAF3zcuQ$o$e*G*G;P}AHE$WiO3S533p9hpSL|Xu
zDWy@Tr}pi08j5cyRC1bp^3DqT&WDm&BBe!W%6r=f)^1uJS_L_lByh+(Cw|9wex1C%
z?;T5(_-`wV=_dc%DqY#@Tcxy0sKF|wRYIkeTW}xf^t*Oy-L;NmCTo?jx<8U4$L>r1
zP}Q<GNe?G~(k2~Fy6>9CyC?cxc^yjna<j+7IaAYi3u~r=(u_kZSf4%4w#;JIG3&rp
z!*6Q0okU+6&g7snlnKoiXzH_t#EEKX6ufjAkp`@~aN^C@j&1`+)3RUp1#7cY!0(5y
zTD^M5)tWhT<}S8`Sm*~Iv3V%1(y_HvlfA#)ULL~90s9x;N#bPy{6z8#0@E!+buY)m
zBMlyf;z4{pPhFmLZ(hVret4gewmxZVQ?I+&OMF^5Ii&};iagF{3#|v`eL-+&?QKSr
ze<`gqlWjV)$y}~`6p?oR<;QcieRbBjIUCCLkE)=TUZ(=YH&Yq2Gs3>(W7^%<MzN)n
za9LP30p?m5dxW>MH)xL03E#kab_l%NSiJS&^*Z7A;BVHI@M^Gi>$f|BMR}3uSL|D!
zY||XbXT$c|#fz8DwRF^hU5<cD{?Xf#!cnffF_yQ0V_=SBPX!eAI?84i@q$T7Y#5J|
z!gUO^e}CcED5$k??8|Bg$m{}%&MV7HFTM1%l9J+O(Do3mT)BzJmO*K3p9fbadUgc2
z8zqF3m;NHO4EvNq%Y@2QTo|>6R;|vAZsZDv!k(3XAcxR;plow5LA2Ttm(LMUb@^EF
zOXN@=#XY6?!MI@1cMgi;o_AJ9($!tOc=4A?M|ftJE?uHViE`&5#GkQ&Jw^(fJ$rVv
zbtcZoogD}SZWA7b<<s)Ac%Fgx;KXTk-KcnJT3Y(g*eZJ++Rmaar!*v)Q4y2GxX@UH
z6nVl_YU;8aJ>YKkwld2nCnu-ahL3<>#8#BLRpYR96d1Z2B^Zmg@Wj3>Ut3CMTd6LJ
zm(@!~pYGeYcYUO_)X;sc^z`aAFq}racI~uXWn#8qV6~O9m}4X4ayG5&3k>Z}U@umb
z5Zd}2Y3;$dIaLY$U9{zuRuP8^a8$0ReJmN3E<kX2j?UaZvG7vgol7^DTN)1^WP0O?
z!Z6*O6*&y`(4-)YHJ-Rry?Tw~=;A)~!V-nK-9D9;K2i_e>gS@9;&et9u7~^QK%nH0
zf=iIV`$XWni*}{pi5v><!|lbf&<%m-c3P<}kbfZhL>KVDKMwjE(XCmtdL`i$(fjoi
z>tJdI{#dx;A)l+m$3t?ou%WWjjvYHTU$9_7bO)PCcLg4#P$wduNpMGjL0ze21Gi$A
zpTDFb>6G>g(&p3E=F;5^iB0v^TW@WpTj>RFb6{%m8ZT`)DJk?`G3}l1u`5D7!Je_P
z_sxcI@GkfdM<!npzM4F)!qC>DgyRVhqa~E^)X2!lm}U7{A6_?sV{<sfB}L~;#<P#8
z?<)A!qwd>82hH$;f`T~;+H}?@?L8d$`d@kFm4IO6JC6EJ@um5E(Z1ReAD6qYL87Pb
zi+hZ9SlY1fjw~C;B+Ly<H#%ENNvYPPq@?J2(NWpamL5yxkimI_ie*)+=A4OwqYCr+
z{9F`vCs<l<THujcq`QL3HLqK@UUj+~GXvvzg&C>)7n=?Ny+WS9ZqT4Xk>$Yzr-kRC
zEA-`n+sD%kWwE30YYxBR%oaJVb{djLJHjUddyO)-^!bv`rwC_Jj+UBHB3Q%Q=Bd||
zFoOo*cfwnfEi++)*F#oeqJ`g+@R?)NS=lgY@r%|fEKIu$qYSMzGcB#vY$ExGGwt`L
zuL>Vs2^_I8?dTS>in~_=amO?-!NG(%46H}@aW#j7Ih@G0<V2Q1+4lj5iP014nl)=b
z$BFSiPP^|)ORF)ZckkY2+~iR37V=2v*+iPvq*<v$RJxX%bf~EOOHZ3~&f##ii(oq8
z4TXvuWOG_g{<?Tm3;OYsCr>`1Z{NOJ^y;NXb4m{WM!Knw;k{IFIbc8MAk)moxFLMf
z4L98IpD@PY?qOWpH(<bEO`wdUem>F+r0abNKD%@n%AssW@_a@FI$f7fmp+yHM}~o<
zWi@KlSecfV=C=p6l&LZ4H;}#q@>--c@VyT&KzW`3{}bxoRd8E>h2uK!c2-=13V^DN
zMN=3fR2%pDD2xe#<JdPv6{fNe%a<c<wtSS7CLg}_<U=!paCe2h4t_WB^sz9~59K+J
zQ{v|Z6K^Tb;=R6vgCflEpsNR796ZcPJb7Mg;qZFOhB3@KHcVcR+i*M|UXKWF>k0j%
zigQ}t89u3DoD^kfo`wDzg)4H=?(w^fV^|3FGq2-buFS~zUg8fatg8Zg6!!U&XUgx^
zt=oS2atN*#^k>@&2&NV(AHJ+@G^;aL5-wCY*3t5sSs0dIL*an~2j4>eH!6%(&w=+T
z2+K@T1k+0b+*$EYn2={$iQk{fb%4BEh4VV@`Rh8JaajE@?%{Wo@L42#l<fOR1ge9c
zFm>tgx#ylcQZZ(mM%I*VgkFe%$F@MA*yJAy1+wA$ShZ?t4~kauLa8Szd{8cSQgSef
zsBmA=A(#g!c*2$jZm8(+@0BlSEl^?LrlnaJ{GPGO7oHYql%lYlTAF-l&eBT|)22<6
z0h}k8TwY%8AHZd)YqM{i2oBBt@-06JQE}*5au~d>&(R%*TSXXWx-gY@L?nK#3=Lih
z>nl1rXv{2{vW$tuP3yC-vf6ts;LnBjfGy21t@^245-7LJe>93;b+6=TNVSS*Ws^X3
zE>hVFURB;I2~;)-R5iBBrm?E>R$2*EHMUBtx~hU#HVGVX^SGk#r&YGfR+YC(0zae#
z4%pbfuflN`hxZTz%PPu`)0VfnmeMv6E%*RCwfaidjKzzWjv*dB>*dt`6}A_3d*B`&
z9((iAp)vcpcXGSn2GX?BHY7eLN6xFs$th#(jR|;s4z8&;ZK5y--JQ6;vVb((NmHA{
z?sK_)@V=LxeCu$kGm6*dbjeH~oTB{BYWA96;IR7-URs5@sbH>mLN^XM+#HLK&}2}L
z4GNQ|x%R0rw;nd)HT3(#uXziW$U*xraOWh#3(f_#u8S8ho2xKyEw@2;4^huch}Poa
z@ZR~Da?Y$>yYARG-gslL=;S18>dZ|^=ML!gx?T#;dM+M(2+wiojY|!Q9XCnHGt@bT
zVjuaz!{N2Uq_MHrhGJ-S*^WK`(3#l6P;4Q38i!q7Lp{uupx89Aaq?Ti<%H;&2K7qi
zBAv>>L6zGb{rxRA%`wy1Y{5`a0fyAr#w+sU(PEGnau72QBY%?cRl4Poh=*=5?%uug
zBZSlf;eAO1>5HQBMUUY5&R1|0UpHN+29uhacIMo<bDMd2>ADaNWirBj2jRBy;wlPv
z?wom2VPR+u`EnfW@|9Cx7a0gwRjbkHxiu5Y!Qh{#e)Ew<tYPVZYY1h^=Dt@u8t}@B
z^7<|~Z-Yw2hjOmx&Kezs?>VkJ{gPWQ=AaFO=727A=l1Q}#*_9}N)PNJ>iiBt8i!8?
zIbZO>g9j(CS+k~VAP{;4odwaEIj$Bij*E1l?v13+5gvKCojP@DR$RP)B#mV?WdiEn
z0Dfw0diYO5uQW85>%+mZlxaD1?}`o`{0UD_)WK^Fk<%&fm6YrCC?Y1n!P{GabRxVS
zTY8n7vnOLOrqi>e2i4ei4yt)-kyEmWTA9NroA_Y;?{n)Pfh<RWx1$2Cc7=vf#<-0w
z^8JZ#z}I4`+=mO0GraXd@c&1=f8_C^zEc=w|Fp0$_#RA(Xxz=n=L<1J`SJ>Rq_mT>
zvf2)yA?MMkKBmE)DWr9f&Y^CnVz?`~Yqy|LqsCpmSGg?R(e~+(xNj0L9U}(6+%`1j
zivfR%HzcME9U^g^cz<Ydkz34aU==koOKt>ZBl3}*8sKL<*Ynw>o40NI_6y201zB(P
zHe6(ND-EnK4QF(#R;@;J!BivBe9~m-#Sp@W<*>J>$S;eM^rtL;Baiikh5MEQYaS_^
z;d{WwhQdE8a*DZt^Zhsq82yBjH%AO?8pHPWf<a>i?s~s<1>C=X|Nd^!>Z?0kdq{fF
z%flPyoBTb9*FqSd#x^uB-BAdeI-r<|wxPw#BR_xHcpBPc)aeZGH^9TR?zc3+rWn(;
zDE6g-f(<1Y{<ZKc(<{g0dnhozF+L|eN|a4yM?CUdi=k;M$mkxL0_~ItrQQ@`D(X(+
zQz9?Y2#W-#0X|MSzodbVrQF|G-q7;4k~4=-G(O3D75|9vI4m#nHq}d9UZG)yu7Zha
zz?AbM;=d;^W(!Lm;&Cp$G_y-_ap-+s=ve^W1&5O%_0iOm&zCrP=wzaJpqF0M?^S$F
zNuxIu4%pZr{@xmz3W_ieIE{n`xry4Tua{Fyh#!Dw;c2~fD-H8`3y<fgrlv0EeG8>I
zwPVMQC$ld5DjwZYfN3s&2c_*y-W0WTUFpq>HNZKHTn&KjYtW$a3NOxvF_=H{ioh$}
zIG>He|A_HE0#2RC5plLrRm3bZg0=C|VAy5~N&Uvs5%6k5+lSG|Z$uO65Tc!eW*7?>
z&h5nAt<g+J-=+?s)KvGBdGnV3$IH_b4r|^6C?C`)V{GXY`79S|@eYj|6fTrbC$>nT
zlJL6^8r{xkN$@<K?k$2JtV2gF&=C#Wp-gT|%hyEtw+sAU@LdjHb8X0Xu#AsUI{iz-
zPwc~cqID?edPd{tsN@3`m09y?M!qwkDP~>vIn%ZqW9bp*=_r3ALfl38{8;?q#AzV&
zQA~f*HqXjxw?RE_EFWNZp~Qaz-^Uvgtt*-$3WvdWr7maFa2wI1ejLm1XvOuhNE*^x
z$VQ!9)Tsf|xR^q^c4}^>n&_lA&$G}=PsY*Mg4$@QQKM#i%D)|;TUh8E)^Y{8orkWp
zi6vS59a)==FdEuo_@=X5P$zr|_R_&VylK(|+$Y$~&6Mp1Q>cUZ(K-_yErZmhQ~J5(
zXZ?(f#!V>04){LB22wk*43BL54H84jTqhW+pk*KXpR$Y0yPIGDuTe=pYMz-%<u6Ml
zJ9=3yep8qV^k8I|O2dlYcvTvN8AO&7>(r?`)VrgWh~L4&F~*6|PGp+F{q)c>@=fxl
zWlHijyJ{UMUv$*=;^T!OT?M=b2KN{}YV_i1EG@nHKgcf@Y~rkC%Iwr8nnsG4GTw=o
z%ix^Ph_n=*wG9)`4>Y_d=8i-MqXhQ({L7)e0=e>jT3gpc>d+v>^IJB3E@3M9t``?S
z9C{HRFM4sL4^a1)*oD22jiW8XVGESeIS0dki4i95Y^Pea)b@0^Ut=?qv4s^8yYR9u
z7hb)3_1^*VKAiI?oyF`}0<B-^R60}F71UWBQe$_|;Pi~@IeQDZeLR8$nua8B$9`i=
zQ1B{nl?09n3Ha2|j){^~6{1Q4KXM6F_BGp&Ts>9Q`!Pu%+Sqsl@lm$%p8KIHrx&|;
zne2b<n~1#F)q=P1eyZ~Df>)P`o(~y=>(0|tIQ!z8!m*BYcZ*j4B!pj&m(xbgm@%Wi
z=)CE?VJ{ETXWM_!9nU+uq<>QH1&fBVUBt_Dw|VLHavi*M<`R{w#mD9m{#&N-%Xr$A
zg!k{nQ}p_hHSx#EtGeQpgI8g5-xgeP>nu<34S7b|m*c4Gh45-->Ab$mrZPfPW<!Yd
zL&)Q0D}S5DlZW20>?XL%OSpsbv~*q{-gnx!PQ{P-l@^a2{|7nskxuk_uxP!ywfR~4
zSU(o8Kd$A?&|g_P@V^mX{NWZxIqxSQ9XJ~&J0m{K#^uA~YVV)mlIE8xr*MR?(Oatu
ze-~F~>%U&^J{iw#`s2=ED0J(0VSK1*SwIHYty@=%t30<Mh<UuG`#y`>MBBXD1v18h
z;{6E27(pb@FL+^n4Z<Fgk#WKp&#T33#X7%<S1w*BZh){mTAbfs?Ok59-(%zQ+2y;L
zS9PNb?BBn?7V*)AJVtam@s*TYn=H!5Yzp8X(2@3Gc;96S^~rd`zeW!C;j`hP%?0Oi
z`M`e5YtUsjK=>=MJ$x~}$>*Q19YQ0UPuZ4>HlCX>Ay0o_HVm23fb%Yc*PEi_>qeoJ
zl(gIV3$ixATLCY3UCi=}=w%TfF<1qkdy)5VElxfkCHSVO$Q=T1ZuRQb^Lc&yHNmN8
zSG;ut1`M{}t|=z|H2GbRF>lphnW2luUR~6vQF|%r%{iTT#JcS4!S9pLX=)5g&j%dF
zAd3Z*sf}pWVCWlWQ6K4F4d=mu0|z~zPE`2<kKQsVDRJ2li#k3!=SDlz;$k<8v?mM8
z#}&4r=YK>g4>C}uvDq{~M59B2s_0N#284iN<vflN{@TK<9}%1}LelY{4})e|ob?gZ
zOwOOKApRE%lTWW_{(r#>!;m-k?>}gP7Zw|4*J+D-O#9xy@bubs`t|F#1Eb@rQ1M0F
zJA9K~`eOcmU>;>3McLyv?pWR!`brdkF7?u#MB`LW{=YAL5)^?&8sWVteK2{g!7z=Z
zOuGpTjF6d;2g800eq|aZd5oqh%u*ReTT&9bnAg|WQisoMT8wBR;e{G$**$!k_x>%a
zoL^G1|1x$k-;PDln5F~MW1I}0bF@{QZLTOR+}jtRZEOZlhj~HYzJsRISk$R(IFK&F
zTuXmxSbw5(y@Q#+mpgZEAJL*k%XY|UD7LYLok)Eh#K@KU-$z5?zc8dDbqb;56xNqe
z#0{C69aYUB_`8L^YLqsY<m6Dk8ncZbjvu$NSw>|WofAD9vG^nHW4xzxH&7!)GS6=Q
zuVSR~NJ$BO6$tpt7@pu^=aZq{Oj~99+)cs&`E$vXNqjcuBk&(#bZqMfTN4iW{r<mT
zm}`}e#^^B;U)){_Wqn*!<m?bVqro%qsu|8JBw1`~-MaM@XhOiwK;a`rXC1JwQMmqN
zBXTXNY#yZB<GOx>!?<48)Qmdx(uX<J`1kDDGo3umIZ;}gv#GGK^gFTe%+VKI3JShH
z$>xXbF5nMrvxp}%GHh(#yk*o-L_=B5OHNLH!G2CBl1`u7=}wsM#+t8}MMrauwD3HI
zS*MW=4P`SQbNmHH9hpv<PXa|7RD_@LI3`Kov}sds>cn?wLunZC%QU7>BQ*NX1s8aO
zmr~^tbTTtLJ9{W~ilEEN$;&q!j*k!PC-)ZW{SbFdOBmr_LhoH@Xy&X>)7oG#=~eRj
zw}+?d|8vUegMO{wAKWRNIyGRA(g|Kg$MGuvW%3CfXcuTN1)U^Y^ZAnJ!b`_32krhJ
X4@L8Nfq1a*00000NkvXXu0mjf(4zFt

literal 0
HcmV?d00001


From 05958103bcf8aa923cd74dac154dc77cf76df1cd Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Sun, 11 Feb 2018 22:58:21 +1000
Subject: [PATCH 09/26] Testing foreign key validation before frictionless
 pr#124.

---
 package.json                          |   6 +-
 src/renderer/frictionless.js          | 253 ++++++++++++++++++++------
 src/renderer/frictionlessUtilities.js |  13 +-
 src/renderer/mixins/RelationKeys.vue  |   2 +-
 src/renderer/partials/ForeignKeys.vue |   2 +-
 src/renderer/store/modules/hots.js    |  18 +-
 src/renderer/store/modules/tabs.js    |   3 +
 yarn.lock                             |  54 +++++-
 8 files changed, 280 insertions(+), 71 deletions(-)

diff --git a/package.json b/package.json
index ddf264841..17e8275a1 100644
--- a/package.json
+++ b/package.json
@@ -78,6 +78,7 @@
     "bootstrap.native": "^2.0.17",
     "components-font-awesome": "^4.7.0",
     "csv": "^2.0.0",
+    "csvtojson": "^1.1.9",
     "datapackage": "^1.0.5",
     "escape-regexp": "^0.0.1",
     "etl": "^0.5.8",
@@ -96,12 +97,13 @@
     "node-fs-extra": "^0.8.2",
     "pikaday": "^1.6.1",
     "pouchdb-adapter-idb": "^6.3.4",
+    "promisepipe": "^2.1.1",
     "request": "^2.81.0",
     "rxjs": "^5.5.2",
     "slug": "^0.9.1",
     "sortablejs": "^1.6.0",
     "svgo": "^1.0.0",
-    "tableschema": "^1.5.1",
+    "tableschema": "^1.6.0",
     "temp": "^0.8.3",
     "unzipper": "^0.8.11",
     "vee-validate": "^2.0.3",
@@ -138,7 +140,7 @@
     "devtron": "^1.1.0",
     "electron": "^1.7.11",
     "electron-builder": "^19.27.2",
-    "electron-debug": "^1.5.0",
+    "electron-debug": "~1.4.0",
     "electron-devtools-installer": "^2.0.1",
     "eslint": "^4.10.0",
     "eslint-config-standard": "^10.2.1",
diff --git a/src/renderer/frictionless.js b/src/renderer/frictionless.js
index 569a392d0..42ac91c57 100644
--- a/src/renderer/frictionless.js
+++ b/src/renderer/frictionless.js
@@ -1,20 +1,23 @@
 import {Table, Schema} from 'tableschema'
 import {HotRegister} from '@/hot.js'
 import store from '@/store/modules/hots.js'
-import {includeHeadersInData, hasAllColumnNames} from '@/frictionlessUtilities.js'
+import tabStore from '@/store/modules/tabs.js'
+import {includeHeadersInData, hasAllColumnNames, hasAllColumnTypes} from '@/frictionlessUtilities.js'
 import {allTablesAllColumnsFromSchema$} from '@/rxSubject.js'
+import stringify from 'csv-stringify'
+import csv from 'csvtojson'
+import promisePipe from 'promisepipe'
 
-async function initDataAndInferSchema(data) {
+async function inferSchema(data) {
   const schema = await Schema.load({})
-  await schema.infer(data)
-  return schema
-}
-
-async function initDataAgainstSchema(data, schema) {
-  // provide schema rather than infer
+  // workaround for schema.infer stripping headers
+  let dataClone = [...data]
+  let headers = dataClone.shift()
   // frictionless default for csv dialect is that tables DO have headers
-  let table = await Table.load(data, {schema: schema, headers: 0})
-  return table
+  // await schema.infer(data, {headers: 0})
+  await schema.infer(dataClone, {headers: headers})
+  console.log(schema)
+  return schema
 }
 
 function storeData(hotId, schema) {
@@ -29,7 +32,7 @@ export async function guessColumnProperties() {
   let id = hot.guid
   let data = includeHeadersInData(hot)
   // let activeHot = HotRegister.getActiveHotIdData()
-  let schema = await initDataAndInferSchema(data)
+  let schema = await inferSchema(data)
   let isStored = storeData(id, schema)
   allTablesAllColumnsFromSchema$.next(store.getters.getAllHotTablesColumnProperties(store.state, store.getters)())
   let message = isStored
@@ -38,37 +41,143 @@ export async function guessColumnProperties() {
   return message
 }
 
-function checkRow(rowNumber, row, schema, errorCollector) {
+function checkRow(rowNumber, row, schema, tableRows, errorCollector) {
+  console.log(tableRows)
+  console.log(row)
+  // if row contains foreign relation objects cast the original
   try {
-    schema.castRow(row)
-  } catch (err) {
-    if (err.multiple) {
-      for (const error of err.errors) {
-        let columnNumber = error.columnNumber || 'N/A'
-        errorCollector.push({columnNumber: columnNumber, rowNumber: rowNumber, message: error.message, name: error.name})
-      }
+    if (_.isArray(tableRows)) {
+      schema.castRow(tableRows[rowNumber - 1])
     } else {
-      let columnNumber = err.columnNumber || 'N/A'
-      errorCollector.push({columnNumber: columnNumber, rowNumber: rowNumber, message: err.message, name: err.name})
+      schema.castRow(row)
     }
+  } catch (err) {
+    errorHandler(err, rowNumber, errorCollector)
   }
 }
 
-async function checkForSchema(data, hotId) {
+async function buildSchema(data, hotId) {
+  let schema = await inferSchema(data)
   let hotTab = store.state.hotTabs[hotId]
-  let schema = await initDataAndInferSchema(data)
   schema.descriptor.fields = hotTab.columnProperties
   schema.descriptor.primaryKey = hotTab.tableProperties.primaryKeys
   schema.descriptor.foreignKeys = hotTab.tableProperties.foreignKeys
-  store.mutations.initMissingValues(store.state, store.state.hotTabs[hotId])
+  store.mutations.initMissingValues(store.state, hotTab)
   schema.descriptor.missingValues = hotTab.tableProperties.missingValues
-  let table = await initDataAgainstSchema(data, schema)
+  return schema
+}
+
+async function createFrictionlessTable(data, schema) {
+  // provide schema rather than infer
+  // frictionless default for csv dialect is that tables DO have headers
+  let dataClone = [...data]
+  let headers = dataClone.shift()
+  let table = await Table.load(dataClone, {
+    schema: schema,
+    headers: headers
+  })
   table.schema.commit()
   return table
 }
 
+async function collateForeignKeys(localHotId, callback) {
+  console.log('inside collate foreign keys...')
+  const foreignKeys = store.state.hotTabs[localHotId].tableProperties.foreignKeys
+  if (typeof foreignKeys === 'undefined') {
+    return false
+  }
+  console.log('foreign keys detected...')
+  let relations = {}
+  for (const foreignKey of foreignKeys) {
+    let foreignHotId = getHotIdFromForeignKeyForeignTable(foreignKey.reference.resource, localHotId)
+    // foreign keys must also have column properties set
+    if (!hasColumnProperties(foreignHotId, callback)) {
+      relations = false
+      break
+    }
+    let data = getForeignKeyData(foreignHotId)
+    // let json = []
+    let schema = await buildSchema(data, foreignHotId)
+    let table = await createFrictionlessTable(data, schema)
+    let rows = await table.read({keyed: true})
+    console.log('have schema fields')
+    console.log(schema.fields)
+    console.log('have foreign table')
+    console.log(table)
+    console.log('have rows')
+    console.log(rows)
+    // send data and headers separately to csvToJson
+    // let headers = data.shift()
+    // console.log('calling csv to json...')
+    // await csvToJson(data, json, schema, foreignKey.reference.fields, schema.fields)
+    // if (_.has(relations, foreignKey.reference.resource)) {
+    //   console.log(`Warning: Relations key ${foreignKey.reference.resource} already exists.`)
+    // }
+    relations[foreignKey.reference.resource] = rows
+  }
+  return relations
+}
+
+function getForeignKeyData(foreignHotId) {
+  let hot = HotRegister.getInstance(foreignHotId)
+  return includeHeadersInData(hot)
+}
+
+function getHotIdFromForeignKeyForeignTable(title, hotId) {
+  // check for fk in same table
+  if (title === '') {
+    return hotId
+  }
+  let tabId = tabStore.getters.findTabIdFromTitle(tabStore.state, tabStore.getters)(title)
+  return store.getters.getSyncHotIdFromTabId(store.state, store.getters)(tabId)
+}
+
+// async function csvToJson(inputData, json, schema, headers, schemaFields) {
+//   let stream = stringify()
+//   // send headers immediately to stream
+//   stream.write(headers)
+//   let indicies = []
+//   for (const [index, field] of schemaFields.entries()) {
+//     if (indicies.length === headers.length) {
+//       break
+//     }
+//     if (_.indexOf(headers, field.name) !== -1) {
+//       indicies.push(index)
+//     }
+//   }
+//   console.log('indicies are:')
+//   console.log(indicies)
+//   for (let row of inputData) {
+//     // ensure foreign key values include field properties if valid
+//     // let castRow
+//     try {
+//       // castRow = schema.castRow(row)
+//       // castRow = row
+//       let filteredRow = row.filter(function(elem, index, array) {
+//         return schemaFields[index]
+//       })
+//     } catch (error) {
+//       console.log('Cast row failed', error)
+//       castRow = row
+//     }
+//     stream.write(castRow)
+//   }
+//   stream.end()
+//   try {
+//     await promisePipe(csv({checkType: true}).fromStream(stream).on('json', (row) => {
+//       json.push(row)
+//     }).on('done', () => {
+//       console.log('completed csv to json.')
+//       console.log(json)
+//     }))
+//   } catch (error) {
+//     throw new Error('There was a problem converting csv to json.', error)
+//   }
+// }
+
 function isRowBlank(row) {
-  return row.filter(Boolean).length === 0
+  let isRowBlank = row.filter(Boolean)
+  return isRowBlank.length === 0
 }
 
 function blankCellCount(row) {
@@ -100,13 +209,57 @@ function checkHeaderErrors(headers, errorCollector, hasColHeaders) {
   }
 }
 
+export async function validateActiveDataAgainstSchema(callback) {
+  let hot = HotRegister.getActiveInstance()
+  let hotId = hot.guid
+  if (!hasColumnProperties(hotId, callback)) {
+    return
+  }
+  const data = includeHeadersInData(hot)
+  const errorCollector = []
+  const hasColHeaders = hot.hasColHeaders()
+  // ensure headers not lost from data
+  const headers = data[0]
+  checkHeaderErrors(headers, errorCollector, hasColHeaders)
+  let schema = await buildSchema(data, hotId)
+  let table = await createFrictionlessTable(data, schema)
+  let relations = await collateForeignKeys(hotId, callback)
+  let tableRows
+  try {
+    tableRows = await table.read()
+  } catch (error) {
+    console.log('cast errors on read')
+    console.log(error)
+  }
+  const stream = await table.iter({keyed: false, extended: true, stream: true, cast: true, relations: relations})
+  stream.on('data', (row) => {
+    // TODO: consider better way to accommodate or remove - need headers/column names so this logic may be redundant
+    let rowNumber = hasColHeaders
+      ? row[0]
+      : row[0] + 1
+    if (isRowBlank(row[2])) {
+      errorCollector.push({rowNumber: rowNumber, message: `Row ${rowNumber} is completely blank`, name: 'Blank Row'})
+    }
+    checkRow(rowNumber, row[2], table.schema, tableRows, errorCollector)
+  })
+  stream.on('error', (error) => {
+    console.log(error)
+    errorHandler(error, 'N/A', errorCollector)
+    // ensure error sent back
+    stream.end()
+  })
+  stream.on('end', () => {
+    callback(errorCollector)
+  })
+}
+
 function hasColumnProperties(hotId, callb) {
   let columnProperties = store.state.hotTabs[hotId].columnProperties
   if (!columnProperties || columnProperties.length === 0) {
     callb([
       {
         rowNumber: 0,
-        message: `Column properties must be set.`,
+        message: `Column properties, including the column properties of any foreign keys, must be set.`,
         name: 'No Column Properties'
       }
     ])
@@ -116,39 +269,33 @@ function hasColumnProperties(hotId, callb) {
     callb([
       {
         rowNumber: 0,
-        message: `Every Column property must have a 'name'.`,
+        message: `Every Column property, including the column properties of any foreign keys, must have a 'name'.`,
         name: 'Missing Column Property names'
       }
     ])
     return false
   }
+  // if (!hasAllColumnTypes(hotId, columnProperties)) {
+  //   callb([
+  //     {
+  //       rowNumber: 0,
+  //       message: `Every Column property, including the column properties of any foreign keys, must have a 'type'.`,
+  //       name: 'Missing Column Property types'
+  //     }
+  //   ])
+  //   return false
+  // }
   return true
 }
 
-export async function validateActiveDataAgainstSchema(callback) {
-  let hot = HotRegister.getActiveInstance()
-  let id = hot.guid
-  if (!hasColumnProperties(id, callback)) {
-    return
-  }
-  let data = includeHeadersInData(hot)
-  const errorCollector = []
-  const hasColHeaders = hot.hasColHeaders()
-  checkHeaderErrors(data[0], errorCollector, hasColHeaders)
-  let table = await checkForSchema(data, id)
-  // don't cast at stream, wait until row to cast otherwise not all errors will be reported.
-  const stream = await table.iter({extended: true, stream: true, cast: false, relations: true})
-  stream.on('data', (row) => {
-    // TODO: consider better way to accommodate or remove - need headers/column names so this logic may be redundant
-    let rowNumber = hasColHeaders
-      ? row[0]
-      : row[0] + 1
-    if (isRowBlank(row[2])) {
-      errorCollector.push({rowNumber: rowNumber, message: `Row ${rowNumber} is completely blank`, name: 'Blank Row'})
+function errorHandler(err, rowNumber, errorCollector) {
+  if (err.multiple) {
+    for (const error of err.errors) {
+      let columnNumber = error.columnNumber || 'N/A'
+      errorCollector.push({columnNumber: columnNumber, rowNumber: rowNumber, message: error.message, name: error.name})
     }
-    checkRow(rowNumber, row[2], table.schema, errorCollector)
-  })
-  stream.on('end', () => {
-    callback(errorCollector)
-  })
+  } else {
+    let columnNumber = err.columnNumber || 'N/A'
+    errorCollector.push({columnNumber: columnNumber, rowNumber: rowNumber, message: err.message, name: err.name})
+  }
 }
diff --git a/src/renderer/frictionlessUtilities.js b/src/renderer/frictionlessUtilities.js
index 571bb0680..fcc38e8dd 100644
--- a/src/renderer/frictionlessUtilities.js
+++ b/src/renderer/frictionlessUtilities.js
@@ -11,6 +11,15 @@ export function includeHeadersInData(hot) {
 
 export function hasAllColumnNames(hotId, columnProperties) {
   let names = store.getters.getAllHotColumnNamesFromHotId(store.state, store.getters)(hotId)
-  let validNames = _.without(names, undefined, null, '')
-  return validNames.length === columnProperties.length
+  return hasAllValidColumnProperty(names, columnProperties)
+}
+
+export function hasAllColumnTypes(hotId, columnProperties) {
+  let types = store.getters.getAllHotColumnTypesFromHotId(store.state, store.getters)(hotId)
+  return hasAllValidColumnProperty(types, columnProperties)
+}
+
+function hasAllValidColumnProperty(values, columnProperties) {
+  let validValues = _.without(values, undefined, null, '')
+  return validValues.length === columnProperties.length
 }
diff --git a/src/renderer/mixins/RelationKeys.vue b/src/renderer/mixins/RelationKeys.vue
index 7115a56af..50190c0b4 100644
--- a/src/renderer/mixins/RelationKeys.vue
+++ b/src/renderer/mixins/RelationKeys.vue
@@ -21,7 +21,7 @@ export default {
     }
   },
   computed: {
-    ...mapGetters(['getActiveTab', 'getAllHotTablesColumnNames', 'getAllTabTitles', 'getTabObjects', 'getHotIdFromTabId', 'getTabId', 'getAllForeignKeys', 'tabTitle'])
+    ...mapGetters(['getActiveTab', 'getAllHotTablesColumnNames', 'getAllTabTitles', 'getTabObjects', 'getHotIdFromTabId', 'getSyncHotIdFromTabId', 'getAllForeignKeys', 'tabTitle'])
   },
   methods: {
     ...mapMutations(['pushForeignKeysLocalFieldsForTable', 'pushForeignKeysForeignFieldsForTable', 'pushForeignKeysForeignTableForTable']),
diff --git a/src/renderer/partials/ForeignKeys.vue b/src/renderer/partials/ForeignKeys.vue
index 907b7c1c4..c44a958d2 100644
--- a/src/renderer/partials/ForeignKeys.vue
+++ b/src/renderer/partials/ForeignKeys.vue
@@ -157,7 +157,7 @@ export default {
       let tabId = _.findKey(this.allTabTableNames, function(o) {
         return o === tableName
       })
-      let hotId = this.getTabId(tabId)
+      let hotId = this.getSyncHotIdFromTabId(tabId)
       return hotId
     },
     getSelectedLocalKeys: function(index) {
diff --git a/src/renderer/store/modules/hots.js b/src/renderer/store/modules/hots.js
index 77ac9e308..22a81f052 100644
--- a/src/renderer/store/modules/hots.js
+++ b/src/renderer/store/modules/hots.js
@@ -59,15 +59,23 @@ const getters = {
     return hotIdColumnNames
   },
   getAllHotColumnNamesFromHotId: (state, getters) => (hotId) => {
+    return getters.getAllHotColumnPropertyFromHotId(state, getters)({hotId: hotId, key: 'name'})
+  },
+  getAllHotColumnTypesFromHotId: (state, getters) => (hotId) => {
+    return getters.getAllHotColumnPropertyFromHotId(state, getters)({hotId: hotId, key: 'type'})
+  },
+  getAllHotColumnPropertyFromHotId: (state, getters) => (property) => {
+    const hotId = property.hotId
+    const propertyKey = property.key
     if (!state.hotTabs[hotId].columnProperties) {
       state.hotTabs[hotId].columnProperties = []
       // return
     }
-    let names = state.hotTabs[hotId].columnProperties.map(column => {
-      let name = column.name
-      return column.name
+    let values = state.hotTabs[hotId].columnProperties.map(column => {
+      let value = column[propertyKey]
+      return column[propertyKey]
     })
-    return names
+    return values
   },
   getHotIdFromTabId: (state, getters) => (tabId) => {
     return new Promise((resolve, reject) => {
@@ -82,7 +90,7 @@ const getters = {
       }
     })
   },
-  getTabId: (state, getters) => (tabId) => {
+  getSyncHotIdFromTabId: (state, getters) => (tabId) => {
     let hotId = _.findKey(state.hotTabs, {tabId: tabId})
     return hotId
   },
diff --git a/src/renderer/store/modules/tabs.js b/src/renderer/store/modules/tabs.js
index b96c6c412..ad14f5506 100644
--- a/src/renderer/store/modules/tabs.js
+++ b/src/renderer/store/modules/tabs.js
@@ -48,6 +48,9 @@ const getters = {
       allTabTitles[tabId] = object.title
     })
     return allTabTitles
+  },
+  findTabIdFromTitle: (state, getters) => (title) => {
+    return _.findKey(state.tabObjects, function(o) { return o.title === title })
   }
 }
 
diff --git a/yarn.lock b/yarn.lock
index a362710a1..c0ce64545 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -491,6 +491,13 @@ axios@^0.16.1:
     follow-redirects "^1.2.3"
     is-buffer "^1.1.5"
 
+axios@^0.17.1:
+  version "0.17.1"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.17.1.tgz#2d8e3e5d0bdbd7327f91bc814f5c57660f81824d"
+  dependencies:
+    follow-redirects "^1.2.5"
+    is-buffer "^1.1.5"
+
 babel-code-frame@^6.22.0, babel-code-frame@^6.26.0:
   version "6.26.0"
   resolved "https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b"
@@ -2684,6 +2691,13 @@ csv@^2.0.0:
     csv-stringify "^2.0.0"
     stream-transform "^1.0.0"
 
+csvtojson@^1.1.9:
+  version "1.1.9"
+  resolved "https://registry.yarnpkg.com/csvtojson/-/csvtojson-1.1.9.tgz#e641ae72f7bc2fa3f9aaf127e021fc89447c1cd1"
+  dependencies:
+    lodash "^4.17.3"
+    strip-bom "1.0.0"
+
 currently-unhandled@^0.4.1:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea"
@@ -3142,9 +3156,9 @@ electron-chromedriver@~1.7.1:
     electron-download "^4.1.0"
     extract-zip "^1.6.5"
 
-electron-debug@^1.5.0:
-  version "1.5.0"
-  resolved "https://registry.yarnpkg.com/electron-debug/-/electron-debug-1.5.0.tgz#d88c02146efb7fc5ae1b21eac56fbe4987eae50c"
+electron-debug@~1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/electron-debug/-/electron-debug-1.4.0.tgz#bec7005522220a9d0622153352e1bbff0f37af2e"
   dependencies:
     electron-is-dev "^0.3.0"
     electron-localshortcut "^3.0.0"
@@ -4019,6 +4033,10 @@ find-up@^2.0.0, find-up@^2.1.0:
   dependencies:
     locate-path "^2.0.0"
 
+first-chunk-stream@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/first-chunk-stream/-/first-chunk-stream-1.0.0.tgz#59bfb50cd905f60d7c394cd3d9acaab4e6ad934e"
+
 flat-cache@^1.2.1:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-1.3.0.tgz#d3030b32b38154f4e3b7e9c709f490f7ef97c481"
@@ -4045,6 +4063,12 @@ follow-redirects@^1.2.3:
   dependencies:
     debug "^3.1.0"
 
+follow-redirects@^1.2.5:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.4.1.tgz#d8120f4518190f55aac65bb6fc7b85fcd666d6aa"
+  dependencies:
+    debug "^3.1.0"
+
 for-in@^1.0.1, for-in@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"
@@ -7482,6 +7506,10 @@ promise@^7.1.1:
   dependencies:
     asap "~2.0.3"
 
+promisepipe@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/promisepipe/-/promisepipe-2.1.1.tgz#c3046d608ad34dd635cc7ffd7e6d16ff7ac3cc83"
+
 property-is-enumerable-x@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/property-is-enumerable-x/-/property-is-enumerable-x-1.1.0.tgz#7ca48917476cd0914b37809bfd05776a0d942f6f"
@@ -8705,6 +8733,13 @@ strip-ansi@^4.0.0:
   dependencies:
     ansi-regex "^3.0.0"
 
+strip-bom@1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-1.0.0.tgz#85b8862f3844b5a6d5ec8467a93598173a36f794"
+  dependencies:
+    first-chunk-stream "^1.0.0"
+    is-utf8 "^0.2.0"
+
 strip-bom@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz#6219a85616520491f35788bdbf1447a99c7e6b0e"
@@ -8877,11 +8912,11 @@ tableschema@^1.2.1:
     stream-to-async-iterator "^0.2.0"
     tv4 "^1.2.7"
 
-tableschema@^1.5.1:
-  version "1.5.1"
-  resolved "https://registry.yarnpkg.com/tableschema/-/tableschema-1.5.1.tgz#cf030be7ac8dd7802f6a86e9826e34eaf59512a4"
+tableschema@^1.6.0:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/tableschema/-/tableschema-1.6.0.tgz#2ddf30b537b63856d27f315e3dd80abd71f365ea"
   dependencies:
-    axios "^0.16.1"
+    axios "^0.17.1"
     csv "^1.1.1"
     d3-time-format "^0.3.2"
     es6-error "^4.0.2"
@@ -8890,6 +8925,7 @@ tableschema@^1.5.1:
     regenerator-runtime "^0.11.0"
     stream-to-async-iterator "^0.2.0"
     tv4 "^1.2.7"
+    validator "^9.2.0"
 
 tapable@^0.2.7:
   version "0.2.8"
@@ -9516,6 +9552,10 @@ validate.io-undefined@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/validate.io-undefined/-/validate.io-undefined-1.0.3.tgz#7e27fcbb315b841e78243431897671929e20b7f4"
 
+validator@^9.2.0:
+  version "9.4.0"
+  resolved "https://registry.yarnpkg.com/validator/-/validator-9.4.0.tgz#c503ef88f7e6b8fb7688599267b309482d81ae60"
+
 validator@~7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/validator/-/validator-7.0.0.tgz#c74deb8063512fac35547938e6f0b1504a282fd2"

From bee29aad8a3771f0fe03f73ab62809cf37f2de6f Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Mon, 12 Feb 2018 00:07:35 +1000
Subject: [PATCH 10/26] #302. Completed foreign key validation before
 frictionless pr#124.

---
 src/renderer/frictionless.js | 111 ++++++-----------------------------
 1 file changed, 19 insertions(+), 92 deletions(-)

diff --git a/src/renderer/frictionless.js b/src/renderer/frictionless.js
index 42ac91c57..7f7c5581e 100644
--- a/src/renderer/frictionless.js
+++ b/src/renderer/frictionless.js
@@ -42,15 +42,9 @@ export async function guessColumnProperties() {
 }
 
 function checkRow(rowNumber, row, schema, tableRows, errorCollector) {
-  console.log(tableRows)
-  console.log(row)
   // if row contains foreign relation objects cast the original
   try {
-    if (_.isArray(tableRows)) {
-      schema.castRow(tableRows[rowNumber - 1])
-    } else {
-      schema.castRow(row)
-    }
+    schema.castRow(row)
   } catch (err) {
     errorHandler(err, rowNumber, errorCollector)
   }
@@ -81,12 +75,10 @@ async function createFrictionlessTable(data, schema) {
 }
 
 async function collateForeignKeys(localHotId, callback) {
-  console.log('inside collate foreign keys...')
   const foreignKeys = store.state.hotTabs[localHotId].tableProperties.foreignKeys
   if (typeof foreignKeys === 'undefined') {
     return false
   }
-  console.log('foreign keys detected...')
   let relations = {}
   for (const foreignKey of foreignKeys) {
     let foreignHotId = getHotIdFromForeignKeyForeignTable(foreignKey.reference.resource, localHotId)
@@ -96,23 +88,9 @@ async function collateForeignKeys(localHotId, callback) {
       break
     }
     let data = getForeignKeyData(foreignHotId)
-    // let json = []
     let schema = await buildSchema(data, foreignHotId)
     let table = await createFrictionlessTable(data, schema)
     let rows = await table.read({keyed: true})
-    console.log('have schema fields')
-    console.log(schema.fields)
-    console.log('have foreign table')
-    console.log(table)
-    console.log('have rows')
-    console.log(rows)
-    // send data and headers separately to csvToJson
-    // let headers = data.shift()
-    // console.log('calling csv to json...')
-    // await csvToJson(data, json, schema, foreignKey.reference.fields, schema.fields)
-    // if (_.has(relations, foreignKey.reference.resource)) {
-    //   console.log(`Warning: Relations key ${foreignKey.reference.resource} already exists.`)
-    // }
     relations[foreignKey.reference.resource] = rows
   }
   return relations
@@ -132,49 +110,6 @@ function getHotIdFromForeignKeyForeignTable(title, hotId) {
   return store.getters.getSyncHotIdFromTabId(store.state, store.getters)(tabId)
 }
 
-// async function csvToJson(inputData, json, schema, headers, schemaFields) {
-//   let stream = stringify()
-//   // send headers immediately to stream
-//   stream.write(headers)
-//   let indicies = []
-//   for (const [index, field] of schemaFields.entries()) {
-//     if (indicies.length === headers.length) {
-//       break
-//     }
-//     if (_.indexOf(headers, field.name) !== -1) {
-//       indicies.push(index)
-//     }
-//   }
-//   console.log('indicies are:')
-//   console.log(indicies)
-//   for (let row of inputData) {
-//     // ensure foreign key values include field properties if valid
-//     // let castRow
-//     try {
-//       // castRow = schema.castRow(row)
-//       // castRow = row
-//       let filteredRow = row.filter(function(elem, index, array) {
-//         return schemaFields[index]
-//       })
-//     } catch (error) {
-//       console.log('Cast row failed', error)
-//       castRow = row
-//     }
-//     stream.write(castRow)
-//   }
-//   stream.end()
-//   try {
-//     await promisePipe(csv({checkType: true}).fromStream(stream).on('json', (row) => {
-//       json.push(row)
-//     }).on('done', () => {
-//       console.log('completed csv to json.')
-//       console.log(json)
-//     }))
-//   } catch (error) {
-//     throw new Error('There was a problem converting csv to json.', error)
-//   }
-// }
-
 function isRowBlank(row) {
   let isRowBlank = row.filter(Boolean)
   return isRowBlank.length === 0
@@ -223,15 +158,17 @@ export async function validateActiveDataAgainstSchema(callback) {
   checkHeaderErrors(headers, errorCollector, hasColHeaders)
   let schema = await buildSchema(data, hotId)
   let table = await createFrictionlessTable(data, schema)
-  let relations = await collateForeignKeys(hotId, callback)
-  let tableRows
-  try {
-    tableRows = await table.read()
-  } catch (error) {
-    console.log('cast errors on read')
-    console.log(error)
-  }
-  const stream = await table.iter({keyed: false, extended: true, stream: true, cast: true, relations: relations})
+  // wait for frictionless pr#124 and uncomment
+  let relations = false
+  // try {
+  //   relations = await collateForeignKeys(hotId, callback)
+  // } catch (error) {
+  //   errorCollector.push({rowNumber: 0,
+  //     message: `There was a problem validating 1 or more foreign tables. Validate foreign tables first.`,
+  //     name: 'Invalid foreign table(s)'
+  //   })
+  // }
+  const stream = await table.iter({keyed: false, extended: true, stream: true, cast: false, relations: relations})
   stream.on('data', (row) => {
     // TODO: consider better way to accommodate or remove - need headers/column names so this logic may be redundant
     let rowNumber = hasColHeaders
@@ -240,14 +177,14 @@ export async function validateActiveDataAgainstSchema(callback) {
     if (isRowBlank(row[2])) {
       errorCollector.push({rowNumber: rowNumber, message: `Row ${rowNumber} is completely blank`, name: 'Blank Row'})
     }
-    checkRow(rowNumber, row[2], table.schema, tableRows, errorCollector)
-  })
-  stream.on('error', (error) => {
-    console.log(error)
-    errorHandler(error, 'N/A', errorCollector)
-    // ensure error sent back
-    stream.end()
+    checkRow(rowNumber, row[2], table.schema, errorCollector)
   })
+  // stream.on('error', (error) => {
+  //   console.log(error)
+  //   errorHandler(error, 'N/A', errorCollector)
+  //   // ensure error sent back
+  //   stream.end()
+  // })
   stream.on('end', () => {
     callback(errorCollector)
   })
@@ -275,16 +212,6 @@ function hasColumnProperties(hotId, callb) {
     ])
     return false
   }
-  // if (!hasAllColumnTypes(hotId, columnProperties)) {
-  //   callb([
-  //     {
-  //       rowNumber: 0,
-  //       message: `Every Column property, including the column properties of any foreign keys, must have a 'type'.`,
-  //       name: 'Missing Column Property types'
-  //     }
-  //   ])
-  //   return false
-  // }
   return true
 }
 

From 738cb799f0d5f4d811dc6a01ca3631a81bef9e70 Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Mon, 12 Feb 2018 00:14:37 +1000
Subject: [PATCH 11/26] removed redundancies.

---
 package.json                 |  2 --
 src/renderer/frictionless.js |  5 +----
 yarn.lock                    | 22 ----------------------
 3 files changed, 1 insertion(+), 28 deletions(-)

diff --git a/package.json b/package.json
index 17e8275a1..b087afa54 100644
--- a/package.json
+++ b/package.json
@@ -78,7 +78,6 @@
     "bootstrap.native": "^2.0.17",
     "components-font-awesome": "^4.7.0",
     "csv": "^2.0.0",
-    "csvtojson": "^1.1.9",
     "datapackage": "^1.0.5",
     "escape-regexp": "^0.0.1",
     "etl": "^0.5.8",
@@ -97,7 +96,6 @@
     "node-fs-extra": "^0.8.2",
     "pikaday": "^1.6.1",
     "pouchdb-adapter-idb": "^6.3.4",
-    "promisepipe": "^2.1.1",
     "request": "^2.81.0",
     "rxjs": "^5.5.2",
     "slug": "^0.9.1",
diff --git a/src/renderer/frictionless.js b/src/renderer/frictionless.js
index 7f7c5581e..19c470e92 100644
--- a/src/renderer/frictionless.js
+++ b/src/renderer/frictionless.js
@@ -2,11 +2,8 @@ import {Table, Schema} from 'tableschema'
 import {HotRegister} from '@/hot.js'
 import store from '@/store/modules/hots.js'
 import tabStore from '@/store/modules/tabs.js'
-import {includeHeadersInData, hasAllColumnNames, hasAllColumnTypes} from '@/frictionlessUtilities.js'
+import {includeHeadersInData, hasAllColumnNames} from '@/frictionlessUtilities.js'
 import {allTablesAllColumnsFromSchema$} from '@/rxSubject.js'
-import stringify from 'csv-stringify'
-import csv from 'csvtojson'
-import promisePipe from 'promisepipe'
 
 async function inferSchema(data) {
   const schema = await Schema.load({})
diff --git a/yarn.lock b/yarn.lock
index c0ce64545..d8f496dfb 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2691,13 +2691,6 @@ csv@^2.0.0:
     csv-stringify "^2.0.0"
     stream-transform "^1.0.0"
 
-csvtojson@^1.1.9:
-  version "1.1.9"
-  resolved "https://registry.yarnpkg.com/csvtojson/-/csvtojson-1.1.9.tgz#e641ae72f7bc2fa3f9aaf127e021fc89447c1cd1"
-  dependencies:
-    lodash "^4.17.3"
-    strip-bom "1.0.0"
-
 currently-unhandled@^0.4.1:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea"
@@ -4033,10 +4026,6 @@ find-up@^2.0.0, find-up@^2.1.0:
   dependencies:
     locate-path "^2.0.0"
 
-first-chunk-stream@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/first-chunk-stream/-/first-chunk-stream-1.0.0.tgz#59bfb50cd905f60d7c394cd3d9acaab4e6ad934e"
-
 flat-cache@^1.2.1:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-1.3.0.tgz#d3030b32b38154f4e3b7e9c709f490f7ef97c481"
@@ -7506,10 +7495,6 @@ promise@^7.1.1:
   dependencies:
     asap "~2.0.3"
 
-promisepipe@^2.1.1:
-  version "2.1.1"
-  resolved "https://registry.yarnpkg.com/promisepipe/-/promisepipe-2.1.1.tgz#c3046d608ad34dd635cc7ffd7e6d16ff7ac3cc83"
-
 property-is-enumerable-x@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/property-is-enumerable-x/-/property-is-enumerable-x-1.1.0.tgz#7ca48917476cd0914b37809bfd05776a0d942f6f"
@@ -8733,13 +8718,6 @@ strip-ansi@^4.0.0:
   dependencies:
     ansi-regex "^3.0.0"
 
-strip-bom@1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-1.0.0.tgz#85b8862f3844b5a6d5ec8467a93598173a36f794"
-  dependencies:
-    first-chunk-stream "^1.0.0"
-    is-utf8 "^0.2.0"
-
 strip-bom@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz#6219a85616520491f35788bdbf1447a99c7e6b0e"

From b04b9ab247a933ec0cef2db2006d93f0ec2d9a5d Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Mon, 12 Feb 2018 17:09:17 +1000
Subject: [PATCH 12/26] Added contributors using sources as template. Updated
 style to fit labels. Updated validation rules to accommodate path.

---
 src/renderer/mixins/PackageTooltip.vue      |   4 +
 src/renderer/partials/Contributors.vue      | 181 ++++++++++++++++++++
 src/renderer/partials/PackageProperties.vue |  16 +-
 src/renderer/partials/SideNav.vue           |   2 +-
 static/css/contributors.styl                |  35 ++++
 5 files changed, 230 insertions(+), 8 deletions(-)
 create mode 100644 src/renderer/partials/Contributors.vue
 create mode 100644 static/css/contributors.styl

diff --git a/src/renderer/mixins/PackageTooltip.vue b/src/renderer/mixins/PackageTooltip.vue
index bb458f488..701b0eacb 100644
--- a/src/renderer/mixins/PackageTooltip.vue
+++ b/src/renderer/mixins/PackageTooltip.vue
@@ -33,6 +33,10 @@ export default {
     'tooltipPackageLicenses': {
       components: {externalLink},
       template: `<div id="tooltip-package-licenses">The <externalLink text="Open License" url="http://frictionlessdata.io/specs/data-package/#licenses"/> under which this data package is provided</div>`
+    },
+    'tooltipPackageContributors': {
+      components: {externalLink},
+      template: `<div id="tooltip-package-contributors">Each <externalLink text="Contributor" url="http://frictionlessdata.io/specs/data-package/#contributors"/> must have a title and may have path and/or email</div>`
     }
   }
 }
diff --git a/src/renderer/partials/Contributors.vue b/src/renderer/partials/Contributors.vue
new file mode 100644
index 000000000..d486d0d53
--- /dev/null
+++ b/src/renderer/partials/Contributors.vue
@@ -0,0 +1,181 @@
+<template>
+  <div id="contributors">
+    <div v-for="(contributor,index) in getContributors" :key="index" class="contributor col-sm-12">
+      <div class="inputs-container">
+        <div v-for="prop in Object.keys(contributor)" class="input-group">
+          <span class="input-group-addon input-sm">{{prop}}</span>
+          <input :class="{ 'form-control input-sm': true, 'validate-danger': errors.has(getValidationProp(prop) + index) }" :value="contributor[prop]" @input="setContributorProp(index, prop, $event.target.value)" type="text" :id="prop + index" v-validate="contributorValidationRules(prop, index)" :name="getValidationProp(prop) + index"/>
+          <div v-show="errors.has(getValidationProp(prop) + index)" class="row help validate-danger">
+            {{ errors.first(getValidationProp(prop) + index)}}
+          </div>
+        </div>
+      </div>
+      <button v-show="getContributors.length > 1" type="button" class="btn btn-danger btn-sm" @click="removeContributor(index)">
+        <span class="glyphicon glyphicon-minus"/>
+      </button>
+    </div>
+    <div class="button-container">
+      <button type="button" class="add-contributor btn btn-primary btn-sm" @click="addContributor()">
+        <span class="glyphicon glyphicon-plus"/>Add contributor
+      </button>
+    </div>
+  </div>
+</template>
+<script>
+import {
+  mapMutations,
+  mapState,
+  mapGetters
+} from 'vuex'
+import SideNav from './SideNav'
+import AsyncComputed from 'vue-async-computed'
+import ValidationRules from '../mixins/ValidationRules'
+import VeeValidate from 'vee-validate'
+import Vue from 'vue'
+Vue.use(AsyncComputed)
+export default {
+  name: 'contributors',
+  mixins: [ValidationRules],
+  data() {
+    return {
+      contributors: []
+    }
+  },
+  props: ['setProperty', 'getProperty', 'getPropertyGivenHotId'],
+  extends: SideNav,
+  computed: {
+    ...mapGetters(['getActiveTab']),
+    regexForPath() {
+      // no ../ or nulls or absolute paths allowed
+      return /^(([.](?![.])|[^/.:]+)+[/]*)+$/
+    },
+    roles() {
+      return ['author', 'publisher', 'maintainer', 'wrangler', 'contributor']
+    }
+  },
+  asyncComputed: {
+    getContributors: {
+      async get() {
+        let tab = this.getActiveTab
+        let contributors = await this.getContributorsFromTab(tab)
+        return contributors
+      },
+      watch() {
+        return this.contributors
+      }
+    }
+  },
+  methods: {
+    ...mapMutations([
+      'pushPackageProperty'
+    ]),
+    getValidationProp: function(prop) {
+      return prop === 'path' ? 'urlpath' : prop
+    },
+    removeContributor: function(index) {
+      let contributors = this.getProperty('contributors')
+      contributors.splice(index, 1)
+      this.setProperty('contributors', contributors)
+      this.contributors = contributors
+    },
+    addContributor: function() {
+      let contributors = this.getProperty('contributors') || []
+      contributors.push(this.emptyContributor())
+      this.setProperty('contributors', contributors)
+      this.contributors = contributors
+    },
+    emptyContributor: function() {
+      return {'title': '', 'path': '', 'email': '', 'role': '', 'organization': ''}
+    },
+    getContributorsFromTab: async function(tab) {
+      let hotId = await this.waitForHotIdFromTabId(tab)
+      let contributors = this.getPropertyGivenHotId('contributors', hotId)
+      return contributors
+    },
+    initContributors: async function(tab) {
+      let contributors = await this.getContributorsFromTab(tab)
+      if (!contributors) {
+        const vueAddContributor = this.addContributor
+        _.delay(function() {
+          vueAddContributor()
+        }, 100)
+      }
+    },
+    setContributorProp: function(index, prop, value) {
+      this.setProperty(`contributors[${index}][${prop}]`, value)
+      let contributors = this.getProperty('contributors') || []
+      this.contributors = contributors
+      if (prop === 'path') {
+        this.validateUrlOrPath(index, value)
+      }
+    },
+    validateUrlOrPath: async function(index, value) {
+      let prop = 'path'
+      let field = `${prop}${index}`
+      try {
+        let hasValidUrl = await this.validateUrl(field, value)
+        let hasValidPath = await this.validatePath(field, value)
+        this.$validator.detach(field)
+        if (!hasValidUrl && !hasValidPath) {
+          this.$validator.errors.add({field: field, msg: 'The path field must be a valid email or path.'})
+        }
+      } catch (err) {
+        console.log('Problem with validation', err)
+      }
+    },
+    validateUrl: async function(field, value) {
+      // keep url:true as string for validation to work correctly
+      return this.validate(field, value, 'url:true')
+    },
+    validatePath: async function(field, value) {
+      return this.validate(field, value, {
+        regex: this.regexForPath
+      })
+    },
+    validate: async function(field, value, rules) {
+      let isValid = true
+      // ensure there are no other fields by this name
+      this.$validator.detach(field)
+      await this.$validator.attach({
+        name: field,
+        rules: rules
+      })
+      isValid = await this.$validator.validate(field, value)
+      this.$validator.detach(field)
+      // console.log(isValid)
+      return isValid
+    },
+    contributorValidationRules: function(prop, index) {
+      switch (prop) {
+        case 'email':
+          return 'email'
+        case 'title':
+          return 'required'
+        case 'path':
+          return 'url'
+        case 'role':
+          return {
+            in: this.roles
+          }
+        default:
+          return ''
+      }
+    }
+  },
+  mounted: function() {
+    let tab = this.getActiveTab
+    this.initContributors(tab)
+  },
+  watch: {
+    getActiveTab: function(tab) {
+      this.initContributors(tab)
+    }
+  }
+}
+</script>
+<style lang="styl" scoped>
+@import '~static/css/contributors'
+</style>
+<style lang="styl" scoped>
+@import '~static/css/validationrules'
+</style>
diff --git a/src/renderer/partials/PackageProperties.vue b/src/renderer/partials/PackageProperties.vue
index 82ce8850e..51ccaa936 100644
--- a/src/renderer/partials/PackageProperties.vue
+++ b/src/renderer/partials/PackageProperties.vue
@@ -18,6 +18,7 @@
 import SideNav from '@/partials/SideNav'
 import licenses from '@/partials/Licenses'
 import sources from '@/partials/Sources'
+import contributors from '@/partials/Contributors'
 import PackageTooltip from '@/mixins/PackageTooltip'
 import ValidationRules from '@/mixins/ValidationRules'
 import {
@@ -31,35 +32,32 @@ export default {
   mixins: [ValidationRules, PackageTooltip],
   components: {
     licenses,
-    sources
+    sources,
+    contributors
   },
   data() {
     return {
       formprops: [{
         label: 'Name*',
         key: 'name',
-        type: 'input',
         tooltipId: 'tooltip-package-name',
         tooltipView: 'tooltipPackageName'
       },
       {
         label: 'Id',
         key: 'id',
-        type: 'input',
         tooltipId: 'tooltip-package-id',
         tooltipView: 'tooltipPackageId'
       },
       {
         label: 'Title',
         key: 'title',
-        type: 'input',
         tooltipId: 'tooltip-package-title',
         tooltipView: 'tooltipPackageTitle'
       },
       {
         label: 'Description',
         key: 'description',
-        type: 'markdown',
         tooltipId: 'tooltip-package-description',
         tooltipView: 'tooltipPackageDescription'
       },
@@ -67,14 +65,12 @@ export default {
       {
         label: 'Version',
         key: 'version',
-        type: 'input',
         tooltipId: 'tooltip-package-version',
         tooltipView: 'tooltipPackageVersion'
       },
       {
         label: 'Source(s)',
         key: 'sources',
-        type: 'dropdown',
         tooltipId: 'tooltip-package-sources',
         tooltipView: 'tooltipPackageSources'
       },
@@ -83,6 +79,12 @@ export default {
         key: 'licenses',
         tooltipId: 'tooltip-package-licenses',
         tooltipView: 'tooltipPackageLicenses'
+      },
+      {
+        label: 'Contributor(s)',
+        key: 'contributors',
+        tooltipId: 'tooltip-package-contributors',
+        tooltipView: 'tooltipPackageContributors'
       }
       ]
     }
diff --git a/src/renderer/partials/SideNav.vue b/src/renderer/partials/SideNav.vue
index 38764c14c..208802b73 100644
--- a/src/renderer/partials/SideNav.vue
+++ b/src/renderer/partials/SideNav.vue
@@ -33,7 +33,7 @@ export default {
   },
   methods: {
     isSharedComponent: function(key) {
-      let isShared = ['sources', 'licenses', 'primaryKeys', 'foreignKeys'].indexOf(key) !== -1
+      let isShared = ['sources', 'licenses', 'primaryKeys', 'foreignKeys', 'contributors'].indexOf(key) !== -1
       return isShared
     },
     propertyGetObjectGivenHotId: function(key, hotId) {
diff --git a/static/css/contributors.styl b/static/css/contributors.styl
new file mode 100644
index 000000000..b2bb37067
--- /dev/null
+++ b/static/css/contributors.styl
@@ -0,0 +1,35 @@
+#contributors
+  margin-bottom 10px
+  .contributor
+    margin-right 0
+    padding-right 0
+    padding-left 0
+    margin-bottom 5px
+    display block
+    width 100%
+    .inputs-container
+      display inline-block
+      width 90%
+      float left
+      .input-group
+        width 100%
+        input
+          width 100%
+          padding-left 10px !important
+        .input-group-addon
+          width 30%
+          text-align left
+    button
+      display inline-block
+      height 100% !important
+      min-height 100% !important
+      margin 30px 0px 30px 5px
+      padding 5px 8px 5px 8px
+      float right
+  .button-container
+    width 100%
+    overflow hidden
+    display block
+    button.add-contributor
+      display block
+      float right

From 2f8747ce6e481da2b7c869a88392cc0cc5eb58bd Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Mon, 12 Feb 2018 17:14:05 +1000
Subject: [PATCH 13/26] added contributors to export requirements.

---
 src/renderer/frictionlessDataPackage.js | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/renderer/frictionlessDataPackage.js b/src/renderer/frictionlessDataPackage.js
index f7b7de753..61fd75720 100644
--- a/src/renderer/frictionlessDataPackage.js
+++ b/src/renderer/frictionlessDataPackage.js
@@ -60,6 +60,7 @@ function hasAllPackageRequirements(requiredMessages) {
       requiredMessages.push(`Package property, 'name' must be set.`)
     }
     addSourcesRequirements(packageProperties, requiredMessages, 'package')
+    addContributorsRequirements(packageProperties, requiredMessages, 'package')
   }
   return requiredMessages.length === 0
 }
@@ -153,6 +154,22 @@ function addSourcesRequirements(properties, requiredMessages, entityName) {
   }
 }
 
+function addContributorsRequirements(properties, requiredMessages, entityName) {
+  if (typeof properties.contributors === 'undefined') {
+    return
+  }
+  for (let contributor of properties.contributors) {
+    if (hasAllEmptyValues(contributor)) {
+      _.pull(properties.contributors, contributor)
+    } else if (!contributor.title || contributor.title.trim() === '') {
+      requiredMessages.push(`At least 1 ${entityName} contributor does not have a title.`)
+      return false
+    } else {
+      // console.log('contributor is valid')
+    }
+  }
+}
+
 function hasAllEmptyValues(propertyObject) {
   let isEmpty = true
   _.forOwn(propertyObject, function(value, key) {

From 49a80b6722638f88d3b9446372903a374571bbbe Mon Sep 17 00:00:00 2001
From: Stephen-Gates <Stephen.Gates@me.com>
Date: Mon, 12 Feb 2018 20:55:22 +1000
Subject: [PATCH 14/26] refine contributor help tip

---
 src/renderer/mixins/PackageTooltip.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/renderer/mixins/PackageTooltip.vue b/src/renderer/mixins/PackageTooltip.vue
index 701b0eacb..17c770114 100644
--- a/src/renderer/mixins/PackageTooltip.vue
+++ b/src/renderer/mixins/PackageTooltip.vue
@@ -36,7 +36,7 @@ export default {
     },
     'tooltipPackageContributors': {
       components: {externalLink},
-      template: `<div id="tooltip-package-contributors">Each <externalLink text="Contributor" url="http://frictionlessdata.io/specs/data-package/#contributors"/> must have a title and may have path and/or email</div>`
+      template: `<div id="tooltip-package-contributors">Each <externalLink text="Contributor" url="http://frictionlessdata.io/specs/data-package/#contributors"/> must have a title and may contain path, email, role and organization</div>`
     }
   }
 }

From 4b5627198554f6d91aa69dbbb549ba3a1775f4a2 Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Tue, 13 Feb 2018 00:16:28 +1000
Subject: [PATCH 15/26] Testing changes for pr.

---
 package.json                 |  2 +-
 src/renderer/frictionless.js | 47 ++++++++++++++++++++----------------
 yarn.lock                    | 17 ++++++-------
 3 files changed, 35 insertions(+), 31 deletions(-)

diff --git a/package.json b/package.json
index b087afa54..475b74c4b 100644
--- a/package.json
+++ b/package.json
@@ -101,7 +101,7 @@
     "slug": "^0.9.1",
     "sortablejs": "^1.6.0",
     "svgo": "^1.0.0",
-    "tableschema": "^1.6.0",
+    "tableschema": "../tableschema-js",
     "temp": "^0.8.3",
     "unzipper": "^0.8.11",
     "vee-validate": "^2.0.3",
diff --git a/src/renderer/frictionless.js b/src/renderer/frictionless.js
index 19c470e92..c2a7c32f4 100644
--- a/src/renderer/frictionless.js
+++ b/src/renderer/frictionless.js
@@ -13,7 +13,6 @@ async function inferSchema(data) {
   // frictionless default for csv dialect is that tables DO have headers
   // await schema.infer(data, {headers: 0})
   await schema.infer(dataClone, {headers: headers})
-  console.log(schema)
   return schema
 }
 
@@ -108,8 +107,7 @@ function getHotIdFromForeignKeyForeignTable(title, hotId) {
 }
 
 function isRowBlank(row) {
-  let isRowBlank = row.filter(Boolean)
-  return isRowBlank.length === 0
+  return row.filter(Boolean).length === 0
 }
 
 function blankCellCount(row) {
@@ -157,31 +155,38 @@ export async function validateActiveDataAgainstSchema(callback) {
   let table = await createFrictionlessTable(data, schema)
   // wait for frictionless pr#124 and uncomment
   let relations = false
-  // try {
-  //   relations = await collateForeignKeys(hotId, callback)
-  // } catch (error) {
-  //   errorCollector.push({rowNumber: 0,
-  //     message: `There was a problem validating 1 or more foreign tables. Validate foreign tables first.`,
-  //     name: 'Invalid foreign table(s)'
-  //   })
-  // }
-  const stream = await table.iter({keyed: false, extended: true, stream: true, cast: false, relations: relations})
+  try {
+    relations = await collateForeignKeys(hotId, callback)
+  } catch (error) {
+    errorCollector.push({rowNumber: 0,
+      message: `There was a problem validating 1 or more foreign tables. Validate foreign tables first.`,
+      name: 'Invalid foreign table(s)'
+    })
+  }
+  const stream = await table.iter({keyed: false, extended: true, stream: true, cast: false, forceCast: true, relations: relations})
   stream.on('data', (row) => {
     // TODO: consider better way to accommodate or remove - need headers/column names so this logic may be redundant
     let rowNumber = hasColHeaders
       ? row[0]
       : row[0] + 1
-    if (isRowBlank(row[2])) {
-      errorCollector.push({rowNumber: rowNumber, message: `Row ${rowNumber} is completely blank`, name: 'Blank Row'})
+    if (row[2] instanceof Error) {
+      let err = row[2]
+      errorHandler(err, rowNumber, errorCollector)
+    } else {
+      if (isRowBlank(row[2])) {
+        errorCollector.push({rowNumber: rowNumber, message: `Row ${rowNumber} is completely blank`, name: 'Blank Row'})
+      }
+      // TODO: once frictionless release allows forceCast remove this call & the corresponding method
+    // checkRow(rowNumber, row[2], table.schema, errorCollector)
     }
-    checkRow(rowNumber, row[2], table.schema, errorCollector)
   })
-  // stream.on('error', (error) => {
-  //   console.log(error)
-  //   errorHandler(error, 'N/A', errorCollector)
-  //   // ensure error sent back
-  //   stream.end()
-  // })
+  stream.on('error', (error) => {
+    console.log(error)
+    const rowNumber = error.rowNumber ? error.rowNumber : 'N/A'
+    errorHandler(error, rowNumber, errorCollector)
+    // ensure error sent back
+    stream.end()
+  })
   stream.on('end', () => {
     callback(errorCollector)
   })
diff --git a/yarn.lock b/yarn.lock
index d8f496dfb..6e17877de 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -8876,11 +8876,10 @@ table@^4.0.1:
     slice-ansi "1.0.0"
     string-width "^2.1.1"
 
-tableschema@^1.2.1:
-  version "1.4.2"
-  resolved "https://registry.yarnpkg.com/tableschema/-/tableschema-1.4.2.tgz#d0077ef955358982c66f8caeba8f5bcac87e2963"
+tableschema@../tableschema-js:
+  version "1.6.0"
   dependencies:
-    axios "^0.16.1"
+    axios "^0.17.1"
     csv "^1.1.1"
     d3-time-format "^0.3.2"
     es6-error "^4.0.2"
@@ -8889,12 +8888,13 @@ tableschema@^1.2.1:
     regenerator-runtime "^0.11.0"
     stream-to-async-iterator "^0.2.0"
     tv4 "^1.2.7"
+    validator "^9.2.0"
 
-tableschema@^1.6.0:
-  version "1.6.0"
-  resolved "https://registry.yarnpkg.com/tableschema/-/tableschema-1.6.0.tgz#2ddf30b537b63856d27f315e3dd80abd71f365ea"
+tableschema@^1.2.1:
+  version "1.4.2"
+  resolved "https://registry.yarnpkg.com/tableschema/-/tableschema-1.4.2.tgz#d0077ef955358982c66f8caeba8f5bcac87e2963"
   dependencies:
-    axios "^0.17.1"
+    axios "^0.16.1"
     csv "^1.1.1"
     d3-time-format "^0.3.2"
     es6-error "^4.0.2"
@@ -8903,7 +8903,6 @@ tableschema@^1.6.0:
     regenerator-runtime "^0.11.0"
     stream-to-async-iterator "^0.2.0"
     tv4 "^1.2.7"
-    validator "^9.2.0"
 
 tapable@^0.2.7:
   version "0.2.8"

From 9fa49e6a3654a2634224736ab5b01324646e1861 Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Tue, 13 Feb 2018 11:44:21 +1000
Subject: [PATCH 16/26] Once sources or contributors are empty, remove.

---
 src/renderer/frictionlessDataPackage.js | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/renderer/frictionlessDataPackage.js b/src/renderer/frictionlessDataPackage.js
index 61fd75720..dfa5f4904 100644
--- a/src/renderer/frictionlessDataPackage.js
+++ b/src/renderer/frictionlessDataPackage.js
@@ -152,6 +152,10 @@ function addSourcesRequirements(properties, requiredMessages, entityName) {
       // console.log('source is valid')
     }
   }
+  if (properties.sources.length < 1) {
+    properties.sources = null
+    _.unset(properties, 'sources')
+  }
 }
 
 function addContributorsRequirements(properties, requiredMessages, entityName) {
@@ -168,6 +172,10 @@ function addContributorsRequirements(properties, requiredMessages, entityName) {
       // console.log('contributor is valid')
     }
   }
+  if (properties.contributors.length < 1) {
+    properties.contributors = null
+    _.unset(properties, 'contributors')
+  }
 }
 
 function hasAllEmptyValues(propertyObject) {

From d23fea00b1e280e28df46c402ba3a18970637ee5 Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Tue, 13 Feb 2018 11:53:50 +1000
Subject: [PATCH 17/26] Sources and Contributors are optional. Ensure that
 input can be completely reset.

---
 src/renderer/partials/Contributors.vue | 8 +-------
 src/renderer/partials/Sources.vue      | 8 +-------
 2 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/src/renderer/partials/Contributors.vue b/src/renderer/partials/Contributors.vue
index d486d0d53..a4b9fd8d5 100644
--- a/src/renderer/partials/Contributors.vue
+++ b/src/renderer/partials/Contributors.vue
@@ -10,7 +10,7 @@
           </div>
         </div>
       </div>
-      <button v-show="getContributors.length > 1" type="button" class="btn btn-danger btn-sm" @click="removeContributor(index)">
+      <button type="button" class="btn btn-danger btn-sm" @click="removeContributor(index)">
         <span class="glyphicon glyphicon-minus"/>
       </button>
     </div>
@@ -94,12 +94,6 @@ export default {
     },
     initContributors: async function(tab) {
       let contributors = await this.getContributorsFromTab(tab)
-      if (!contributors) {
-        const vueAddContributor = this.addContributor
-        _.delay(function() {
-          vueAddContributor()
-        }, 100)
-      }
     },
     setContributorProp: function(index, prop, value) {
       this.setProperty(`contributors[${index}][${prop}]`, value)
diff --git a/src/renderer/partials/Sources.vue b/src/renderer/partials/Sources.vue
index 288c80af1..799a52c51 100644
--- a/src/renderer/partials/Sources.vue
+++ b/src/renderer/partials/Sources.vue
@@ -10,7 +10,7 @@
           </div>
         </div>
       </div>
-      <button v-show="getSources.length > 1" type="button" class="btn btn-danger btn-sm" @click="removeSource(index)">
+      <button type="button" class="btn btn-danger btn-sm" @click="removeSource(index)">
         <span class="glyphicon glyphicon-minus"/>
       </button>
     </div>
@@ -88,12 +88,6 @@ export default {
     },
     initSources: async function(tab) {
       let sources = await this.getSourcesFromTab(tab)
-      if (!sources) {
-        const vueAddSource = this.addSource
-        _.delay(function() {
-          vueAddSource()
-        }, 100)
-      }
     },
     setSourceProp: function(index, prop, value) {
       this.setProperty(`sources[${index}][${prop}]`, value)

From 674ee9514eb059300aeb97df28e8d3e53075a1d6 Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Tue, 13 Feb 2018 13:08:02 +1000
Subject: [PATCH 18/26] Fixes #439. Updated dependency. Ensure directory
 created with promise. Made better use of vuex to handle correctly outside
 vue.

---
 package.json                  |  2 +-
 src/renderer/importPackage.js | 19 ++++++++++---------
 yarn.lock                     | 23 +----------------------
 3 files changed, 12 insertions(+), 32 deletions(-)

diff --git a/package.json b/package.json
index b087afa54..e4e5d74cc 100644
--- a/package.json
+++ b/package.json
@@ -82,6 +82,7 @@
     "escape-regexp": "^0.0.1",
     "etl": "^0.5.8",
     "exports-loader": "^0.6.4",
+    "fs-extra": "^5.0.0",
     "handsontable": "^0.34.5",
     "imports-loader": "^0.7.1",
     "jquery": "^3.2.1",
@@ -93,7 +94,6 @@
     "lodash": "^4.17.4",
     "markdown-it": "^8.4.0",
     "moment": "^2.19.1",
-    "node-fs-extra": "^0.8.2",
     "pikaday": "^1.6.1",
     "pouchdb-adapter-idb": "^6.3.4",
     "request": "^2.81.0",
diff --git a/src/renderer/importPackage.js b/src/renderer/importPackage.js
index ff8b41932..f161359ad 100644
--- a/src/renderer/importPackage.js
+++ b/src/renderer/importPackage.js
@@ -1,14 +1,15 @@
 import fs from 'fs-extra'
 import path from 'path'
-import hotStore from '@/store/modules/hots.js'
-import tabStore from '@/store/modules/tabs.js'
+import store from '@/store'
 import unzipper from 'unzipper'
 import etl from 'etl'
 import {ipcRenderer as ipc} from 'electron'
 
 export async function unzipFile(zipSource, storeCallback) {
   try {
-    let processedProperties = await unzipFileToDir(zipSource, createUnzipDestination(zipSource))
+    let destination = createUnzipDestination(zipSource)
+    await fs.ensureDir(destination)
+    let processedProperties = await unzipFileToDir(zipSource, destination)
     storeCallback(processedProperties)
   } catch (err) {
     console.log(`Error processing zip source: ${zipSource}`, err)
@@ -51,7 +52,7 @@ async function processStream(entry, processed, fileDestination) {
       await fs.ensureFile(fileDestination)
       await unzippedEntryToFile(entry, fileDestination)
       let textMd = await stringify(fileDestination)
-      await setProvenance(textMd)
+      setProvenance(textMd)
       processed.md.push(fileDestination)
       break
     default:
@@ -83,8 +84,8 @@ async function stringify(filename) {
   return text
 }
 
-async function setProvenance(text) {
-  hotStore.mutations.pushProvenance(hotStore.state, text)
+function setProvenance(text) {
+  store.commit('pushProvenance', text)
 }
 
 async function processJsonFile(processed, unzipDestination) {
@@ -118,7 +119,7 @@ async function getHotIdsFromFilenames(processed, unzipDestination) {
     if (!tabId) {
       throw new Error(`There was a problem matching ${fileDestination} with an opened tab.`)
     }
-    let hotId = _.findKey(hotStore.state.hotTabs, {tabId: tabId})
+    let hotId = _.findKey(store.getters.getHotTabs, {tabId: tabId})
     // ensure csv path accounts for parent folders zipped up
     let re = new RegExp('^' + processed.parentFolders + '/')
     let resourcePathname = _.replace(pathname, re, '')
@@ -129,11 +130,11 @@ async function getHotIdsFromFilenames(processed, unzipDestination) {
 
 async function getTabIdFromFilename(filename) {
   return new Promise((resolve, reject) => {
-    let tabId = _.findKey(tabStore.state.tabObjects, {filename: filename})
+    let tabId = _.findKey(store.getters.getTabObjects, {filename: filename})
     if (!tabId) {
       // wait for tabs to be ready
       _.delay(function(filename) {
-        resolve(_.findKey(tabStore.state.tabObjects, {filename: filename}))
+        resolve(_.findKey(store.getters.getTabObjects, {filename: filename}))
       }, 500, filename)
     } else {
       resolve(tabId)
diff --git a/yarn.lock b/yarn.lock
index d8f496dfb..9a597c1e1 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -5464,10 +5464,6 @@ jsonfile@^4.0.0:
   optionalDependencies:
     graceful-fs "^4.1.6"
 
-jsonfile@~1.1.0:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-1.1.1.tgz#da4fd6ad77f1a255203ea63c7bc32dc31ef64433"
-
 jsonify@~0.0.0:
   version "0.0.0"
   resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73"
@@ -6253,10 +6249,6 @@ mixin-deep@^1.2.0:
     for-in "^1.0.2"
     is-extendable "^1.0.1"
 
-mkdirp@0.3.x:
-  version "0.3.5"
-  resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.5.tgz#de3e5f8961c88c787ee1368df849ac4413eca8d7"
-
 mkdirp@0.5.0:
   version "0.5.0"
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.0.tgz#1d73076a6df986cd9344e15e71fcc05a4c9abf12"
@@ -6398,10 +6390,6 @@ ncname@1.0.x:
   dependencies:
     xml-char-classes "^1.0.0"
 
-ncp@~0.4.2:
-  version "0.4.2"
-  resolved "https://registry.yarnpkg.com/ncp/-/ncp-0.4.2.tgz#abcc6cbd3ec2ed2a729ff6e7c1fa8f01784a8574"
-
 ndjson@^1.4.0:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/ndjson/-/ndjson-1.5.0.tgz#ae603b36b134bcec347b452422b0bf98d5832ec8"
@@ -6442,15 +6430,6 @@ node-forge@0.6.33:
   version "0.6.33"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.6.33.tgz#463811879f573d45155ad6a9f43dc296e8e85ebc"
 
-node-fs-extra@^0.8.2:
-  version "0.8.2"
-  resolved "https://registry.yarnpkg.com/node-fs-extra/-/node-fs-extra-0.8.2.tgz#09fb2b60d30f7d703e361ecb626a91404f17097a"
-  dependencies:
-    jsonfile "~1.1.0"
-    mkdirp "0.3.x"
-    ncp "~0.4.2"
-    rimraf "~2.2.0"
-
 node-gyp@^3.6.2:
   version "3.6.2"
   resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-3.6.2.tgz#9bfbe54562286284838e750eac05295853fa1c60"
@@ -8041,7 +8020,7 @@ rimraf@2, rimraf@^2.5.1, rimraf@^2.5.2, rimraf@^2.5.4, rimraf@^2.6.0, rimraf@^2.
   dependencies:
     glob "^7.0.5"
 
-rimraf@^2.2.8, rimraf@~2.2.0, rimraf@~2.2.6:
+rimraf@^2.2.8, rimraf@~2.2.6:
   version "2.2.8"
   resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.2.8.tgz#e439be2aaee327321952730f99a8929e4fc50582"
 

From 0d2ef5aff838fddddc90595f111926627725f3e1 Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Tue, 13 Feb 2018 14:23:27 +1000
Subject: [PATCH 19/26] Fixes #451. remove redundant methods. Changed dropdown
 and added default.

---
 src/renderer/partials/Contributors.vue | 78 +++++++-------------------
 src/renderer/partials/Sources.vue      |  2 +
 2 files changed, 22 insertions(+), 58 deletions(-)

diff --git a/src/renderer/partials/Contributors.vue b/src/renderer/partials/Contributors.vue
index a4b9fd8d5..46de47e7d 100644
--- a/src/renderer/partials/Contributors.vue
+++ b/src/renderer/partials/Contributors.vue
@@ -4,7 +4,12 @@
       <div class="inputs-container">
         <div v-for="prop in Object.keys(contributor)" class="input-group">
           <span class="input-group-addon input-sm">{{prop}}</span>
-          <input :class="{ 'form-control input-sm': true, 'validate-danger': errors.has(getValidationProp(prop) + index) }" :value="contributor[prop]" @input="setContributorProp(index, prop, $event.target.value)" type="text" :id="prop + index" v-validate="contributorValidationRules(prop, index)" :name="getValidationProp(prop) + index"/>
+          <select v-if="prop === 'role'" class="form-control input-sm" :value="contributor[prop]" @input="setContributorProp(index, prop, $event.target.value)" :id="prop" >
+            <option v-for="role in roles" :key="role" v-bind:value="role">
+              {{ role}}
+            </option>
+          </select>
+          <input v-else :class="{ 'form-control input-sm': true, 'validate-danger': errors.has(getValidationProp(prop) + index) }" :value="contributor[prop]" @input="setContributorProp(index, prop, $event.target.value)" type="text" :id="prop + index" v-validate="contributorValidationRules(prop, index)" :name="getValidationProp(prop) + index"/>
           <div v-show="errors.has(getValidationProp(prop) + index)" class="row help validate-danger">
             {{ errors.first(getValidationProp(prop) + index)}}
           </div>
@@ -51,13 +56,20 @@ export default {
     },
     roles() {
       return ['author', 'publisher', 'maintainer', 'wrangler', 'contributor']
+    },
+    defaultRole() {
+      return 'contributor'
     }
   },
   asyncComputed: {
     getContributors: {
       async get() {
-        let tab = this.getActiveTab
-        let contributors = await this.getContributorsFromTab(tab)
+        let contributors = this.getContributorsFromPackageProperties() || []
+        for (const [index, contributor] of contributors.entries()) {
+          if (contributor.role.trim() === '') {
+            this.setProperty(`contributors[${index}]role`, this.defaultRole)
+          }
+        }
         return contributors
       },
       watch() {
@@ -87,57 +99,17 @@ export default {
     emptyContributor: function() {
       return {'title': '', 'path': '', 'email': '', 'role': '', 'organization': ''}
     },
-    getContributorsFromTab: async function(tab) {
-      let hotId = await this.waitForHotIdFromTabId(tab)
-      let contributors = this.getPropertyGivenHotId('contributors', hotId)
+    getContributorsFromPackageProperties: function() {
+      let contributors = this.getProperty('contributors')
       return contributors
     },
-    initContributors: async function(tab) {
-      let contributors = await this.getContributorsFromTab(tab)
+    initContributors: function() {
+      this.contributors = this.getContributorsFromPackageProperties()
     },
     setContributorProp: function(index, prop, value) {
       this.setProperty(`contributors[${index}][${prop}]`, value)
       let contributors = this.getProperty('contributors') || []
       this.contributors = contributors
-      if (prop === 'path') {
-        this.validateUrlOrPath(index, value)
-      }
-    },
-    validateUrlOrPath: async function(index, value) {
-      let prop = 'path'
-      let field = `${prop}${index}`
-      try {
-        let hasValidUrl = await this.validateUrl(field, value)
-        let hasValidPath = await this.validatePath(field, value)
-        this.$validator.detach(field)
-        if (!hasValidUrl && !hasValidPath) {
-          this.$validator.errors.add({field: field, msg: 'The path field must be a valid email or path.'})
-        }
-      } catch (err) {
-        console.log('Problem with validation', err)
-      }
-    },
-    validateUrl: async function(field, value) {
-      // keep url:true as string for validation to work correctly
-      return this.validate(field, value, 'url:true')
-    },
-    validatePath: async function(field, value) {
-      return this.validate(field, value, {
-        regex: this.regexForPath
-      })
-    },
-    validate: async function(field, value, rules) {
-      let isValid = true
-      // ensure there are no other fields by this name
-      this.$validator.detach(field)
-      await this.$validator.attach({
-        name: field,
-        rules: rules
-      })
-      isValid = await this.$validator.validate(field, value)
-      this.$validator.detach(field)
-      // console.log(isValid)
-      return isValid
     },
     contributorValidationRules: function(prop, index) {
       switch (prop) {
@@ -147,23 +119,13 @@ export default {
           return 'required'
         case 'path':
           return 'url'
-        case 'role':
-          return {
-            in: this.roles
-          }
         default:
           return ''
       }
     }
   },
   mounted: function() {
-    let tab = this.getActiveTab
-    this.initContributors(tab)
-  },
-  watch: {
-    getActiveTab: function(tab) {
-      this.initContributors(tab)
-    }
+    this.initContributors()
   }
 }
 </script>
diff --git a/src/renderer/partials/Sources.vue b/src/renderer/partials/Sources.vue
index 799a52c51..4c63ea74b 100644
--- a/src/renderer/partials/Sources.vue
+++ b/src/renderer/partials/Sources.vue
@@ -54,6 +54,7 @@ export default {
     getSources: {
       async get() {
         let tab = this.getActiveTab
+        // TODO: may need distinction here for package vs tables
         let sources = await this.getSourcesFromTab(tab)
         return sources
       },
@@ -86,6 +87,7 @@ export default {
       let sources = this.getPropertyGivenHotId('sources', hotId)
       return sources
     },
+    // TODO: fix this redundant method
     initSources: async function(tab) {
       let sources = await this.getSourcesFromTab(tab)
     },

From 719b1b3f5f1e89d16a336005a3c1c491b7412153 Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Tue, 13 Feb 2018 15:07:05 +1000
Subject: [PATCH 20/26] corrected store use.

---
 src/renderer/headerRow.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/renderer/headerRow.js b/src/renderer/headerRow.js
index 69870bfe1..9bbcd9de3 100644
--- a/src/renderer/headerRow.js
+++ b/src/renderer/headerRow.js
@@ -1,6 +1,6 @@
 import {ipcRenderer as ipc} from 'electron'
 import {allTablesAllColumnNames$} from '@/rxSubject.js'
-import store from '@/store/modules/hots.js'
+import store from '@/store'
 import {HotRegister} from '@/hot.js'
 import {pushAllTabTitlesSubscription} from '@/store/modules/tabs.js'
 
@@ -56,12 +56,12 @@ function updateHot(hot, data, header) {
 }
 
 function updateAllColumnsName(values) {
-  store.mutations.pushAllColumnsProperty(store.state, {
+  store.commit('pushAllColumnsProperty', {
     hotId: HotRegister.getActiveInstance().guid,
     key: 'name',
     values: values
   })
   // do not allow getter to cache as does not seem to pick up change
-  allTablesAllColumnNames$.next(store.getters.getAllHotTablesColumnNames(store.state, store.getters)())
+  allTablesAllColumnNames$.next(store.getters.getAllHotTablesColumnNames())
   pushAllTabTitlesSubscription()
 }

From 0e41a60729da2aa5c72dc9f8bdc20a2f85f870f5 Mon Sep 17 00:00:00 2001
From: Stephen-Gates <Stephen.Gates@me.com>
Date: Wed, 14 Feb 2018 13:11:44 +1000
Subject: [PATCH 21/26] update enum error message

---
 src/renderer/mixins/ValidationRules.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/renderer/mixins/ValidationRules.vue b/src/renderer/mixins/ValidationRules.vue
index 1e6d72cf5..170190df3 100644
--- a/src/renderer/mixins/ValidationRules.vue
+++ b/src/renderer/mixins/ValidationRules.vue
@@ -45,7 +45,7 @@ export default {
             regex: 'The name field format is invalid. It must consist only of lowercase alphanumeric characters plus ".", "-" and "_".'
           },
           enum: {
-            required: 'Separate valid values with a comma.'
+            required: 'Quote each "valid value" and separate with a comma.'
           },
           pattern: {
             required: 'There must be a pattern present.'

From f7c5490f095ba5b36aa67063fdfba5b809893d6a Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Wed, 14 Feb 2018 16:44:13 +1000
Subject: [PATCH 22/26] Fixes #282. Fixes #218.

---
 src/main/file.js                           | 18 +++---
 src/renderer/components/CustomFormat.vue   | 50 +++++++++------
 src/renderer/data-actions.js               | 35 +++++++++--
 src/renderer/dialect.js                    | 20 ++++++
 src/renderer/exportPackage.js              |  3 +-
 src/renderer/file-formats.js               | 30 ++++-----
 src/renderer/frictionless.js               |  1 -
 src/renderer/frictionlessDataPackage.js    |  4 +-
 src/renderer/importPackage.js              | 72 ++++++++++++++--------
 src/renderer/index.js                      | 15 ++---
 src/renderer/partials/ColumnProperties.vue |  4 ++
 src/renderer/store/modules/hots.js         |  3 +
 12 files changed, 171 insertions(+), 84 deletions(-)
 create mode 100644 src/renderer/dialect.js

diff --git a/src/main/file.js b/src/main/file.js
index 23f4f9720..9107e8e7a 100644
--- a/src/main/file.js
+++ b/src/main/file.js
@@ -8,12 +8,12 @@ function makeCustomFormat(separator, delimiter) {
   return {
     label: 'Custom',
     filters: [],
-    options: {
-      separator: separator,
-      delimiter: delimiter
+    dialect: {
+      delimiter: delimiter,
+      quoteChar: quoteChar
     },
-    mime_type: 'text/plain',
-    default_extension: 'txt'
+    mediatype: 'text/plain',
+    format: 'txt'
   }
 }
 
@@ -27,7 +27,7 @@ function saveAsCustom() {
   })
   ipc.once('formatSelected', function(event, data) {
     dialog.close()
-    let format = makeCustomFormat(data.separator, data.delimiter)
+    let format = makeCustomFormat(data.delimiter, data.quoteChar)
     saveFileAs(format, currentWindow)
   })
   dialog.loadURL(`http://localhost:9080/#/customformat`)
@@ -120,8 +120,8 @@ function openFile(format) {
   })
 }
 
-ipc.on('openFileIntoTab', (event, arg) => {
-  readFile(arg)
+ipc.on('openFileIntoTab', (event, arg1, arg2) => {
+  readFile(arg1, arg2)
 })
 
 function readFile(filename, format) {
@@ -131,7 +131,7 @@ function readFile(filename, format) {
   }
   Fs.readFile(filename, 'utf-8', function(err, data) {
     if (err) {
-      console.log(err.stack)
+      console.log(err)
     } else {
       createWindowTabWithFormattedDataFile(data, format, filename)
       // enableSave()
diff --git a/src/renderer/components/CustomFormat.vue b/src/renderer/components/CustomFormat.vue
index 710a47068..95f9f6e13 100644
--- a/src/renderer/components/CustomFormat.vue
+++ b/src/renderer/components/CustomFormat.vue
@@ -1,29 +1,28 @@
 <template>
   <div id="container" class="container-fluid">
-  <form id="custom-format">
+  <form id="custom-format" v-on:submit.prevent="submit()">
     <div class="form-wrapper">
 
       <div class="form-group">
-        <label for="separator">Separator</label>
-        <input type="text" class="form-control" id="separator" name="separator" value=",">
+        <label for="separator">Delimiter</label>
+        <input type="text" class="form-control" id="delimiter" name="delimiter" :value="getDelimiter" @input="setDelimiter($event.target.value)" >
       </div>
 
       <div class="form-group">
-        <label for="delimiter">Text Delimiter</label>
-        <input type="text" class="form-control" id="delimiter" name="delimiter" value='"'>
+        <label for="delimiter">Quote Character</label>
+        <input type="text" class="form-control" id="quoteChar" name="quoteChar" :value="getQuoteChar" @input="setQuoteChar($event.target.value)">
       </div>
 
     </div>
 
     <div class="well">
       <button id="submit" class="btn btn-default">Go</button>
-      <button id="cancel" class="btn btn-default">Cancel</button>
+      <button id="cancel" class="btn btn-default" @click="cancel()">Cancel</button>
     </div>
   </form>
   </div>
 </template>
 <script>
-window.$ = window.jQuery = require('jquery/dist/jquery.js')
 const {
   shell
 } = require('electron')
@@ -31,20 +30,35 @@ var ipc = require('electron').ipcRenderer
 require('bootstrap/dist/js/bootstrap.min.js')
 export default {
   name: 'customformat',
-  methods: {},
-  mounted: function() {
-    $('#cancel').click(function(e) {
-      window.close()
-    })
-
-    $('#submit').click(function(e) {
-      e.preventDefault()
+  computed: {
+    getDelimiter() {
+      return ','
+      // TODO : couple to vuex
+    },
+    getQuoteChar() {
+      return ','
+      // TODO : couple to vuex
+    }
+  },
+  methods: {
+    setDelimiter(delimiter) {
+      // TODO: complete
+    },
+    setQuoteChar(quoteChar) {
+      // TODO: complete
+    },
+    submit() {
       let format = {
-        separator: $('#separator').val(),
-        delimiter: $('#delimiter').val()
+        delimiter: document.querySelector('#delimiter').val(),
+        quoteChar: document.querySelector('#quoteChar').val()
       }
       ipc.send('formatSelected', format)
-    })
+    },
+    cancel() {
+      window.close()
+    }
+  },
+  mounted: function() {
   }
 }
 </script>
diff --git a/src/renderer/data-actions.js b/src/renderer/data-actions.js
index b0c9970e0..f4730df4d 100644
--- a/src/renderer/data-actions.js
+++ b/src/renderer/data-actions.js
@@ -1,21 +1,31 @@
-import tabStore from '../renderer/store/modules/tabs.js'
+import tabStore from '@/store/modules/tabs.js'
 import fs from 'fs'
 import {fixRaggedRows} from '@/ragged-rows.js'
 import {includeHeadersInData} from '@/frictionlessUtilities.js'
 import {toggleHeaderNoFeedback} from '@/headerRow.js'
+import {pushCsvDialect} from '@/dialect.js'
 // import parse from 'csv-parse/lib/sync'
 // import stringify from 'csv-stringify'
+// TODO : replace jQuery with node 'csv' library's stringify and transform
 const $ = global.jQuery = require('jquery/dist/jquery.js')
 require('jquery-csv/src/jquery.csv.js')
+var parse = require('csv-parse/lib/sync')
+var stringify = require('csv-stringify/lib/sync')
 
+// { delimiter: ',', lineTerminator, quoteChar, doubleQuote, escapeChar, nullSequence, skipInitialSpace, header, caseSensitiveHeader, csvddfVersion }
+const frictionlessToCsvmapper = {delimiter: 'delimiter', lineTerminator: 'rowDelimiter', quoteChar: 'quote', escapeChar: 'escape', skipInitialSpace: 'ltrim'}
 export function loadDataIntoHot(hot, data, format) {
   let arrays
   // if no format specified, default to csv
   if (typeof format === 'undefined' || !format) {
-    arrays = $.csv.toArrays(data)
+    arrays = parse(data)
   } else {
-    arrays = $.csv.toArrays(data, format.options)
+    let csvOptions = dialectToCsvOptions(format.dialect)
+    // TODO: update to stream
+    arrays = parse(data, csvOptions)
+    pushCsvDialect(hot.guid, format)
   }
+
   fixRaggedRows(arrays)
   hot.loadData(arrays)
   hot.render()
@@ -48,9 +58,24 @@ export function saveDataToFile(hot, format, filename, callback) {
   let data
   // if no format specified, default to csv
   if (typeof format === 'undefined' || !format) {
-    data = $.csv.fromArrays(arrays)
+    // TODO: update to stream
+    data = stringify(arrays)
   } else {
-    data = $.csv.fromArrays(arrays, format.options)
+    let csvOptions = dialectToCsvOptions(format.dialect)
+    data = stringify(arrays, csvOptions)
+    pushCsvDialect(hot.guid, format)
   }
   fs.writeFile(filename, data, callback)
 }
+
+function dialectToCsvOptions(dialect) {
+  let csvOptions = {}
+  if (dialect) {
+    _.forEach(frictionlessToCsvmapper, function(csvKey, frictionlessKey) {
+      if (_.has(dialect, frictionlessKey)) {
+        csvOptions[csvKey] = dialect[frictionlessKey]
+      }
+    })
+  }
+  return csvOptions
+}
diff --git a/src/renderer/dialect.js b/src/renderer/dialect.js
new file mode 100644
index 000000000..7ff7cb21e
--- /dev/null
+++ b/src/renderer/dialect.js
@@ -0,0 +1,20 @@
+import store from '@/store'
+
+export function pushCsvDialect(guid, formatOriginal = {}) {
+  let format = {}
+  _.assign(format, formatOriginal)
+  // TODO : neaten up to merge existing values and then push all up
+  if (format.dialect) {
+    _.unset(format.dialect, 'objectMode')
+    _.merge(format.dialect, {header: true})
+    _.forEach(format.dialect, function(value, key) {
+      store.commit('pushTableProperty', {hotId: guid, key: `dialect.${key}`, value: value})
+    })
+  }
+  if (format.mediatype) {
+    store.commit('pushTableProperty', {hotId: guid, key: 'mediatype', value: format.mediatype})
+  }
+  if (format.format) {
+    store.commit('pushTableProperty', {hotId: guid, key: 'format', value: format.format})
+  }
+}
diff --git a/src/renderer/exportPackage.js b/src/renderer/exportPackage.js
index d3124a80c..4187e15c7 100644
--- a/src/renderer/exportPackage.js
+++ b/src/renderer/exportPackage.js
@@ -7,7 +7,8 @@ import hotStore from '@/store/modules/hots.js'
 import {extractNameFromFile} from '@/store/tabStoreUtilities.js'
 const Dialog = remote.dialog
 
-export function createZipFile(json) {
+export function createZipFile(text) {
+  let json = JSON.stringify(text, null, 4)
   Dialog.showSaveDialog({
     filters: [
       {
diff --git a/src/renderer/file-formats.js b/src/renderer/file-formats.js
index 45c9c69aa..92cdac593 100644
--- a/src/renderer/file-formats.js
+++ b/src/renderer/file-formats.js
@@ -13,12 +13,12 @@ const fileFormats = {
         extensions: ['csv']
       }
     ],
-    options: {
-      separator: ',',
-      delimiter: '"'
+    dialect: {
+      delimiter: ',',
+      quoteChar: '"'
     },
-    mime_type: 'text/csv',
-    default_extension: 'csv'
+    mediatype: 'text/csv',
+    format: 'csv'
   },
   tsv: {
     label: 'Tab separated...',
@@ -34,12 +34,12 @@ const fileFormats = {
         extensions: ['dat']
       }
     ],
-    options: {
-      separator: '\t',
-      delimiter: '"'
+    dialect: {
+      delimiter: '\t',
+      quoteChar: '"'
     },
-    mime_type: 'text/tab-separated-values',
-    default_extension: 'tsv'
+    mediatype: 'text/tab-separated-values',
+    format: 'tsv'
   },
   semicolon: {
     label: 'Semicolon separated...',
@@ -49,12 +49,12 @@ const fileFormats = {
         extensions: ['csv']
       }
     ],
-    options: {
-      separator: ';',
-      delimiter: '"'
+    dialect: {
+      delimiter: ';',
+      quoteChar: '"'
     },
-    mime_type: 'text/csv',
-    default_extension: 'csv'
+    mediatype: 'text/csv',
+    format: 'csv'
   }
 }
 
diff --git a/src/renderer/frictionless.js b/src/renderer/frictionless.js
index 19c470e92..98067539c 100644
--- a/src/renderer/frictionless.js
+++ b/src/renderer/frictionless.js
@@ -13,7 +13,6 @@ async function inferSchema(data) {
   // frictionless default for csv dialect is that tables DO have headers
   // await schema.infer(data, {headers: 0})
   await schema.infer(dataClone, {headers: headers})
-  console.log(schema)
   return schema
 }
 
diff --git a/src/renderer/frictionlessDataPackage.js b/src/renderer/frictionlessDataPackage.js
index dfa5f4904..55e39369b 100644
--- a/src/renderer/frictionlessDataPackage.js
+++ b/src/renderer/frictionlessDataPackage.js
@@ -1,4 +1,4 @@
-import {Resource, Package, validate} from 'datapackage'
+import {Resource, Package} from 'datapackage'
 import {HotRegister} from '@/hot.js'
 import tabStore from '@/store/modules/tabs.js'
 import hotStore from '@/store/modules/hots.js'
@@ -22,7 +22,7 @@ export async function createDataPackage() {
         errorMessages.push('There is a problem with at least 1 package property. Please check and try again.')
         return errorMessages
       }
-      createZipFile(JSON.stringify(dataPackage.descriptor))
+      createZipFile(dataPackage.descriptor)
     }
   } catch (err) {
     if (err) {
diff --git a/src/renderer/importPackage.js b/src/renderer/importPackage.js
index f161359ad..a90fa4461 100644
--- a/src/renderer/importPackage.js
+++ b/src/renderer/importPackage.js
@@ -4,6 +4,7 @@ import store from '@/store'
 import unzipper from 'unzipper'
 import etl from 'etl'
 import {ipcRenderer as ipc} from 'electron'
+import {Resource, Package} from 'datapackage'
 
 export async function unzipFile(zipSource, storeCallback) {
   try {
@@ -23,24 +24,36 @@ function createUnzipDestination(zipSource) {
 }
 
 async function unzipFileToDir(zipSource, unzipDestination) {
-  let processed = {json: [], csv: [], md: []}
+  let processed = {json: [], resource: [], md: []}
   await fs.createReadStream(zipSource).pipe(unzipper.Parse()).pipe(etl.map(async entry => {
     let fileDestination = `${unzipDestination}/${entry.path}`
     await processStream(entry, processed, fileDestination)
   })).promise()
   validateMdFile(processed)
   // wait until all tabs opened before processing data package json
-  let processedProperties = await processJsonFile(processed, unzipDestination)
+  if (processed.json.length !== 1) {
+    throw new Error('There must be 1, and only 1, json file.')
+  }
+  let dataPackageJson = await getDataPackageJson(processed)
+  let csvPathHotIds = await processResources(dataPackageJson, unzipDestination, processed)
+  let processedProperties = await processJson(dataPackageJson, csvPathHotIds, unzipDestination)
   return processedProperties
 }
 
+async function getDataPackageJson(processed) {
+  let filename = processed.json[0]
+  let text = await stringify(filename)
+  let dataPackageJson = JSON.parse(text)
+  return dataPackageJson
+}
+
 async function processStream(entry, processed, fileDestination) {
   switch (path.extname(entry.path)) {
     case '.csv':
+    case '.tsv':
       await fs.ensureFile(fileDestination)
       await unzippedEntryToFile(entry, fileDestination)
-      await ipc.send('openFileIntoTab', fileDestination)
-      processed.csv.push(entry.path)
+      processed.resource.push(entry.path)
       break
     case '.json':
       await fs.ensureFile(fileDestination)
@@ -88,17 +101,31 @@ function setProvenance(text) {
   store.commit('pushProvenance', text)
 }
 
-async function processJsonFile(processed, unzipDestination) {
-  if (processed.json.length !== 1) {
-    throw new Error('There must be 1, and only 1, json file.')
-  }
+async function processResources(dataPackageJson, unzipDestination, processed) {
+  let resourcePaths = await getAllResourcePaths(dataPackageJson, unzipDestination)
   let csvPathHotIds = await getHotIdsFromFilenames(processed, unzipDestination)
-  let filename = processed.json[0]
-  let text = await stringify(filename)
-  let datapackageJson = JSON.parse(text)
-  validateResourcesAndDataFiles(getAllResourcePaths(datapackageJson), _.keys(csvPathHotIds))
-  let processedProperties = processJson(datapackageJson, csvPathHotIds)
-  return processedProperties
+  validateResourcesAndDataFiles(resourcePaths, _.keys(csvPathHotIds))
+  return csvPathHotIds
+}
+
+async function getAllResourcePaths(dataPackageJson, unzipDestination) {
+  let resourcePaths = []
+  for (let dataResource of dataPackageJson.resources) {
+    let fileDestination = `${unzipDestination}/${dataResource.path}`
+    let format = dataResourcetoFormat(dataResource)
+    await ipc.send('openFileIntoTab', fileDestination, format)
+    resourcePaths.push(dataResource.path)
+  }
+  return resourcePaths
+}
+
+function dataResourcetoFormat(dataResource) {
+  let format = {}
+  _.assign(format, dataResource)
+  for (const key of ['missingValues', 'name', 'path', 'profile', 'schema']) {
+    _.unset(format, key)
+  }
+  return format
 }
 
 function validateResourcesAndDataFiles(resourcePaths, csvPaths) {
@@ -112,7 +139,7 @@ function validateResourcesAndDataFiles(resourcePaths, csvPaths) {
 async function getHotIdsFromFilenames(processed, unzipDestination) {
   let dataPackageJson = processed.json[0]
   let csvTabs = {}
-  for (let pathname of processed.csv) {
+  for (let pathname of processed.resource) {
     let fileDestination = `${unzipDestination}/${pathname}`
     let tabId = await getTabIdFromFilename(fileDestination)
     // every processed csv should have a matching tab
@@ -142,15 +169,8 @@ async function getTabIdFromFilename(filename) {
   })
 }
 
-function getAllResourcePaths(datapackageJson) {
-  let resourcePaths = datapackageJson.resources.map(function(dataResource) {
-    return dataResource.path
-  })
-  return resourcePaths
-}
-
-function processJson(datapackageJson, csvPathHotIds) {
-  let allTableProperties = datapackageJson.resources
+function processJson(dataPackageJson, csvPathHotIds, unzipDestination) {
+  let allTableProperties = dataPackageJson.resources
   let allColumnPropertiesByHotId = {}
   let allTablePropertiesByHotId = {}
   for (let tableProperties of allTableProperties) {
@@ -163,9 +183,9 @@ function processJson(datapackageJson, csvPathHotIds) {
     _.unset(tableProperties, 'schema')
     allTablePropertiesByHotId[csvPathHotIds[tableProperties.path]] = tableProperties
   }
-  _.unset(datapackageJson, 'resources')
+  _.unset(dataPackageJson, 'resources')
   return {
-    package: datapackageJson,
+    package: dataPackageJson,
     tables: allTablePropertiesByHotId,
     columns: allColumnPropertiesByHotId
   }
diff --git a/src/renderer/index.js b/src/renderer/index.js
index f32b09f91..d3457bab5 100644
--- a/src/renderer/index.js
+++ b/src/renderer/index.js
@@ -19,10 +19,10 @@ export function addHotContainerListeners(container) {
     var f = e.dataTransfer.files[0]
     fs.readFile(f.path, 'utf-8', function(err, data) {
       if (err) {
-        console.log(err.stack)
+        console.log(err)
       }
       // if we're dragging a file in, default the format to comma-separated
-      loadData(hot, data, file.formats.csv.options)
+      loadData(hot, data, file.formats.csv)
     })
   }
 
@@ -48,11 +48,12 @@ ipc.on('getCSV', function(e, format) {
   let hot = HotRegister.getActiveInstance()
   var data
   // if no format specified, default to csv
-  if (typeof format === 'undefined') {
-    data = $.csv.fromArrays(hot.getData())
-  } else {
-    data = $.csv.fromArrays(hot.getData(), format.options)
-  }
+  // TODO: update to node csv and check dialect mappings
+  // if (typeof format === 'undefined') {
+  //   data = $.csv.fromArrays(hot.getData())
+  // } else {
+  //   data = $.csv.fromArrays(hot.getData(), format.options)
+  // }
   ipc.send('sendCSV', data)
 })
 
diff --git a/src/renderer/partials/ColumnProperties.vue b/src/renderer/partials/ColumnProperties.vue
index 9f5c8f50f..78bd9f940 100644
--- a/src/renderer/partials/ColumnProperties.vue
+++ b/src/renderer/partials/ColumnProperties.vue
@@ -314,6 +314,10 @@ export default {
       return true
     },
     setConstraintValue: function(key, value) {
+      // TODO: split at double quote and comma
+      // if (key === 'enum') {
+      //   value = _.toArray(value)
+      // }
       this.constraintInputKeyValues[key] = value
       this.pushConstraintInputKeyValues()
     },
diff --git a/src/renderer/store/modules/hots.js b/src/renderer/store/modules/hots.js
index 22a81f052..92a267527 100644
--- a/src/renderer/store/modules/hots.js
+++ b/src/renderer/store/modules/hots.js
@@ -108,6 +108,9 @@ const getters = {
   getPackageProperty: (state, getters) => (property) => {
     return state.packageProperties[property.key]
   },
+  getPackageProperties: state => {
+    return state.packageProperties
+  },
   getConstraint: (state, getters) => (property) => {
     let hotColumnProperties = getHotColumnPropertiesFromPropertyObject(property)
     let constraints = hotColumnProperties['constraints']

From f996f7b4d4ec6823fafe35f6d0b049cb84ca67fd Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Wed, 14 Feb 2018 17:49:44 +1000
Subject: [PATCH 23/26] updated tableschema version.

---
 package.json | 2 +-
 yarn.lock    | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/package.json b/package.json
index e4e5d74cc..01cb42c2e 100644
--- a/package.json
+++ b/package.json
@@ -101,7 +101,7 @@
     "slug": "^0.9.1",
     "sortablejs": "^1.6.0",
     "svgo": "^1.0.0",
-    "tableschema": "^1.6.0",
+    "tableschema": "^1.7.0",
     "temp": "^0.8.3",
     "unzipper": "^0.8.11",
     "vee-validate": "^2.0.3",
diff --git a/yarn.lock b/yarn.lock
index 9a597c1e1..41ed4d59f 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -8869,9 +8869,9 @@ tableschema@^1.2.1:
     stream-to-async-iterator "^0.2.0"
     tv4 "^1.2.7"
 
-tableschema@^1.6.0:
-  version "1.6.0"
-  resolved "https://registry.yarnpkg.com/tableschema/-/tableschema-1.6.0.tgz#2ddf30b537b63856d27f315e3dd80abd71f365ea"
+tableschema@^1.7.0:
+  version "1.7.0"
+  resolved "https://registry.yarnpkg.com/tableschema/-/tableschema-1.7.0.tgz#79d8ab6cc0a45ddf57779fe8ff32f1cdd63d471e"
   dependencies:
     axios "^0.17.1"
     csv "^1.1.1"

From c8a6fff1c6f4a8386896ccdfbc9c3c187b77c242 Mon Sep 17 00:00:00 2001
From: Stephen-Gates <Stephen.Gates@me.com>
Date: Wed, 14 Feb 2018 21:06:42 +1000
Subject: [PATCH 24/26] fix error message for missing headers

---
 src/renderer/frictionlessDataPackage.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/renderer/frictionlessDataPackage.js b/src/renderer/frictionlessDataPackage.js
index 55e39369b..97c192e35 100644
--- a/src/renderer/frictionlessDataPackage.js
+++ b/src/renderer/frictionlessDataPackage.js
@@ -132,7 +132,7 @@ function hasAllResourceRequirements(hot, requiredMessages) {
     requiredMessages.push(`Column properties must be set.`)
   } else {
     if (!hasAllColumnNames(hot.guid, columnProperties)) {
-      requiredMessages.push(`Column property names cannot be empty.`)
+      requiredMessages.push(`Column property names cannot be empty - set a Header Row`)
     }
   }
   return requiredMessages.length === 0

From b186220d3558062a4808e95c0ef046ca4cab4350 Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Wed, 14 Feb 2018 22:08:00 +1000
Subject: [PATCH 25/26] Removed debug log. Fixed #301. Fixed #302.

---
 src/renderer/store/modules/hots.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/renderer/store/modules/hots.js b/src/renderer/store/modules/hots.js
index 7e76c464d..c63bdee1e 100644
--- a/src/renderer/store/modules/hots.js
+++ b/src/renderer/store/modules/hots.js
@@ -217,7 +217,6 @@ const mutations = {
     }
     _.merge(columnProperties, hotIdSchema.schema.descriptor.fields)
     state.hotTabs[hotId].columnProperties = columnProperties
-    console.log(state.hotTabs[hotId].columnProperties)
     return state.hotTabs[hotId].columnProperties
   },
   initColumnProperties(state, hotTab) {

From 9f9ac27d9c5ff33e2b3857004e5e039548c13684 Mon Sep 17 00:00:00 2001
From: Matthew Mulholland <matt@redboxresearchdata.com.au>
Date: Wed, 14 Feb 2018 23:48:08 +1000
Subject: [PATCH 26/26] Bumped version for next release.

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 01cb42c2e..25de83143 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "DataCurator",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "author": " <matt@redboxresearchdata.com.au>",
   "description": "Data Curator is a simple desktop CSV editor to help describe, validate and share usable open data",
   "license": "MIT",