diff --git a/package.json b/package.json index 5f4393141..416d430fd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "DataCurator", - "version": "0.15.0", + "version": "0.16.0", "author": " ", "description": "Data Curator is a simple desktop CSV editor to help describe, validate and share usable open data", "license": "MIT", @@ -99,7 +99,7 @@ "sortablejs": "^1.6.0", "spectron-fake-dialog": "^0.0.1", "svgo": "^1.0.5", - "tableschema": "^1.7.0", + "tableschema": "^1.9.1", "tmp": "^0.0.33", "unzipper": "^0.8.12", "vee-validate": "^2.0.4", diff --git a/src/main/excel.js b/src/main/excel.js index 52f744208..5e46c96fc 100644 --- a/src/main/excel.js +++ b/src/main/excel.js @@ -15,8 +15,8 @@ export function importExcel() { if (fileNames === undefined) { return } var fileName = fileNames[0] var workbook = XLSX.readFile(fileName) - var first_sheet_name = workbook.SheetNames[0] - var worksheet = workbook.Sheets[first_sheet_name] + // var first_sheet_name = workbook.SheetNames[0] + // var worksheet = workbook.Sheets[first_sheet_name] let shortcutsSubMenu = getSubMenuFromMenu('File', 'Open Excel Sheet...') shortcutsSubMenu.enabled = false @@ -30,7 +30,7 @@ export function importExcel() { closeWindowSafely(browserWindow) }) ipc.once('worksheetSelected', function(e, sheet_name) { - let data = XLSX.utils.sheet_to_csv(workbook.Sheets[sheet_name]) + var data = XLSX.utils.sheet_to_json(workbook.Sheets[sheet_name], {header: 1}) closeWindowSafely(browserWindow) createWindowTabWithData(data) }) diff --git a/src/main/menu.js b/src/main/menu.js index c7c7b099b..26f40b87c 100644 --- a/src/main/menu.js +++ b/src/main/menu.js @@ -1,13 +1,13 @@ -import {openFile, saveFileAs, saveFile, importDataPackage} from './file.js' -import {showUrlDialog} from './url.js' -import {createWindowTab, focusMainWindow} from './windows.js' -import {importExcel} from './excel.js' -import {showKeyboardHelp} from './help.js' -import {fileFormats} from '../renderer/file-formats.js' -import {shell, BrowserWindow, Menu} from 'electron' +import { openFile, saveFileAs, saveFile, importDataPackage } from './file.js' +import { showUrlDialog } from './url.js' +import { createWindowTab, focusMainWindow } from './windows.js' +import { importExcel } from './excel.js' +import { showKeyboardHelp } from './help.js' +import { fileFormats } from '../renderer/file-formats.js' +import { shell, BrowserWindow, Menu } from 'electron' class AppMenu { - initTemplate() { + initTemplate () { const webContents = this.webContents this.template = [ { @@ -16,7 +16,7 @@ class AppMenu { { label: 'New', accelerator: 'CmdOrCtrl+N', - click() { + click () { createWindowTab() } }, { @@ -27,7 +27,7 @@ class AppMenu { }, { label: 'Open Excel Sheet...', enabled: true, - click() { + click () { importExcel() } // Placeholder for future feature @@ -59,12 +59,12 @@ class AppMenu { // }, { // label: 'Settings', // enabled: false - // }, { + // }, { type: 'separator' }, { label: 'Save', accelerator: 'CmdOrCtrl+S', - click() { + click () { saveFile() }, id: 'save', @@ -90,7 +90,7 @@ class AppMenu { { label: 'Undo', accelerator: 'CmdOrCtrl+Z', - click() { + click () { webContents().send('editUndo') } }, { @@ -98,7 +98,7 @@ class AppMenu { accelerator: process.platform === 'darwin' ? 'Shift+CmdOrCtrl+Z' : 'CmdOrCtrl+Y', - click() { + click () { webContents().send('editRedo') } }, { @@ -130,13 +130,13 @@ class AppMenu { }, { label: 'Insert Row Above', accelerator: 'CmdOrCtrl+I', - click() { + click () { webContents().send('insertRowAbove') } }, { label: 'Insert Row Below', accelerator: 'CmdOrCtrl+K', - click() { + click () { webContents().send('insertRowBelow') } }, { @@ -144,25 +144,25 @@ class AppMenu { }, { label: 'Insert Column Before', accelerator: 'CmdOrCtrl+J', - click() { + click () { webContents().send('insertColumnLeft') } }, { label: 'Insert Column After', accelerator: 'CmdOrCtrl+L', - click() { + click () { webContents().send('insertColumnRight') } }, { type: 'separator' }, { label: 'Remove Row(s)', - click() { + click () { webContents().send('removeRows') } }, { label: 'Remove Column(s)', - click() { + click () { webContents().send('removeColumns') } } @@ -173,24 +173,48 @@ class AppMenu { submenu: [ { label: 'Find', - accelerator: 'CmdOrCtrl+F' + accelerator: 'CmdOrCtrl+F', + click: function () { + webContents().send('showSidePanel', 'findReplace') + } + }, { + type: 'separator' }, { label: 'Find Next', accelerator: 'CmdOrCtrl+G', - enabled: false + enabled: false, + click: function () { + webContents().send('clickFindButton', 'findNext') + } }, { label: 'Find Previous', accelerator: 'Shift+CmdOrCtrl+G', - enabled: false + enabled: false, + click: function () { + webContents().send('clickFindButton', 'findPrevious') + } }, { type: 'separator' + }, { + label: 'Replace Previous', + accelerator: 'CmdOrCtrl+E', + enabled: false, + click: function () { + webContents().send('clickFindButton', 'replacePrevious') + } }, { label: 'Replace Next', accelerator: 'Alt+CmdOrCtrl+E', - enabled: false + enabled: false, + click: function () { + webContents().send('clickFindButton', 'replaceNext') + } }, { label: 'Replace All', - enabled: false + enabled: false, + click: function () { + webContents().send('clickFindButton', 'replaceAll') + } } ] }, @@ -202,7 +226,7 @@ class AppMenu { accelerator: 'Shift+CmdOrCtrl+H', type: 'checkbox', checked: false, - click(menuItem) { + click (menuItem) { // revert 'checked' toggle so only controlled by header row event menuItem.checked = !menuItem.checked webContents().send('toggleActiveHeaderRow') @@ -212,7 +236,7 @@ class AppMenu { label: 'Case Sensitive Header Row', type: 'checkbox', checked: false, - click(menuItem) { + click (menuItem) { // revert 'checked' toggle so only controlled by event menuItem.checked = !menuItem.checked webContents().send('toggleCaseSensitiveHeader') @@ -237,7 +261,7 @@ class AppMenu { // }, { label: 'Guess Column Properties', accelerator: 'Shift+CmdOrCtrl+G', - click: function() { + click: function () { webContents().send('guessColumnProperties') } }, { @@ -245,25 +269,25 @@ class AppMenu { }, { label: 'Set Column Properties', accelerator: process.env.NODE_ENV !== 'development' ? 'Shift+CmdOrCtrl+C' : 'Alt+CmdOrCtrl+C', - click() { + click () { webContents().send('triggerMenuButton', 'Column') } }, { label: 'Set Table Properties', accelerator: 'Shift+CmdOrCtrl+T', - click() { + click () { webContents().send('triggerMenuButton', 'Table') } }, { label: 'Set Provenance Information', accelerator: 'Shift+CmdOrCtrl+P', - click() { + click () { webContents().send('triggerMenuButton', 'Provenance') } }, { label: 'Set Data Package Properties', accelerator: 'Shift+CmdOrCtrl+D', - click() { + click () { webContents().send('triggerMenuButton', 'Package') } }, { @@ -271,7 +295,7 @@ class AppMenu { }, { label: 'Validate Table', accelerator: 'Shift+CmdOrCtrl+V', - click() { + click () { webContents().send('validateTable') } }, { @@ -279,7 +303,7 @@ class AppMenu { }, { label: 'Export Data Package...', accelerator: 'Shift+CmdOrCtrl+X', - click() { + click () { webContents().send('triggerMenuButton', 'Export') } } @@ -346,19 +370,19 @@ class AppMenu { label: 'Keyboard Shortcuts', accelerator: 'CmdOrCtrl+/', enabled: true, - click() { + click () { showKeyboardHelp() } }, { type: 'separator' }, { label: 'Support Forum', - click() { + click () { shell.openExternal('https://ask.theodi.org.au/c/projects/data-curator') } }, { label: 'Report Issues', - click() { + click () { shell.openExternal('https://github.com/ODIQueensland/data-curator/blob/develop/.github/CONTRIBUTING.md') } } @@ -374,13 +398,14 @@ class AppMenu { ] } - buildAllMenusForFileFormats() { + buildAllMenusForFileFormats () { for (const format in fileFormats) { this.openSubMenu.push(this.buildMenuForFileFormats(format, openFile, 'CmdOrCtrl+O')) this.saveSubMenu.push(this.buildMenuForFileFormats(format, saveFileAs, 'Shift+CmdOrCtrl+S')) } } - buildMenuForFileFormats(format, clickFn, csvAccelerator) { + + buildMenuForFileFormats (format, clickFn, csvAccelerator) { const option = { label: fileFormats[format].label, click: ((format => () => { @@ -393,31 +418,31 @@ class AppMenu { return option } - buildOpenDataPackageMenu() { + buildOpenDataPackageMenu () { this.openDataPackageSubMenu = [{ - label: 'zip from URL....', + label: 'zip from URL...', enabled: true, - click() { + click () { // downloadDataPackageJson() showUrlDialog() } }, { label: 'zip from file...', enabled: true, - click() { + click () { importDataPackage() } }, { label: 'json from URL...', enabled: true, - click() { + click () { // downloadDataPackageJson() showUrlDialog() } }] } - updateMenuForDarwin() { + updateMenuForDarwin () { const webContents = this.webContents if (process.platform === 'darwin') { this.template.unshift({ @@ -425,7 +450,7 @@ class AppMenu { submenu: [ { label: 'About Data Curator', - click: function() { + click: function () { webContents().send('showSidePanel', 'about') } // Placeholder for future feature @@ -462,7 +487,7 @@ class AppMenu { } } - updateMenuForNonDarwin() { + updateMenuForNonDarwin () { const webContents = this.webContents if (process.platform !== 'darwin') { let subTemplate = this.getSubTemplateFromLabel('Window') @@ -470,14 +495,14 @@ class AppMenu { type: 'separator' }, { label: 'About Data Curator', - click: function() { + click: function () { webContents().send('showSidePanel', 'about') } }) } } - updateMenuForProduction() { + updateMenuForProduction () { if (process.env.NODE_ENV !== 'production') { this.template.push({ label: 'Developer', @@ -492,23 +517,23 @@ class AppMenu { } } - getSubTemplateFromLabel(label) { + getSubTemplateFromLabel (label) { return this.template.find(x => x.label === label) } - webContents() { + webContents () { // use .fromId rather than .focusedWindow as latter does not apply if app minimized // use .fromId rather than .getAllWindows[0] as if child window present and main window minimized won't work let browserWindow = focusMainWindow() return browserWindow.webContents } - initContainers() { + initContainers () { this.openSubMenu = [] this.saveSubMenu = [] } - constructor() { + constructor () { this.initContainers() this.buildAllMenusForFileFormats() this.buildOpenDataPackageMenu() @@ -520,32 +545,62 @@ class AppMenu { } } -export function getMenu(menuLabel) { +export function getMenu (menuLabel) { let menu = Menu.getApplicationMenu().items.find(x => x.label === menuLabel) return menu } -export function getSubMenuFromMenu(menuLabel, subMenuLabel) { - let menu = Menu.getApplicationMenu().items.find(x => x.label === menuLabel) +export function enableAllSubMenuItemsFromMenuLabel (menuLabel) { + let menu = getMenu(menuLabel) + enableAllSubMenuItemsFromMenuObject(menu) +} + +export function enableAllSubMenuItemsFromMenuObject (menu) { + menu.submenu.items.forEach(function (x) { + x.enabled = !!x.label + }) +} + +export function disableAllSubMenuItemsFromMenuLabel (menuLabel) { + let menu = getMenu(menuLabel) + disableAllSubMenuItemsFromMenuObject(menu) +} + +export function disableAllSubMenuItemsFromMenuObject (menu) { + menu.submenu.items.forEach(function (x) { + if (typeof x.label !== 'undefined') { + x.enabled = false + } + }) +} + +export function getSubMenuLabelsFromMenu (menuLabel) { + let menu = getMenu(menuLabel) + let subMenuLabels = menu.submenu.items.map(x => x.label) + return subMenuLabels +} + +export function getSubMenuFromMenu (menuLabel, subMenuLabel) { + let menu = getMenu(menuLabel) let subMenu = menu.submenu.items.find(x => x.label === subMenuLabel) return subMenu } -export function enableSubMenuItemsFromMenuObject(menu, labels) { +export function enableSubMenuItemsFromMenuObject (menu, labels) { for (const label of labels) { const subMenu = menu.submenu.items.find(x => x.label === label) subMenu.enabled = true } } -export function disableSubMenuItemsFromMenuObject(menu, labels) { +export function disableSubMenuItemsFromMenuObject (menu, labels) { for (const label of labels) { const subMenu = menu.submenu.items.find(x => x.label === label) subMenu.enabled = false } } -export function clickLabelsOnMenu(args) { +export function clickLabelsOnMenu (args) { let menu = Menu.getApplicationMenu().items.find(x => x.label === args[0]) menu.click() let returnLabel = menu.label diff --git a/src/main/rendererToMain.js b/src/main/rendererToMain.js index 0e4dca730..711bdc47a 100644 --- a/src/main/rendererToMain.js +++ b/src/main/rendererToMain.js @@ -1,7 +1,15 @@ -import {ipcMain as ipc} from 'electron' +import {ipcMain as ipc, dialog} from 'electron' import {showErrors} from './errors.js' -import {getSubMenuFromMenu, clickLabelsOnMenu} from './menu' +import { + getMenu, + getSubMenuFromMenu, + clickLabelsOnMenu, + disableAllSubMenuItemsFromMenuObject, + enableSubMenuItemsFromMenuObject, + enableAllSubMenuItemsFromMenuLabel +} from './menu' import {focusMainWindow, closeSecondaryWindow} from './windows.js' +import {loadPackageJson, loadResourceDataFromPackageUrl} from './url.js' ipc.on('toggleSaveMenu', (event, arg) => { let saveSubMenu = getSubMenuFromMenu('File', 'Save') @@ -39,3 +47,41 @@ ipc.on('focusMainWindow', (event, arg) => { ipc.on('closeSecondaryWindow', (event, arg) => { closeSecondaryWindow(arg) }) + +ipc.on('closedFindReplace', (event, arg) => { + let menu = getMenu('Find') + disableAllSubMenuItemsFromMenuObject(menu) + enableSubMenuItemsFromMenuObject(menu, ['Find']) +}) + +ipc.on('openedFindReplace', (event, arg) => { + enableAllSubMenuItemsFromMenuLabel('Find') +}) + +ipc.on('loadPackageUrl', async function(event, index, hotId, url) { + const mainWindow = focusMainWindow() + const dataPackage = await loadPackageJson(url) + if (dataPackage) { + mainWindow.webContents.send('packageUrlLoaded', index, hotId, url, dataPackage.descriptor) + } +}) + +ipc.on('loadPackageUrlResourcesAsFkRelations', async function(event, url, resourceName) { + try { + const rows = await loadResourceDataFromPackageUrl(url, resourceName) + event.returnValue = rows + } catch (error) { + const errorMessage = 'There was a problem collating data from url resources' + const mainWindow = focusMainWindow() + dialog.showMessageBox(mainWindow, { + type: 'error', + title: 'Problem loading data package url', + message: errorMessage + }) + console.log(errorMessage, error) + } +}) + +function sendStopLoadingPackageFeedback() { + mainWindow.webContents.send('stopLoadingPackageFeedback') +} diff --git a/src/main/url.js b/src/main/url.js index 0c07f3c49..b5d3ef706 100644 --- a/src/main/url.js +++ b/src/main/url.js @@ -3,7 +3,7 @@ import fs from 'fs-extra' import path from 'path' import {ipcMain as ipc, dialog} from 'electron' import {focusOrNewSecondaryWindow, focusMainWindow, closeWindowSafely} from './windows' -import {getSubMenuFromMenu, disableSubMenuItemsFromMenuObject, enableSubMenuItemsFromMenuObject} from './menu.js' +import {getSubMenuFromMenu, disableAllSubMenuItemsFromMenuObject, enableAllSubMenuItemsFromMenuObject} from './menu.js' import {Package} from 'datapackage' import tmp from 'tmp' import _ from 'lodash' @@ -14,12 +14,12 @@ tmp.setGracefulCleanup() // TODO: handle errors by rejecting promises and throwing back up stack export function showUrlDialog() { - let labels = ['zip from URL....', 'zip from file...', 'json from URL...'] + // let labels = ['zip from URL....', 'zip from file...', 'json from URL...'] let menu = getSubMenuFromMenu('File', 'Open Data Package') - disableSubMenuItemsFromMenuObject(menu, labels) + disableAllSubMenuItemsFromMenuObject(menu) let browserWindow = focusOrNewSecondaryWindow('urldialog', {width: 300, height: 150, modal: true, alwaysOnTop: true}) browserWindow.on('closed', () => { - enableSubMenuItemsFromMenuObject(menu, labels) + enableAllSubMenuItemsFromMenuObject(menu) }) browserWindow.webContents.on('did-finish-load', () => { ipc.once('urlCancelled', () => { @@ -98,6 +98,13 @@ async function loadPackageFromJsonUrl(urlText) { mainWindow.webContents.send('closeAndshowLoadingScreen', 'Loading package URL..') const dataPackageJson = await loadPackageJson(urlText, mainWindow) if (!dataPackageJson) { + dialog.showMessageBox(mainWindow, { + type: 'warning', + title: `Unable to load Data Package`, + message: + `The data package, ${urlText}, could not be loaded. + If the data package is a URL, please check that the URL exists.` + }) mainWindow.webContents.send('closeLoadingScreen') return } @@ -106,7 +113,11 @@ async function loadPackageFromJsonUrl(urlText) { mainWindow.webContents.send('closeLoadingScreen') return } - await loadResources(dataPackageJson, mainWindow) + try { + await loadResources(dataPackageJson, mainWindow) + } catch (error) { + console.log('There was a problem loading package from json', error) + } } function showInvalidMessage(urlText, mainWindow) { @@ -130,21 +141,12 @@ function showUrlPathNotSupportedMessage(urlText, mainWindow) { } // datapackage-js does not support loading url in browser -async function loadPackageJson(json, mainWindow) { +export async function loadPackageJson(json) { try { const dataPackage = await Package.load(json) return dataPackage } catch (error) { console.log(`There was a problem loading the package: ${json}`, error) - dialog.showMessageBox(mainWindow, { - type: 'warning', - title: `Unable to load Data Package`, - message: -`The data package, ${json}, could not be loaded. -If the data package is a URL, please check that the URL exists.` - }) - // throw new Error() - // return Promise.reject(error) } } @@ -163,3 +165,17 @@ async function loadResources(dataPackageJson, mainWindow) { mainWindow.webContents.send('addTabWithFormattedDataAndDescriptor', dataWithHeaders, format, dataResource.descriptor) } } + +export async function loadResourceDataFromPackageUrl(url, resourceName) { + const dataPackage = await loadPackageJson(url) + const rowOfObjects = [] + if (dataPackage && _.indexOf(dataPackage.resourceNames, resourceName) > -1) { + const dataResource = dataPackage.getResource(resourceName) + const data = await dataResource.read() + const headers = dataResource.headers + for (const row of data) { + rowOfObjects.push(_.zipObject(headers, row)) + } + } + return rowOfObjects +} diff --git a/src/renderer/components/Home.vue b/src/renderer/components/Home.vue index 728d09250..d7db06b4f 100644 --- a/src/renderer/components/Home.vue +++ b/src/renderer/components/Home.vue @@ -302,7 +302,7 @@ export default { tooltipView: 'tooltipValidate' }, { - name: 'Find', + name: 'Find and Replace', id: 'find', image: 'static/img/find.svg', tooltipId: 'tooltip-find', @@ -713,6 +713,9 @@ export default { closeSideNav: function() { this.enableTransition = false this.sideNavStatus = 'closed' + if (this.sideNavView == 'findReplace') { + ipc.send('closedFindReplace') + } this.sideNavView = '' }, openSideNav: function() { @@ -741,6 +744,11 @@ export default { if (toolbarMenu) { this.sideNavPosition = toolbarMenu.sideNavPosition this.sideNavView = toolbarMenu.sideNavView + if (this.sideNavView !== 'findReplace') { + ipc.send('closedFindReplace') + } else { + ipc.send('openedFindReplace') + } this.sideNavViewTitle = toolbarMenu.name this.openSideNav() } @@ -752,9 +760,9 @@ export default { } if (this.sideNavStatus === 'closed') { this.enableTransition = false - } else if (this.sideNavPosition === 'left' && (index !== -1 && menuName !== 'Find')) { + } else if (this.sideNavPosition === 'left' && (index !== -1 && menuName !== 'Find and Replace')) { this.enableTransition = false - } else if (this.sideNavPosition === 'right' && (index === -1 || menuName === 'Find')) { + } else if (this.sideNavPosition === 'right' && (index === -1 || menuName === 'Find and Replace')) { this.enableTransition = false } else { this.updateTransitions(index) @@ -766,6 +774,9 @@ export default { findMenuByName: function(name) { return this.toolbarMenus.find(x => x.name === name) }, + findMenuBySideNavView: function(sideNavView) { + return this.toolbarMenus.find(x => typeof x.sideNavView !== 'undefined' && x.sideNavView === sideNavView) + }, updateActiveColumn: function(selected) { if (selected) { this.currentColumnIndex = selected[1] @@ -812,11 +823,18 @@ export default { return `tab${tabId}` }, triggerSideNav: function(properties) { - this.updateToolbarMenuForSideNav(-1) - this.sideNavPosition = properties.sideNavPosition || 'left' - this.sideNavView = properties.sideNavView - this.sideNavViewTitle = properties.title || properties.sideNavView - this.openSideNav() + const menu = this.findMenuBySideNavView(properties.sideNavView) + if (menu) { + this.triggerMenuButton(menu.name) + } else { + this.updateToolbarMenuForSideNav(-1) + // Find is a menu button so ensure closed signal fires + ipc.send('closedFindReplace') + this.sideNavPosition = properties.sideNavPosition || 'left' + this.sideNavView = properties.sideNavView + this.sideNavViewTitle = properties.title || properties.sideNavView + this.openSideNav() + } }, triggerMenuButton: function(menuName) { let index = _.findIndex(this.toolbarMenus, function(o) { @@ -1101,7 +1119,9 @@ export default { // console.log(`sync calc: ${plugin.getSyncCalculationLimit()}`) // console.log(`first row: ${plugin.getFirstVisibleRow()}`) // console.log(`second row: ${plugin.getLastVisibleRow()}`) - ipc.send('hasHeaderRow', hot.hasColHeaders()) + if (hot) { + ipc.send('hasHeaderRow', hot.hasColHeaders()) + } ipc.send('hasCaseSensitiveHeader', isCaseSensitive(hotId)) }) onNextHotIdFromTabRx(getHotIdFromTabIdFunction()) @@ -1120,6 +1140,7 @@ export default { vueValidateTable() }) this.pushDefaultPackageProperties() + ipc.send('closedFindReplace') }, updated: function() { if (this.loadingDataMessage && this.loadingDataMessage.length > 0) { diff --git a/src/renderer/components/KeyboardHelp.vue b/src/renderer/components/KeyboardHelp.vue index 714dc192e..b28fa15de 100644 --- a/src/renderer/components/KeyboardHelp.vue +++ b/src/renderer/components/KeyboardHelp.vue @@ -192,7 +192,7 @@ - @@ -259,7 +266,7 @@ - Undo the last edit command + Undo the last edit command. Replace commands cannot be undone Ctrl Z Command ⌘ Z @@ -283,7 +290,7 @@ Ctrl V Command ⌘ V - - - - +
@@ -436,10 +420,11 @@ - + +
Minimize - create a new table of dataMinimize - minimize the Data Curator window Command ⌘ M
@@ -463,11 +449,13 @@ + Keyboard Shortcuts Ctrl / diff --git a/src/renderer/data-actions.js b/src/renderer/data-actions.js index b49d72ca9..57cca578b 100644 --- a/src/renderer/data-actions.js +++ b/src/renderer/data-actions.js @@ -18,24 +18,28 @@ export function loadDataIntoHot(hot, data, format) { } export function loadCsvDataIntoHot(hot, data, format) { - let arrays - // if no format specified, default to csv - if (typeof format === 'undefined' || !format) { - arrays = parse(data) - } else { - let csvOptions = dialectToCsvOptions(format.dialect) - // let csv parser handle the line terminators - _.unset(csvOptions, 'rowDelimiter') - // TODO: update to stream - arrays = parse(data, csvOptions) - pushCsvFormat(hot.guid, format) - } + try { + let arrays + // if no format specified, default to csv + if (typeof format === 'undefined' || !format) { + arrays = parse(data) + } else { + let csvOptions = dialectToCsvOptions(format.dialect) + // let csv parser handle the line terminators + _.unset(csvOptions, 'rowDelimiter') + // TODO: update to stream + arrays = parse(data, csvOptions) + pushCsvFormat(hot.guid, format) + } - fixRaggedRows(arrays) - hot.loadData(arrays) - hot.render() - // frictionless csv header default = true - toggleHeaderNoFeedback(hot) + fixRaggedRows(arrays) + hot.loadData(arrays) + hot.render() + // frictionless csv header default = true + toggleHeaderNoFeedback(hot) + } catch (error) { + console.log('There was a problem loading data', error) + } } export function loadArrayDataIntoHot(hot, arrays, format) { diff --git a/src/renderer/frictionless.js b/src/renderer/frictionless.js index 70562c70d..3eb14c8be 100644 --- a/src/renderer/frictionless.js +++ b/src/renderer/frictionless.js @@ -4,6 +4,8 @@ import store from '@/store/modules/hots.js' import tabStore from '@/store/modules/tabs.js' import {includeHeadersInData, hasAllColumnNames, getValidNames} from '@/frictionlessUtilities.js' import {allTablesAllColumnsFromSchema$} from '@/rxSubject.js' +import {ipcRenderer as ipc} from 'electron' +import {Package} from 'datapackage' async function inferSchema(data) { const schema = await Schema.load({}) @@ -88,21 +90,35 @@ async function collateForeignKeys(localHotId, callback) { } let relations = {} for (const foreignKey of foreignKeys) { - let foreignHotId = getHotIdFromForeignKeyForeignTable(foreignKey.reference.resource, localHotId) - // foreign keys must also have column properties set - if (!hasColumnProperties(foreignHotId, callback)) { - relations = false - break + let rows + if (foreignKey.reference.package) { + rows = await collatePackageForeignKeys(foreignKey) + } else { + let foreignHotId = getHotIdFromForeignKeyForeignTable(foreignKey.reference.resource, localHotId) + // foreign keys must also have column properties set + if (!hasColumnProperties(foreignHotId, callback)) { + relations = false + break + } + let data = getForeignKeyData(foreignHotId) + let schema = await buildSchema(data, foreignHotId) + let table = await createFrictionlessTable(data, schema) + rows = await table.read({keyed: true}) } - let data = getForeignKeyData(foreignHotId) - let schema = await buildSchema(data, foreignHotId) - let table = await createFrictionlessTable(data, schema) - let rows = await table.read({keyed: true}) relations[foreignKey.reference.resource] = rows } return relations } +async function collatePackageForeignKeys(foreignKey) { + try { + let rows = await ipc.sendSync('loadPackageUrlResourcesAsFkRelations', foreignKey.reference.package, foreignKey.reference.resource) + return rows + } catch (error) { + console.log('There was an error in creating package foreign keys', error) + } +} + function getForeignKeyData(foreignHotId) { let hot = HotRegister.getInstance(foreignHotId) return includeHeadersInData(hot) @@ -165,6 +181,7 @@ export async function validateActiveDataAgainstSchema(callback) { try { relations = await collateForeignKeys(hotId, callback) } catch (error) { + console.log(error) errorCollector.push({message: `There was a problem validating 1 or more foreign tables. Validate foreign tables first.`, name: 'Invalid foreign table(s)'}) } const stream = await table.iter({ diff --git a/src/renderer/hot.js b/src/renderer/hot.js index 811a8b9ae..3bf18fbc1 100644 --- a/src/renderer/hot.js +++ b/src/renderer/hot.js @@ -74,6 +74,7 @@ const HotRegister = { } }, afterSetDataAtCell() { + // console.log('data set at cell.') afterSetDataAtCell$.next(true) }, enterMoves({shiftKey}) { diff --git a/src/renderer/importPackage.js b/src/renderer/importPackage.js index 136769e20..36ebaaf1f 100644 --- a/src/renderer/importPackage.js +++ b/src/renderer/importPackage.js @@ -41,10 +41,6 @@ async function unzipFileToDir(zipSource, unzipDestination, isTransient) { return processedProperties } -function handleZippedFolder() { - -} - async function getDataPackageJson(processed) { let filename = processed.json[0] let text = await stringify(filename) @@ -122,7 +118,6 @@ async function getAllResourcePaths(dataPackageJson, unzipDestination, processed) } else { fileDestination = path.join(unzipDestination, dataResource.path) } - console.log(`file destination is ${fileDestination}`) let format = dataResourceToFormat(dataResource) await ipc.send('openFileIntoTab', fileDestination, format) resourcePaths.push(dataResource.path) @@ -153,7 +148,6 @@ async function getHotIdsFromFilenames(processed, unzipDestination, isTransient = let re = new RegExp('^' + processed.parentFolders + '/') let resourcePathname = _.replace(pathname, re, '') csvTabs[`${resourcePathname}`] = hotId - console.log(`is transient: ${isTransient}`) if (isTransient) { store.commit('resetTabFilename', tabId) } diff --git a/src/renderer/mixins/RelationKeys.vue b/src/renderer/mixins/RelationKeys.vue index ba04c4319..7f00e6a5b 100644 --- a/src/renderer/mixins/RelationKeys.vue +++ b/src/renderer/mixins/RelationKeys.vue @@ -32,10 +32,12 @@ export default { return _.without(allTablesAllNames[hotId], '', null, undefined) } }, + mounted: async function() { - let vueUpdateSubscriptions = this.updateSubscriptions + let self = this this.$subscribeTo(allTablesAllColumnNames$, async function(allTablesAllColumnNames) { - await vueUpdateSubscriptions(allTablesAllColumnNames) + const hotId = await self.getHotIdFromTabId(self.getActiveTab) + await self.updateSubscriptions(allTablesAllColumnNames, hotId) }) this.initTableHeaderKeys() } diff --git a/src/renderer/partials/ColumnProperties.vue b/src/renderer/partials/ColumnProperties.vue index 0aa98451d..bf5b2cdd4 100644 --- a/src/renderer/partials/ColumnProperties.vue +++ b/src/renderer/partials/ColumnProperties.vue @@ -2,9 +2,11 @@