From ba204a79340e6284d177c6d26c5959d54cb31101 Mon Sep 17 00:00:00 2001 From: The Dumb Terminal Date: Tue, 24 Jan 2023 23:06:57 +0000 Subject: [PATCH] ready for v6 release --- CHANGELOG.md | 5 ++ package.json | 5 +- src/converter.js | 15 ++--- .../expected.json | 5 ++ .../input.json | 2 +- .../additionalProperties/expected.json | 5 ++ .../additionalProperties/input.json | 21 ++++++ test/integration/converter.js | 29 +++++++++ test/unit/converter.js | 65 +++++++------------ 9 files changed, 99 insertions(+), 53 deletions(-) rename test/integration/continueOnError/{emptyObject => additionalProperties}/expected.json (71%) rename test/integration/continueOnError/{emptyObject => additionalProperties}/input.json (91%) create mode 100644 test/integration/continueOnErrorAndPreventAdditionalObjectProperties/additionalProperties/expected.json create mode 100644 test/integration/continueOnErrorAndPreventAdditionalObjectProperties/additionalProperties/input.json diff --git a/CHANGELOG.md b/CHANGELOG.md index be56c44..b305bed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## v6.0.0 (24/01/2023) + +- Objects with no properties convert to JSON fields. +- Updated dependencies. + ## v5.1.0 (11/08/2022) - JSBQ CLI output now compatible with Google's bq CLI. diff --git a/package.json b/package.json index fb80336..739dca0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jsonschema-bigquery", - "version": "5.1.0", + "version": "6.0.0", "description": "Convert JSON schema to Google BigQuery schema", "main": "src/converter.js", "scripts": { @@ -24,7 +24,8 @@ "json", "schema", "google", - "bigquery" + "bigquery", + "convert" ], "author": { "name": "thedumbterminal", diff --git a/src/converter.js b/src/converter.js index df7ad9f..b105635 100644 --- a/src/converter.js +++ b/src/converter.js @@ -166,8 +166,9 @@ converter._array = (name, node) => { } converter._object = (name, node, mode) => { - let result - const hasProperties = _.isPlainObject(node.properties) + let result = { + fields: [], + } let fieldType = 'RECORD' try { if ( @@ -179,16 +180,14 @@ converter._object = (name, node, mode) => { node ) } - if (!hasProperties) { + if ( + !_.isPlainObject(node.properties) || + Object.keys(node.properties).length === 0 + ) { // Big Query can handle semi-structured data : // https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json#loading_semi-structured_json_data fieldType = 'JSON' return converter._scalar(name, fieldType, mode, node.description) - } else if (Object.keys(node.properties).length === 0) { - throw new SchemaError( - 'Record fields must have one or more child fields', - node - ) } result = converter._scalar(name, fieldType, mode, node.description) diff --git a/test/integration/continueOnError/emptyObject/expected.json b/test/integration/continueOnError/additionalProperties/expected.json similarity index 71% rename from test/integration/continueOnError/emptyObject/expected.json rename to test/integration/continueOnError/additionalProperties/expected.json index 7360bb4..ab3f2e3 100644 --- a/test/integration/continueOnError/emptyObject/expected.json +++ b/test/integration/continueOnError/additionalProperties/expected.json @@ -10,6 +10,11 @@ "name": "street_address", "type": "STRING", "mode": "NULLABLE" + }, + { + "name": "country", + "type": "JSON", + "mode": "NULLABLE" } ] } diff --git a/test/integration/continueOnError/emptyObject/input.json b/test/integration/continueOnError/additionalProperties/input.json similarity index 91% rename from test/integration/continueOnError/emptyObject/input.json rename to test/integration/continueOnError/additionalProperties/input.json index ead6836..ba5dd21 100644 --- a/test/integration/continueOnError/emptyObject/input.json +++ b/test/integration/continueOnError/additionalProperties/input.json @@ -14,7 +14,7 @@ "properties": {} } }, - "additionalProperties": false + "additionalProperties": true } }, "additionalProperties": false diff --git a/test/integration/continueOnErrorAndPreventAdditionalObjectProperties/additionalProperties/expected.json b/test/integration/continueOnErrorAndPreventAdditionalObjectProperties/additionalProperties/expected.json new file mode 100644 index 0000000..a3ed612 --- /dev/null +++ b/test/integration/continueOnErrorAndPreventAdditionalObjectProperties/additionalProperties/expected.json @@ -0,0 +1,5 @@ +{ + "schema": { + "fields": [] + } +} diff --git a/test/integration/continueOnErrorAndPreventAdditionalObjectProperties/additionalProperties/input.json b/test/integration/continueOnErrorAndPreventAdditionalObjectProperties/additionalProperties/input.json new file mode 100644 index 0000000..4566042 --- /dev/null +++ b/test/integration/continueOnErrorAndPreventAdditionalObjectProperties/additionalProperties/input.json @@ -0,0 +1,21 @@ +{ + "id": "http://yourdomain.com/schemas/myschema.json", + "description": "Complex example empty Object", + "type": "object", + "properties": { + "address": { + "type": "object", + "properties": { + "street_address": { + "type": "string" + }, + "country": { + "type": "object", + "properties": {} + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true +} diff --git a/test/integration/converter.js b/test/integration/converter.js index d1a701a..d68b46c 100644 --- a/test/integration/converter.js +++ b/test/integration/converter.js @@ -53,4 +53,33 @@ describe('converter integration', () => { }) }) }) + + describe('continueOnError and preventAdditionalObjectProperties options', () => { + const sampleDir = + './test/integration/continueOnErrorAndPreventAdditionalObjectProperties' + // eslint-disable-next-line mocha/no-setup-in-describe + const testDirs = fs.readdirSync(sampleDir) + + // eslint-disable-next-line mocha/no-setup-in-describe + testDirs.forEach((dir) => { + describe(dir, () => { + let expected + let result + + before(() => { + const inJson = require(`../../${sampleDir}/${dir}/input.json`) + expected = require(`../../${sampleDir}/${dir}/expected.json`) + const options = { + continueOnError: true, + preventAdditionalObjectProperties: true, + } + result = converter.run(inJson, options, 'p', 't') + }) + + it('converts to big query', () => { + assert.deepStrictEqual(result, expected) + }) + }) + }) + }) }) diff --git a/test/unit/converter.js b/test/unit/converter.js index 6f0c1e9..fd6f2a8 100644 --- a/test/unit/converter.js +++ b/test/unit/converter.js @@ -117,6 +117,29 @@ describe('converter unit', () => { }) }) + context( + 'with the "preventAdditionalObjectProperties" and "continueOnError" options', + () => { + let result + + beforeEach(() => { + converter._options = { + preventAdditionalObjectProperties: true, + continueOnError: true, + } + const node = { + properties: {}, + } + result = converter._object('test', node, 'NULLABLE') + }) + + it('skips the field', () => { + const expected = { fields: [] } + assert.deepStrictEqual(result, expected) + }) + } + ) + context('with no properties', () => { it('converts to JSON when properties not defined', () => { const expected = { @@ -129,48 +152,6 @@ describe('converter unit', () => { assert.deepStrictEqual(result, expected) }) }) - - context('with zero properties', () => { - it('does not allow objects to have zero properties defined', () => { - const node = { - properties: {}, - } - assert.throws(() => { - converter._object('test', node, 'NULLABLE') - }, /Record fields must have one or more child fields/) - }) - }) - - context('with no properties continueOnError', () => { - beforeEach(() => { - converter._options = { - continueOnError: true, - } - }) - - it('does not allow objects to not have properties defined', () => { - assert.doesNotThrow(() => { - converter._object('test', {}, 'NULLABLE') - }, /No properties defined for object/) - }) - }) - - context('with zero properties continueOnError', () => { - beforeEach(() => { - converter._options = { - continueOnError: true, - } - }) - - it('does not allow objects to have zero properties defined', () => { - const node = { - properties: {}, - } - assert.doesNotThrow(() => { - converter._object('test', node, 'NULLABLE') - }, /Record fields must have one or more child fields/) - }) - }) }) describe('run()', () => {