Merge pull request #646 from k1LoW/jsonschema

Fix schema.json for generating JSON Schema of schema.json

Makefile

@@ -16,7 +16,7 @@ TBLS ?= ./tbls
 
 default: test
 
-ci: depsdev build db test testdoc testdoc_hide_auto_increment test_too_many_tables test_json test_ext_subcommand doc
+ci: depsdev build db test testdoc testdoc_hide_auto_increment test_too_many_tables test_json test_ext_subcommand test_jsonschema doc
 
 ci_windows: depsdev build db_sqlite testdoc_sqlite
 
@@ -147,6 +147,12 @@ test_ext_subcommand: build
 	env PATH="${PWD}/testdata/bin:${PATH}" TBLS_DSN=pg://postgres:pgpass@localhost:55432/testdb?sslmode=disable $(TBLS) echo | grep 'TBLS_DSN=pg://postgres:pgpass@localhost:55432/testdb?sslmode=disable' > /dev/null
 	echo hello | env PATH="${PWD}/testdata/bin:${PATH}" $(TBLS) echo -c ./testdata/ext_subcommand_tbls.yml | grep 'STDIN=hello' > /dev/null
 
+test_jsonschema:
+	cd scripts/jsonschema && go run main.go | diff -u ../../spec/tbls.schema.json_schema.json -
+
+generate_jsonschema:
+	cd scripts/jsonschema && go run main.go > ../../spec/tbls.schema.json_schema.json
+
 generate_test_json: build
 	sqlite3 $(PWD)/filter_tables.sqlite3 < testdata/ddl/filter_tables.sql
 	$(TBLS) out sq://$(PWD)/filter_tables.sqlite3 -t json > testdata/filter_tables.json
@@ -158,9 +164,9 @@ build:
 	go build -tags timetzdata -ldflags="$(BUILD_LDFLAGS)"
 
 depsdev:
-	go install github.com/linyows/git-semv/cmd/git-semv@v1.2.0
-	go install github.com/Songmu/ghch/cmd/ghch@v0.10.2
-	go install github.com/xo/usql@v0.9.5
+	go install github.com/linyows/git-semv/cmd/git-semv@latest
+	go install github.com/Songmu/ghch/cmd/ghch@latest
+	go install github.com/xo/usql@latest
 	go install github.com/Songmu/gocredits/cmd/gocredits@latest
 	go install github.com/securego/gosec/v2/cmd/gosec@latest
 

sample/clickhouse/schema.json

@@ -1 +1 @@
-{"name":"testdb","desc":"","tables":[{"name":"id_value_dictionary","type":"Dictionary","comment":"","columns":[{"name":"id","type":"UInt64","nullable":false,"default":null,"comment":""},{"name":"value","type":"String","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[],"triggers":[],"def":"CREATE DICTIONARY testdb.id_value_dictionary (`id` UInt64, `value` String) PRIMARY KEY id SOURCE(CLICKHOUSE(TABLE 'source_table')) LIFETIME(MIN 0 MAX 1000) LAYOUT(FLAT())"},{"name":"materialized_view","type":"MaterializedView","comment":"","columns":[{"name":"name1","type":"UInt64","nullable":false,"default":null,"comment":""},{"name":"name2","type":"Nullable(String)","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[],"triggers":[],"def":"CREATE MATERIALIZED VIEW testdb.materialized_view (`name1` UInt64, `name2` Nullable(String)) ENGINE = Memory AS SELECT name1, name2 FROM testdb.table_name ORDER BY name1 DESC","referenced_tables":["table_name"]},{"name":"numbers_table","type":"SystemNumbers","comment":"","columns":[{"name":"number","type":"UInt64","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[],"triggers":[],"def":"CREATE TABLE testdb.numbers_table (`number` UInt64) AS numbers(100)"},{"name":"source_table","type":"MergeTree","comment":"","columns":[{"name":"id","type":"UInt64","nullable":false,"default":null,"comment":""},{"name":"value","type":"String","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[{"name":"sorting key","type":"SORTING KEY","def":"ORDER BY (id)","table":"source_table","referenced_table":null,"columns":["id"],"referenced_columns":null,"comment":""},{"name":"primary key","type":"PRIMARY KEY","def":"PRIMARY KEY (id)","table":"source_table","referenced_table":null,"columns":["id"],"referenced_columns":null,"comment":""}],"triggers":[],"def":"CREATE TABLE testdb.source_table (`id` UInt64, `value` String) ENGINE = MergeTree PRIMARY KEY id ORDER BY id SETTINGS index_granularity = 8192"},{"name":"t1","type":"Memory","comment":"","columns":[{"name":"x","type":"String","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[],"triggers":[],"def":"CREATE TABLE testdb.t1 (`x` String) ENGINE = Memory"},{"name":"table_name","type":"MergeTree","comment":"comment for table","columns":[{"name":"name1","type":"UInt64","nullable":false,"default":null,"comment":"comment for column 1"},{"name":"name2","type":"Nullable(String)","nullable":false,"default":"DEFAULT 'column 2'","comment":"comment for column 2"},{"name":"name3","type":"LowCardinality(String)","nullable":false,"default":"MATERIALIZED upper(name2)","comment":"comment for column 3"},{"name":"name4","type":"SimpleAggregateFunction(sum, Float64)","nullable":false,"default":null,"comment":""},{"name":"name5","type":"DateTime","nullable":false,"default":"DEFAULT now()","comment":""},{"name":"name6","type":"String","nullable":false,"default":"ALIAS formatReadableSize(name1)","comment":""},{"name":"name7","type":"String","nullable":false,"default":"MATERIALIZED hex(name1)","comment":""},{"name":"name8","type":"FixedString(4)","nullable":false,"default":"DEFAULT unhex(name7)","comment":""}],"indexes":[{"name":"idx1","def":"bloom_filter(0.01)","table":"table_name","columns":["name1"],"comment":""},{"name":"idx2","def":"minmax","table":"table_name","columns":["name1 * 2"],"comment":""},{"name":"idx3","def":"set(1000)","table":"table_name","columns":["name1 * length(name2)"],"comment":""}],"constraints":[{"name":"partition key","type":"PARTITION KEY","def":"PARTITION BY ((name1, name3, name5))","table":"table_name","referenced_table":null,"columns":["name1","name3","name5"],"referenced_columns":null,"comment":""},{"name":"sorting key","type":"SORTING KEY","def":"ORDER BY (name1, name5)","table":"table_name","referenced_table":null,"columns":["name1","name5"],"referenced_columns":null,"comment":""},{"name":"primary key","type":"PRIMARY KEY","def":"PRIMARY KEY (name1, name5)","table":"table_name","referenced_table":null,"columns":["name1","name5"],"referenced_columns":null,"comment":""},{"name":"sampling key","type":"SAMPLING KEY","def":"SAMPLE BY (name1)","table":"table_name","referenced_table":null,"columns":["name1"],"referenced_columns":null,"comment":""}],"triggers":[],"def":"CREATE TABLE testdb.table_name (`name1` UInt64 COMMENT 'comment for column 1', `name2` Nullable(String) DEFAULT 'column 2' COMMENT 'comment for column 2' CODEC(ZSTD(1)), `name3` LowCardinality(String) MATERIALIZED upper(name2) COMMENT 'comment for column 3', `name4` SimpleAggregateFunction(sum, Float64) TTL name5 + toIntervalDay(1), `name5` DateTime DEFAULT now(), `name6` String ALIAS formatReadableSize(name1), `name7` String MATERIALIZED hex(name1), `name8` FixedString(4) DEFAULT unhex(name7), INDEX idx1 name1 TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx2 name1 * 2 TYPE minmax GRANULARITY 3, INDEX idx3 name1 * length(name2) TYPE set(1000) GRANULARITY 4, PROJECTION projection_name_1 (SELECT name1, name2, name3 ORDER BY name1)) ENGINE = MergeTree PARTITION BY (name1, name3, name5) PRIMARY KEY (name1, name5) ORDER BY (name1, name5) SAMPLE BY name1 SETTINGS index_granularity = 8192 COMMENT 'comment for table'"},{"name":"view","type":"View","comment":"","columns":[{"name":"name1","type":"UInt64","nullable":false,"default":null,"comment":""},{"name":"name2","type":"Nullable(String)","nullable":false,"default":null,"comment":""},{"name":"name4","type":"SimpleAggregateFunction(sum, Float64)","nullable":false,"default":null,"comment":""},{"name":"name5","type":"DateTime","nullable":false,"default":null,"comment":""},{"name":"name8","type":"FixedString(4)","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[],"triggers":[],"def":"CREATE VIEW testdb.view (`name1` UInt64, `name2` Nullable(String), `name4` SimpleAggregateFunction(sum, Float64), `name5` DateTime, `name8` FixedString(4)) AS SELECT * FROM testdb.table_name"}],"relations":[],"functions":[{"name":"linear_equation","return_type":"","arguments":"","type":""}],"driver":{"name":"clickhouse","database_version":"24.4.4.85","meta":{"dict":{"Functions":"Stored procedures and functions"}}}}
+{"name":"testdb","tables":[{"name":"id_value_dictionary","type":"Dictionary","columns":[{"name":"id","type":"UInt64","nullable":false},{"name":"value","type":"String","nullable":false}],"def":"CREATE DICTIONARY testdb.id_value_dictionary (`id` UInt64, `value` String) PRIMARY KEY id SOURCE(CLICKHOUSE(TABLE 'source_table')) LIFETIME(MIN 0 MAX 1000) LAYOUT(FLAT())"},{"name":"materialized_view","type":"MaterializedView","columns":[{"name":"name1","type":"UInt64","nullable":false},{"name":"name2","type":"Nullable(String)","nullable":false}],"def":"CREATE MATERIALIZED VIEW testdb.materialized_view (`name1` UInt64, `name2` Nullable(String)) ENGINE = Memory AS SELECT name1, name2 FROM testdb.table_name ORDER BY name1 DESC","referenced_tables":["table_name"]},{"name":"numbers_table","type":"SystemNumbers","columns":[{"name":"number","type":"UInt64","nullable":false}],"def":"CREATE TABLE testdb.numbers_table (`number` UInt64) AS numbers(100)"},{"name":"source_table","type":"MergeTree","columns":[{"name":"id","type":"UInt64","nullable":false},{"name":"value","type":"String","nullable":false}],"constraints":[{"name":"sorting key","type":"SORTING KEY","def":"ORDER BY (id)","table":"source_table","columns":["id"]},{"name":"primary key","type":"PRIMARY KEY","def":"PRIMARY KEY (id)","table":"source_table","columns":["id"]}],"def":"CREATE TABLE testdb.source_table (`id` UInt64, `value` String) ENGINE = MergeTree PRIMARY KEY id ORDER BY id SETTINGS index_granularity = 8192"},{"name":"t1","type":"Memory","columns":[{"name":"x","type":"String","nullable":false}],"def":"CREATE TABLE testdb.t1 (`x` String) ENGINE = Memory"},{"name":"table_name","type":"MergeTree","comment":"comment for table","columns":[{"name":"name1","type":"UInt64","nullable":false,"comment":"comment for column 1"},{"name":"name2","type":"Nullable(String)","nullable":false,"default":"DEFAULT 'column 2'","comment":"comment for column 2"},{"name":"name3","type":"LowCardinality(String)","nullable":false,"default":"MATERIALIZED upper(name2)","comment":"comment for column 3"},{"name":"name4","type":"SimpleAggregateFunction(sum, Float64)","nullable":false},{"name":"name5","type":"DateTime","nullable":false,"default":"DEFAULT now()"},{"name":"name6","type":"String","nullable":false,"default":"ALIAS formatReadableSize(name1)"},{"name":"name7","type":"String","nullable":false,"default":"MATERIALIZED hex(name1)"},{"name":"name8","type":"FixedString(4)","nullable":false,"default":"DEFAULT unhex(name7)"}],"indexes":[{"name":"idx1","def":"bloom_filter(0.01)","table":"table_name","columns":["name1"]},{"name":"idx2","def":"minmax","table":"table_name","columns":["name1 * 2"]},{"name":"idx3","def":"set(1000)","table":"table_name","columns":["name1 * length(name2)"]}],"constraints":[{"name":"partition key","type":"PARTITION KEY","def":"PARTITION BY ((name1, name3, name5))","table":"table_name","columns":["name1","name3","name5"]},{"name":"sorting key","type":"SORTING KEY","def":"ORDER BY (name1, name5)","table":"table_name","columns":["name1","name5"]},{"name":"primary key","type":"PRIMARY KEY","def":"PRIMARY KEY (name1, name5)","table":"table_name","columns":["name1","name5"]},{"name":"sampling key","type":"SAMPLING KEY","def":"SAMPLE BY (name1)","table":"table_name","columns":["name1"]}],"def":"CREATE TABLE testdb.table_name (`name1` UInt64 COMMENT 'comment for column 1', `name2` Nullable(String) DEFAULT 'column 2' COMMENT 'comment for column 2' CODEC(ZSTD(1)), `name3` LowCardinality(String) MATERIALIZED upper(name2) COMMENT 'comment for column 3', `name4` SimpleAggregateFunction(sum, Float64) TTL name5 + toIntervalDay(1), `name5` DateTime DEFAULT now(), `name6` String ALIAS formatReadableSize(name1), `name7` String MATERIALIZED hex(name1), `name8` FixedString(4) DEFAULT unhex(name7), INDEX idx1 name1 TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx2 name1 * 2 TYPE minmax GRANULARITY 3, INDEX idx3 name1 * length(name2) TYPE set(1000) GRANULARITY 4, PROJECTION projection_name_1 (SELECT name1, name2, name3 ORDER BY name1)) ENGINE = MergeTree PARTITION BY (name1, name3, name5) PRIMARY KEY (name1, name5) ORDER BY (name1, name5) SAMPLE BY name1 SETTINGS index_granularity = 8192 COMMENT 'comment for table'"},{"name":"view","type":"View","columns":[{"name":"name1","type":"UInt64","nullable":false},{"name":"name2","type":"Nullable(String)","nullable":false},{"name":"name4","type":"SimpleAggregateFunction(sum, Float64)","nullable":false},{"name":"name5","type":"DateTime","nullable":false},{"name":"name8","type":"FixedString(4)","nullable":false}],"def":"CREATE VIEW testdb.view (`name1` UInt64, `name2` Nullable(String), `name4` SimpleAggregateFunction(sum, Float64), `name5` DateTime, `name8` FixedString(4)) AS SELECT * FROM testdb.table_name"}],"functions":[{"name":"linear_equation","return_type":"","arguments":"","type":""}],"driver":{"name":"clickhouse","database_version":"24.4.4.113","meta":{"dict":{"Functions":"Stored procedures and functions"}}}}

sample/dynamodb/schema.json

@@ -1 +1 @@
-{"name":"Amazon DynamoDB (ap-northeast-1)","desc":"","tables":[{"name":"Forum","type":"BASIC TABLE","comment":"","columns":[{"name":"Name","type":"S","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[{"name":"Primary Key","type":"Partition key","def":"[{ AttributeName: \"Name\", KeyType: \"HASH\" }]","table":null,"referenced_table":null,"columns":["Name"],"referenced_columns":null,"comment":""}],"triggers":[],"def":""},{"name":"ProductCatalog","type":"BASIC TABLE","comment":"","columns":[{"name":"Id","type":"N","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[{"name":"Primary Key","type":"Partition key","def":"[{ AttributeName: \"Id\", KeyType: \"HASH\" }]","table":null,"referenced_table":null,"columns":["Id"],"referenced_columns":null,"comment":""}],"triggers":[],"def":""},{"name":"Reply","type":"BASIC TABLE","comment":"","columns":[{"name":"Id","type":"S","nullable":false,"default":null,"comment":""},{"name":"ReplyDateTime","type":"S","nullable":false,"default":null,"comment":""},{"name":"PostedBy","type":"S","nullable":false,"default":null,"comment":""}],"indexes":[{"name":"PostedBy-index","def":"LocalSecondaryIndex { [{ AttributeName: \"Id\", KeyType: \"HASH\" } { AttributeName: \"PostedBy\", KeyType: \"RANGE\" }], { ProjectionType: \"KEYS_ONLY\" } }","table":null,"columns":null,"comment":""}],"constraints":[{"name":"Primary Key","type":"Partition key and sort key","def":"[{ AttributeName: \"Id\", KeyType: \"HASH\" } { AttributeName: \"ReplyDateTime\", KeyType: \"RANGE\" }]","table":null,"referenced_table":null,"columns":["Id","ReplyDateTime"],"referenced_columns":null,"comment":""}],"triggers":[],"def":""},{"name":"Thread","type":"BASIC TABLE","comment":"","columns":[{"name":"ForumName","type":"S","nullable":false,"default":null,"comment":""},{"name":"Subject","type":"S","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[{"name":"Primary Key","type":"Partition key and sort key","def":"[{ AttributeName: \"ForumName\", KeyType: \"HASH\" } { AttributeName: \"Subject\", KeyType: \"RANGE\" }]","table":null,"referenced_table":null,"columns":["ForumName","Subject"],"referenced_columns":null,"comment":""}],"triggers":[],"def":""}],"relations":[{"table":"Thread","columns":["ForumName"],"cardinality":"Zero or more","parent_table":"Forum","parent_columns":["Name"],"parent_cardinality":"Exactly one","def":"Thread-\u003eForum","virtual":true}],"functions":null,"driver":{"name":"dynamodb","database_version":"","meta":{"dict":{"Column":"Attribute","Columns":"Attributes","Constraints":"Primary Key","Indexes":"Secondary Indexes"}}}}
+{"name":"Amazon DynamoDB (ap-northeast-1)","tables":[{"name":"Forum","type":"BASIC TABLE","columns":[{"name":"Name","type":"S","nullable":false}],"constraints":[{"name":"Primary Key","type":"Partition key","def":"[{ AttributeName: \"Name\", KeyType: \"HASH\" }]","table":null,"columns":["Name"]}]},{"name":"ProductCatalog","type":"BASIC TABLE","columns":[{"name":"Id","type":"N","nullable":false}],"constraints":[{"name":"Primary Key","type":"Partition key","def":"[{ AttributeName: \"Id\", KeyType: \"HASH\" }]","table":null,"columns":["Id"]}]},{"name":"Reply","type":"BASIC TABLE","columns":[{"name":"Id","type":"S","nullable":false},{"name":"ReplyDateTime","type":"S","nullable":false},{"name":"PostedBy","type":"S","nullable":false}],"indexes":[{"name":"PostedBy-index","def":"LocalSecondaryIndex { [{ AttributeName: \"Id\", KeyType: \"HASH\" } { AttributeName: \"PostedBy\", KeyType: \"RANGE\" }], { ProjectionType: \"KEYS_ONLY\" } }","table":null,"columns":null}],"constraints":[{"name":"Primary Key","type":"Partition key and sort key","def":"[{ AttributeName: \"Id\", KeyType: \"HASH\" } { AttributeName: \"ReplyDateTime\", KeyType: \"RANGE\" }]","table":null,"columns":["Id","ReplyDateTime"]}]},{"name":"Thread","type":"BASIC TABLE","columns":[{"name":"ForumName","type":"S","nullable":false},{"name":"Subject","type":"S","nullable":false}],"constraints":[{"name":"Primary Key","type":"Partition key and sort key","def":"[{ AttributeName: \"ForumName\", KeyType: \"HASH\" } { AttributeName: \"Subject\", KeyType: \"RANGE\" }]","table":null,"columns":["ForumName","Subject"]}]}],"relations":[{"table":"Thread","columns":["ForumName"],"cardinality":"zero_or_more","parent_table":"Forum","parent_columns":["Name"],"parent_cardinality":"exactly_one","def":"Thread-\u003eForum","virtual":true}],"driver":{"name":"dynamodb","meta":{"dict":{"Column":"Attribute","Columns":"Attributes","Constraints":"Primary Key","Indexes":"Secondary Indexes"}}}}

sample/mongo/schema.json

@@ -1 +1 @@
-{"name":"","desc":"","tables":[{"name":"test.restaurants","type":"collection","comment":"Count of documents is 149","columns":[{"name":"_id","type":"objectId","nullable":false,"default":null,"comment":""},{"name":"address","type":"document","nullable":false,"default":null,"comment":""},{"name":"borough","type":"string","nullable":false,"default":null,"comment":""},{"name":"cuisine","type":"string","nullable":false,"default":null,"comment":""},{"name":"grades","type":"array","nullable":false,"default":null,"comment":""},{"name":"name","type":"string","nullable":false,"default":null,"comment":""},{"name":"restaurant_id","type":"string","nullable":false,"default":null,"comment":""}],"indexes":[{"name":"_id_","def":"{\"_id\": {\"$numberInt\":\"1\"}}","table":null,"columns":null,"comment":"Non-unique, Version 2"}],"constraints":[],"triggers":[],"def":""}],"relations":[],"functions":null,"driver":{"name":"mongodb","database_version":"","meta":{"dict":{"Column":"Attribute","Columns":"Attributes","Indexes":"Indexes"}}}}
+{"tables":[{"name":"test.restaurants","type":"collection","comment":"Count of documents is 149","columns":[{"name":"_id","type":"objectId","nullable":false},{"name":"address","type":"document","nullable":false},{"name":"borough","type":"string","nullable":false},{"name":"cuisine","type":"string","nullable":false},{"name":"grades","type":"array","nullable":false},{"name":"name","type":"string","nullable":false},{"name":"restaurant_id","type":"string","nullable":false}],"indexes":[{"name":"_id_","def":"{\"_id\": {\"$numberInt\":\"1\"}}","table":null,"columns":null,"comment":"Non-unique, Version 2"}]}],"driver":{"name":"mongodb","meta":{"dict":{"Column":"Attribute","Columns":"Attributes","Indexes":"Indexes"}}}}

schema/cardinality.go

@@ -8,10 +8,10 @@ import (
 type Cardinality string
 
 const (
-	ZeroOrOne          Cardinality = "Zero or one"
-	ExactlyOne         Cardinality = "Exactly one"
-	ZeroOrMore         Cardinality = "Zero or more"
-	OneOrMore          Cardinality = "One or more"
+	ZeroOrOne          Cardinality = "zero_or_one"
+	ExactlyOne         Cardinality = "exactly_one"
+	ZeroOrMore         Cardinality = "zero_or_more"
+	OneOrMore          Cardinality = "one_or_more"
 	UnknownCardinality Cardinality = ""
 )
 
@@ -23,6 +23,13 @@ var cardinalityAliases = map[string]Cardinality{
 	"one or zero":  ZeroOrOne,
 	"zero or many": ZeroOrMore,
 	"one or many":  OneOrMore,
+	"zero_or_one":  ZeroOrOne,
+	"exactly_one":  ExactlyOne,
+	"zero_or_more": ZeroOrMore,
+	"one_or_more":  OneOrMore,
+	"one_or_zero":  ZeroOrOne,
+	"zero_or_many": ZeroOrMore,
+	"one_or_many":  OneOrMore,
 	"many(0)":      ZeroOrMore,
 	"many(1)":      OneOrMore,
 	"0+":           ZeroOrMore,