Feat/metadata json (#1242)

* Moved sequence adaptation uploading to the command dictionary page, started adding support for uploading parameter and channel dictionaries

* Fixed issues with uploading command dictionaries and sequence adaptations

* Added support for parsing channel and parameter dictionaries

* Fixed an issue with uploading a parameter dictionary

* Fixed the form builder from showing on the sequence select screen, fixed a problem with the seq page not loading

* First pass at adding parcels

* Changed sequence editing to use parcels rather than command dictionaries

* Hooked up sequence adaptation id to parcels

* Added support for a single parameter dictionary

* link adaptation code up with parcel

* editor is readonly in table view

* content assist uses adaptation

* check unclosed blocks

* annotate all parser errors

* variable names, but not types are checked

* better detection of command on selection line

* code style

* cleaned up some commented out print statements

* cleaned up some commented out print statements

* reduce block nesting and remove obsolete commented code

* Some code cleanup

* use symbol for all enum type arguments

* wip grammar support for arrays

* support for nested objects

---------

Co-authored-by: Cody Hansen <[email protected]>

codemirror-lang-sequence/src/sequence.grammar

@@ -1,5 +1,5 @@
 @top Sequence {
-  (newLine | whiteSpace)?
+  optSpace
   (
     commentLine* ~maybeComments
     (IdDeclaration | ParameterDeclaration | LocalDeclaration | GenericDirective)
@@ -43,6 +43,10 @@ commentLine {
   LineComment newLine
 }
 
+optSpace {
+  (newLine | whiteSpace)?
+}
+
 Commands {
   (LoadAndGoDirective newLine)?
   commandBlock
@@ -84,11 +88,23 @@ Metadata {
   MetaEntry {
     metadataDirective
     whiteSpace Key { String }
-    whiteSpace Value { String }
+    whiteSpace Value { metaValue }
     newLine
   }+
 }
 
+metaValue {
+  String | Number | Boolean | Null | Array | Object
+}
+
+Object { "{" list<Property>? "}" }
+Array  { "[" list<metaValue>? "]" }
+
+Property { PropertyName optSpace ":" optSpace metaValue }
+PropertyName[isolate] { String }
+
+list<item> { optSpace item (optSpace "," optSpace item)* optSpace }
+
 Models {
   Model {
     modelDirective
@@ -131,9 +147,10 @@ Stem { !stemStart identifier }
     "0x" (hex | "_")+ "n"?
   }
 
-  TRUE { "TRUE" }
-  FALSE { "FALSE" }
+  TRUE { "true" }
+  FALSE { "false" }
   Boolean { TRUE | FALSE }
+  Null { "null" }
 
   LineComment { "#"![\n\r]* }
 

codemirror-lang-sequence/test/cases/parse_tree.txt

@@ -192,7 +192,7 @@ CMD_1 1 2 3
 CMD_2 "hello, it's me"
 @METADATA "bar" "{ \"foo\": 5}"
 @MODEL "a" 5 "c"
-@MODEL "d" TRUE "f"
+@MODEL "d" true "f"
 ==>
 
 Sequence(

codemirror-lang-sequence/test/token.js

@@ -12,6 +12,81 @@ const LINE_COMMENT_TOKEN = 'LineComment';
 const METADATA_TOKEN = 'Metadata';
 const METADATA_ENTRY_TOKEN = 'MetaEntry';
 
+function getMetaType(node) {
+  return node.firstChild.nextSibling.firstChild.name;
+}
+
+function getMetaValue(node, input) {
+  const mv = node.firstChild.nextSibling.firstChild;
+  return JSON.parse(input.slice(mv.from, mv.to));
+}
+
+describe('metadata', () => {
+  it('primitive types', () => {
+    const input = `
+@METADATA "name 1" "string val"
+@METADATA "name 2" false
+@METADATA "name3" 3
+@METADATA "name4" 4e1
+C STEM
+`;
+    const parseTree = SeqLanguage.parser.parse(input);
+    assertNoErrorNodes(input);
+    const topLevelMetaData = parseTree.topNode.getChild(METADATA_TOKEN);
+    const metaEntries = topLevelMetaData.getChildren(METADATA_ENTRY_TOKEN);
+    assert.equal(metaEntries.length, 4);
+    assert.equal(getMetaType(metaEntries[0]), 'String');
+    assert.equal(getMetaType(metaEntries[1]), 'Boolean');
+    assert.equal(getMetaType(metaEntries[2]), 'Number');
+    assert.equal(getMetaType(metaEntries[3]), 'Number');
+  });
+
+  it('structured types', () => {
+    const input = `
+@METADATA "name 1" [ 1,2  , 3 ]
+@METADATA "name 2" ["a",    true  ,
+  2 ]
+
+@METADATA "name 3" {
+   "level1": {
+     "level2": [
+       false,
+       1,
+       "two"
+     ],
+     "level2 nest": {
+      "level3": true
+     }
+   }
+}
+
+          @METADATA  "name 4"         {}
+
+C STEM
+`;
+    const parseTree = SeqLanguage.parser.parse(input);
+    assertNoErrorNodes(input);
+    const topLevelMetaData = parseTree.topNode.getChild(METADATA_TOKEN);
+    const metaEntries = topLevelMetaData.getChildren(METADATA_ENTRY_TOKEN);
+    assert.equal(metaEntries.length, 4);
+    assert.equal(getMetaType(metaEntries[0]), 'Array');
+    assert.equal(getMetaType(metaEntries[1]), 'Array');
+    assert.equal(getMetaType(metaEntries[2]), 'Object');
+    assert.equal(getMetaType(metaEntries[3]), 'Object');
+    assert.deepStrictEqual(getMetaValue(metaEntries[0], input), [1, 2, 3]);
+    assert.deepStrictEqual(getMetaValue(metaEntries[1], input), ['a', true, 2]);
+    assert.deepStrictEqual(getMetaValue(metaEntries[2], input), {
+      level1: {
+        level2: [false, 1, 'two'],
+        'level2 nest': {
+          level3: true,
+        },
+      },
+    });
+    assert.deepStrictEqual(getMetaValue(metaEntries[3], input), {});
+  });
+});
+
 describe('error positions', () => {
   for (const { testname, input, first_error } of [
     {

src/utilities/new-sequence-editor/from-seq-json.ts

@@ -103,7 +103,7 @@ export function seqJsonMetadataToSequence(metadata: Metadata): string {
   const metaDataString = Object.entries(metadata)
     .map(
       ([key, value]: [key: string, value: unknown]) =>
-        `@METADATA ${quoteEscape(key)} ${typeof value === 'string' ? quoteEscape(String(value)) : `"${value}"`}`,
+        `@METADATA ${quoteEscape(key)} ${JSON.stringify(value, null, 2)}`,
     )
     .join('\n');
   return metaDataString.length > 0 ? `${metaDataString}\n` : '';
@@ -127,7 +127,6 @@ export function seqJsonToSequence(seqJson: SeqJson | null): string {
   const sequence: string[] = [];
 
   if (seqJson) {
-
     // ID
     sequence.push(`@ID "${seqJson.id}"\n`);
 

src/utilities/new-sequence-editor/to-seq-json.ts

@@ -517,9 +517,15 @@ function parseMetadata(node: SyntaxNode, text: string): Metadata | undefined {
     }
 
     const keyText = removeQuotes(text.slice(keyNode.from, keyNode.to)) as string;
-    const valueText = removeQuotes(text.slice(valueNode.from, valueNode.to));
 
-    obj[keyText] = valueText;
+    let value = text.slice(valueNode.from, valueNode.to);
+    try {
+      value = JSON.parse(value);
+    } catch (e) {
+      logInfo(`Malformed metadata ${value}`);
+    }
+
+    obj[keyText] = value;
   });
 
   return obj;