Skip to content

Commit

Permalink
Restack 1.1 macros and symbols to meet spec changes (#1013)
Browse files Browse the repository at this point in the history 
* Restack macros and symbols to meet spec changes

* Update ion-tests for symbol and macro restack

* Fix comment in src/test/java/com/amazon/ion/impl/bin/WriteBufferTest.java

Co-authored-by: Tyler Gregg <[email protected]>

---------

Co-authored-by: Tyler Gregg <[email protected]>
  • Loading branch information
@jobarr-amzn @tgregg
jobarr-amzn and tgregg committed Dec 13, 2024
1 parent 7a18550 commit 620228b
Show file tree
Hide file tree
Showing 10 changed files with 113 additions and 119 deletions.
2 changes: 1 addition & 1 deletion ion-tests
Submodule ion-tests updated 10 files
+17 −17 conformance/system_macros/add_symbols.ion
+6 −6 conformance/system_macros/default.ion
+9 −9 conformance/system_macros/make_decimal.ion
+1 −1 conformance/system_macros/make_field.ion
+6 −6 conformance/system_macros/make_string.ion
+6 −6 conformance/system_macros/make_symbol.ion
+17 −17 conformance/system_macros/make_timestamp.ion
+6 −6 conformance/system_macros/set_macros.ion
+6 −6 conformance/system_macros/set_symbols.ion
+4 −4 conformance/system_macros/sum.ion
96 changes: 47 additions & 49 deletions src/main/java/com/amazon/ion/impl/SystemSymbols_1_1.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,55 +24,53 @@ enum class SystemSymbols_1_1(val id: Int, val text: String) {
MACRO_TABLE( /* */ 14, "macro_table"),
SYMBOL_TABLE( /* */ 15, "symbol_table"),
MODULE( /* */ 16, "module"),
RETAIN( /* */ 17, "retain"), // https://github.com/amazon-ion/ion-docs/issues/345
EXPORT( /* */ 18, "export"),
CATALOG_KEY( /* */ 19, "catalog_key"), // https://github.com/amazon-ion/ion-docs/issues/345
IMPORT( /* */ 20, "import"),
THE_EMPTY_SYMBOL( /* */ 21, ""),
LITERAL( /* */ 22, "literal"),
IF_NONE( /* */ 23, "if_none"),
IF_SOME( /* */ 24, "if_some"),
IF_SINGLE( /* */ 25, "if_single"),
IF_MULTI( /* */ 26, "if_multi"),
FOR( /* */ 27, "for"),
DEFAULT( /* */ 28, "default"),
VALUES( /* */ 29, "values"),
ANNOTATE( /* */ 30, "annotate"),
MAKE_STRING( /* */ 31, "make_string"),
MAKE_SYMBOL( /* */ 32, "make_symbol"),
MAKE_BLOB( /* */ 33, "make_blob"),
MAKE_DECIMAL( /* */ 34, "make_decimal"),
MAKE_TIMESTAMP( /* */ 35, "make_timestamp"),
MAKE_LIST( /* */ 36, "make_list"),
MAKE_SEXP( /* */ 37, "make_sexp"),
MAKE_STRUCT( /* */ 38, "make_struct"),
PARSE_ION( /* */ 39, "parse_ion"),
REPEAT( /* */ 40, "repeat"),
DELTA( /* */ 41, "delta"),
FLATTEN( /* */ 42, "flatten"),
SUM( /* */ 43, "sum"),
SET_SYMBOLS( /* */ 44, "set_symbols"),
ADD_SYMBOLS( /* */ 45, "add_symbols"),
SET_MACROS( /* */ 46, "set_macros"),
ADD_MACROS( /* */ 47, "add_macros"),
USE( /* */ 48, "use"),
META( /* */ 49, "meta"),
FLEX_SYMBOL( /* */ 50, "flex_symbol"),
FLEX_INT( /* */ 51, "flex_int"),
FLEX_UINT( /* */ 52, "flex_uint"),
UINT8( /* */ 53, "uint8"),
UINT16( /* */ 54, "uint16"),
UINT32( /* */ 55, "uint32"),
UINT64( /* */ 56, "uint64"),
INT8( /* */ 57, "int8"),
INT16( /* */ 58, "int16"),
INT32( /* */ 59, "int32"),
INT64( /* */ 60, "int64"),
FLOAT16( /* */ 61, "float16"),
FLOAT32( /* */ 62, "float32"),
FLOAT64( /* */ 63, "float64"),
NONE( /* */ 64, "none"),
MAKE_FIELD( /* */ 65, "make_field"),
EXPORT( /* */ 17, "export"),
IMPORT( /* */ 18, "import"),
FLEX_SYMBOL( /* */ 19, "flex_symbol"),
FLEX_INT( /* */ 20, "flex_int"),
FLEX_UINT( /* */ 21, "flex_uint"),
UINT8( /* */ 22, "uint8"),
UINT16( /* */ 23, "uint16"),
UINT32( /* */ 24, "uint32"),
UINT64( /* */ 25, "uint64"),
INT8( /* */ 26, "int8"),
INT16( /* */ 27, "int16"),
INT32( /* */ 28, "int32"),
INT64( /* */ 29, "int64"),
FLOAT16( /* */ 30, "float16"),
FLOAT32( /* */ 31, "float32"),
FLOAT64( /* */ 32, "float64"),
EMPTY_TEXT( /* */ 33, ""),
FOR( /* */ 34, "for"),
LITERAL( /* */ 35, "literal"),
IF_NONE( /* */ 36, "if_none"),
IF_SOME( /* */ 37, "if_some"),
IF_SINGLE( /* */ 38, "if_single"),
IF_MULTI( /* */ 39, "if_multi"),
NONE( /* */ 40, "none"),
VALUES( /* */ 41, "values"),
DEFAULT( /* */ 42, "default"),
META( /* */ 43, "meta"),
REPEAT( /* */ 44, "repeat"),
FLATTEN( /* */ 45, "flatten"),
DELTA( /* */ 46, "delta"),
SUM( /* */ 47, "sum"),
ANNOTATE( /* */ 48, "annotate"),
MAKE_STRING( /* */ 49, "make_string"),
MAKE_SYMBOL( /* */ 50, "make_symbol"),
MAKE_DECIMAL( /* */ 51, "make_decimal"),
MAKE_TIMESTAMP( /* */ 52, "make_timestamp"),
MAKE_BLOB( /* */ 53, "make_blob"),
MAKE_LIST( /* */ 54, "make_list"),
MAKE_SEXP( /* */ 55, "make_sexp"),
MAKE_FIELD( /* */ 56, "make_field"),
MAKE_STRUCT( /* */ 57, "make_struct"),
PARSE_ION( /* */ 58, "parse_ion"),
SET_SYMBOLS( /* */ 59, "set_symbols"),
ADD_SYMBOLS( /* */ 60, "add_symbols"),
SET_MACROS( /* */ 61, "set_macros"),
ADD_MACROS( /* */ 62, "add_macros"),
USE( /* */ 63, "use"),
;

val utf8Bytes = text.encodeToByteArray()
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/amazon/ion/impl/bin/WriteBuffer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1536,7 +1536,7 @@ public int writeFlexSym(int sid) {
*/
public int writeFlexSym(Utf8StringEncoder.Result text) {
if (text.getEncodedLength() == 0) {
return writeFlexSym(SystemSymbols_1_1.THE_EMPTY_SYMBOL);
return writeFlexSym(SystemSymbols_1_1.EMPTY_TEXT);
} else {
int numLengthBytes = writeFlexInt(-text.getEncodedLength());
writeBytes(text.getBuffer(), 0, text.getEncodedLength());
Expand Down
12 changes: 6 additions & 6 deletions src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -450,20 +450,20 @@ class MacroEvaluator {
@JvmStatic
fun forSystemMacro(macro: SystemMacro): ExpansionKind {
return when (macro) {
SystemMacro.IfNone -> IfNone
SystemMacro.IfSome -> IfSome
SystemMacro.IfSingle -> IfSingle
SystemMacro.IfMulti -> IfMulti
SystemMacro.None -> Values // "none" takes no args, so we can treat it as an empty "values" expansion
SystemMacro.Values -> Values
SystemMacro.Annotate -> Annotate
SystemMacro.MakeString -> MakeString
SystemMacro.MakeSymbol -> MakeSymbol
SystemMacro.MakeBlob -> MakeBlob
SystemMacro.MakeDecimal -> MakeDecimal
SystemMacro.MakeTimestamp -> MakeTimestamp
SystemMacro.IfNone -> IfNone
SystemMacro.IfSome -> IfSome
SystemMacro.IfSingle -> IfSingle
SystemMacro.IfMulti -> IfMulti
SystemMacro.Repeat -> Repeat
SystemMacro.MakeBlob -> MakeBlob
SystemMacro.MakeField -> MakeField
SystemMacro.Repeat -> Repeat
SystemMacro.Sum -> Sum
SystemMacro.Delta -> Delta
else -> if (macro.body != null) {
Expand Down
70 changes: 33 additions & 37 deletions src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,24 @@ enum class SystemMacro(
// The real macros
None(0, NONE, emptyList()),
Values(1, VALUES, listOf(zeroToManyTagged("values"))),
Annotate(2, ANNOTATE, listOf(zeroToManyTagged("ann"), exactlyOneTagged("value"))),
MakeString(3, MAKE_STRING, listOf(zeroToManyTagged("text"))),
MakeSymbol(4, MAKE_SYMBOL, listOf(zeroToManyTagged("text"))),
MakeBlob(5, MAKE_BLOB, listOf(zeroToManyTagged("bytes"))),
MakeDecimal(6, MAKE_DECIMAL, listOf(exactlyOneTagged("coefficient"), exactlyOneTagged("exponent"))),
Default(
2, DEFAULT, listOf(zeroToManyTagged("expr"), zeroToManyTagged("default_expr")),
templateBody {
macro(IfNone) { variable(0); variable(1); variable(0) }
}
),
Meta(3, META, listOf(zeroToManyTagged("values")), templateBody { macro(None) {} }),
Repeat(4, REPEAT, listOf(exactlyOneTagged("n"), zeroToManyTagged("value"))),
Flatten(5, FLATTEN, listOf(zeroToManyTagged("values")), null), // TODO: flatten
Delta(6, DELTA, listOf(zeroToManyTagged("deltas"))),
Sum(7, SUM, listOf(exactlyOneTagged("a"), exactlyOneTagged("b"))),

Annotate(8, ANNOTATE, listOf(zeroToManyTagged("ann"), exactlyOneTagged("value"))),
MakeString(9, MAKE_STRING, listOf(zeroToManyTagged("text"))),
MakeSymbol(10, MAKE_SYMBOL, listOf(zeroToManyTagged("text"))),
MakeDecimal(11, MAKE_DECIMAL, listOf(exactlyOneTagged("coefficient"), exactlyOneTagged("exponent"))),
MakeTimestamp(
7, MAKE_TIMESTAMP,
12, MAKE_TIMESTAMP,
listOf(
exactlyOneTagged("year"),
zeroOrOneTagged("month"),
Expand All @@ -46,9 +57,17 @@ enum class SystemMacro(
zeroOrOneTagged("offset_minutes"),
)
),
// TODO: make_list
// TODO: make_sexp
// TODO: make_struct
MakeBlob(13, MAKE_BLOB, listOf(zeroToManyTagged("bytes"))),
MakeList(14, MAKE_LIST, listOf(zeroToManyTagged("sequences")), null), // TODO: make_list
MakeSExp(15, MAKE_SEXP, listOf(zeroToManyTagged("sequences")), null), // TODO: make_sexp
MakeField(
16, MAKE_FIELD,
listOf(
Macro.Parameter("field_name", Macro.ParameterEncoding.FlexSym, Macro.ParameterCardinality.ExactlyOne), exactlyOneTagged("value")
)
),
MakeStruct(17, MAKE_STRUCT, listOf(zeroToManyTagged("structs")), null), // TODO: make_struct
ParseIon(18, PARSE_ION, listOf(zeroToManyTagged("data")), null), // TODO: parse_ion

/**
* ```ion
Expand All @@ -60,7 +79,7 @@ enum class SystemMacro(
* ```
*/
SetSymbols(
11, SET_SYMBOLS, listOf(zeroToManyTagged("symbols")),
19, SET_SYMBOLS, listOf(zeroToManyTagged("symbols")),
templateBody {
annotated(ION, ::sexp) {
symbol(MODULE)
Expand All @@ -87,7 +106,7 @@ enum class SystemMacro(
* ```
*/
AddSymbols(
12, ADD_SYMBOLS, listOf(zeroToManyTagged("symbols")),
20, ADD_SYMBOLS, listOf(zeroToManyTagged("symbols")),
templateBody {
annotated(ION, ::sexp) {
symbol(MODULE)
Expand Down Expand Up @@ -115,7 +134,7 @@ enum class SystemMacro(
* ```
*/
SetMacros(
13, SET_MACROS, listOf(zeroToManyTagged("macros")),
21, SET_MACROS, listOf(zeroToManyTagged("macros")),
templateBody {
annotated(ION, ::sexp) {
symbol(MODULE)
Expand All @@ -142,7 +161,7 @@ enum class SystemMacro(
* ```
*/
AddMacros(
14, ADD_MACROS, listOf(zeroToManyTagged("macros")),
22, ADD_MACROS, listOf(zeroToManyTagged("macros")),
templateBody {
annotated(ION, ::sexp) {
symbol(MODULE)
Expand Down Expand Up @@ -171,7 +190,7 @@ enum class SystemMacro(
* ```
*/
Use(
15, USE, listOf(exactlyOneTagged("catalog_key"), zeroOrOneTagged("version")),
23, USE, listOf(exactlyOneTagged("catalog_key"), zeroOrOneTagged("version")),
templateBody {
val theModule = _Private_Utils.newSymbolToken("the_module")
annotated(ION, ::sexp) {
Expand Down Expand Up @@ -201,29 +220,6 @@ enum class SystemMacro(
}
}
),
// TODO: parse_ion
Repeat(17, REPEAT, listOf(exactlyOneTagged("n"), zeroToManyTagged("value"))),
Delta(18, DELTA, listOf(zeroToManyTagged("deltas"))),
// TODO: flatten
Sum(20, SUM, listOf(exactlyOneTagged("a"), exactlyOneTagged("b"))),
Comment(21, META, listOf(zeroToManyTagged("values")), templateBody { macro(None) {} }),
MakeField(
22, MAKE_FIELD,
listOf(
Macro.Parameter("field_name", Macro.ParameterEncoding.FlexSym, Macro.ParameterCardinality.ExactlyOne), exactlyOneTagged("value")
)
),

Default(
23, DEFAULT, listOf(zeroToManyTagged("expr"), zeroToManyTagged("default_expr")),
templateBody {
macro(IfNone) {
variable(0)
variable(1)
variable(0)
}
}
),
;

val macroName: String get() = this.systemSymbol.text
Expand Down
8 changes: 4 additions & 4 deletions src/test/java/com/amazon/ion/impl/IonReaderContinuableCoreBinaryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1069,7 +1069,7 @@ public void taglessCompactSymbols(boolean constructFromBytes) throws Exception {
0x00, // User macro ID 0
0xF9, 0x6E, 0x61, 0x6D, 0x65, // interpreted as compact symbol (FlexSym with inline text "name")
0x09, // interpreted as compact symbol (FlexSym with SID 4)
0x01, 0x75 // interpreted as compact symbol (special FlexSym)
0x01, 0x81 // interpreted as compact symbol (special FlexSym)
));
try (IonReaderContinuableCoreBinary reader = initializeReader(constructFromBytes, data)) {
assertSequence(
Expand All @@ -1090,7 +1090,7 @@ public void taglessCompactSymbols(boolean constructFromBytes) throws Exception {
@ValueSource(booleans = {true, false})
public void addSymbolsSystemMacro(boolean constructFromBytes) throws Exception {
byte[] data = withIvm(1, bytes(
0xEF, 0x0C, // system macro add_symbols
0xEF, 0x14, // system macro add_symbols
0x02, // AEB: 0b------aa; a=10, expression group
0x01, // FlexInt 0, a delimited expression group
0x93, 0x61, 0x62, 0x63, // 3-byte string, utf-8 "abc"
Expand Down Expand Up @@ -1181,13 +1181,13 @@ private void assertMacroAwareTranscribeProducesEquivalentStream(byte[] data, boo
@ValueSource(booleans = {true, false})
public void encodingLevelTranscodeOfSystemMacroInvocation(boolean constructFromBytes) throws Exception {
byte[] data = withIvm(1, bytes(
0xEF, 0x0C, // system macro add_symbols
0xEF, 0x14, // system macro add_symbols
0x02, // AEB: 0b------aa; a=10, expression group
0x01, // FlexInt 0, a delimited expression group
0x93, 0x61, 0x62, 0x63, // 3-byte string, utf-8 "abc"
0xF0, // delimited end... of expression group
0xE1, // SID single byte
0x42 // SID $66
0x40 // SID $64
));
assertMacroAwareTranscribeProducesEquivalentStream(data, constructFromBytes);
}
Expand Down
26 changes: 13 additions & 13 deletions src/test/java/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5062,15 +5062,15 @@ private void readAnnotationsWithSpecialFlexSyms_1_1(boolean constructFromBytes,
@ParameterizedTest
@ValueSource(strings = {
// Minimal representations
"E7 01 75 60 | One empty-text annotation; value int 0 \n" +
"E7 01 81 60 | One empty-text annotation; value int 0 \n" +
"E7 01 60 60 | One SID 0 annotation; value int 0 \n" +
"E8 01 75 01 60 60 | Two annotations: empty text, SID 0; value int 0 \n" +
"E9 09 01 60 01 75 60 | Variable length = 4 annotations: SID 0, empty text; value int 0 \n",
"E8 01 81 01 60 60 | Two annotations: empty text, SID 0; value int 0 \n" +
"E9 09 01 60 01 81 60 | Variable length = 4 annotations: SID 0, empty text; value int 0 \n",
// Overpadded representations
"E7 02 00 75 60 | One overpadded empty-text annotation; value int 0 \n" +
"E7 02 00 81 60 | One overpadded empty-text annotation; value int 0 \n" +
"E7 04 00 00 60 60 | One overpadded SID 0 annotation; value int 0 \n" +
"E8 08 00 00 00 75 02 00 60 60 | Two overpadded annotations: empty text, SID 0; value int 0 \n" +
"E9 90 00 00 00 00 01 60 01 75 60 | Variable overpadded length = 4 annotations: SID 0, empty text; value int 0 \n"
"E8 08 00 00 00 81 02 00 60 60 | Two overpadded annotations: empty text, SID 0; value int 0 \n" +
"E9 90 00 00 00 00 01 60 01 81 60 | Variable overpadded length = 4 annotations: SID 0, empty text; value int 0 \n"
})
public void readAnnotationsWithSpecialFlexSyms_1_1(String inputBytes) throws Exception {
readAnnotationsWithSpecialFlexSyms_1_1(true, inputBytes);
Expand Down Expand Up @@ -5308,17 +5308,17 @@ public void assertStructWithEmptyInlineFieldNamesCorrectlyParsed(boolean constru
// Empty field name in fixed-length SID struct
"D4 | Struct Length = 4 \n" +
"01 | switch to FlexSym encoding \n" +
"01 75 | FlexSym empty text \n" +
"01 81 | FlexSym empty text \n" +
"6F | false",
// Empty field name in variable-length SID struct
"FD | Variable length SID struct \n" +
"09 | Length = 4 \n" +
"01 | switch to FlexSym encoding \n" +
"01 75 | FlexSym empty text \n" +
"01 81 | FlexSym empty text \n" +
"6F | false",
// Empty field name in delimited struct
"F3 | Delimited struct \n" +
"01 75 | FlexSym empty text \n" +
"01 81 | FlexSym empty text \n" +
"6F | false \n" +
"01 F0 | End delimited struct"
})
Expand Down Expand Up @@ -5990,13 +5990,13 @@ private void invokeValuesUsingSystemMacroOpcodeHelper(boolean constructFromBytes
public void addSymbolsSystemMacro(boolean constructFromBytes) throws Exception {
int[] data = new int[] {
0xE0, 0x01, 0x01, 0xEA, // Ion 1.1 IVM
0xEF, 0x0C, // system macro add_symbols
0xEF, 0x14, // system macro add_symbols
0x02, // AEB: 0b------aa; a=10, expression group
0x01, // FlexInt 0, a delimited expression group
0x93, 0x61, 0x62, 0x63, // 3-byte string, utf-8 "abc"
0xF0, // delimited end... of expression group
0xE1, // SID single byte
0x42 // SID $66
0x40 // SID $64
};
try (IonReader reader = readerFor(constructFromBytes,data)) {
assertEquals(IonType.SYMBOL, reader.next());
Expand All @@ -6008,13 +6008,13 @@ public void addSymbolsSystemMacro(boolean constructFromBytes) throws Exception {
public void addSymbolsSystemMacroWhenNotEntirelyBuffered() throws Exception {
int[] data = new int[] {
/* 0 - 3 */ // 0xEA, 0x01, 0x01, 0xE0, // implicitly provided by BinaryIonAppender
/* 4 - 5 */ 0xEF, 0x0C, // system macro add_symbols
/* 4 - 5 */ 0xEF, 0x14, // system macro add_symbols
/* 6 - 7 */ 0x02, 0x01, // AEB: 0b------aa; a=10, FlexInt 0, a delimited expression group
/* 8 - 12 */ 0x93, 0x66, 0x6F, 0x72, // "for"
/* 13 - 16 */ 0x93, 0x66, 0x75, 0x72, // "fur"
/* 17 - 21 */ 0x94, 0x66, 0x6F, 0x75, 0x72, // "four"
/* 22 - 22 */ 0xF0, // delimited end... of expression group
/* 23 - 24 */ 0xE1, 0x41 + 3, // SID single byte ${usid 1} => "four"
/* 23 - 24 */ 0xE1, 0x3F + 3, // SID single byte ${usid 1} => "four"
};
byte[] bytes = new TestUtils.BinaryIonAppender(1).append(data).toByteArray();
totalBytesInStream = bytes.length;
Expand Down
Loading

0 comments on commit 620228b

Please sign in to comment.