Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Restack 1.1 macros and symbols to meet spec changes #1013

Merged
merged 3 commits into from
Dec 12, 2024
Merged

Restack 1.1 macros and symbols to meet spec changes #1013

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b05276a
Restack macros and symbols to meet spec changes
jobarr-amzn Dec 11, 2024
38770b9
Update ion-tests for symbol and macro restack
jobarr-amzn Dec 12, 2024
a665276
Fix comment in src/test/java/com/amazon/ion/impl/bin/WriteBufferTest.…
jobarr-amzn Dec 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ion-tests
Open in desktop
Submodule ion-tests updated 10 files
+17 −17 conformance/system_macros/add_symbols.ion
+6 −6 conformance/system_macros/default.ion
+9 −9 conformance/system_macros/make_decimal.ion
+1 −1 conformance/system_macros/make_field.ion
+6 −6 conformance/system_macros/make_string.ion
+6 −6 conformance/system_macros/make_symbol.ion
+17 −17 conformance/system_macros/make_timestamp.ion
+6 −6 conformance/system_macros/set_macros.ion
+6 −6 conformance/system_macros/set_symbols.ion
+4 −4 conformance/system_macros/sum.ion
96 changes: 47 additions & 49 deletions src/main/java/com/amazon/ion/impl/SystemSymbols_1_1.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,55 +24,53 @@ enum class SystemSymbols_1_1(val id: Int, val text: String) {
MACRO_TABLE( /* */ 14, "macro_table"),
SYMBOL_TABLE( /* */ 15, "symbol_table"),
MODULE( /* */ 16, "module"),
RETAIN( /* */ 17, "retain"), // https://github.com/amazon-ion/ion-docs/issues/345
EXPORT( /* */ 18, "export"),
CATALOG_KEY( /* */ 19, "catalog_key"), // https://github.com/amazon-ion/ion-docs/issues/345
IMPORT( /* */ 20, "import"),
THE_EMPTY_SYMBOL( /* */ 21, ""),
LITERAL( /* */ 22, "literal"),
IF_NONE( /* */ 23, "if_none"),
IF_SOME( /* */ 24, "if_some"),
IF_SINGLE( /* */ 25, "if_single"),
IF_MULTI( /* */ 26, "if_multi"),
FOR( /* */ 27, "for"),
DEFAULT( /* */ 28, "default"),
VALUES( /* */ 29, "values"),
ANNOTATE( /* */ 30, "annotate"),
MAKE_STRING( /* */ 31, "make_string"),
MAKE_SYMBOL( /* */ 32, "make_symbol"),
MAKE_BLOB( /* */ 33, "make_blob"),
MAKE_DECIMAL( /* */ 34, "make_decimal"),
MAKE_TIMESTAMP( /* */ 35, "make_timestamp"),
MAKE_LIST( /* */ 36, "make_list"),
MAKE_SEXP( /* */ 37, "make_sexp"),
MAKE_STRUCT( /* */ 38, "make_struct"),
PARSE_ION( /* */ 39, "parse_ion"),
REPEAT( /* */ 40, "repeat"),
DELTA( /* */ 41, "delta"),
FLATTEN( /* */ 42, "flatten"),
SUM( /* */ 43, "sum"),
SET_SYMBOLS( /* */ 44, "set_symbols"),
ADD_SYMBOLS( /* */ 45, "add_symbols"),
SET_MACROS( /* */ 46, "set_macros"),
ADD_MACROS( /* */ 47, "add_macros"),
USE( /* */ 48, "use"),
META( /* */ 49, "meta"),
FLEX_SYMBOL( /* */ 50, "flex_symbol"),
FLEX_INT( /* */ 51, "flex_int"),
FLEX_UINT( /* */ 52, "flex_uint"),
UINT8( /* */ 53, "uint8"),
UINT16( /* */ 54, "uint16"),
UINT32( /* */ 55, "uint32"),
UINT64( /* */ 56, "uint64"),
INT8( /* */ 57, "int8"),
INT16( /* */ 58, "int16"),
INT32( /* */ 59, "int32"),
INT64( /* */ 60, "int64"),
FLOAT16( /* */ 61, "float16"),
FLOAT32( /* */ 62, "float32"),
FLOAT64( /* */ 63, "float64"),
NONE( /* */ 64, "none"),
MAKE_FIELD( /* */ 65, "make_field"),
EXPORT( /* */ 17, "export"),
IMPORT( /* */ 18, "import"),
FLEX_SYMBOL( /* */ 19, "flex_symbol"),
FLEX_INT( /* */ 20, "flex_int"),
FLEX_UINT( /* */ 21, "flex_uint"),
UINT8( /* */ 22, "uint8"),
UINT16( /* */ 23, "uint16"),
UINT32( /* */ 24, "uint32"),
UINT64( /* */ 25, "uint64"),
INT8( /* */ 26, "int8"),
INT16( /* */ 27, "int16"),
INT32( /* */ 28, "int32"),
INT64( /* */ 29, "int64"),
FLOAT16( /* */ 30, "float16"),
FLOAT32( /* */ 31, "float32"),
FLOAT64( /* */ 32, "float64"),
EMPTY_TEXT( /* */ 33, ""),
FOR( /* */ 34, "for"),
LITERAL( /* */ 35, "literal"),
IF_NONE( /* */ 36, "if_none"),
IF_SOME( /* */ 37, "if_some"),
IF_SINGLE( /* */ 38, "if_single"),
IF_MULTI( /* */ 39, "if_multi"),
NONE( /* */ 40, "none"),
VALUES( /* */ 41, "values"),
DEFAULT( /* */ 42, "default"),
META( /* */ 43, "meta"),
REPEAT( /* */ 44, "repeat"),
FLATTEN( /* */ 45, "flatten"),
DELTA( /* */ 46, "delta"),
SUM( /* */ 47, "sum"),
ANNOTATE( /* */ 48, "annotate"),
MAKE_STRING( /* */ 49, "make_string"),
MAKE_SYMBOL( /* */ 50, "make_symbol"),
MAKE_DECIMAL( /* */ 51, "make_decimal"),
MAKE_TIMESTAMP( /* */ 52, "make_timestamp"),
MAKE_BLOB( /* */ 53, "make_blob"),
MAKE_LIST( /* */ 54, "make_list"),
MAKE_SEXP( /* */ 55, "make_sexp"),
MAKE_FIELD( /* */ 56, "make_field"),
MAKE_STRUCT( /* */ 57, "make_struct"),
PARSE_ION( /* */ 58, "parse_ion"),
SET_SYMBOLS( /* */ 59, "set_symbols"),
ADD_SYMBOLS( /* */ 60, "add_symbols"),
SET_MACROS( /* */ 61, "set_macros"),
ADD_MACROS( /* */ 62, "add_macros"),
USE( /* */ 63, "use"),
;

val utf8Bytes = text.encodeToByteArray()
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/amazon/ion/impl/bin/WriteBuffer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1536,7 +1536,7 @@ public int writeFlexSym(int sid) {
*/
public int writeFlexSym(Utf8StringEncoder.Result text) {
if (text.getEncodedLength() == 0) {
return writeFlexSym(SystemSymbols_1_1.THE_EMPTY_SYMBOL);
return writeFlexSym(SystemSymbols_1_1.EMPTY_TEXT);
} else {
int numLengthBytes = writeFlexInt(-text.getEncodedLength());
writeBytes(text.getBuffer(), 0, text.getEncodedLength());
Expand Down
12 changes: 6 additions & 6 deletions src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -450,20 +450,20 @@ class MacroEvaluator {
@JvmStatic
fun forSystemMacro(macro: SystemMacro): ExpansionKind {
return when (macro) {
SystemMacro.IfNone -> IfNone
SystemMacro.IfSome -> IfSome
SystemMacro.IfSingle -> IfSingle
SystemMacro.IfMulti -> IfMulti
SystemMacro.None -> Values // "none" takes no args, so we can treat it as an empty "values" expansion
SystemMacro.Values -> Values
SystemMacro.Annotate -> Annotate
SystemMacro.MakeString -> MakeString
SystemMacro.MakeSymbol -> MakeSymbol
SystemMacro.MakeBlob -> MakeBlob
SystemMacro.MakeDecimal -> MakeDecimal
SystemMacro.MakeTimestamp -> MakeTimestamp
SystemMacro.IfNone -> IfNone
SystemMacro.IfSome -> IfSome
SystemMacro.IfSingle -> IfSingle
SystemMacro.IfMulti -> IfMulti
SystemMacro.Repeat -> Repeat
SystemMacro.MakeBlob -> MakeBlob
SystemMacro.MakeField -> MakeField
SystemMacro.Repeat -> Repeat
SystemMacro.Sum -> Sum
SystemMacro.Delta -> Delta
else -> if (macro.body != null) {
Expand Down
70 changes: 33 additions & 37 deletions src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,24 @@ enum class SystemMacro(
// The real macros
None(0, NONE, emptyList()),
Values(1, VALUES, listOf(zeroToManyTagged("values"))),
Annotate(2, ANNOTATE, listOf(zeroToManyTagged("ann"), exactlyOneTagged("value"))),
MakeString(3, MAKE_STRING, listOf(zeroToManyTagged("text"))),
MakeSymbol(4, MAKE_SYMBOL, listOf(zeroToManyTagged("text"))),
MakeBlob(5, MAKE_BLOB, listOf(zeroToManyTagged("bytes"))),
MakeDecimal(6, MAKE_DECIMAL, listOf(exactlyOneTagged("coefficient"), exactlyOneTagged("exponent"))),
Default(
2, DEFAULT, listOf(zeroToManyTagged("expr"), zeroToManyTagged("default_expr")),
templateBody {
macro(IfNone) { variable(0); variable(1); variable(0) }
}
),
Meta(3, META, listOf(zeroToManyTagged("values")), templateBody { macro(None) {} }),
Repeat(4, REPEAT, listOf(exactlyOneTagged("n"), zeroToManyTagged("value"))),
Flatten(5, FLATTEN, listOf(zeroToManyTagged("values")), null), // TODO: flatten
Delta(6, DELTA, listOf(zeroToManyTagged("deltas"))),
Sum(7, SUM, listOf(exactlyOneTagged("a"), exactlyOneTagged("b"))),

Annotate(8, ANNOTATE, listOf(zeroToManyTagged("ann"), exactlyOneTagged("value"))),
MakeString(9, MAKE_STRING, listOf(zeroToManyTagged("text"))),
MakeSymbol(10, MAKE_SYMBOL, listOf(zeroToManyTagged("text"))),
MakeDecimal(11, MAKE_DECIMAL, listOf(exactlyOneTagged("coefficient"), exactlyOneTagged("exponent"))),
MakeTimestamp(
7, MAKE_TIMESTAMP,
12, MAKE_TIMESTAMP,
listOf(
exactlyOneTagged("year"),
zeroOrOneTagged("month"),
Expand All @@ -46,9 +57,17 @@ enum class SystemMacro(
zeroOrOneTagged("offset_minutes"),
)
),
// TODO: make_list
// TODO: make_sexp
// TODO: make_struct
MakeBlob(13, MAKE_BLOB, listOf(zeroToManyTagged("bytes"))),
MakeList(14, MAKE_LIST, listOf(zeroToManyTagged("sequences")), null), // TODO: make_list
MakeSExp(15, MAKE_SEXP, listOf(zeroToManyTagged("sequences")), null), // TODO: make_sexp
Comment on lines +61 to +62
Copy link
Contributor

@popematt popematt Dec 12, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Non-blocking—the nulls are unnecessary.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll fix that, thanks.

MakeField(
16, MAKE_FIELD,
listOf(
Macro.Parameter("field_name", Macro.ParameterEncoding.FlexSym, Macro.ParameterCardinality.ExactlyOne), exactlyOneTagged("value")
)
),
MakeStruct(17, MAKE_STRUCT, listOf(zeroToManyTagged("structs")), null), // TODO: make_struct
ParseIon(18, PARSE_ION, listOf(zeroToManyTagged("data")), null), // TODO: parse_ion

/**
* ```ion
Expand All @@ -60,7 +79,7 @@ enum class SystemMacro(
* ```
*/
SetSymbols(
11, SET_SYMBOLS, listOf(zeroToManyTagged("symbols")),
19, SET_SYMBOLS, listOf(zeroToManyTagged("symbols")),
templateBody {
annotated(ION, ::sexp) {
symbol(MODULE)
Expand All @@ -87,7 +106,7 @@ enum class SystemMacro(
* ```
*/
AddSymbols(
12, ADD_SYMBOLS, listOf(zeroToManyTagged("symbols")),
20, ADD_SYMBOLS, listOf(zeroToManyTagged("symbols")),
templateBody {
annotated(ION, ::sexp) {
symbol(MODULE)
Expand Down Expand Up @@ -115,7 +134,7 @@ enum class SystemMacro(
* ```
*/
SetMacros(
13, SET_MACROS, listOf(zeroToManyTagged("macros")),
21, SET_MACROS, listOf(zeroToManyTagged("macros")),
templateBody {
annotated(ION, ::sexp) {
symbol(MODULE)
Expand All @@ -142,7 +161,7 @@ enum class SystemMacro(
* ```
*/
AddMacros(
14, ADD_MACROS, listOf(zeroToManyTagged("macros")),
22, ADD_MACROS, listOf(zeroToManyTagged("macros")),
templateBody {
annotated(ION, ::sexp) {
symbol(MODULE)
Expand Down Expand Up @@ -171,7 +190,7 @@ enum class SystemMacro(
* ```
*/
Use(
15, USE, listOf(exactlyOneTagged("catalog_key"), zeroOrOneTagged("version")),
23, USE, listOf(exactlyOneTagged("catalog_key"), zeroOrOneTagged("version")),
templateBody {
val theModule = _Private_Utils.newSymbolToken("the_module")
annotated(ION, ::sexp) {
Expand Down Expand Up @@ -201,29 +220,6 @@ enum class SystemMacro(
}
}
),
// TODO: parse_ion
Repeat(17, REPEAT, listOf(exactlyOneTagged("n"), zeroToManyTagged("value"))),
Delta(18, DELTA, listOf(zeroToManyTagged("deltas"))),
// TODO: flatten
Sum(20, SUM, listOf(exactlyOneTagged("a"), exactlyOneTagged("b"))),
Comment(21, META, listOf(zeroToManyTagged("values")), templateBody { macro(None) {} }),
MakeField(
22, MAKE_FIELD,
listOf(
Macro.Parameter("field_name", Macro.ParameterEncoding.FlexSym, Macro.ParameterCardinality.ExactlyOne), exactlyOneTagged("value")
)
),

Default(
23, DEFAULT, listOf(zeroToManyTagged("expr"), zeroToManyTagged("default_expr")),
templateBody {
macro(IfNone) {
variable(0)
variable(1)
variable(0)
}
}
),
;

val macroName: String get() = this.systemSymbol.text
Expand Down
8 changes: 4 additions & 4 deletions src/test/java/com/amazon/ion/impl/IonReaderContinuableCoreBinaryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1069,7 +1069,7 @@ public void taglessCompactSymbols(boolean constructFromBytes) throws Exception {
0x00, // User macro ID 0
0xF9, 0x6E, 0x61, 0x6D, 0x65, // interpreted as compact symbol (FlexSym with inline text "name")
0x09, // interpreted as compact symbol (FlexSym with SID 4)
0x01, 0x75 // interpreted as compact symbol (special FlexSym)
0x01, 0x81 // interpreted as compact symbol (special FlexSym)
));
try (IonReaderContinuableCoreBinary reader = initializeReader(constructFromBytes, data)) {
assertSequence(
Expand All @@ -1090,7 +1090,7 @@ public void taglessCompactSymbols(boolean constructFromBytes) throws Exception {
@ValueSource(booleans = {true, false})
public void addSymbolsSystemMacro(boolean constructFromBytes) throws Exception {
byte[] data = withIvm(1, bytes(
0xEF, 0x0C, // system macro add_symbols
0xEF, 0x14, // system macro add_symbols
0x02, // AEB: 0b------aa; a=10, expression group
0x01, // FlexInt 0, a delimited expression group
0x93, 0x61, 0x62, 0x63, // 3-byte string, utf-8 "abc"
Expand Down Expand Up @@ -1181,13 +1181,13 @@ private void assertMacroAwareTranscribeProducesEquivalentStream(byte[] data, boo
@ValueSource(booleans = {true, false})
public void encodingLevelTranscodeOfSystemMacroInvocation(boolean constructFromBytes) throws Exception {
byte[] data = withIvm(1, bytes(
0xEF, 0x0C, // system macro add_symbols
0xEF, 0x14, // system macro add_symbols
0x02, // AEB: 0b------aa; a=10, expression group
0x01, // FlexInt 0, a delimited expression group
0x93, 0x61, 0x62, 0x63, // 3-byte string, utf-8 "abc"
0xF0, // delimited end... of expression group
0xE1, // SID single byte
0x42 // SID $66
0x40 // SID $64
));
assertMacroAwareTranscribeProducesEquivalentStream(data, constructFromBytes);
}
Expand Down
26 changes: 13 additions & 13 deletions src/test/java/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5062,15 +5062,15 @@ private void readAnnotationsWithSpecialFlexSyms_1_1(boolean constructFromBytes,
@ParameterizedTest
@ValueSource(strings = {
// Minimal representations
"E7 01 75 60 | One empty-text annotation; value int 0 \n" +
"E7 01 81 60 | One empty-text annotation; value int 0 \n" +
"E7 01 60 60 | One SID 0 annotation; value int 0 \n" +
"E8 01 75 01 60 60 | Two annotations: empty text, SID 0; value int 0 \n" +
"E9 09 01 60 01 75 60 | Variable length = 4 annotations: SID 0, empty text; value int 0 \n",
"E8 01 81 01 60 60 | Two annotations: empty text, SID 0; value int 0 \n" +
"E9 09 01 60 01 81 60 | Variable length = 4 annotations: SID 0, empty text; value int 0 \n",
// Overpadded representations
"E7 02 00 75 60 | One overpadded empty-text annotation; value int 0 \n" +
"E7 02 00 81 60 | One overpadded empty-text annotation; value int 0 \n" +
"E7 04 00 00 60 60 | One overpadded SID 0 annotation; value int 0 \n" +
"E8 08 00 00 00 75 02 00 60 60 | Two overpadded annotations: empty text, SID 0; value int 0 \n" +
"E9 90 00 00 00 00 01 60 01 75 60 | Variable overpadded length = 4 annotations: SID 0, empty text; value int 0 \n"
"E8 08 00 00 00 81 02 00 60 60 | Two overpadded annotations: empty text, SID 0; value int 0 \n" +
"E9 90 00 00 00 00 01 60 01 81 60 | Variable overpadded length = 4 annotations: SID 0, empty text; value int 0 \n"
})
public void readAnnotationsWithSpecialFlexSyms_1_1(String inputBytes) throws Exception {
readAnnotationsWithSpecialFlexSyms_1_1(true, inputBytes);
Expand Down Expand Up @@ -5308,17 +5308,17 @@ public void assertStructWithEmptyInlineFieldNamesCorrectlyParsed(boolean constru
// Empty field name in fixed-length SID struct
"D4 | Struct Length = 4 \n" +
"01 | switch to FlexSym encoding \n" +
"01 75 | FlexSym empty text \n" +
"01 81 | FlexSym empty text \n" +
"6F | false",
// Empty field name in variable-length SID struct
"FD | Variable length SID struct \n" +
"09 | Length = 4 \n" +
"01 | switch to FlexSym encoding \n" +
"01 75 | FlexSym empty text \n" +
"01 81 | FlexSym empty text \n" +
"6F | false",
// Empty field name in delimited struct
"F3 | Delimited struct \n" +
"01 75 | FlexSym empty text \n" +
"01 81 | FlexSym empty text \n" +
"6F | false \n" +
"01 F0 | End delimited struct"
})
Expand Down Expand Up @@ -5990,13 +5990,13 @@ private void invokeValuesUsingSystemMacroOpcodeHelper(boolean constructFromBytes
public void addSymbolsSystemMacro(boolean constructFromBytes) throws Exception {
int[] data = new int[] {
0xE0, 0x01, 0x01, 0xEA, // Ion 1.1 IVM
0xEF, 0x0C, // system macro add_symbols
0xEF, 0x14, // system macro add_symbols
0x02, // AEB: 0b------aa; a=10, expression group
0x01, // FlexInt 0, a delimited expression group
0x93, 0x61, 0x62, 0x63, // 3-byte string, utf-8 "abc"
0xF0, // delimited end... of expression group
0xE1, // SID single byte
0x42 // SID $66
0x40 // SID $64
};
try (IonReader reader = readerFor(constructFromBytes,data)) {
assertEquals(IonType.SYMBOL, reader.next());
Expand All @@ -6008,13 +6008,13 @@ public void addSymbolsSystemMacro(boolean constructFromBytes) throws Exception {
public void addSymbolsSystemMacroWhenNotEntirelyBuffered() throws Exception {
int[] data = new int[] {
/* 0 - 3 */ // 0xEA, 0x01, 0x01, 0xE0, // implicitly provided by BinaryIonAppender
/* 4 - 5 */ 0xEF, 0x0C, // system macro add_symbols
/* 4 - 5 */ 0xEF, 0x14, // system macro add_symbols
/* 6 - 7 */ 0x02, 0x01, // AEB: 0b------aa; a=10, FlexInt 0, a delimited expression group
/* 8 - 12 */ 0x93, 0x66, 0x6F, 0x72, // "for"
/* 13 - 16 */ 0x93, 0x66, 0x75, 0x72, // "fur"
/* 17 - 21 */ 0x94, 0x66, 0x6F, 0x75, 0x72, // "four"
/* 22 - 22 */ 0xF0, // delimited end... of expression group
/* 23 - 24 */ 0xE1, 0x41 + 3, // SID single byte ${usid 1} => "four"
/* 23 - 24 */ 0xE1, 0x3F + 3, // SID single byte ${usid 1} => "four"
};
byte[] bytes = new TestUtils.BinaryIonAppender(1).append(data).toByteArray();
totalBytesInStream = bytes.length;
Expand Down
Loading
Loading