From 263be46e85c4c3aba58aa199bf790960290db7ce Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Fri, 12 Jan 2024 21:50:16 -0800 Subject: [PATCH] Adds macro compiler --- .../java/com/amazon/ion/impl/macro/Macro.kt | 40 ++- .../amazon/ion/impl/macro/MacroCompiler.kt | 240 ++++++++++++++++++ .../amazon/ion/impl/macro/MacroSignature.kt | 16 -- .../ion/impl/macro/TemplateBodyExpression.kt | 81 ++++++ .../ion/impl/macro/TemplateExpression.kt | 48 ---- .../java/com/amazon/ion/FakeSymbolToken.java | 24 ++ .../ion/impl/macro/MacroCompilerTest.kt | 225 ++++++++++++++++ 7 files changed, 601 insertions(+), 73 deletions(-) create mode 100644 src/main/java/com/amazon/ion/impl/macro/MacroCompiler.kt delete mode 100644 src/main/java/com/amazon/ion/impl/macro/MacroSignature.kt create mode 100644 src/main/java/com/amazon/ion/impl/macro/TemplateBodyExpression.kt delete mode 100644 src/main/java/com/amazon/ion/impl/macro/TemplateExpression.kt create mode 100644 src/test/java/com/amazon/ion/impl/macro/MacroCompilerTest.kt diff --git a/src/main/java/com/amazon/ion/impl/macro/Macro.kt b/src/main/java/com/amazon/ion/impl/macro/Macro.kt index 88d3ce8333..71827e1b4c 100644 --- a/src/main/java/com/amazon/ion/impl/macro/Macro.kt +++ b/src/main/java/com/amazon/ion/impl/macro/Macro.kt @@ -1,23 +1,45 @@ package com.amazon.ion.impl.macro -import java.math.BigDecimal - /** * Marker interface for Macros */ -sealed interface Macro +sealed interface Macro { + val signature: List + + data class Parameter(val variableName: String, val type: ParameterEncoding, val grouped: Boolean) + + enum class ParameterEncoding(val ionTextName: String) { + Tagged("any"), + // TODO: List all of the possible tagless encodings + } +} /** - * Represents a template macro. A template macro is defined by a name, a signature, and a list of template expressions. + * Represents a template macro. A template macro is defined by a signature, and a list of template expressions. + * A template macro only gains a name and/or ID when it is added to a macro table. */ -data class TemplateMacro(val name: String, val f: BigDecimal, val signature: MacroSignature, val body: List) : Macro +data class TemplateMacro(override val signature: List, val body: List) : Macro { + private val cachedHashCode by lazy { signature.hashCode() * 31 + body.hashCode() } + override fun hashCode(): Int = cachedHashCode + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is TemplateMacro) return false + // Check the hashCode as a quick check before we dive into the actual data. + if (cachedHashCode != other.cachedHashCode) return false + if (signature != other.signature) return false + if (body != other.body) return false + return true + } +} /** * Macros that are built in, rather than being defined by a template. */ -enum class SystemMacro : Macro { - Stream, // A stream is technically not a macro, but we can implement it as a macro that is the identity function. - Annotate, - MakeString, +enum class SystemMacro(override val signature: List) : Macro { + // TODO: replace these placeholders + Stream(emptyList()), // A stream is technically not a macro, but we can implement it as a macro that is the identity function. + Annotate(emptyList()), + MakeString(listOf(Macro.Parameter("text", Macro.ParameterEncoding.Tagged, grouped = true))), // TODO: Other system macros } diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroCompiler.kt b/src/main/java/com/amazon/ion/impl/macro/MacroCompiler.kt new file mode 100644 index 0000000000..431aace2b3 --- /dev/null +++ b/src/main/java/com/amazon/ion/impl/macro/MacroCompiler.kt @@ -0,0 +1,240 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ion.impl.macro + +import com.amazon.ion.* +import com.amazon.ion.impl.macro.TemplateBodyExpression.* +import com.amazon.ion.util.confirm + +/** + * [MacroCompiler] wraps an [IonReader]. When directed to do so, it will take over advancing and getting values from the + * reader in order to read one [TemplateMacro]. + * + * This is currently implemented using [IonReader], but it could be adapted to work with + * [IonReaderContinuableCore][com.amazon.ion.impl.IonReaderContinuableCore]. + */ +class MacroCompiler(private val reader: IonReader) { + // TODO: Make sure that we can throw exceptions if there's an over-sized value. + + /** The name of the macro that was read. Returns `null` if no macro name is available. */ + var macroName: String? = null + private set // Only mutable internally + + private val signature: MutableList = mutableListOf() + private val expressions: MutableList = mutableListOf() + + /** + * Compiles a template macro definition from the reader. Caller is responsible for positioning the reader at—but not + * stepped into—the macro template s-expression. + * + * TODO: if we switch the macro compiler to use a continuable reader, change the return type of this + * to a compiler state enum, and add a separate function to get the compiled macro once it is ready. + */ + fun compileMacro(): TemplateMacro { + macroName = null + signature.clear() + expressions.clear() + + confirm(reader.type == IonType.SEXP) { "macro compilation expects a sexp starting with the keyword `macro`" } + reader.confirmNoAnnotations("a macro definition sexp") + reader.readContainer { + confirm(reader.next() == IonType.SYMBOL && reader.stringValue() == "macro") { "macro compilation expects a sexp starting with the keyword `macro`" } + + nextAndCheckType(IonType.SYMBOL, "macro name") + confirmNoAnnotations("macro name") + // TODO: Enforce 'identifier' syntax subset of symbol + // Possibly add support for macro definitions without names? + macroName = symbolValue().assumeText() + + nextAndCheckType(IonType.SEXP, "macro signature") + confirmNoAnnotations("macro signature") + readSignature() + + forEachRemaining { compileTemplateBodyExpression(isQuoted = false) } + } + return TemplateMacro(signature.toList(), expressions.toList()) + } + + /** + * Reads the macro signature, populating parameters in [signature]. + * Caller is responsible for making sure that the reader is positioned on (but not stepped into) the signature sexp. + */ + private fun readSignature() { + reader.forEachInContainer { + when (it) { + IonType.SYMBOL -> addParameter(grouped = false) + IonType.LIST -> { + confirmNoAnnotations(location = "grouped parameter enclosing list") + readContainer { + nextAndCheckType(IonType.SYMBOL, "parameter name") + addParameter(grouped = true) + confirm(next() == null) { "grouped parameter list must enclose only one variable name" } + } + } + else -> throw IonException("parameter must be a symbol or a list; found ${reader.type}") + } + } + } + + /** + * Adds a parameter to the macro signature. + * Caller is responsible for making sure that the reader is positioned on a parameter name. + */ + private fun addParameter(grouped: Boolean) { + val annotations = reader.typeAnnotations + confirm(annotations.isEmptyOr(Macro.ParameterEncoding.Tagged.ionTextName)) { "unsupported parameter encoding ${annotations.toList()}" } + val parameterName = reader.symbolValue().assumeText() + confirm(signature.none { it.variableName == parameterName }) { "redeclaration of parameter '$parameterName'" } + signature.add(Macro.Parameter(parameterName, Macro.ParameterEncoding.Tagged, grouped)) + } + + /** + * Compiles the current value on the reader into a [TemplateBodyExpression] and adds it to [expressions]. + * Caller is responsible for ensuring that the reader is positioned on a value. + * + * If called when the reader is not positioned on any value, throws [IllegalStateException]. + */ + private fun compileTemplateBodyExpression(isQuoted: Boolean) { + // TODO: Could typeAnnotations ever be an array of nulls? + // NOTE: `toList()` does not allocate for an empty list. + val annotations = reader.typeAnnotationSymbols.toList() + + if (reader.isNullValue) { + expressions.add(NullValue(annotations, reader.type)) + } else when (reader.type) { + IonType.BOOL -> expressions.add(BoolValue(annotations, reader.booleanValue())) + IonType.INT -> expressions.add( + when (reader.integerSize!!) { + IntegerSize.INT, + IntegerSize.LONG -> IntValue(annotations, reader.longValue()) + IntegerSize.BIG_INTEGER -> BigIntValue(annotations, reader.bigIntegerValue()) + } + ) + IonType.FLOAT -> expressions.add(FloatValue(annotations, reader.doubleValue())) + IonType.DECIMAL -> expressions.add(DecimalValue(annotations, reader.decimalValue())) + IonType.TIMESTAMP -> expressions.add(TimestampValue(annotations, reader.timestampValue())) + IonType.STRING -> expressions.add(StringValue(annotations, reader.stringValue())) + IonType.BLOB -> expressions.add(BlobValue(annotations, reader.newBytes())) + IonType.CLOB -> expressions.add(ClobValue(annotations, reader.newBytes())) + IonType.SYMBOL -> { + if (isQuoted) { + expressions.add(SymbolValue(annotations, reader.symbolValue())) + } else { + val name = reader.stringValue() + reader.confirmNoAnnotations("on variable reference '$name'") + val index = signature.indexOfFirst { it.variableName == name } + confirm(index >= 0) { "variable '$name' is not recognized" } + expressions.add(Variable(index)) + } + } + IonType.LIST -> compileSequence(isQuoted) { start, end -> ListValue(annotations, start, end) } + IonType.SEXP -> { + if (isQuoted) { + compileSequence(isQuoted = true) { start, end -> SExpValue(annotations, start, end) } + } else { + reader.confirmNoAnnotations(location = "a macro invocation") + compileMacroInvocation() + } + } + IonType.STRUCT -> compileStruct(annotations, isQuoted) + // IonType.NULL, IonType.DATAGRAM, null + else -> TODO("Unreachable.") + } + } + + /** + * Compiles a struct in a macro template. + * When calling, the reader should be positioned at the struct, but not stepped into it. + * If this function returns normally, it will be stepped out of the struct. + * Caller will need to call [IonReader.next] to get the next value. + */ + private fun compileStruct(annotations: List, isQuoted: Boolean) { + val start = expressions.size + expressions.add(Placeholder) + val templateStructIndex = mutableMapOf>() + reader.forEachInContainer { + expressions.add(FieldName(fieldNameSymbol)) + fieldNameSymbol.text?.let { + val valueIndex = expressions.size + // Default is an array list with capacity of 1, since the most common case is that a field name occurs once. + templateStructIndex.getOrPut(it) { ArrayList(1) } += valueIndex + } + compileTemplateBodyExpression(isQuoted) + } + val end = expressions.lastIndex + expressions[start] = StructValue(annotations, start, end, templateStructIndex) + } + + /** + * Compiles a list or sexp in a macro template. + * When calling, the reader should be positioned at the sequence, but not stepped into it. + * If this function returns normally, it will be stepped out of the sequence. + * Caller will need to call [IonReader.next] to get the next value. + */ + private inline fun compileSequence(isQuoted: Boolean, newTemplateBodySequence: (Int, Int) -> TemplateBodyExpression) { + val seqStart = expressions.size + expressions.add(Placeholder) + reader.forEachInContainer { compileTemplateBodyExpression(isQuoted) } + val seqEnd = expressions.lastIndex + expressions[seqStart] = newTemplateBodySequence(seqStart, seqEnd) + } + + /** + * Compiles a macro invocation in a macro template. + * When calling, the reader should be positioned at the sexp, but not stepped into it. + * If this function returns normally, it will be stepped out of the sexp. + * Caller will need to call [IonReader.next] to get the next value. + */ + private fun compileMacroInvocation() { + reader.stepIn() + val macroRef = when (reader.next()) { + IonType.SYMBOL -> { + val macroName = reader.stringValue() + // TODO: Once we have a macro table, validate name exists in current macro table. + if (macroName == "quote") null else MacroRef.ByName(macroName) + } + // TODO: When we have an ID for the macro "quote", add handling for it here. + // TODO: Once we have a macro table, validate that id exists in current macro table. + IonType.INT -> MacroRef.ById(reader.longValue()) + else -> throw IonException("macro invocation must start with an id (int) or identifier (symbol); found ${reader.type ?: "nothing"}\"") + } + + if (macroRef == null) { + // It's the "quote" macro; skip compiling a macro invocation and just treat all contents as literals + reader.forEachRemaining { compileTemplateBodyExpression(isQuoted = true) } + } else { + val macroStart = expressions.size + expressions.add(Placeholder) + reader.forEachRemaining { compileTemplateBodyExpression(isQuoted = false) } + val macroEnd = expressions.lastIndex + expressions[macroStart] = + MacroInvocation(macroRef, macroStart, macroEnd) + } + reader.stepOut() + } + + // Helper functions + + /** Utility method for checking that annotations are empty or a single array with the given annotations */ + private fun Array.isEmptyOr(text: String): Boolean = isEmpty() || (size == 1 && this[0] == text) + + /** Throws [IonException] if any annotations are on the current value in this [IonReader]. */ + private fun IonReader.confirmNoAnnotations(location: String) { + confirm(typeAnnotations.isEmpty()) { "found annotations on $location" } + } + + /** Moves to the next type and throw [IonException] if it is not the `expected` [IonType]. */ + private fun IonReader.nextAndCheckType(expected: IonType, location: String) { + confirm(next() == expected) { "$location must be a $expected; found ${type ?: "nothing"}" } + } + + /** Steps into a container, executes [block], and steps out. */ + private inline fun IonReader.readContainer(block: IonReader.() -> Unit) { stepIn(); block(); stepOut() } + + /** Executes [block] for each remaining value at the current reader depth. */ + private inline fun IonReader.forEachRemaining(block: IonReader.(IonType) -> Unit) { while (next() != null) { block(type) } } + + /** Steps into a container, executes [block] for each value at that reader depth, and steps out. */ + private inline fun IonReader.forEachInContainer(block: IonReader.(IonType) -> Unit) = readContainer { forEachRemaining(block) } +} diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroSignature.kt b/src/main/java/com/amazon/ion/impl/macro/MacroSignature.kt deleted file mode 100644 index d4bb7c1ca3..0000000000 --- a/src/main/java/com/amazon/ion/impl/macro/MacroSignature.kt +++ /dev/null @@ -1,16 +0,0 @@ -package com.amazon.ion.impl.macro - -/** - * Represents the signature of a macro. - */ -@JvmInline value class MacroSignature(val parameters: List) { - - data class Parameter(val variableName: String, val type: Encoding, val grouped: Boolean) - - enum class Encoding(val ionTextName: String?) { - Tagged(null), - Any("any"), - Int8("int8"), - // TODO: List all of the possible tagless encodings - } -} diff --git a/src/main/java/com/amazon/ion/impl/macro/TemplateBodyExpression.kt b/src/main/java/com/amazon/ion/impl/macro/TemplateBodyExpression.kt new file mode 100644 index 0000000000..48007c45bf --- /dev/null +++ b/src/main/java/com/amazon/ion/impl/macro/TemplateBodyExpression.kt @@ -0,0 +1,81 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ion.impl.macro + +import com.amazon.ion.IonType +import com.amazon.ion.SymbolToken +import com.amazon.ion.Timestamp +import java.math.BigDecimal +import java.math.BigInteger + +/** + * Represents an expression in the body of a template. + * + * We cannot use [`IonValue`](com.amazon.ion.IonValue) for this because `IonValue` requires references to parent + * containers and to an IonSystem which makes it impractical for reading and writing macros definitions. Furthermore, + * there is information we need to capture that cannot be expressed in the IonValue model, such as macro invocations + * and variable references. + * + * A template body is compiled into a list of expressions, without nesting, for ease and efficiency of evaluating + * e-expressions. Because of this, the container types do not have other values nested in them; rather they contain a + * range that indicates which of the following expressions are part of that container. + */ +sealed interface TemplateBodyExpression { + // TODO: Special Forms (if_void, for, ...)? + + /** + * A temporary placeholder that is used only while a macro is partially compiled. + */ + object Placeholder : TemplateBodyExpression + + // Scalars + data class NullValue(val annotations: List = emptyList(), val type: IonType) : TemplateBodyExpression + data class BoolValue(val annotations: List = emptyList(), val value: Boolean) : TemplateBodyExpression + data class IntValue(val annotations: List = emptyList(), val value: Long) : TemplateBodyExpression + data class BigIntValue(val annotations: List = emptyList(), val value: BigInteger) : TemplateBodyExpression + data class FloatValue(val annotations: List = emptyList(), val value: Double) : TemplateBodyExpression + data class DecimalValue(val annotations: List = emptyList(), val value: BigDecimal) : TemplateBodyExpression + data class TimestampValue(val annotations: List = emptyList(), val value: Timestamp) : TemplateBodyExpression + data class StringValue(val annotations: List = emptyList(), val value: String) : TemplateBodyExpression + data class SymbolValue(val annotations: List = emptyList(), val value: SymbolToken) : TemplateBodyExpression + // We must override hashcode and equals in the lob types because `value` is a `byte[]` + data class BlobValue(val annotations: List = emptyList(), val value: ByteArray) : TemplateBodyExpression { + override fun hashCode(): Int = annotations.hashCode() * 31 + value.contentHashCode() + override fun equals(other: Any?): Boolean = other is BlobValue && annotations == other.annotations && value.contentEquals(other.value) + } + data class ClobValue(val annotations: List = emptyList(), val value: ByteArray) : TemplateBodyExpression { + override fun hashCode(): Int = annotations.hashCode() * 31 + value.contentHashCode() + override fun equals(other: Any?): Boolean = other is ClobValue && annotations == other.annotations && value.contentEquals(other.value) + } + + /** + * An Ion List that could contain variables or macro invocations. + * + * @property startInclusive the index of the first expression of the list (i.e. this instance) + * @property endInclusive the index of the last expression contained in the list + */ + data class ListValue(val annotations: List = emptyList(), val startInclusive: Int, val endInclusive: Int) : TemplateBodyExpression + + /** + * An Ion SExp that could contain variables or macro invocations. + */ + data class SExpValue(val annotations: List = emptyList(), val startInclusive: Int, val endInclusive: Int) : TemplateBodyExpression + + /** + * An Ion Struct that could contain variables or macro invocations. + */ + data class StructValue(val annotations: List = emptyList(), val startInclusive: Int, val endInclusive: Int, val templateStructIndex: Map>) : TemplateBodyExpression + + data class FieldName(val value: SymbolToken) : TemplateBodyExpression + + /** + * A reference to a variable that needs to be expanded. + */ + data class Variable(val signatureIndex: Int) : TemplateBodyExpression + + /** + * A macro invocation that needs to be expanded. + */ + data class MacroInvocation(val macro: MacroRef, val startInclusive: Int, val endInclusive: Int) : TemplateBodyExpression +} diff --git a/src/main/java/com/amazon/ion/impl/macro/TemplateExpression.kt b/src/main/java/com/amazon/ion/impl/macro/TemplateExpression.kt deleted file mode 100644 index 43e30bc0bc..0000000000 --- a/src/main/java/com/amazon/ion/impl/macro/TemplateExpression.kt +++ /dev/null @@ -1,48 +0,0 @@ -package com.amazon.ion.impl.macro - -import com.amazon.ion.SymbolToken -import com.amazon.ion.impl.macro.ionelement.api.IonElement - -/** - * Represents an expression in the body of a template. - */ -sealed interface TemplateExpression { - // TODO: Special Forms (if_void, for, ...)? - - /** - * A value that is taken literally. I.e. there can be no variable substitutions or macros inside here. Usually, - * these will just be scalars, but it is possible for it to be a container type. - * - * We cannot use [`IonValue`](com.amazon.ion.IonValue) for this because `IonValue` requires references to parent - * containers and to an IonSystem which makes it impractical for reading and writing macros definitions. - * For now, we are using [IonElement], but that is not a good permanent solution. It is copy/pasted into this repo - * to avoid a circular dependency, and the dependencies are shaded, so it is not suitable for a public API. - */ - @JvmInline value class LiteralValue(val value: IonElement) : TemplateExpression - - /** - * An Ion List that could contain variables or macro invocations. - */ - data class ListValue(val annotations: List, val expressionRange: IntRange) : TemplateExpression - - /** - * An Ion SExp that could contain variables or macro invocations. - */ - data class SExpValue(val annotations: List, val expressionRange: IntRange) : TemplateExpression - - /** - * An Ion Struct that could contain variables or macro invocations. - */ - data class StructValue(val annotations: List, val expressionRange: IntRange, val templateStructIndex: Map) - - /** - * A reference to a variable that needs to be expanded. - */ - // @JvmInline value - class Variable(val signatureIndex: Int) : TemplateExpression - - /** - * A macro invocation that needs to be expanded. - */ - data class MacroInvocation(val macro: MacroRef, val argumentExpressionsRange: IntRange) : TemplateExpression -} diff --git a/src/test/java/com/amazon/ion/FakeSymbolToken.java b/src/test/java/com/amazon/ion/FakeSymbolToken.java index 426e95a40e..769f01b668 100644 --- a/src/test/java/com/amazon/ion/FakeSymbolToken.java +++ b/src/test/java/com/amazon/ion/FakeSymbolToken.java @@ -45,4 +45,28 @@ public int getSid() { return mySid; } + + @Override + public String toString() + { + return "SymbolToken::{text:" + myText + ",id:" + mySid + "}"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || !(o instanceof SymbolToken)) return false; + + SymbolToken other = (SymbolToken) o; + if(getText() == null || other.getText() == null){ + return getText() == other.getText(); + } + return getText().equals(other.getText()); + } + + @Override + public int hashCode() { + if(getText() != null) return getText().hashCode(); + return 0; + } } diff --git a/src/test/java/com/amazon/ion/impl/macro/MacroCompilerTest.kt b/src/test/java/com/amazon/ion/impl/macro/MacroCompilerTest.kt new file mode 100644 index 0000000000..a5df125f18 --- /dev/null +++ b/src/test/java/com/amazon/ion/impl/macro/MacroCompilerTest.kt @@ -0,0 +1,225 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.amazon.ion.impl.macro + +import com.amazon.ion.* +import com.amazon.ion.impl.macro.Macro.* +import com.amazon.ion.impl.macro.Macro.ParameterEncoding.* +import com.amazon.ion.impl.macro.MacroRef.* +import com.amazon.ion.impl.macro.TemplateBodyExpression.* +import com.amazon.ion.system.IonSystemBuilder +import java.math.BigDecimal +import java.math.BigInteger +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.Arguments +import org.junit.jupiter.params.provider.MethodSource +import org.junit.jupiter.params.provider.ValueSource + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MacroCompilerTest { + + val ion = IonSystemBuilder.standard().build() + + private data class MacroSourceAndTemplate(val source: String, val template: TemplateMacro) : Arguments { + override fun get(): Array = arrayOf(source, template.signature, template.body) + } + + private fun annotations(vararg a: String): List = a.map { FakeSymbolToken(it, -1) } + + private infix fun String.shouldCompileTo(macro: TemplateMacro) = MacroSourceAndTemplate(this, macro) + + private fun testCases() = listOf( + "(macro empty ())" shouldCompileTo TemplateMacro(emptyList(), emptyList()), + "(macro identity (x) x)" shouldCompileTo TemplateMacro( + listOf(Parameter("x", Tagged, grouped = false)), + listOf(Variable(0)), + ), + "(macro identity (any::x) x)" shouldCompileTo TemplateMacro( + listOf(Parameter("x", Tagged, grouped = false)), + listOf(Variable(0)), + ), + "(macro pi () 3.141592653589793)" shouldCompileTo TemplateMacro( + signature = emptyList(), + body = listOf(DecimalValue(emptyList(), BigDecimal("3.141592653589793"))) + ), + "(macro group_test ([x]))" shouldCompileTo TemplateMacro( + signature = listOf(Parameter("x", Tagged, grouped = true)), + body = emptyList() + ), + // Outer 'values' call allows multiple expressions in the body + // The second `values` is a macro call that has a single argument: the variable `x` + // The `quote` call causes the third (inner) `(values x)` to be an uninterpreted s-expression. + """(macro quote_test (x) (values (values x) (quote (values x))))""" shouldCompileTo TemplateMacro( + signature = listOf(Parameter("x", Tagged, grouped = false)), + body = listOf( + MacroInvocation(ByName("values"), startInclusive = 0, endInclusive = 5), + MacroInvocation(ByName("values"), startInclusive = 1, endInclusive = 2), + Variable(0), + SExpValue(emptyList(), startInclusive = 3, endInclusive = 5), + SymbolValue(emptyList(), FakeSymbolToken("values", -1)), + SymbolValue(emptyList(), FakeSymbolToken("x", -1)), + ), + ), + "(macro each_type () null true 1 ${"9".repeat(50)} 1e0 1d0 2024-01-16T \"foo\" (quote bar) [] (quote ()) {} {{}} {{\"\"}} )" shouldCompileTo TemplateMacro( + signature = emptyList(), + body = listOf( + NullValue(type = IonType.NULL), + BoolValue(value = true), + IntValue(value = 1), + BigIntValue(value = BigInteger("9".repeat(50))), + FloatValue(value = 1.0), + DecimalValue(value = BigDecimal.ONE), + TimestampValue(value = Timestamp.valueOf("2024-01-16T")), + StringValue(value = "foo"), + SymbolValue(value = FakeSymbolToken("bar", -1)), + ListValue(startInclusive = 9, endInclusive = 9), + SExpValue(startInclusive = 10, endInclusive = 10), + StructValue(startInclusive = 11, endInclusive = 11, templateStructIndex = emptyMap()), + BlobValue(value = ByteArray(0)), + ClobValue(value = ByteArray(0)) + ) + ), + """(macro foo () (values 42 "hello" false))""" shouldCompileTo TemplateMacro( + signature = emptyList(), + body = listOf( + MacroInvocation(ByName("values"), startInclusive = 0, endInclusive = 3), + IntValue(value = 42), + StringValue(value = "hello"), + BoolValue(value = false), + ) + ), + """(macro invoke_by_id () (12 true false))""" shouldCompileTo TemplateMacro( + signature = emptyList(), + body = listOf( + MacroInvocation(ById(12), startInclusive = 0, endInclusive = 2), + BoolValue(value = true), + BoolValue(value = false), + ) + ), + "(macro foo (x y z) [100, [200, a::b::300], x, {y: [true, false, z]}])" shouldCompileTo TemplateMacro( + signature = listOf( + Parameter("x", Tagged, grouped = false), + Parameter("y", Tagged, grouped = false), + Parameter("z", Tagged, grouped = false) + ), + body = listOf( + ListValue(startInclusive = 0, endInclusive = 11), + IntValue(value = 100), + ListValue(startInclusive = 2, endInclusive = 4), + IntValue(value = 200), + IntValue(annotations("a", "b"), value = 300), + Variable(0), + StructValue(startInclusive = 6, endInclusive = 11, templateStructIndex = mapOf("y" to listOf(8))), + FieldName(FakeSymbolToken("y", -1)), + ListValue(startInclusive = 8, endInclusive = 11), + BoolValue(value = true), + BoolValue(value = false), + Variable(2), + ) + ) + ) + + @ParameterizedTest(name = "{0}") + @MethodSource("testCases") + fun assertMacroCompilation(source: String, signature: List, body: List) { + val reader = ion.newReader(source) + val compiler = MacroCompiler(reader) + reader.next() + val macroDef = compiler.compileMacro() + val expectedDef = TemplateMacro(signature, body) + assertEquals(expectedDef, macroDef) + } + + @Test + fun `test reading a list of macros`() { + // This test case is essentially the same as the last one, except that it puts all the macro definitions into + // one Ion list, and then compiles them sequentially from that list. + // If this test fails, do not bother trying to fix it until all cases in the parameterized test are passing. + val source = "[${testCases().joinToString(",") { it.source }}]" + val templates = testCases().map { it.template }.iterator() + + val reader = ion.newReader(source) + val compiler = MacroCompiler(reader) + // Advance and step into list + reader.next(); reader.stepIn() + while (reader.next() != null) { + val macroDef = compiler.compileMacro() + val expectedDef = templates.next() + assertEquals(expectedDef, macroDef) + } + reader.stepOut() + reader.close() + } + + @Test + fun `macro compiler should return the correct name`() { + val reader = ion.newReader( + """ + (macro foo (x) 1) + (macro bar (y) 2) + """ + ) + val compiler = MacroCompiler(reader) + assertNull(compiler.macroName) + reader.next() + compiler.compileMacro() + assertEquals("foo", compiler.macroName) + reader.next() + compiler.compileMacro() + assertEquals("bar", compiler.macroName) + } + + // macro with invalid variable + // try compiling something that is not a sexp + // macro missing keyword + // macro has invalid name + // macro has annotations + + @ParameterizedTest + @ValueSource( + strings = [ + // There should be exactly one thing wrong in each of these samples. + + // Problems up to and including the macro name + "[macro, pi, (), 3.141592653589793]", // Macro def must be a sexp + "foo::(macro pi () 3.141592653589793)", // Macros cannot be annotated + """("macro" pi () 3.141592653589793)""", // 'macro' must be a symbol + "(pi () 3.141592653589793)", // doesn't start with 'macro' + "(macaroon pi () 3.141592653589793)", // doesn't start with 'macro' + "(macro pi::pi () 3.141592653589793)", // Illegal annotation on macro name + "(macro () 3.141592653589793)", // No macro name + "(macro 2.5 () 3.141592653589793)", // Macro name is not a symbol + """(macro "pi"() 3.141592653589793)""", // Macro name is not a symbol + "(macro \$0 () 3.141592653589793)", // Macro name must have known text + + // Problems in the signature + "(macro identity x x)", // Missing sexp around signature + "(macro identity [x] x)", // Using list instead of sexp for signature + "(macro identity any::(x) x)", // Signature sexp should not be annotated + "(macro identity (foo::x) x)", // Unknown type in signature + "(macro identity (any::[x]) x)", // Annotation should be on parameter name, not the grouping indicator + "(macro identity ([]) x)", // Grouping indicator must have one symbol in it + "(macro identity ([x, y]) x)", // Grouping indicator must have one symbol in it + "(macro identity (x x) x)", // Repeated parameter name + """(macro identity ("x") x)""", // Parameter name must be a symbol, not a string + + // Problems in the body + "(macro transform (x) y)", // Unknown variable + "(macro transform (x) foo::x)", // Variable cannot be annotated + "(macro transform (x) foo::(quote x))", // Macro invocation cannot be annotated + """(macro transform (x) ("quote" x))""", // Macro invocation must start with a symbol or integer id + ] + ) + fun assertCompilationFails(source: String) { + val reader = ion.newReader(source) + reader.next() + val compiler = MacroCompiler(reader) + assertThrows { compiler.compileMacro() } + } +}