From d9e4c57ac2433594278395874d8259ccd99fd4b7 Mon Sep 17 00:00:00 2001 From: Matthew Pope <81593196+popematt@users.noreply.github.com> Date: Thu, 9 May 2024 21:08:07 -0700 Subject: [PATCH] Adds BufferedAppendableFastAppendable (#845) --- .../impl/BufferedAppendableFastAppendable.kt | 52 +++++++++++++++++++ ... => BufferedOutputStreamFastAppendable.kt} | 16 ++++-- .../ion/impl/bin/IonManagedWriter_1_1.kt | 25 ++++++++- .../ion/impl/bin/ManagedWriterOptions_1_1.kt | 2 +- src/test/java/com/amazon/ion/Ion11Test.kt | 36 ++++++++++++- 5 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 src/main/java/com/amazon/ion/impl/BufferedAppendableFastAppendable.kt rename src/main/java/com/amazon/ion/impl/{BlockBufferingOutputStreamFastAppendable.kt => BufferedOutputStreamFastAppendable.kt} (95%) diff --git a/src/main/java/com/amazon/ion/impl/BufferedAppendableFastAppendable.kt b/src/main/java/com/amazon/ion/impl/BufferedAppendableFastAppendable.kt new file mode 100644 index 000000000..6afed56d9 --- /dev/null +++ b/src/main/java/com/amazon/ion/impl/BufferedAppendableFastAppendable.kt @@ -0,0 +1,52 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.impl + +import com.amazon.ion.impl.bin.* +import com.amazon.ion.util.* +import java.io.Closeable +import java.io.Flushable + +/** + * A [_Private_FastAppendable] that buffers data to a [StringBuilder]. Only when + * [flush] is called is the data written to the wrapped [Appendable]. + * + * This is necessary for cases where an [IonManagedWriter_1_1] over Ion text needs to emit encoding directives that are + * not known in advance. The [AppendableFastAppendable] class has no buffering, so system and user values would be + * emitted in the wrong order. + * + * Once [IonManagedWriter_1_1] supports an auto-flush feature, then this class will have very little practical + * difference from [AppendableFastAppendable] for the case where no system values are needed. + * + * TODO: + * - Add proper tests + * + * @see BufferedOutputStreamFastAppendable + * @see AppendableFastAppendable + */ +internal class BufferedAppendableFastAppendable( + private val wrapped: Appendable, + private val buffer: StringBuilder = StringBuilder() +) : _Private_FastAppendable, Flushable, Closeable, Appendable by buffer { + + override fun appendAscii(c: Char) { append(c) } + override fun appendAscii(csq: CharSequence?) { append(csq) } + override fun appendAscii(csq: CharSequence?, start: Int, end: Int) { append(csq, start, end) } + override fun appendUtf16(c: Char) { append(c) } + + override fun appendUtf16Surrogate(leadSurrogate: Char, trailSurrogate: Char) { + append(leadSurrogate) + append(trailSurrogate) + } + + override fun close() { + flush() + if (wrapped is Closeable) wrapped.close() + } + + override fun flush() { + wrapped.append(buffer) + if (wrapped is Flushable) wrapped.flush() + buffer.setLength(0) + } +} diff --git a/src/main/java/com/amazon/ion/impl/BlockBufferingOutputStreamFastAppendable.kt b/src/main/java/com/amazon/ion/impl/BufferedOutputStreamFastAppendable.kt similarity index 95% rename from src/main/java/com/amazon/ion/impl/BlockBufferingOutputStreamFastAppendable.kt rename to src/main/java/com/amazon/ion/impl/BufferedOutputStreamFastAppendable.kt index 960d1a8f4..46fe52c70 100644 --- a/src/main/java/com/amazon/ion/impl/BlockBufferingOutputStreamFastAppendable.kt +++ b/src/main/java/com/amazon/ion/impl/BufferedOutputStreamFastAppendable.kt @@ -21,8 +21,11 @@ import java.io.OutputStream * * TODO: * - Add proper tests + * + * @see BufferedAppendableFastAppendable + * @see OutputStreamFastAppendable */ -internal class BlockBufferingOutputStreamFastAppendable( +internal class BufferedOutputStreamFastAppendable( private val out: OutputStream, private val allocator: BlockAllocator, /** @@ -54,9 +57,13 @@ internal class BlockBufferingOutputStreamFastAppendable( } override fun close() { - flush() - blocks.onEach { it.close() }.clear() - index = Int.MIN_VALUE + try { + flush() + } finally { + blocks.onEach { it.close() }.clear() + index = Int.MIN_VALUE + out.close() + } } override fun flush() { @@ -66,6 +73,7 @@ internal class BlockBufferingOutputStreamFastAppendable( } index = 0 current = blocks[index] + out.flush() } override fun write(b: Int) { diff --git a/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt b/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt index 33c09c7d6..7f1d1c4b3 100644 --- a/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt +++ b/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt @@ -40,7 +40,7 @@ internal class IonManagedWriter_1_1( textOptions as _Private_IonTextWriterBuilder val appender = { - val bufferedOutput = BlockBufferingOutputStreamFastAppendable(output, BlockAllocatorProviders.basicProvider().vendAllocator(4096)) + val bufferedOutput = BufferedOutputStreamFastAppendable(output, BlockAllocatorProviders.basicProvider().vendAllocator(4096)) _Private_IonTextAppender.forFastAppendable(bufferedOutput, Charsets.UTF_8) } @@ -58,6 +58,29 @@ internal class IonManagedWriter_1_1( ) } + @JvmStatic + fun textWriter(output: Appendable, managedWriterOptions: ManagedWriterOptions_1_1, textOptions: IonTextWriterBuilder): IonManagedWriter_1_1 { + textOptions as _Private_IonTextWriterBuilder + + val appender = { + val bufferedOutput = BufferedAppendableFastAppendable(output) + _Private_IonTextAppender.forFastAppendable(bufferedOutput, Charsets.UTF_8) + } + + return IonManagedWriter_1_1( + userData = IonRawTextWriter_1_1( + options = textOptions, + output = appender(), + ), + systemData = IonRawTextWriter_1_1( + options = textOptions, + output = appender(), + ), + options = managedWriterOptions.copy(internEncodingDirectiveSymbols = false), + onClose = {}, + ) + } + @JvmStatic fun binaryWriter(output: OutputStream, managedWriterOptions: ManagedWriterOptions_1_1, binaryOptions: _Private_IonBinaryWriterBuilder_1_1): IonManagedWriter_1_1 { // TODO: Add autoflush diff --git a/src/main/java/com/amazon/ion/impl/bin/ManagedWriterOptions_1_1.kt b/src/main/java/com/amazon/ion/impl/bin/ManagedWriterOptions_1_1.kt index f3d4a7f57..b44330517 100644 --- a/src/main/java/com/amazon/ion/impl/bin/ManagedWriterOptions_1_1.kt +++ b/src/main/java/com/amazon/ion/impl/bin/ManagedWriterOptions_1_1.kt @@ -12,7 +12,7 @@ data class ManagedWriterOptions_1_1( * For binary, almost certainly want this to be true, and for text, it's * more readable if it's false. */ - val internEncodingDirectiveSymbols: Boolean = false, + val internEncodingDirectiveSymbols: Boolean, val symbolInliningStrategy: SymbolInliningStrategy, val delimitedContainerStrategy: DelimitedContainerStrategy, ) : SymbolInliningStrategy by symbolInliningStrategy, DelimitedContainerStrategy by delimitedContainerStrategy { diff --git a/src/test/java/com/amazon/ion/Ion11Test.kt b/src/test/java/com/amazon/ion/Ion11Test.kt index 95b90505a..9dc772001 100644 --- a/src/test/java/com/amazon/ion/Ion11Test.kt +++ b/src/test/java/com/amazon/ion/Ion11Test.kt @@ -9,8 +9,7 @@ import com.amazon.ion.TestUtils.GOOD_IONTESTS_FILES import com.amazon.ion.TestUtils.TEXT_ONLY_FILTER import com.amazon.ion.TestUtils.hexStringToByteArray import com.amazon.ion.TestUtils.testdataFiles -import com.amazon.ion.impl.bin.DelimitedContainerStrategy -import com.amazon.ion.impl.bin.SymbolInliningStrategy +import com.amazon.ion.impl.bin.* import com.amazon.ion.system.IonBinaryWriterBuilder import com.amazon.ion.system.IonSystemBuilder import com.amazon.ion.system.IonTextWriterBuilder @@ -88,6 +87,27 @@ class Ion11Test { } } + @ParameterizedTest(name = "{0}") + @MethodSource("ionData") + fun writeIon11TextToAppendable(name: String, ion: ByteArray) { + val textOptions = IonTextWriterBuilder + .standard() + .withNewLineType(IonTextWriterBuilder.NewLineType.LF) + + textTestAppendable(ion) { + IonManagedWriter_1_1.textWriter( + output = it, + managedWriterOptions = ManagedWriterOptions_1_1( + internEncodingDirectiveSymbols = false, + // Test using NEVER_INLINE to make sure that the BufferedAppendableFastAppendable works correctly. + symbolInliningStrategy = SymbolInliningStrategy.NEVER_INLINE, + delimitedContainerStrategy = DelimitedContainerStrategy.ALWAYS_DELIMITED + ), + textOptions = textOptions, + ) + } + } + @ParameterizedTest(name = "{0}") @MethodSource("ionData") fun writeIon11TextWithSymtab(name: String, ion: ByteArray) { @@ -143,6 +163,18 @@ class Ion11Test { assertEquals(data, loadedData.toList()) } + fun textTestAppendable(ion: ByteArray, writerFn: (Appendable) -> IonWriter) { + val data: List = ION.loader.load(ion).map { it } + val appendable = StringBuilder() + val writer = writerFn(appendable) + data.forEach { it.writeTo(writer) } + writer.close() + println(appendable.toString()) + val loadedData = ION.loader.load(appendable.toString()) + println(loadedData) + assertEquals(data, loadedData.toList()) + } + fun binaryTest(ion: ByteArray, writerFn: (OutputStream) -> IonWriter) { val data: List = ION.loader.load(ion).map { it } val baos = ByteArrayOutputStream()