From 8589c4f47f4e0d2aa930f96d7ed56316b22660e3 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 3 Dec 2024 14:15:52 -0800 Subject: [PATCH] Allows values to be macro-aware transcoded value-by-value, adds support for creating macro-aware readers from InputStream, and improves testing. --- .../com/amazon/ion/MacroAwareIonReader.kt | 27 +- .../impl/IonReaderContinuableCoreBinary.java | 79 ++++- .../ion/impl/_Private_IonReaderBuilder.java | 61 ++-- .../com/amazon/ion/Ion_1_1_RoundTripTest.kt | 2 +- .../EncodingDirectiveCompilationTest.java | 324 +++++++++++++++--- .../IonReaderContinuableCoreBinaryTest.java | 2 +- 6 files changed, 405 insertions(+), 90 deletions(-) diff --git a/src/main/java/com/amazon/ion/MacroAwareIonReader.kt b/src/main/java/com/amazon/ion/MacroAwareIonReader.kt index b05a7b09d..5d06a333c 100644 --- a/src/main/java/com/amazon/ion/MacroAwareIonReader.kt +++ b/src/main/java/com/amazon/ion/MacroAwareIonReader.kt @@ -11,9 +11,27 @@ import java.io.IOException interface MacroAwareIonReader : Closeable { /** - * Performs a macro-aware transcode of the stream being read by this reader. + * Performs a macro-aware transcode of all values in the stream. This is + * shorthand for calling [prepareTranscodeTo], then calling [transcodeNext] + * repetitively until it returns `false`. + * @param writer the writer to which the reader's stream will be transcoded. + */ + @Throws(IOException::class) + fun transcodeAllTo(writer: MacroAwareIonWriter) + + /** + * Prepares the reader to perform a macro-aware transcode to the given + * writer. This must be called before calling [transcodeNext], but is not + * necessary if calling [transcodeAllTo]. + * @param writer the writer to which the reader's stream will be transcoded. + */ + fun prepareTranscodeTo(writer: MacroAwareIonWriter) + + /** + * Performs a macro-aware transcode of the next value read by this reader + * to the writer previously provided to a call to [prepareTranscodeTo]. * For Ion 1.0 streams, this functions similarly to providing a system-level - * [IonReader] to [IonWriter.writeValues]. For Ion 1.1 streams, the transcoded + * [IonReader] to [IonWriter.writeValue]. For Ion 1.1 streams, the transcoded * stream will include the same symbol tables, encoding directives, and * e-expression invocations as the source stream. In both cases, the * transcoded stream will be data-model equivalent to the source stream. @@ -34,8 +52,9 @@ interface MacroAwareIonReader : Closeable { * To get a [MacroAwareIonReader] use `_Private_IonReaderBuilder.buildMacroAware`. * To get a [MacroAwareIonWriter] use [IonEncodingVersion.textWriterBuilder] or * [IonEncodingVersion.binaryWriterBuilder]. - * @param writer the writer to which the reader's stream will be transcoded. + * @return true if a value was transcoded; false if the end of the stream was reached. + * @throws IOException if thrown during writing. */ @Throws(IOException::class) - fun transcodeTo(writer: MacroAwareIonWriter) + fun transcodeNext(): Boolean } diff --git a/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java b/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java index 58bc5abd1..77372976a 100644 --- a/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java +++ b/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java @@ -160,6 +160,9 @@ class IonReaderContinuableCoreBinary extends IonCursorBinary implements IonReade // Indicates whether the reader is currently evaluating an e-expression. protected boolean isEvaluatingEExpression = false; + // The writer that will perform a macro-aware transcode, if requested. + private MacroAwareIonWriter macroAwareTranscoder = null; + /** * Constructs a new reader from the given byte array. * @param configuration the configuration to use. The buffer size and oversized value configuration are unused, as @@ -1799,30 +1802,34 @@ private boolean evaluateNext() { } @Override - public void transcodeTo(MacroAwareIonWriter writer) throws IOException { + public void transcodeAllTo(MacroAwareIonWriter writer) throws IOException { + prepareTranscodeTo(writer); + while (transcodeNext()); + } + + @Override + public void prepareTranscodeTo(MacroAwareIonWriter writer) { registerIvmNotificationConsumer((major, minor) -> { resetEncodingContext(); // Which IVM to write is inherent to the writer implementation. // We don't have a single implementation that writes both formats. writer.startEncodingSegmentWithIonVersionMarker(); }); - while (transcodeNextTo(writer) != Event.NEEDS_DATA); + macroAwareTranscoder = writer; } - /** - * Transcodes the next value, and any encoding directives that may precede it, - * to the given writer. - * @param writer the writer to which the value will be transcoded. - * @return the result of the operation. - * @throws IOException if thrown during writing. - */ - Event transcodeNextTo(MacroAwareIonWriter writer) throws IOException { + @Override + public boolean transcodeNext() throws IOException { + if (macroAwareTranscoder == null) { + throw new IllegalArgumentException("prepareTranscodeTo must be called before transcodeNext."); + } // NOTE: this method is structured very similarly to nextValue(). During performance analysis, we should // see if the methods can be unified without sacrificing hot path performance. Performance of this method // is not considered critical. lobBytesRead = 0; while (true) { if (parent == null || state != State.READING_VALUE) { + boolean isEncodingDirective = false; if (state != State.READING_VALUE && state != State.COMPILING_MACRO) { boolean isEncodingDirectiveFromEExpression = isEvaluatingEExpression; encodingDirectiveReader.readEncodingDirective(); @@ -1832,17 +1839,22 @@ Event transcodeNextTo(MacroAwareIonWriter writer) throws IOException { // If the encoding directive was expanded from an e-expression, that expression has already been // written. In that case, just make sure the writer is using the new context. Otherwise, also write // the encoding directive. - writer.startEncodingSegmentWithEncodingDirective( + macroAwareTranscoder.startEncodingSegmentWithEncodingDirective( encodingDirectiveReader.newMacros, encodingDirectiveReader.isMacroTableAppend, encodingDirectiveReader.newSymbols, encodingDirectiveReader.isSymbolTableAppend, isEncodingDirectiveFromEExpression ); + isEncodingDirective = true; } if (isEvaluatingEExpression) { if (evaluateNext()) { - continue; + if (isEncodingDirective) { + continue; + } + // This is the end of a top-level macro invocation that expanded to a user value. + return true; } } else { event = super.nextValue(); @@ -1854,6 +1866,7 @@ Event transcodeNextTo(MacroAwareIonWriter writer) throws IOException { } } else if (isEvaluatingEExpression) { if (evaluateNext()) { + // This is the end of a contained macro invocation; continue iterating through the parent container. continue; } } else { @@ -1861,9 +1874,10 @@ Event transcodeNextTo(MacroAwareIonWriter writer) throws IOException { } if (valueTid != null && valueTid.isMacroInvocation) { expressionArgsReader.beginEvaluatingMacroInvocation(macroEvaluator); - macroEvaluatorIonReader.transcodeArgumentsTo(writer); + macroEvaluatorIonReader.transcodeArgumentsTo(macroAwareTranscoder); isEvaluatingEExpression = true; if (evaluateNext()) { + // This macro invocation expands to nothing; continue iterating until a user value is found. continue; } if (parent == null && isPositionedOnEvaluatedEncodingDirective()) { @@ -1878,15 +1892,44 @@ Event transcodeNextTo(MacroAwareIonWriter writer) throws IOException { } break; } - if (event != Event.NEEDS_DATA) { - if (minorVersion > 0 && isPositionedOnSymbolTable()) { + if (event == Event.NEEDS_DATA || event == Event.END_CONTAINER) { + return false; + } + transcodeValueLiteral(); + return true; + } + + /** + * Transcodes a value literal to the macroAwareTranscoder. The caller must ensure that the reader is positioned + * on a value literal (i.e. a scalar or container value not expanded from an e-expression) before calling this + * method. + * @throws IOException if thrown by the writer during transcoding. + */ + private void transcodeValueLiteral() throws IOException { + if (parent == null && isPositionedOnSymbolTable()) { + if (minorVersion > 0) { // TODO finalize handling of Ion 1.0-style symbol tables in Ion 1.1: https://github.com/amazon-ion/ion-java/issues/1002 throw new IonException("Macro-aware transcoding of Ion 1.1 data containing Ion 1.0-style symbol tables not yet supported."); } - // The reader is now positioned on an actual encoding value. Write the value. - writer.writeValue(asIonReader); + // Ion 1.0 symbol tables are transcoded verbatim for now; this may change depending on the resolution to + // https://github.com/amazon-ion/ion-java/issues/1002. + macroAwareTranscoder.writeValue(asIonReader); + } else if (event == Event.START_CONTAINER && !isNullValue()) { + // Containers need to be transcoded recursively to avoid expanding macro invocations at any depth. + if (isInStruct()) { + macroAwareTranscoder.setFieldNameSymbol(getFieldNameSymbol()); + } + macroAwareTranscoder.setTypeAnnotationSymbols(asIonReader.getTypeAnnotationSymbols()); + macroAwareTranscoder.stepIn(getEncodingType()); + super.stepIntoContainer(); + while (transcodeNext()); // TODO make this iterative. + super.stepOutOfContainer(); + macroAwareTranscoder.stepOut(); + } else { + // The reader is now positioned on a scalar literal. Write the value. + // Note: writeValue will include any field name and/or annotations on the scalar. + macroAwareTranscoder.writeValue(asIonReader); } - return event; } @Override diff --git a/src/main/java/com/amazon/ion/impl/_Private_IonReaderBuilder.java b/src/main/java/com/amazon/ion/impl/_Private_IonReaderBuilder.java index 3344b4732..3ee99ff4b 100644 --- a/src/main/java/com/amazon/ion/impl/_Private_IonReaderBuilder.java +++ b/src/main/java/com/amazon/ion/impl/_Private_IonReaderBuilder.java @@ -183,26 +183,26 @@ public void close() throws IOException { } @FunctionalInterface - interface IonReaderFromBytesFactoryText { - IonReader makeReader(IonCatalog catalog, byte[] ionData, int offset, int length, _Private_LocalSymbolTableFactory lstFactory); + interface IonReaderFromBytesFactoryText { + T makeReader(IonCatalog catalog, byte[] ionData, int offset, int length, _Private_LocalSymbolTableFactory lstFactory); } @FunctionalInterface - interface IonReaderFromBytesFactoryBinary { - IonReader makeReader(_Private_IonReaderBuilder builder, byte[] ionData, int offset, int length); + interface IonReaderFromBytesFactoryBinary { + T makeReader(_Private_IonReaderBuilder builder, byte[] ionData, int offset, int length); } - static IonReader buildReader( + static T buildReader( _Private_IonReaderBuilder builder, byte[] ionData, int offset, int length, - IonReaderFromBytesFactoryBinary binary, - IonReaderFromBytesFactoryText text + IonReaderFromBytesFactoryBinary binary, + IonReaderFromBytesFactoryText text ) { if (IonStreamUtils.isGzip(ionData, offset, length)) { try { - return buildReader( + return (T) buildReader( builder, new GZIPInputStream(new ByteArrayInputStream(ionData, offset, length)), _Private_IonReaderFactory::makeReaderBinary, @@ -257,20 +257,20 @@ private static boolean startsWithGzipHeader(byte[] buffer, int length) { } @FunctionalInterface - interface IonReaderFromInputStreamFactoryText { - IonReader makeReader(IonCatalog catalog, InputStream source, _Private_LocalSymbolTableFactory lstFactory); + interface IonReaderFromInputStreamFactoryText { + T makeReader(IonCatalog catalog, InputStream source, _Private_LocalSymbolTableFactory lstFactory); } @FunctionalInterface - interface IonReaderFromInputStreamFactoryBinary { - IonReader makeReader(_Private_IonReaderBuilder builder, InputStream source, byte[] alreadyRead, int alreadyReadOff, int alreadyReadLen); + interface IonReaderFromInputStreamFactoryBinary { + T makeReader(_Private_IonReaderBuilder builder, InputStream source, byte[] alreadyRead, int alreadyReadOff, int alreadyReadLen); } - static IonReader buildReader( + static T buildReader( _Private_IonReaderBuilder builder, InputStream source, - IonReaderFromInputStreamFactoryBinary binary, - IonReaderFromInputStreamFactoryText text + IonReaderFromInputStreamFactoryBinary binary, + IonReaderFromInputStreamFactoryText text ) { if (source == null) { throw new NullPointerException("Cannot build a reader from a null InputStream."); @@ -358,10 +358,31 @@ public IonTextReader build(String ionText) { * @return a new MacroAwareIonReader instance. */ public MacroAwareIonReader buildMacroAware(byte[] ionData) { - // TODO make this work for text too. - if (!IonStreamUtils.isIonBinary(ionData)) { - throw new UnsupportedOperationException("MacroAwareIonReader is not yet implemented for text data."); - } - return new IonReaderContinuableCoreBinary(getBufferConfiguration(), ionData, 0, ionData.length); + return buildReader( + this, + ionData, + 0, + ionData.length, + (builder, data, offset, length) -> new IonReaderContinuableCoreBinary(builder.getBufferConfiguration(), data, offset,length), + (catalog, data, offset, length, factory) -> { + throw new UnsupportedOperationException("MacroAwareIonReader is not yet implemented for text data."); + } + ); + } + + /** + * Creates a new {@link MacroAwareIonReader} over the given data. + * @param ionData the data to read. + * @return a new MacroAwareIonReader instance. + */ + public MacroAwareIonReader buildMacroAware(InputStream ionData) { + return buildReader( + this, + ionData, + (builder, source, alreadyRead, alreadyReadOff, alreadyReadLen) -> new IonReaderContinuableCoreBinary(builder.getBufferConfiguration(), source, alreadyRead, alreadyReadOff, alreadyReadLen), + (catalog, source, factory) -> { + throw new UnsupportedOperationException("MacroAwareIonReader is not yet implemented for text data."); + } + ); } } diff --git a/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt b/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt index 52f7b5e3a..1a1764188 100644 --- a/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt +++ b/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt @@ -262,7 +262,7 @@ class Ion_1_1_RoundTripTest { val reader: MacroAwareIonReader = (IonReaderBuilder.standard() as _Private_IonReaderBuilder).buildMacroAware(ion) val writer: MacroAwareIonWriter = ION_1_1.textWriterBuilder().build(actual) as MacroAwareIonWriter - reader.transcodeTo(writer) + reader.transcodeAllTo(writer) reader.close() writer.close() diff --git a/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java b/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java index 3178f4514..1abe53967 100644 --- a/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java +++ b/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java @@ -29,7 +29,6 @@ import org.hamcrest.Description; import org.hamcrest.Matcher; import org.hamcrest.TypeSafeMatcher; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -300,7 +299,7 @@ IonReader newReader(byte[] input) { @Override MacroAwareIonReader newMacroAwareReader(byte[] input) { - throw new UnsupportedOperationException("Building MacroAwareIonReader from InputStream not yet supported."); + return ((_Private_IonReaderBuilder) IonReaderBuilder.standard()).buildMacroAware(new ByteArrayInputStream(input)); } }, BYTE_ARRAY { @@ -564,9 +563,7 @@ public void structMacroWithOneOptionalInvoked(InputType inputType, StreamType st } } - @ParameterizedTest(name = "{0},{1}") - @MethodSource("allCombinations") - public void macroInvocationWithinStruct(InputType inputType, StreamType streamType) throws Exception { + private byte[] macroInvocationWithinStruct(StreamType streamType, SortedMap expectedMacroTable) { ByteArrayOutputStream out = new ByteArrayOutputStream(); IonRawWriter_1_1 writer = streamType.newWriter(out); writer.writeIVM(); @@ -586,7 +583,6 @@ public void macroInvocationWithinStruct(InputType inputType, StreamType streamTy endMacroTable(writer); endEncodingDirective(writer); - SortedMap expectedMacroTable = new TreeMap<>(); expectedMacroTable.put("People", new TemplateMacro( Arrays.asList( new Macro.Parameter("$ID", Macro.ParameterEncoding.Tagged, Macro.ParameterCardinality.ExactlyOne), @@ -616,7 +612,14 @@ public void macroInvocationWithinStruct(InputType inputType, StreamType streamTy writer.stepOut(); writer.stepOut(); - byte[] data = getBytes(writer, out); + return getBytes(writer, out); + } + + @ParameterizedTest(name = "{0},{1}") + @MethodSource("allCombinations") + public void macroInvocationWithinStruct(InputType inputType, StreamType streamType) throws Exception { + SortedMap expectedMacroTable = new TreeMap<>(); + byte[] data = macroInvocationWithinStruct(streamType, expectedMacroTable); try (IonReader reader = inputType.newReader(data)) { assertEquals(IonType.STRUCT, reader.next()); @@ -638,6 +641,179 @@ public void macroInvocationWithinStruct(InputType inputType, StreamType streamTy } } + /** + * Performs a macro-aware transcode by repetitively calling {@link MacroAwareIonReader#transcodeNext()}. + * @param data the data to transcode. + * @param inputType the input type for the data to transcode. + * @param outputFormat the output format for the transcoded data. + * @param numberOfValues the number of values to transcode. + * @param assertEnd true if, after transcoding the requested number of values, this method should assert that + * calling `transcodeNext()` one more time would result in stream end (i.e., return `false`). + * @return a stream containing the transcoded data. + * @throws Exception if thrown during transcoding. + */ + private ByteArrayOutputStream macroAwareTranscodeValueByValue( + byte[] data, + InputType inputType, + StreamType outputFormat, + int numberOfValues, + boolean assertEnd + ) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try ( + MacroAwareIonReader reader = inputType.newMacroAwareReader(data); + MacroAwareIonWriter rewriter = outputFormat.newMacroAwareWriter(out) + ) { + reader.prepareTranscodeTo(rewriter); + for (int i = 0; i < numberOfValues; i++) { + assertTrue(reader.transcodeNext()); + } + if (assertEnd) { + assertFalse(reader.transcodeNext()); + } + } + return out; + } + + // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added + @ParameterizedTest(name = "{0},{1}") + @MethodSource("allCombinations") + public void nestedInvocationMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { + byte[] data = macroInvocationWithinStruct(StreamType.BINARY, new TreeMap<>()); + + ByteArrayOutputStream out = macroAwareTranscodeValueByValue(data, inputType, outputFormat, 1, false); + + verifyStream(data, out, outputFormat, + substringCount("$ion_1_1", 1), + substringCount(SystemSymbols_1_1.ION_ENCODING, 2), + substringCount(SystemSymbols_1_1.SYMBOL_TABLE, 2), + substringCount(SystemSymbols_1_1.MACRO_TABLE, 1), + substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 0), + substringCount(SystemSymbols_1_1.ADD_MACROS, 0), + substringCount(SystemSymbols_1_1.SET_SYMBOLS, 0), + substringCount(SystemSymbols_1_1.SET_MACROS, 0), + substringCount("(:People", 1) + ); + } + + // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added + @ParameterizedTest(name = "{0},{1}") + @MethodSource("allCombinations") + public void multipleNestedInvocationMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { + ByteArrayOutputStream source = new ByteArrayOutputStream(); + IonRawWriter_1_1 writer = StreamType.BINARY.newWriter(source); + writer.writeIVM(); + + writeSymbolTableEExpression(false, writer, "foo", "bar", "baz", "zar"); + + writer.stepInStruct(true); + writer.writeFieldName(FIRST_LOCAL_SYMBOL_ID); // foo + writer.stepInEExp(SystemMacro.Values); + writer.stepInExpressionGroup(false); + writer.writeInt(1); + writer.writeInt(2); + writer.stepOut(); + writer.stepOut(); + writer.writeFieldName(FIRST_LOCAL_SYMBOL_ID + 1); // bar + writer.stepInEExp(SystemMacro.Values); + writer.stepInExpressionGroup(false); + writer.writeInt(3); + writer.writeInt(4); + writer.stepOut(); + writer.stepOut(); + writer.writeFieldName(FIRST_LOCAL_SYMBOL_ID + 2); // baz + writer.writeAnnotations(FIRST_LOCAL_SYMBOL_ID); // foo + writer.writeInt(5); + writer.writeFieldName(FIRST_LOCAL_SYMBOL_ID + 3); // zar + writer.writeAnnotations(FIRST_LOCAL_SYMBOL_ID + 1); // bar + writer.stepInStruct(true); + writer.stepOut(); + writer.stepOut(); + writer.writeInt(123); + + byte[] data = getBytes(writer, source); + ByteArrayOutputStream out = macroAwareTranscodeValueByValue(data, inputType, outputFormat, 2, true); + + verifyStream(data, out, outputFormat, + substringCount("$ion_1_1", 1), + substringCount(SystemSymbols_1_1.ION_ENCODING, 0), + substringCount(SystemSymbols_1_1.SYMBOL_TABLE, 0), + substringCount(SystemSymbols_1_1.MACRO_TABLE, 0), + substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 0), + substringCount(SystemSymbols_1_1.ADD_MACROS, 0), + substringCount(SystemSymbols_1_1.SET_SYMBOLS, 1), + substringCount(SystemSymbols_1_1.SET_MACROS, 0), + substringCount(SystemSymbols_1_1.VALUES, 2) + ); + } + + private byte[] zeroArgMacroThatExpandsToEncodingDirective(StreamType outputFormat) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonRawWriter_1_1 writer = outputFormat.newWriter(out); + writer.writeIVM(); + + Map symbols = initializeSymbolTable(writer, "foo", "bar"); + + startEncodingDirective(writer); + startMacroTable(writer); + startMacro(writer, symbols, "abcdef"); + writeMacroSignature(writer, symbols); // empty signature + // The body: an encoding directive that sets the symbol table to ["abc", "def"] + startEncodingDirective(writer); + writeEncodingDirectiveSymbolTable(writer, "abc", "def"); + endEncodingDirective(writer); + endMacro(writer); + endMacroTable(writer); + endEncodingDirective(writer); + + SortedMap expectedMacroTable = new TreeMap<>(); + expectedMacroTable.put("abcdef", new TemplateMacro( + Collections.emptyList(), + Arrays.asList( + new Expression.SExpValue(Collections.singletonList(new FakeSymbolToken(SystemSymbols_1_1.ION_ENCODING.name(), SystemSymbols_1_1.ION_ENCODING.getId())), 0, 5), + new Expression.SExpValue(Collections.emptyList(), 1, 5), + new Expression.SymbolValue(Collections.emptyList(), new FakeSymbolToken(SystemSymbols_1_1.SYMBOL_TABLE.name(), SystemSymbols_1_1.SYMBOL_TABLE.getId())), + new Expression.ListValue(Collections.emptyList(), 3, 5), + new Expression.StringValue(Collections.emptyList(), "abc"), + new Expression.StringValue(Collections.emptyList(), "def") + ) + )); + + outputFormat.startMacroInvocationByName(writer, "abcdef", expectedMacroTable); + writer.stepOut(); + writer.writeSymbol(FIRST_LOCAL_SYMBOL_ID + 1); // def + writer.writeSymbol(FIRST_LOCAL_SYMBOL_ID); // abc + + return getBytes(writer, out); + } + + @ParameterizedTest(name = "{0},{1}") + @MethodSource("allCombinations") + public void zeroArgMacroThatExpandsToEncodingDirective(InputType inputType, StreamType streamType) throws Exception { + byte[] data = zeroArgMacroThatExpandsToEncodingDirective(streamType); + try (IonReader reader = inputType.newReader(data)) { + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("def", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("abc", reader.stringValue()); + assertNull(reader.next()); + } + } + + // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added + @ParameterizedTest(name = "{0},{1}") + @MethodSource("allCombinations") + public void zeroArgMacroThatExpandsToEncodingDirectiveMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { + byte[] data = zeroArgMacroThatExpandsToEncodingDirective(StreamType.BINARY); + ByteArrayOutputStream out = macroAwareTranscodeValueByValue(data, inputType, outputFormat, 2, true); + + verifyStream("def abc".getBytes(StandardCharsets.UTF_8), out, outputFormat, + substringCount("$ion_1_1", 1), + substringCount(SystemSymbols_1_1.ION_ENCODING, 3), // Initial symbols, directive with macro, macro body with encoding directive + substringCount("(:abcdef)", 1) + ); + } + @ParameterizedTest(name = "{0},{1}") @MethodSource("allCombinations") public void macroInvocationWithOptionalSuppressedBeforeEndWithinStruct(InputType inputType, StreamType streamType) throws Exception { @@ -952,6 +1128,31 @@ static SubstringCountMatcher substringCount(SystemSymbols_1_1 sub, int count) { return new SubstringCountMatcher(sub.getText(), count); } + /** + * Verifies a stream has the characteristics described by the arguments to this method and that it is data-model + * equivalent to the expected output. + * @param expectedOutput the expected output. + * @param actualOutput the actual output. + * @param streamType the StreamType to which the source data will be transcoded. + * @param expectations a list of expectations for the text representation of the transcoded data. + */ + @SafeVarargs + private static void verifyStream( + byte[] expectedOutput, + ByteArrayOutputStream actualOutput, + StreamType streamType, + Matcher... expectations + ) throws Exception { + if (streamType == StreamType.TEXT) { + String rewritten = actualOutput.toString(StandardCharsets.UTF_8.name()); + assertThat(rewritten, allOf(expectations)); + } + IonSystem system = IonSystemBuilder.standard().build(); + IonDatagram actual = system.getLoader().load(actualOutput.toByteArray()); + IonDatagram expected = system.getLoader().load(expectedOutput); + assertEquals(expected, actual); + } + /** * Performs a macro-aware transcode of the given data, verifying that the resulting stream has the * characteristics described by the arguments to this method and that it is data-model equivalent @@ -961,7 +1162,8 @@ static SubstringCountMatcher substringCount(SystemSymbols_1_1 sub, int count) { * @param streamType the StreamType to which the source data will be transcoded. * @param expectations a list of expectations for the text representation of the transcoded data. */ - private void verifyMacroAwareTranscode( + @SafeVarargs + private static void verifyMacroAwareTranscode( byte[] data, InputType inputType, StreamType streamType, @@ -972,22 +1174,17 @@ private void verifyMacroAwareTranscode( MacroAwareIonReader reader = inputType.newMacroAwareReader(data); MacroAwareIonWriter rewriter = streamType.newMacroAwareWriter(out); ) { - reader.transcodeTo(rewriter); - } - if (streamType == StreamType.TEXT) { - String rewritten = out.toString(StandardCharsets.UTF_8.name()); - assertThat(rewritten, allOf(expectations)); + reader.transcodeAllTo(rewriter); } - IonSystem system = IonSystemBuilder.standard().build(); - IonDatagram actual = system.getLoader().load(out.toByteArray()); - IonDatagram expected = system.getLoader().load(data); - assertEquals(expected, actual); + verifyStream(data, out, streamType, expectations); } - @Test // TODO parameterize for all combinations once support for macro-aware text reading is added - public void macroInvocationsNestedWithinParameterMacroAwareTranscode() throws Exception { + // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added + @ParameterizedTest(name = "{0},{1}") + @MethodSource("allCombinations") + public void macroInvocationsNestedWithinParameterMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { byte[] data = macroInvocationsNestedWithinParameter(StreamType.BINARY); - verifyMacroAwareTranscode(data, InputType.BYTE_ARRAY, StreamType.TEXT, + verifyMacroAwareTranscode(data, inputType, outputFormat, substringCount("$ion_1_1", 1), substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 0), substringCount(SystemSymbols_1_1.ADD_MACROS, 0), @@ -1151,10 +1348,12 @@ public void macroInvocationInMacroDefinition(InputType inputType, StreamType str } } - @Test // TODO parameterize for all combinations once support for macro-aware text reading is added - public void macroInvocationInMacroDefinitionMacroAwareTranscode() throws Exception { + // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added + @ParameterizedTest(name = "{0},{1}") + @MethodSource("allCombinations") + public void macroInvocationInMacroDefinitionMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { byte[] data = macroInvocationInMacroDefinition(StreamType.BINARY); - verifyMacroAwareTranscode(data, InputType.BYTE_ARRAY, StreamType.TEXT, + verifyMacroAwareTranscode(data, inputType, outputFormat, substringCount("$ion_1_1", 1), substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 0), substringCount(SystemSymbols_1_1.ADD_MACROS, 0), @@ -1359,10 +1558,12 @@ public void macroInvocationsProduceEncodingDirectivesThatModifySymbolTable(Input } } - @Test // TODO parameterize for all combinations once support for macro-aware text reading is added - public void macroInvocationsProduceEncodingDirectivesThatModifySymbolTableMacroAwareTranscode() throws Exception { + // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added + @ParameterizedTest(name = "{0},{1}") + @MethodSource("allCombinations") + public void macroInvocationsProduceEncodingDirectivesThatModifySymbolTableMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { byte[] data = macroInvocationsProduceEncodingDirectivesThatModifySymbolTable(StreamType.BINARY); - verifyMacroAwareTranscode(data, InputType.BYTE_ARRAY, StreamType.TEXT, + verifyMacroAwareTranscode(data, inputType, outputFormat, substringCount("$ion_1_1", 1), substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 1), substringCount(SystemSymbols_1_1.ADD_MACROS, 0), @@ -1458,10 +1659,12 @@ public void macroInvocationsProduceEncodingDirectivesThatModifyMacroTable(InputT } } - @Test // TODO parameterize for all combinations once support for macro-aware text reading is added - public void macroInvocationsProduceEncodingDirectivesThatModifyMacroTableMacroAwareTranscode() throws Exception { + // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added + @ParameterizedTest(name = "{0},{1}") + @MethodSource("allCombinations") + public void macroInvocationsProduceEncodingDirectivesThatModifyMacroTableMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { byte[] data = macroInvocationsProduceEncodingDirectivesThatModifyMacroTable(StreamType.BINARY); - verifyMacroAwareTranscode(data, InputType.BYTE_ARRAY, StreamType.TEXT, + verifyMacroAwareTranscode(data, inputType, outputFormat, substringCount("$ion_1_1", 1), substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 1), substringCount(SystemSymbols_1_1.ADD_MACROS, 2), @@ -1471,6 +1674,25 @@ public void macroInvocationsProduceEncodingDirectivesThatModifyMacroTableMacroAw ); } + // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added + @ParameterizedTest(name = "{0},{1}") + @MethodSource("allCombinations") + public void multiValuePartialMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { + byte[] data = macroInvocationsProduceEncodingDirectivesThatModifyMacroTable(StreamType.BINARY); + ByteArrayOutputStream out = macroAwareTranscodeValueByValue(data, inputType, outputFormat, 2, false); + + verifyStream("Pi 3.14159".getBytes(StandardCharsets.UTF_8), out, outputFormat, + substringCount("$ion_1_1", 1), + substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 1), + substringCount(SystemSymbols_1_1.ADD_MACROS, 1), + substringCount(SystemSymbols_1_1.SET_SYMBOLS, 0), + substringCount(SystemSymbols_1_1.SET_MACROS, 0), + substringCount(SystemSymbols_1_1.ION_ENCODING, 0), + substringCount("(:Pi)", 1), + substringCount("(:foo)", 0) + ); + } + @ParameterizedTest(name = "{0},{1}") @MethodSource("allCombinations") public void multipleListsWithinSymbolTableDeclaration(InputType inputType, StreamType streamType) throws Exception { @@ -1527,10 +1749,12 @@ public void emptyMacroAppendToEmptyTable(InputType inputType, StreamType streamT } } - @Test // TODO parameterize for all combinations once support for macro-aware text reading is added - public void emptyMacroAppendToEmptyTableMacroAwareTranscode() throws Exception { + // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added + @ParameterizedTest(name = "{0},{1}") + @MethodSource("allCombinations") + public void emptyMacroAppendToEmptyTableMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { byte[] data = emptyMacroAppendToEmptyTable(StreamType.BINARY); - verifyMacroAwareTranscode(data, InputType.BYTE_ARRAY, StreamType.TEXT, + verifyMacroAwareTranscode(data, inputType, outputFormat, substringCount("$ion_1_1", 1), substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 0), substringCount(SystemSymbols_1_1.ADD_MACROS, 0), @@ -1586,10 +1810,12 @@ public void emptyMacroAppendToNonEmptyTable(InputType inputType, StreamType stre } } - @Test // TODO parameterize for all combinations once support for macro-aware text reading is added - public void emptyMacroAppendToNonEmptyTableMacroAwareTranscode() throws Exception { + // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added + @ParameterizedTest(name = "{0},{1}") + @MethodSource("allCombinations") + public void emptyMacroAppendToNonEmptyTableMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { byte[] data = emptyMacroAppendToNonEmptyTable(StreamType.BINARY); - verifyMacroAwareTranscode(data, InputType.BYTE_ARRAY, StreamType.TEXT, + verifyMacroAwareTranscode(data, inputType, outputFormat, substringCount("$ion_1_1", 1), substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 0), substringCount(SystemSymbols_1_1.ADD_MACROS, 0), @@ -1646,10 +1872,12 @@ public void invokeUnqualifiedSystemMacroInTDL(InputType inputType, StreamType st } } - @Test // TODO parameterize for all combinations once support for macro-aware text reading is added - public void invokeUnqualifiedSystemMacroInTDLMacroAwareTranscode() throws Exception { + // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added + @ParameterizedTest(name = "{0},{1}") + @MethodSource("allCombinations") + public void invokeUnqualifiedSystemMacroInTDLMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { byte[] data = invokeUnqualifiedSystemMacroInTDL(StreamType.BINARY); - verifyMacroAwareTranscode(data, InputType.BYTE_ARRAY, StreamType.TEXT, + verifyMacroAwareTranscode(data, inputType, outputFormat, substringCount("$ion_1_1", 1), substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 0), substringCount(SystemSymbols_1_1.ADD_MACROS, 0), @@ -1659,8 +1887,10 @@ public void invokeUnqualifiedSystemMacroInTDLMacroAwareTranscode() throws Except ); } - @Test // TODO parameterize for all combinations once support for macro-aware text reading is added - public void multipleIonVersionMarkersMacroAwareTranscode() throws Exception { + // TODO also parameterize for StreamType inputFormat support for macro-aware text reading is added + @ParameterizedTest(name = "{0},{1}") + @MethodSource("allCombinations") + public void multipleIonVersionMarkersMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); IonRawWriter_1_1 writer = StreamType.BINARY.newWriter(out); Map symbols = new HashMap<>(); @@ -1671,7 +1901,7 @@ public void multipleIonVersionMarkersMacroAwareTranscode() throws Exception { writeSymbolTableAppendEExpression(writer, symbols, "bar"); // bar writer.writeSymbol(SystemSymbols_1_1.size() + FIRST_LOCAL_SYMBOL_ID); byte[] data = getBytes(writer, out); - verifyMacroAwareTranscode(data, InputType.BYTE_ARRAY, StreamType.TEXT, + verifyMacroAwareTranscode(data, inputType, outputFormat, substringCount("$ion_1_1", 2), substringCount(SystemSymbols_1_1.ADD_SYMBOLS, 2), substringCount(SystemSymbols_1_1.ADD_MACROS, 0), @@ -1681,8 +1911,10 @@ public void multipleIonVersionMarkersMacroAwareTranscode() throws Exception { ); } - @Test // TODO finalize handling of Ion 1.0-style symbol tables in Ion 1.1: https://github.com/amazon-ion/ion-java/issues/1002 - public void ion10SymbolTableMacroAwareTranscode() throws Exception { + // TODO finalize handling of Ion 1.0-style symbol tables in Ion 1.1: https://github.com/amazon-ion/ion-java/issues/1002 + @ParameterizedTest(name = "{0},{1}") + @MethodSource("allCombinations") + public void ion10SymbolTableMacroAwareTranscode(InputType inputType, StreamType outputFormat) throws Exception { byte[] data = bytes( 0xE0, 0x01, 0x01, 0xEA, // Ion 1.1 IVM 0xE4, 0x07, // $ion_symbol_table:: @@ -1695,11 +1927,11 @@ public void ion10SymbolTableMacroAwareTranscode() throws Exception { ); ByteArrayOutputStream out = new ByteArrayOutputStream(); try ( - MacroAwareIonReader reader = InputType.BYTE_ARRAY.newMacroAwareReader(data); - MacroAwareIonWriter rewriter = StreamType.BINARY.newMacroAwareWriter(out); + MacroAwareIonReader reader = inputType.newMacroAwareReader(data); + MacroAwareIonWriter rewriter = outputFormat.newMacroAwareWriter(out); ) { // This may at some point be supported. - assertThrows(IonException.class, () -> reader.transcodeTo(rewriter)); + assertThrows(IonException.class, () -> reader.transcodeAllTo(rewriter)); } } diff --git a/src/test/java/com/amazon/ion/impl/IonReaderContinuableCoreBinaryTest.java b/src/test/java/com/amazon/ion/impl/IonReaderContinuableCoreBinaryTest.java index 37487506a..4c10ca3b7 100644 --- a/src/test/java/com/amazon/ion/impl/IonReaderContinuableCoreBinaryTest.java +++ b/src/test/java/com/amazon/ion/impl/IonReaderContinuableCoreBinaryTest.java @@ -1165,7 +1165,7 @@ private void assertMacroAwareTranscribeProducesEquivalentStream(byte[] data, boo IonReaderContinuableCoreBinary reader = initializeReader(constructFromBytes, data); MacroAwareIonWriter writer = (MacroAwareIonWriter) IonEncodingVersion.ION_1_1.textWriterBuilder().build(sb); ) { - reader.transcodeTo(writer); + reader.transcodeAllTo(writer); } IonSystem system = IonSystemBuilder.standard().build(); IonDatagram actual = system.getLoader().load(sb.toString());