From 5389910a15df61c0c0f1f27810d2c4cfb47d0a21 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Wed, 27 Sep 2023 13:48:36 +0100 Subject: [PATCH 01/41] [C#] Generate DTOs from SBE IR for non-perf-sensitive usecases. In some applications performance is not cricital. Some users would like to use SBE across their whole "estate", but don't want the "sharp edges" associated with using flyweight codecs, e.g., accidental escape. In this commit, I've added a first cut of DTO generation for C# and a simple test based on the Car Example. The DTOs support encoding and decoding via the generated codecs using `EncodeInto(CodecT codec)` and `DecodeFrom(CodecT codec)` methods. Currently there is no support for equality/comparison or read-only views over the data; although, these have been requested. Here are some points that we may or may not wish to change in the future: 1. Non-present (due to the encoded version) string/array data and repeating groups are represented as `null` rather than empty. 2. Non-present primitive values are represented as their associated null value rather than using nullable types. 3. Non-present bitsets are represented as `0`. 4. DTOs are generated via a separate `CodeGenerator` rather than a flag to the existing C# `CodeGenerator`. --- build.gradle | 16 +- csharp/sbe-dll/DirectBuffer.cs | 7 +- csharp/sbe-tests/DtoTests.cs | 110 ++ .../generation/csharp/CSharpDtoGenerator.java | 1076 +++++++++++++++++ .../sbe/generation/csharp/CSharpDtos.java | 35 + .../generation/csharp/CSharpGenerator.java | 109 +- .../sbe/generation/csharp/CSharpUtil.java | 116 +- 7 files changed, 1353 insertions(+), 116 deletions(-) create mode 100644 csharp/sbe-tests/DtoTests.cs create mode 100644 sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java create mode 100644 sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtos.java diff --git a/build.gradle b/build.gradle index 1811fc9913..7dbd8d24fc 100644 --- a/build.gradle +++ b/build.gradle @@ -735,7 +735,7 @@ tasks.register('generateCSharpCodecsWithXIncludes', JavaExec) { 'sbe-samples/src/main/resources/example-extension-schema.xml'] } -tasks.register('generateCSharpCodecsTests', JavaExec) { +tasks.register('generateCSharpTestCodecs', JavaExec) { mainClass.set('uk.co.real_logic.sbe.SbeTool') classpath = project(':sbe-tool').sourceSets.main.runtimeClasspath systemProperties( @@ -754,9 +754,21 @@ tasks.register('generateCSharpCodecsTests', JavaExec) { 'sbe-benchmarks/src/main/resources/fix-message-samples.xml'] } +tasks.register('generateCSharpTestDtos', JavaExec) { + mainClass.set('uk.co.real_logic.sbe.SbeTool') + classpath = project(':sbe-tool').sourceSets.main.runtimeClasspath + systemProperties( + 'sbe.output.dir': 'csharp/sbe-generated', + 'sbe.target.language': 'uk.co.real_logic.sbe.generation.csharp.CSharpDtos', + 'sbe.xinclude.aware': 'true', + 'sbe.validation.xsd': validationXsdPath) + args = ['sbe-samples/src/main/resources/example-extension-schema.xml'] +} + tasks.register('generateCSharpCodecs') { description = 'Generate csharp codecs' - dependsOn 'generateCSharpCodecsTests', + dependsOn 'generateCSharpTestCodecs', + 'generateCSharpTestDtos', 'generateCSharpCodecsWithXIncludes' } diff --git a/csharp/sbe-dll/DirectBuffer.cs b/csharp/sbe-dll/DirectBuffer.cs index 09eb64ec7e..87f08b96b5 100644 --- a/csharp/sbe-dll/DirectBuffer.cs +++ b/csharp/sbe-dll/DirectBuffer.cs @@ -694,7 +694,6 @@ public int SetBytes(int index, ReadOnlySpan src) /// /// Writes a string into the underlying buffer, encoding using the provided . - /// If there is not enough room in the buffer for the bytes it will throw IndexOutOfRangeException. /// /// encoding to use to write the bytes from the string /// source string @@ -702,13 +701,9 @@ public int SetBytes(int index, ReadOnlySpan src) /// count of bytes written public unsafe int SetBytesFromString(Encoding encoding, string src, int index) { - int available = _capacity - index; int byteCount = encoding.GetByteCount(src); - if (byteCount > available) - { - ThrowHelper.ThrowIndexOutOfRangeException(_capacity); - } + CheckLimit(index + byteCount); fixed (char* ptr = src) { diff --git a/csharp/sbe-tests/DtoTests.cs b/csharp/sbe-tests/DtoTests.cs new file mode 100644 index 0000000000..99fbcadc74 --- /dev/null +++ b/csharp/sbe-tests/DtoTests.cs @@ -0,0 +1,110 @@ +using Extension; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Org.SbeTool.Sbe.Dll; + +namespace Org.SbeTool.Sbe.Tests +{ + [TestClass] + public class DtoTests + { + [TestMethod] + public void ShouldRoundTripCar() + { + var inputByteArray = new byte[1024]; + var inputBuffer = new DirectBuffer(inputByteArray); + EncodeCar(inputBuffer); + var decoder = new Car(); + decoder.WrapForDecode(inputBuffer, 0, Car.BlockLength, Car.SchemaVersion); + var decoderString = decoder.ToString(); + var dto = new CarDto(); + dto.DecodeFrom(decoder); + var outputByteArray = new byte[1024]; + var outputBuffer = new DirectBuffer(outputByteArray); + var encoder = new Car(); + encoder.WrapForEncode(outputBuffer, 0); + dto.EncodeInto(encoder); + var dtoString = dto.ToString(); + CollectionAssert.AreEqual(inputByteArray, outputByteArray); + Assert.AreEqual(decoderString, dtoString); + } + + private static void EncodeCar(DirectBuffer buffer) + { + var car = new Car(); + car.WrapForEncode(buffer, 0); + car.SerialNumber = 1234; + car.ModelYear = 2013; + car.Available = BooleanType.T; + car.Code = Model.A; + car.SetVehicleCode("ABCDEF"); + + for (int i = 0, size = Car.SomeNumbersLength; i < size; i++) + { + car.SetSomeNumbers(i, (uint)i); + } + + car.Extras = OptionalExtras.CruiseControl | OptionalExtras.SportsPack; + + car.CupHolderCount = 119; + + car.Engine.Capacity = 2000; + car.Engine.NumCylinders = 4; + car.Engine.SetManufacturerCode("ABC"); + car.Engine.Efficiency = 35; + car.Engine.BoosterEnabled = BooleanType.T; + car.Engine.Booster.BoostType = BoostType.NITROUS; + car.Engine.Booster.HorsePower = 200; + + var fuelFigures = car.FuelFiguresCount(3); + fuelFigures.Next(); + fuelFigures.Speed = 30; + fuelFigures.Mpg = 35.9f; + fuelFigures.SetUsageDescription("this is a description"); + + fuelFigures.Next(); + fuelFigures.Speed = 55; + fuelFigures.Mpg = 49.0f; + fuelFigures.SetUsageDescription("this is a description"); + + fuelFigures.Next(); + fuelFigures.Speed = 75; + fuelFigures.Mpg = 40.0f; + fuelFigures.SetUsageDescription("this is a description"); + + Car.PerformanceFiguresGroup perfFigures = car.PerformanceFiguresCount(2); + perfFigures.Next(); + perfFigures.OctaneRating = 95; + + Car.PerformanceFiguresGroup.AccelerationGroup acceleration = perfFigures.AccelerationCount(3).Next(); + acceleration.Mph = 30; + acceleration.Seconds = 4.0f; + + acceleration.Next(); + acceleration.Mph = 60; + acceleration.Seconds = 7.5f; + + acceleration.Next(); + acceleration.Mph = 100; + acceleration.Seconds = 12.2f; + + perfFigures.Next(); + perfFigures.OctaneRating = 99; + acceleration = perfFigures.AccelerationCount(3).Next(); + + acceleration.Mph = 30; + acceleration.Seconds = 3.8f; + + acceleration.Next(); + acceleration.Mph = 60; + acceleration.Seconds = 7.1f; + + acceleration.Next(); + acceleration.Mph = 100; + acceleration.Seconds = 11.8f; + + car.SetManufacturer("Ford"); + car.SetModel("Fiesta"); + car.SetActivationCode("1234"); + } + } +} \ No newline at end of file diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java new file mode 100644 index 0000000000..a0f1e21d9d --- /dev/null +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java @@ -0,0 +1,1076 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * Copyright (C) 2017 MarketFactory, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.generation.csharp; + +import uk.co.real_logic.sbe.PrimitiveType; +import uk.co.real_logic.sbe.generation.CodeGenerator; +import uk.co.real_logic.sbe.generation.Generators; +import uk.co.real_logic.sbe.ir.Ir; +import uk.co.real_logic.sbe.ir.Signal; +import uk.co.real_logic.sbe.ir.Token; +import org.agrona.Verify; +import org.agrona.generation.OutputManager; + +import java.io.IOException; +import java.io.Writer; +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; + +import static uk.co.real_logic.sbe.generation.csharp.CSharpUtil.*; +import static uk.co.real_logic.sbe.ir.GenerationUtil.collectFields; +import static uk.co.real_logic.sbe.ir.GenerationUtil.collectGroups; +import static uk.co.real_logic.sbe.ir.GenerationUtil.collectVarData; + +/** + * DTO generator for the CSharp programming language. + */ +@SuppressWarnings("CodeBlock2Expr") +public class CSharpDtoGenerator implements CodeGenerator +{ + private static final String INDENT = " "; + private static final String BASE_INDENT = INDENT; + + private final Ir ir; + private final OutputManager outputManager; + + /** + * Create a new C# DTO {@link CodeGenerator}. + * + * @param ir for the messages and types. + * @param outputManager for generating the DTOs to. + */ + public CSharpDtoGenerator(final Ir ir, final OutputManager outputManager) + { + Verify.notNull(ir, "ir"); + Verify.notNull(outputManager, "outputManager"); + + this.ir = ir; + this.outputManager = outputManager; + } + + /** + * {@inheritDoc} + */ + public void generate() throws IOException + { + generateDtosForTypes(); + + for (final List tokens : ir.messages()) + { + final Token msgToken = tokens.get(0); + final String codecClassName = formatClassName(msgToken.name()); + final String className = formatDtoClassName(msgToken.name()); + + final List messageBody = tokens.subList(1, tokens.size() - 1); + int offset = 0; + + final StringBuilder sb = new StringBuilder(); + + final List fields = new ArrayList<>(); + offset = collectFields(messageBody, offset, fields); + generateFields(sb, fields, BASE_INDENT + INDENT); + + final List groups = new ArrayList<>(); + offset = collectGroups(messageBody, offset, groups); + generateGroups(sb, codecClassName, groups, BASE_INDENT + INDENT); + + final List varData = new ArrayList<>(); + collectVarData(messageBody, offset, varData); + generateVarData(sb, varData, BASE_INDENT + INDENT); + + generateDecodeFrom(sb, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); + generateEncodeInto(sb, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); + generateDisplay(sb, codecClassName, "WrapForEncode", null, BASE_INDENT + INDENT); + + try (Writer out = outputManager.createOutput(className)) + { + out.append(generateFileHeader(ir.applicableNamespace(), "using System.Collections.Generic;\n")); + out.append(generateDocumentation(BASE_INDENT, msgToken)); + + out.append(BASE_INDENT).append("public sealed partial class ").append(className).append("\n") + .append(BASE_INDENT).append("{") + .append(sb) + .append(BASE_INDENT).append("}\n") + .append("}\n"); + } + } + } + + private void generateGroups( + final StringBuilder sb, + final String parentMessageClassName, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token groupToken = tokens.get(i); + if (groupToken.signal() != Signal.BEGIN_GROUP) + { + throw new IllegalStateException("tokens must begin with BEGIN_GROUP: token=" + groupToken); + } + final String groupName = groupToken.name(); + final String groupClassName = formatDtoClassName(groupName); + + sb.append("\n") + .append(generateDocumentation(indent, groupToken)) + .append(indent).append("public List<").append(groupClassName).append("> ") + .append(formatPropertyName(groupName)) + .append(" { get; set; } = new List<").append(groupClassName).append(">();\n"); + + sb.append("\n") + .append(generateDocumentation(indent, groupToken)) + .append(indent).append("public sealed partial class ").append(groupClassName).append("\n") + .append(indent).append("{"); + + i++; + i += tokens.get(i).componentTokenCount(); + + final List fields = new ArrayList<>(); + i = collectFields(tokens, i, fields); + generateFields(sb, fields, indent + INDENT); + + final List groups = new ArrayList<>(); + i = collectGroups(tokens, i, groups); + final String codecClassName = parentMessageClassName + "." + formatClassName(groupName) + "Group"; + generateGroups(sb, codecClassName, groups, indent + INDENT); + + final List varData = new ArrayList<>(); + i = collectVarData(tokens, i, varData); + generateVarData(sb, varData, indent + INDENT); + + generateDecodeFrom(sb, codecClassName, fields, groups, varData, indent + INDENT); + generateEncodeInto(sb, codecClassName, fields, groups, varData, indent + INDENT); + + sb.append(indent).append("}\n"); + } + } + + private void generateCompositeDecodeFrom( + final StringBuilder sb, + final String codecClassName, + final List tokens, + final String indent) + { + sb.append("\n") + .append(indent).append("public void DecodeFrom(").append(codecClassName).append(" codec)\n") + .append(indent).append("{\n"); + + for (int i = 0; i < tokens.size(); ) + { + final Token token = tokens.get(i); + + generateFieldDecodeFrom(sb, token, token, codecClassName, indent + INDENT); + + i += tokens.get(i).componentTokenCount(); + } + + sb.append(indent).append("}\n"); + } + + private void generateCompositeEncodeInto( + final StringBuilder sb, + final String codecClassName, + final List tokens, + final String indent) + { + sb.append("\n") + .append(indent).append("public void EncodeInto(").append(codecClassName).append(" codec)\n") + .append(indent).append("{\n"); + + for (int i = 0; i < tokens.size(); ) + { + final Token token = tokens.get(i); + + generateFieldEncodeInto(sb, token, token, indent + INDENT); + + i += tokens.get(i).componentTokenCount(); + } + + sb.append(indent).append("}\n"); + } + + private void generateDecodeFrom( + final StringBuilder sb, + final String codecClassName, + final List fields, + final List groups, + final List varData, + final String indent) + { + sb.append("\n") + .append(indent).append("public void DecodeFrom(").append(codecClassName).append(" codec)\n") + .append(indent).append("{\n"); + + generateFieldsDecodeFrom(sb, fields, codecClassName, indent + INDENT); + generateGroupsDecodeFrom(sb, groups, indent + INDENT); + generateVarDataDecodeFrom(sb, varData, indent + INDENT); + + sb.append(indent).append("}\n"); + } + + private void generateFieldsDecodeFrom( + final StringBuilder sb, + final List tokens, + final String codecClassName, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token signalToken = tokens.get(i); + if (signalToken.signal() == Signal.BEGIN_FIELD) + { + final Token encodingToken = tokens.get(i + 1); + + generateFieldDecodeFrom(sb, signalToken, encodingToken, codecClassName, indent); + } + } + } + + private void generateFieldDecodeFrom( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String codecClassName, final String indent) + { + switch (typeToken.signal()) + { + case ENCODING: + generatePrimitiveDecodeFrom(sb, fieldToken, typeToken, codecClassName, indent); + break; + + case BEGIN_SET: + generatePropertyDecodeFrom(sb, fieldToken, "0", indent); + break; + + case BEGIN_ENUM: + final String enumName = formatClassName(typeToken.applicableTypeName()); + final String nullValue = formatNamespace(ir.packageName()) + "." + enumName + ".NULL_VALUE"; + generatePropertyDecodeFrom(sb, fieldToken, nullValue, indent); + break; + + case BEGIN_COMPOSITE: + generateComplexDecodeFrom(sb, fieldToken, indent); + break; + + default: + break; + } + } + + private void generatePrimitiveDecodeFrom( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String codecClassName, + final String indent) + { + if (typeToken.isConstantEncoding()) + { + return; + } + + final int arrayLength = typeToken.arrayLength(); + + if (arrayLength == 1) + { + final String nullValue = codecClassName + "." + formatPropertyName(fieldToken.name()) + "NullValue"; + generatePropertyDecodeFrom(sb, fieldToken, nullValue, indent); + } + else if (arrayLength > 1) + { + generateArrayDecodeFrom(sb, fieldToken, typeToken, indent); + } + } + + private void generateArrayDecodeFrom( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) + { + wrapInActingVersionCheck( + sb, + fieldToken, + indent, + (blkSb, blkIndent) -> + { + blkSb.append(blkIndent).append(formattedPropertyName) + .append(" = codec.Get").append(formattedPropertyName).append("();\n"); + }, + (blkSb, blkIndent) -> + { + blkSb.append(blkIndent).append(formattedPropertyName).append(" = null;\n"); + } + ); + } + else + { + final String typeName = cSharpTypeName(typeToken.encoding().primitiveType()); + + wrapInActingVersionCheck( + sb, + fieldToken, + indent, + (blkSb, blkIndent) -> + { + blkSb.append(blkIndent).append(formattedPropertyName) + .append(" = new ").append(typeName).append("[") + .append(typeToken.arrayLength()).append("];\n") + .append(blkIndent).append("codec.").append(formattedPropertyName) + .append(".CopyTo(new Span<").append(typeName).append(">(") + .append(formattedPropertyName).append("));\n"); + }, + (blkSb, blkIndent) -> + { + blkSb.append(blkIndent).append(formattedPropertyName).append(" = null;\n"); + } + ); + } + } + + private void generatePropertyDecodeFrom( + final StringBuilder sb, + final Token fieldToken, + final String nullValue, + final String indent) + { + + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + wrapInActingVersionCheck( + sb, + fieldToken, + indent, + (blkSb, blkIndent) -> + { + blkSb.append(blkIndent).append(formattedPropertyName).append(" = codec.") + .append(formattedPropertyName).append(";\n"); + }, + (blkSb, blkIndent) -> + { + blkSb.append(blkIndent).append(formattedPropertyName).append(" = ").append(nullValue).append(";\n"); + } + ); + } + + private void generateComplexDecodeFrom( + final StringBuilder sb, + final Token fieldToken, + final String indent) + { + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + wrapInActingVersionCheck( + sb, + fieldToken, + indent, + (blkSb, blkIndent) -> + { + blkSb.append(blkIndent).append(formattedPropertyName).append(" = new ") + .append(formatDtoClassName(propertyName)).append("();\n") + .append(blkIndent).append(formattedPropertyName).append(".DecodeFrom(codec.") + .append(formattedPropertyName).append(");\n"); + }, + (blkSb, blkIndent) -> + { + blkSb.append(blkIndent).append(formattedPropertyName).append(" = null;\n"); + } + ); + } + + private void generateGroupsDecodeFrom( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token groupToken = tokens.get(i); + if (groupToken.signal() != Signal.BEGIN_GROUP) + { + throw new IllegalStateException("tokens must begin with BEGIN_GROUP: token=" + groupToken); + } + final String groupName = groupToken.name(); + final String formattedPropertyName = formatPropertyName(groupName); + final String groupDtoClassName = formatDtoClassName(groupName); + final String groupCodecVarName = groupName + "Codec"; + + sb.append("\n") + .append(indent) + .append(formattedPropertyName).append(" = new List<").append(groupDtoClassName).append(">();\n"); + + wrapInActingVersionCheck( + sb, + groupToken, + indent, + (blkSb, blkIndent) -> + { + blkSb.append(blkIndent).append("var ").append(groupCodecVarName).append(" = codec.") + .append(formattedPropertyName).append(";\n") + .append(blkIndent).append("while (").append(groupCodecVarName).append(".HasNext)\n") + .append(blkIndent).append("{\n") + .append(blkIndent).append(INDENT) + .append("var element = new ").append(groupDtoClassName).append("();\n") + .append(blkIndent).append(INDENT) + .append("element.DecodeFrom(").append(groupCodecVarName).append(".Next());\n") + .append(blkIndent).append(INDENT) + .append(formattedPropertyName).append(".Add(element);\n") + .append(blkIndent).append("}\n"); + }, + null + ); + + i++; + i += tokens.get(i).componentTokenCount(); + + final List fields = new ArrayList<>(); + i = collectFields(tokens, i, fields); + + final List groups = new ArrayList<>(); + i = collectGroups(tokens, i, groups); + + final List varData = new ArrayList<>(); + i = collectVarData(tokens, i, varData); + } + } + + private void generateVarDataDecodeFrom( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (int i = 0; i < tokens.size(); i++) + { + final Token token = tokens.get(i); + if (token.signal() == Signal.BEGIN_VAR_DATA) + { + final String propertyName = token.name(); + final Token varDataToken = Generators.findFirst("varData", tokens, i); + final String characterEncoding = varDataToken.encoding().characterEncoding(); + + final String formattedPropertyName = formatPropertyName(propertyName); + final String accessor = characterEncoding == null ? + "Get" + formattedPropertyName + "Bytes" : + "Get" + formattedPropertyName; + + wrapInActingVersionCheck( + sb, + token, + indent, + (blkSb, blkIndent) -> + { + blkSb.append(blkIndent).append(formattedPropertyName) + .append(" = codec.").append(accessor).append("();\n"); + }, + (blkSb, blkIndent) -> + { + blkSb.append(blkIndent).append(formattedPropertyName).append(" = null;\n"); + } + ); + } + } + } + + private void wrapInActingVersionCheck( + final StringBuilder sb, + final Token token, + final String indent, + final BiConsumer generatePresentBlock, + final BiConsumer generateNotPresentBlock) + { + final String propertyName = token.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + if (token.version() > 0) + { + sb.append("\n").append(indent).append("if (codec.").append(formattedPropertyName) + .append("InActingVersion())\n") + .append(indent).append("{\n"); + generatePresentBlock.accept(sb, indent + INDENT); + sb.append(indent).append("}\n"); + if (null != generateNotPresentBlock) + { + sb.append(indent).append("else\n") + .append(indent).append("{\n"); + generateNotPresentBlock.accept(sb, indent + INDENT); + sb.append(indent).append("}\n"); + } + } + else + { + generatePresentBlock.accept(sb, indent); + } + } + + private void generateEncodeInto( + final StringBuilder sb, + final String codecClassName, + final List fields, + final List groups, + final List varData, + final String indent) + { + sb.append("\n") + .append(indent).append("public void EncodeInto(").append(codecClassName).append(" codec)\n") + .append(indent).append("{\n"); + + generateFieldsEncodeInto(sb, fields, indent + INDENT); + generateGroupsEncodeInto(sb, groups, indent + INDENT); + generateVarDataEncodeInto(sb, varData, indent + INDENT); + + sb.append(indent).append("}\n"); + } + + private void generateFieldsEncodeInto( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token signalToken = tokens.get(i); + if (signalToken.signal() == Signal.BEGIN_FIELD) + { + final Token encodingToken = tokens.get(i + 1); + generateFieldEncodeInto(sb, signalToken, encodingToken, indent); + } + } + } + + private void generateFieldEncodeInto( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String indent) + { + + switch (typeToken.signal()) + { + case ENCODING: + generatePrimitiveEncodeInto(sb, fieldToken, typeToken, indent); + break; + + case BEGIN_SET: + case BEGIN_ENUM: + generatePropertyEncodeInto(sb, fieldToken, indent); + break; + + case BEGIN_COMPOSITE: + generateComplexEncodeInto(sb, fieldToken, indent); + break; + + default: + break; + } + } + + private void generatePrimitiveEncodeInto( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String indent) + { + if (typeToken.isConstantEncoding()) + { + return; + } + + final int arrayLength = typeToken.arrayLength(); + + if (arrayLength == 1) + { + generatePropertyEncodeInto(sb, fieldToken, indent); + } + else if (arrayLength > 1) + { + generateArrayEncodeInto(sb, fieldToken, typeToken, indent); + } + } + + private void generateArrayEncodeInto( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) + { + sb.append(indent).append("codec.Set").append(formattedPropertyName).append("(") + .append(formattedPropertyName).append(");\n"); + } + else + { + final String typeName = cSharpTypeName(typeToken.encoding().primitiveType()); + + sb.append(indent).append("new Span<").append(typeName).append(">(").append(formattedPropertyName) + .append(").CopyTo(codec.").append(formattedPropertyName).append("AsSpan());\n"); + } + } + + private void generatePropertyEncodeInto( + final StringBuilder sb, + final Token fieldToken, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + sb.append(indent).append("codec.").append(formattedPropertyName).append(" = ") + .append(formattedPropertyName).append(";\n"); + } + + private void generateComplexEncodeInto( + final StringBuilder sb, + final Token fieldToken, + final String indent) + { + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + sb.append(indent).append(formattedPropertyName).append(".EncodeInto(codec.") + .append(formattedPropertyName).append(");\n"); + } + + private void generateGroupsEncodeInto( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token groupToken = tokens.get(i); + if (groupToken.signal() != Signal.BEGIN_GROUP) + { + throw new IllegalStateException("tokens must begin with BEGIN_GROUP: token=" + groupToken); + } + final String groupName = groupToken.name(); + final String formattedPropertyName = formatPropertyName(groupName); + final String groupCodecVarName = groupName + "Codec"; + + sb.append("\n") + .append(indent).append("var ").append(groupCodecVarName) + .append(" = codec.").append(formattedPropertyName) + .append("Count(").append(formattedPropertyName).append(".Count);\n\n") + .append(indent).append("foreach (var group in ").append(formattedPropertyName).append(")\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("group.EncodeInto(").append(groupCodecVarName) + .append(".Next()").append(");\n") + .append(indent).append("}\n\n"); + + i++; + i += tokens.get(i).componentTokenCount(); + + final List fields = new ArrayList<>(); + i = collectFields(tokens, i, fields); + + final List groups = new ArrayList<>(); + i = collectGroups(tokens, i, groups); + + final List varData = new ArrayList<>(); + i = collectVarData(tokens, i, varData); + } + } + + private void generateVarDataEncodeInto( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (final Token token : tokens) + { + if (token.signal() == Signal.BEGIN_VAR_DATA) + { + final String propertyName = token.name(); + + sb.append(indent).append("codec.Set").append(formatPropertyName(propertyName)) + .append("(").append(formatPropertyName(propertyName)).append(");\n"); + } + } + } + + private void generateDisplay( + final StringBuilder sb, + final String codecClassName, + final String wrapMethod, + final String actingVersion, + final String indent) + { + sb.append("\n") + .append(indent).append("public override string ToString()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT) + .append("var buffer = new DirectBuffer(new byte[128], (ignored, newSize) => new byte[newSize]);\n") + .append(indent).append(INDENT).append("var codec = new ").append(codecClassName).append("();\n") + .append(indent).append(INDENT).append("codec."); + sb.append(wrapMethod).append("(buffer, 0"); + if (null != actingVersion) + { + sb.append(", ").append(actingVersion); + } + sb.append(");\n"); + sb.append(indent).append(INDENT).append("EncodeInto(codec);\n") + .append(indent).append(INDENT).append("StringBuilder sb = new StringBuilder();\n") + .append(indent).append(INDENT).append("codec.BuildString(sb);\n") + .append(indent).append(INDENT).append("return sb.ToString();\n") + .append(indent).append("}\n"); + } + + private void generateFields( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token signalToken = tokens.get(i); + if (signalToken.signal() == Signal.BEGIN_FIELD) + { + final Token encodingToken = tokens.get(i + 1); + final String propertyName = signalToken.name(); + + switch (encodingToken.signal()) + { + case ENCODING: + generatePrimitiveProperty(sb, propertyName, signalToken, encodingToken, indent); + break; + + case BEGIN_ENUM: + generateEnumProperty(sb, propertyName, signalToken, encodingToken, indent); + break; + + case BEGIN_SET: + generateBitSetProperty(sb, propertyName, signalToken, encodingToken, indent); + break; + + case BEGIN_COMPOSITE: + generateCompositeProperty(sb, propertyName, signalToken, encodingToken, indent); + break; + + default: + break; + } + } + } + } + + private void generateCompositeProperty( + final StringBuilder sb, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String bitSetName = formatDtoClassName(typeToken.applicableTypeName()); + sb.append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public ").append(bitSetName).append(" ").append(formatPropertyName(propertyName)) + .append(" { get; set; } = new ").append(bitSetName).append("();\n"); + } + + private void generateBitSetProperty( + final StringBuilder sb, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String enumName = formatClassName(typeToken.applicableTypeName()); + + sb.append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public ").append(enumName).append(" ") + .append(formatPropertyName(propertyName)).append(" { get; set; }\n"); + } + + private void generateEnumProperty( + final StringBuilder sb, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String enumName = formatClassName(typeToken.applicableTypeName()); + + if (fieldToken.isConstantEncoding()) + { + final String constValue = fieldToken.encoding().constValue().toString(); + + sb.append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public static ").append(enumName).append(" ") + .append(formatPropertyName(propertyName)).append("\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("get { return ") + .append(formatNamespace(ir.packageName())).append(".").append(constValue) + .append("; }\n") + .append(indent).append("}\n"); + } + else + { + sb.append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public ").append(enumName).append(" ").append(formatPropertyName(propertyName)) + .append(" { get; set; }\n"); + } + } + + private void generatePrimitiveProperty( + final StringBuilder sb, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + if (typeToken.isConstantEncoding()) + { + generateConstPropertyMethods(sb, propertyName, fieldToken, typeToken, indent); + } + else + { + generatePrimitivePropertyMethods(sb, propertyName, fieldToken, typeToken, indent); + } + } + + private void generatePrimitivePropertyMethods( + final StringBuilder sb, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final int arrayLength = typeToken.arrayLength(); + + if (arrayLength == 1) + { + generateSingleValueProperty(sb, propertyName, fieldToken, typeToken, indent); + } + else if (arrayLength > 1) + { + generateArrayProperty(sb, propertyName, fieldToken, typeToken, indent); + } + } + + private void generateArrayProperty( + final StringBuilder sb, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) + { + sb.append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public string ") + .append(formatPropertyName(propertyName)).append(" { get; set; }\n"); + } + else + { + final String typeName = cSharpTypeName(typeToken.encoding().primitiveType()); + + sb.append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public ").append(typeName).append("[] ") + .append(formatPropertyName(propertyName)).append(" { get; set; }\n"); + } + } + + private void generateSingleValueProperty( + final StringBuilder sb, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String typeName = cSharpTypeName(typeToken.encoding().primitiveType()); + + sb.append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public ").append(typeName).append(" ") + .append(formatPropertyName(propertyName)).append(" { get; set; }\n"); + } + + private void generateConstPropertyMethods( + final StringBuilder sb, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) + { + sb.append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public static string ").append(toUpperFirstChar(propertyName)).append("\n") + .append(indent).append("{\n") + .append(indent).append(INDENT) + .append("get { return \"").append(typeToken.encoding().constValue().toString()).append("\"; }\n") + .append(indent).append("}\n"); + } + else + { + final String literalValue = + generateLiteral(typeToken.encoding().primitiveType(), typeToken.encoding().constValue().toString()); + + sb.append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public static ").append(cSharpTypeName(typeToken.encoding().primitiveType())) + .append(" ").append(formatPropertyName(propertyName)).append("\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("get { return ").append(literalValue).append("; }\n") + .append(indent).append("}\n"); + } + } + + private void generateVarData( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token token = tokens.get(i); + if (token.signal() == Signal.BEGIN_VAR_DATA) + { + final String propertyName = token.name(); + final Token varDataToken = Generators.findFirst("varData", tokens, i); + final String characterEncoding = varDataToken.encoding().characterEncoding(); + final String dtoType = characterEncoding == null ? "byte[]" : "string"; + + sb.append("\n") + .append(indent).append("public ").append(dtoType).append(" ") + .append(formatPropertyName(propertyName)).append(" { get; set; }\n"); + } + } + } + + private String formatDtoClassName(final String name) + { + return formatClassName(name + "Dto"); + } + + private void generateDtosForTypes() throws IOException + { + for (final List tokens : ir.types()) + { + switch (tokens.get(0).signal()) + { + case BEGIN_COMPOSITE: + generateComposite(tokens); + break; + + default: + break; + } + } + } + + private void generateComposite(final List tokens) throws IOException + { + final String name = tokens.get(0).applicableTypeName(); + final String className = formatDtoClassName(name); + final String codecClassName = formatClassName(name); + + try (Writer out = outputManager.createOutput(className)) + { + out.append(generateFileHeader(ir.applicableNamespace())); + out.append(generateDocumentation(BASE_INDENT, tokens.get(0))); + + final StringBuilder sb = new StringBuilder(); + + final List compositeTokens = tokens.subList(1, tokens.size() - 1); + generateCompositePropertyElements(sb, compositeTokens, BASE_INDENT + INDENT); + generateCompositeDecodeFrom(sb, codecClassName, compositeTokens, BASE_INDENT + INDENT); + generateCompositeEncodeInto(sb, codecClassName, compositeTokens, BASE_INDENT + INDENT); + generateDisplay(sb, codecClassName, "Wrap", codecClassName + ".SbeSchemaVersion", BASE_INDENT + INDENT); + + out.append(BASE_INDENT).append("public sealed partial class ").append(className).append("\n") + .append(BASE_INDENT).append("{") + .append(sb) + .append(BASE_INDENT).append("}\n") + .append("}\n"); + } + } + + private void generateCompositePropertyElements( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (int i = 0; i < tokens.size(); ) + { + final Token token = tokens.get(i); + final String propertyName = formatPropertyName(token.name()); + + switch (token.signal()) + { + case ENCODING: + generatePrimitiveProperty(sb, propertyName, token, token, indent); + break; + + case BEGIN_ENUM: + generateEnumProperty(sb, propertyName, token, token, indent); + break; + + case BEGIN_SET: + generateBitSetProperty(sb, propertyName, token, token, indent); + break; + + case BEGIN_COMPOSITE: + generateCompositeProperty(sb, propertyName, token, token, indent); + break; + + default: + break; + } + + i += tokens.get(i).componentTokenCount(); + } + } +} diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtos.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtos.java new file mode 100644 index 0000000000..79859d6764 --- /dev/null +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtos.java @@ -0,0 +1,35 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * Copyright 2017 MarketFactory Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.generation.csharp; + +import uk.co.real_logic.sbe.generation.CodeGenerator; +import uk.co.real_logic.sbe.generation.TargetCodeGenerator; +import uk.co.real_logic.sbe.ir.Ir; + +/** + * {@link CodeGenerator} factory for CSharp DTOs. + */ +public class CSharpDtos implements TargetCodeGenerator +{ + /** + * {@inheritDoc} + */ + public CodeGenerator newInstance(final Ir ir, final String outputDir) + { + return new CSharpDtoGenerator(ir, new CSharpNamespaceOutputManager(outputDir, ir.applicableNamespace())); + } +} diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpGenerator.java index 8d15021814..530717c9c4 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpGenerator.java @@ -786,40 +786,6 @@ private CharSequence generateEnumValues(final List tokens, final Token en return sb; } - private CharSequence generateFileHeader(final String packageName) - { - String[] tokens = packageName.split("\\."); - final StringBuilder sb = new StringBuilder(); - for (final String t : tokens) - { - sb.append(toUpperFirstChar(t)).append("."); - } - if (sb.length() > 0) - { - sb.setLength(sb.length() - 1); - } - - tokens = sb.toString().split("-"); - sb.setLength(0); - - for (final String t : tokens) - { - sb.append(toUpperFirstChar(t)); - } - - return String.format( - "// \n" + - "// Generated SBE (Simple Binary Encoding) message codec\n" + - "// \n\n" + - "#pragma warning disable 1591 // disable warning on missing comments\n" + - "using System;\n" + - "using System.Text;\n" + - "using Org.SbeTool.Sbe.Dll;\n\n" + - "namespace %s\n" + - "{\n", - sb); - } - private CharSequence generateClassDeclaration(final String className) { return String.format( @@ -828,20 +794,6 @@ private CharSequence generateClassDeclaration(final String className) className); } - private static String generateDocumentation(final String indent, final Token token) - { - final String description = token.description(); - if (null == description || description.isEmpty()) - { - return ""; - } - - return - indent + "/// \n" + - indent + "/// " + description + "\n" + - indent + "/// \n"; - } - private void generateMetaAttributeEnum() throws IOException { try (Writer out = outputManager.createOutput(META_ATTRIBUTE_ENUM)) @@ -1282,7 +1234,7 @@ private CharSequence generateConstPropertyMethods( final StringBuilder sb = new StringBuilder(); - final String javaTypeName = cSharpTypeName(token.encoding().primitiveType()); + final String csharpTypeName = cSharpTypeName(token.encoding().primitiveType()); final byte[] constantValue = token.encoding().constValue().byteArrayValue(token.encoding().primitiveType()); final CharSequence values = generateByteLiteralList( token.encoding().constValue().byteArrayValue(token.encoding().primitiveType())); @@ -1304,7 +1256,7 @@ private CharSequence generateConstPropertyMethods( indent + INDENT + "{\n" + indent + INDENT + INDENT + "return _%3$sValue[index];\n" + indent + INDENT + "}\n\n", - javaTypeName, + csharpTypeName, toUpperFirstChar(propertyName), propertyName)); @@ -2247,63 +2199,6 @@ private String generateByteOrder(final ByteOrder byteOrder, final int primitiveT return "LittleEndian"; } - private String generateLiteral(final PrimitiveType type, final String value) - { - String literal = ""; - - final String castType = cSharpTypeName(type); - switch (type) - { - case CHAR: - case UINT8: - case INT8: - case INT16: - case UINT16: - literal = "(" + castType + ")" + value; - break; - - case INT32: - literal = value; - break; - - case UINT32: - literal = value + "U"; - break; - - case FLOAT: - if (value.endsWith("NaN")) - { - literal = "float.NaN"; - } - else - { - literal = value + "f"; - } - break; - - case UINT64: - literal = "0x" + Long.toHexString(Long.parseLong(value)) + "UL"; - break; - - case INT64: - literal = value + "L"; - break; - - case DOUBLE: - if (value.endsWith("NaN")) - { - literal = "double.NaN"; - } - else - { - literal = value + "d"; - } - break; - } - - return literal; - } - private void appendGroupInstanceDisplay( final StringBuilder sb, final List fields, diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpUtil.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpUtil.java index cac937c67f..124b352629 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpUtil.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpUtil.java @@ -17,6 +17,8 @@ package uk.co.real_logic.sbe.generation.csharp; import uk.co.real_logic.sbe.PrimitiveType; +import uk.co.real_logic.sbe.generation.Generators; +import uk.co.real_logic.sbe.ir.Token; import java.util.EnumMap; import java.util.Map; @@ -26,6 +28,118 @@ */ public class CSharpUtil { + static String generateLiteral(final PrimitiveType type, final String value) + { + String literal = ""; + + final String castType = cSharpTypeName(type); + switch (type) + { + case CHAR: + case UINT8: + case INT8: + case INT16: + case UINT16: + literal = "(" + castType + ")" + value; + break; + + case INT32: + literal = value; + break; + + case UINT32: + literal = value + "U"; + break; + + case FLOAT: + if (value.endsWith("NaN")) + { + literal = "float.NaN"; + } + else + { + literal = value + "f"; + } + break; + + case UINT64: + literal = "0x" + Long.toHexString(Long.parseLong(value)) + "UL"; + break; + + case INT64: + literal = value + "L"; + break; + + case DOUBLE: + if (value.endsWith("NaN")) + { + literal = "double.NaN"; + } + else + { + literal = value + "d"; + } + break; + } + + return literal; + } + + static CharSequence generateFileHeader(final String packageName, final String... imports) + { + return String.format( + "// \n" + + "// Generated SBE (Simple Binary Encoding) message codec\n" + + "// \n\n" + + "#pragma warning disable 1591 // disable warning on missing comments\n" + + "using System;\n" + + "using System.Text;\n" + + "%1$s" + + "using Org.SbeTool.Sbe.Dll;\n\n" + + "namespace %2$s\n" + + "{\n", + String.join("", imports), + formatNamespace(packageName)); + } + + static String formatNamespace(final String packageName) + { + String[] tokens = packageName.split("\\."); + final StringBuilder sb = new StringBuilder(); + for (final String t : tokens) + { + sb.append(Generators.toUpperFirstChar(t)).append("."); + } + if (sb.length() > 0) + { + sb.setLength(sb.length() - 1); + } + + tokens = sb.toString().split("-"); + sb.setLength(0); + + for (final String t : tokens) + { + sb.append(Generators.toUpperFirstChar(t)); + } + + return sb.toString(); + } + + static String generateDocumentation(final String indent, final Token token) + { + final String description = token.description(); + if (null == description || description.isEmpty()) + { + return ""; + } + + return + indent + "/// \n" + + indent + "/// " + description + "\n" + + indent + "/// \n"; + } + enum Separators { BEGIN_GROUP('['), @@ -89,7 +203,7 @@ public String toString() * @param primitiveType to map. * @return the name of the Java primitive that most closely maps. */ - public static String cSharpTypeName(final PrimitiveType primitiveType) + static String cSharpTypeName(final PrimitiveType primitiveType) { return PRIMITIVE_TYPE_STRING_ENUM_MAP.get(primitiveType); } From 68af4f64b24367d89d635fbff9246fe5eecd7e67 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Thu, 5 Oct 2023 18:45:53 +0100 Subject: [PATCH 02/41] [Java] Start to introduce property based testing (PBT). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In this commit, I've: 1. Added a test dependency on JQwik. 2. Added a JQwik-based generator of arbitrary valid SBE message schemas. This generator exercises a lot of possibilities but not all. In particular, it is missing the generation of constants, min/max, and custom offsets. 3. Added a test that shows the parser parses any arbitrary SBE message schema. Why have I introduced PBT? My aim is to eventually test (an approximation of) the following property: ``` ∀ msgSchema ∈ PossibleMessageSchemas, ∀ bytes ∈ PossibleValidValues(msgSchema), DtoEncode(DtoDecode(bytes)) = bytes ``` I.e., that `DtoEncode` is the inverse of `DtoDecode` and that it preserves the information in an encoded message. --- .gitignore | 3 + build.gradle | 2 + .../sbe/properties/ParserPropertyTest.java | 48 ++ .../sbe/properties/SchemaDomain.java | 360 +++++++++++++++ .../sbe/properties/XmlSchemaWriter.java | 435 ++++++++++++++++++ 5 files changed, 848 insertions(+) create mode 100644 sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java create mode 100644 sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/SchemaDomain.java create mode 100644 sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/XmlSchemaWriter.java diff --git a/.gitignore b/.gitignore index 6769c8db69..4ffe2537bc 100644 --- a/.gitignore +++ b/.gitignore @@ -119,4 +119,7 @@ rust/Cargo.lock .DS_Store /sbe-tool/src/main/golang/uk_co_real_logic_sbe_ir_generated/ +# JQwik +*.jqwik-database + /generated/ diff --git a/build.gradle b/build.gradle index 7dbd8d24fc..c71515f561 100644 --- a/build.gradle +++ b/build.gradle @@ -57,6 +57,7 @@ def checkstyleVersion = '9.3' def hamcrestVersion = '2.2' def mockitoVersion = '4.11.0' def junitVersion = '5.10.2' +def jqwikVersion = '1.8.0' def jmhVersion = '1.37' def agronaVersion = '1.21.2' def agronaVersionRange = '[1.21.2,2.0[' // allow any release >= 1.21.2 and < 2.0.0 @@ -251,6 +252,7 @@ project(':sbe-tool') { testImplementation files('build/classes/java/generated') testImplementation "org.hamcrest:hamcrest:${hamcrestVersion}" testImplementation "org.mockito:mockito-core:${mockitoVersion}" + testImplementation "net.jqwik:jqwik:${jqwikVersion}" testImplementation "org.junit.jupiter:junit-jupiter-params:${junitVersion}" testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${junitVersion}" } diff --git a/sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java b/sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java new file mode 100644 index 0000000000..7abb2b65ca --- /dev/null +++ b/sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java @@ -0,0 +1,48 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties; + +import net.jqwik.api.Arbitrary; +import net.jqwik.api.ForAll; +import net.jqwik.api.Property; +import net.jqwik.api.Provide; +import uk.co.real_logic.sbe.xml.ParserOptions; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import static uk.co.real_logic.sbe.xml.XmlSchemaParser.parse; + +public class ParserPropertyTest +{ + @Property + void shouldParseAnyValidSchema(@ForAll("schemas") final SchemaDomain.MessageSchema schema) throws Exception + { + final String xml = XmlSchemaWriter.writeString(schema); + try (InputStream in = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) + { + parse(in, ParserOptions.DEFAULT); + } + } + + @Provide + Arbitrary schemas() + { + return SchemaDomain.MessageSchema.arbitrary(); + } +} diff --git a/sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/SchemaDomain.java b/sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/SchemaDomain.java new file mode 100644 index 0000000000..1605d7d781 --- /dev/null +++ b/sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/SchemaDomain.java @@ -0,0 +1,360 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties; + +import net.jqwik.api.Arbitraries; +import net.jqwik.api.Arbitrary; +import net.jqwik.api.Combinators; +import uk.co.real_logic.sbe.PrimitiveType; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public final class SchemaDomain +{ + private static final int MAX_COMPOSITE_DEPTH = 3; + private static final int MAX_GROUP_DEPTH = 3; + + private SchemaDomain() + { + } + + static final class EncodedDataTypeSchema implements TypeSchema + { + private final PrimitiveType primitiveType; + private final boolean isEmbedded; + + private EncodedDataTypeSchema( + final PrimitiveType primitiveType, + final boolean isEmbedded + ) + { + this.primitiveType = primitiveType; + this.isEmbedded = isEmbedded; + } + + public PrimitiveType primitiveType() + { + return primitiveType; + } + + @Override + public boolean isEmbedded() + { + return isEmbedded; + } + + @Override + public void accept(final TypeSchemaVisitor visitor) + { + visitor.onEncoded(this); + } + + static Arbitrary arbitrary() + { + return Combinators.combine( + Arbitraries.of(PrimitiveType.values()), + Arbitraries.of(true, false) + ).as(EncodedDataTypeSchema::new); + } + } + + static final class CompositeTypeSchema implements TypeSchema + { + private final List fields; + + private CompositeTypeSchema(final List fields) + { + this.fields = fields; + } + + public List fields() + { + return fields; + } + + static Arbitrary arbitrary(final int depth) + { + return TypeSchema.arbitrary(depth - 1) + .list() + .ofMinSize(1) + .ofMaxSize(3) + .injectDuplicates(0.2) + .map(CompositeTypeSchema::new); + } + + @Override + public void accept(final TypeSchemaVisitor visitor) + { + visitor.onComposite(this); + } + } + + static final class EnumTypeSchema implements TypeSchema + { + private final String encodingType; + private final List validValues; + + private EnumTypeSchema( + final String encodingType, + final List validValues) + { + this.encodingType = encodingType; + this.validValues = validValues; + } + + public String encodingType() + { + return encodingType; + } + + public List validValues() + { + return validValues; + } + + static Arbitrary arbitrary() + { + return Arbitraries.oneOf( + Arbitraries.chars().alpha() + .map(Character::toUpperCase) + .list() + .ofMaxSize(10) + .uniqueElements() + .map(values -> new EnumTypeSchema( + "char", + values.stream().map(String::valueOf).collect(Collectors.toList()) + )), + Arbitraries.integers() + .between(1, 254) + .list() + .ofMaxSize(254) + .uniqueElements() + .map(values -> new EnumTypeSchema( + "uint8", + values.stream().map(String::valueOf).collect(Collectors.toList()) + )) + ); + } + + @Override + public void accept(final TypeSchemaVisitor visitor) + { + visitor.onEnum(this); + } + } + + static final class SetSchema implements TypeSchema + { + private final String encodingType; + private final int choiceCount; + + private SetSchema( + final String encodingType, + final int choiceCount) + { + this.choiceCount = choiceCount; + this.encodingType = encodingType; + } + + public String encodingType() + { + return encodingType; + } + + public int choiceCount() + { + return choiceCount; + } + + @Override + public void accept(final TypeSchemaVisitor visitor) + { + visitor.onSet(this); + } + + static Arbitrary arbitrary() + { + return Combinators.combine( + Arbitraries.of( + "uint8", + "uint16", + "uint32", + "uint64" + ), + Arbitraries.integers().between(1, 4) + ).as(SetSchema::new); + } + } + + interface TypeSchema + { + static Arbitrary arbitrary(final int depth) + { + if (depth == 1) + { + return Arbitraries.oneOf( + EncodedDataTypeSchema.arbitrary(), + EnumTypeSchema.arbitrary(), + SetSchema.arbitrary() + ); + } + else + { + return Arbitraries.oneOf( + CompositeTypeSchema.arbitrary(depth), + EncodedDataTypeSchema.arbitrary(), + EnumTypeSchema.arbitrary(), + SetSchema.arbitrary() + ); + } + } + + default boolean isEmbedded() + { + return false; + } + + void accept(TypeSchemaVisitor visitor); + } + + interface TypeSchemaVisitor + { + void onEncoded(EncodedDataTypeSchema type); + + void onComposite(CompositeTypeSchema type); + + void onEnum(EnumTypeSchema type); + + void onSet(SetSchema type); + } + + static final class VarDataSchema + { + private final Encoding encoding; + + VarDataSchema(final Encoding encoding) + { + this.encoding = encoding; + } + + public Encoding encoding() + { + return encoding; + } + + static Arbitrary arbitrary() + { + return Arbitraries.of(Encoding.values()) + .map(VarDataSchema::new); + } + + enum Encoding + { + ASCII, + BYTES + } + } + + static final class GroupSchema + { + private final List blockFields; + private final List groups; + private final List varData; + + GroupSchema( + final List blockFields, + final List groups, + final List varData) + { + this.blockFields = blockFields; + this.groups = groups; + this.varData = varData; + } + + public List blockFields() + { + return blockFields; + } + + public List groups() + { + return groups; + } + + public List varData() + { + return varData; + } + + static Arbitrary arbitrary(final int depth) + { + final Arbitrary> subGroups = depth == 1 ? + Arbitraries.of(0).map(ignored -> new ArrayList<>()) : + arbitrary(depth - 1).list().ofMaxSize(3); + + return Combinators.combine( + TypeSchema.arbitrary(MAX_COMPOSITE_DEPTH).list().ofMaxSize(5), + subGroups, + VarDataSchema.arbitrary().list().ofMaxSize(3) + ).as(GroupSchema::new); + } + } + + static final class MessageSchema + { + private final List blockFields; + private final List groups; + private final List varData; + + MessageSchema( + final List blockFields, + final List groups, + final List varData + ) + { + this.blockFields = blockFields; + this.groups = groups; + this.varData = varData; + } + + public List blockFields() + { + return blockFields; + } + + public List groups() + { + return groups; + } + + public List varData() + { + return varData; + } + + static Arbitrary arbitrary() + { + return Combinators.combine( + TypeSchema.arbitrary(MAX_COMPOSITE_DEPTH).list().ofMaxSize(10), + GroupSchema.arbitrary(MAX_GROUP_DEPTH).list().ofMaxSize(3), + VarDataSchema.arbitrary().list().ofMaxSize(3) + ).as(MessageSchema::new); + } + } +} diff --git a/sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/XmlSchemaWriter.java b/sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/XmlSchemaWriter.java new file mode 100644 index 0000000000..84ff1b2881 --- /dev/null +++ b/sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/XmlSchemaWriter.java @@ -0,0 +1,435 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties; + +import org.agrona.collections.MutableInteger; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import java.io.File; +import java.io.StringWriter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +import static java.util.Objects.requireNonNull; +import static uk.co.real_logic.sbe.generation.Generators.toLowerFirstChar; + +final class XmlSchemaWriter +{ + private XmlSchemaWriter() + { + } + + public static String writeString(final SchemaDomain.MessageSchema schema) + { + final StringWriter writer = new StringWriter(); + final StreamResult result = new StreamResult(writer); + writeTo(schema, result); + return writer.toString(); + } + + public static void writeFile( + final SchemaDomain.MessageSchema schema, + final File destination) + { + final StreamResult result = new StreamResult(destination); + writeTo(schema, result); + } + + private static void writeTo( + final SchemaDomain.MessageSchema schema, + final StreamResult destination) + { + try + { + final Document document = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument(); + + final Element root = document.createElementNS("http://fixprotocol.io/2016/sbe", "sbe:messageSchema"); + root.setAttribute("id", "42"); + root.setAttribute("package", "uk.co.real_logic.sbe.properties"); + document.appendChild(root); + + final Element topLevelTypes = createTypesElement(document); + root.appendChild(topLevelTypes); + + final HashMap typeToName = new HashMap<>(); + + final TypeSchemaConverter typeSchemaConverter = new TypeSchemaConverter( + document, + topLevelTypes, + typeToName + ); + + appendTypes( + topLevelTypes, + typeSchemaConverter, + schema.blockFields(), + schema.groups()); + + final Element message = document.createElement("sbe:message"); + message.setAttribute("name", "TestMessage"); + message.setAttribute("id", "1"); + root.appendChild(message); + final MutableInteger nextMemberId = new MutableInteger(1); + appendMembers( + document, + typeToName, + schema.blockFields(), + schema.groups(), + schema.varData(), + nextMemberId, + message); + + try + { + final Transformer transformer = TransformerFactory.newInstance().newTransformer(); + + transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); + + final DOMSource source = new DOMSource(document); + + transformer.transform(source, destination); + } + catch (final Exception e) + { + throw new RuntimeException(e); + } + } + catch (final ParserConfigurationException e) + { + throw new RuntimeException(e); + } + } + + private static void appendMembers( + final Document document, + final HashMap typeToName, + final List blockFields, + final List groups, + final List varData, + final MutableInteger nextMemberId, + final Element parent) + { + for (final SchemaDomain.TypeSchema field : blockFields) + { + final int id = nextMemberId.getAndIncrement(); + + final boolean usePrimitiveName = field.isEmbedded() && field instanceof SchemaDomain.EncodedDataTypeSchema; + final String typeName = usePrimitiveName ? + ((SchemaDomain.EncodedDataTypeSchema)field).primitiveType().primitiveName() : + requireNonNull(typeToName.get(field)); + + final Element element = document.createElement("field"); + element.setAttribute("id", Integer.toString(id)); + element.setAttribute("name", "member" + id); + element.setAttribute("type", typeName); + parent.appendChild(element); + } + + for (final SchemaDomain.GroupSchema group : groups) + { + final int id = nextMemberId.getAndIncrement(); + + final Element element = document.createElement("group"); + element.setAttribute("id", Integer.toString(id)); + element.setAttribute("name", "member" + id); + appendMembers( + document, + typeToName, + group.blockFields(), + group.groups(), + group.varData(), + nextMemberId, + element); + parent.appendChild(element); + } + + for (final SchemaDomain.VarDataSchema data : varData) + { + final int id = nextMemberId.getAndIncrement(); + + final Element element = document.createElement("data"); + element.setAttribute("id", Integer.toString(id)); + element.setAttribute("name", "member" + id); + switch (data.encoding()) + { + case ASCII: + element.setAttribute("type", "varStringEncoding"); + break; + case BYTES: + element.setAttribute("type", "varDataEncoding"); + break; + default: + throw new IllegalStateException("Unknown encoding: " + data.encoding()); + } + parent.appendChild(element); + } + } + + private static Element createTypesElement(final Document document) + { + final Element types = document.createElement("types"); + + types.appendChild(createCompositeElement( + document, + "messageHeader", + createTypeElement(document, "blockLength", "uint16"), + createTypeElement(document, "templateId", "uint16"), + createTypeElement(document, "schemaId", "uint16"), + createTypeElement(document, "version", "uint16") + )); + + types.appendChild(createCompositeElement( + document, + "groupSizeEncoding", + createTypeElement(document, "blockLength", "uint16"), + createTypeElement(document, "numInGroup", "uint16") + )); + + final Element varString = createTypeElement(document, "varData", "uint8"); + varString.setAttribute("length", "0"); + varString.setAttribute("characterEncoding", "US-ASCII"); + types.appendChild(createCompositeElement( + document, + "varStringEncoding", + createTypeElement(document, "length", "uint16"), + varString + )); + + final Element varData = createTypeElement(document, "varData", "uint8"); + final Element varDataLength = createTypeElement(document, "length", "uint32"); + varDataLength.setAttribute("maxValue", "1000000"); + varData.setAttribute("length", "0"); + types.appendChild(createCompositeElement( + document, + "varDataEncoding", + varDataLength, + varData + )); + + return types; + } + + private static Element createSetElement( + final Document document, + final String name, + final String encodingType, + final int choiceCount) + { + final Element enumElement = document.createElement("set"); + enumElement.setAttribute("name", name); + enumElement.setAttribute("encodingType", encodingType); + + int caseId = 0; + for (int i = 0; i < choiceCount; i++) + { + final Element choice = document.createElement("choice"); + choice.setAttribute("name", "option" + caseId++); + choice.setTextContent(Integer.toString(i)); + enumElement.appendChild(choice); + } + + return enumElement; + } + + private static Element createEnumElement( + final Document document, + final String name, + final String encodingType, + final List validValues) + { + final Element enumElement = document.createElement("enum"); + enumElement.setAttribute("name", name); + enumElement.setAttribute("encodingType", encodingType); + + int caseId = 0; + for (final String value : validValues) + { + final Element validValue = document.createElement("validValue"); + validValue.setAttribute("name", "Case" + caseId++); + validValue.setTextContent(value); + enumElement.appendChild(validValue); + } + + return enumElement; + } + + private static Element createCompositeElement( + final Document document, + final String name, + final Element... types + ) + { + final Element composite = document.createElement("composite"); + composite.setAttribute("name", name); + + for (final Element type : types) + { + composite.appendChild(type); + } + + return composite; + } + + private static Element createTypeElement( + final Document document, + final String name, + final String primitiveType) + { + final Element blockLength = document.createElement("type"); + blockLength.setAttribute("name", name); + blockLength.setAttribute("primitiveType", primitiveType); + return blockLength; + } + + private static Element createRefElement( + final Document document, + final String name, + final String type) + { + final Element blockLength = document.createElement("ref"); + blockLength.setAttribute("name", name); + blockLength.setAttribute("type", type); + return blockLength; + } + + private static void appendTypes( + final Element topLevelTypes, + final TypeSchemaConverter typeSchemaConverter, + final List blockFields, + final List groups) + { + for (final SchemaDomain.TypeSchema field : blockFields) + { + if (!field.isEmbedded()) + { + topLevelTypes.appendChild(typeSchemaConverter.convert(field)); + } + } + + for (final SchemaDomain.GroupSchema group : groups) + { + appendTypes(topLevelTypes, typeSchemaConverter, group.blockFields(), group.groups()); + } + } + + private static final class TypeSchemaConverter implements SchemaDomain.TypeSchemaVisitor + { + private final Document document; + private final Element topLevelTypes; + private final Map typeToName; + private final Function nextName; + private Element result; + + private TypeSchemaConverter( + final Document document, + final Element topLevelTypes, + final Map typeToName) + { + this.document = document; + this.topLevelTypes = topLevelTypes; + this.typeToName = typeToName; + nextName = ignored -> "Type" + typeToName.size(); + } + + @Override + public void onEncoded(final SchemaDomain.EncodedDataTypeSchema type) + { + result = createTypeElement( + document, + typeToName.computeIfAbsent(type, nextName), + type.primitiveType().primitiveName() + ); + } + + @Override + public void onComposite(final SchemaDomain.CompositeTypeSchema type) + { + final Element[] parts = type.fields().stream() + .map(this::embedOrReference) + .toArray(Element[]::new); + result = createCompositeElement( + document, + typeToName.computeIfAbsent(type, nextName), + parts + ); + } + + @Override + public void onEnum(final SchemaDomain.EnumTypeSchema type) + { + result = createEnumElement( + document, + typeToName.computeIfAbsent(type, nextName), + type.encodingType(), + type.validValues() + ); + } + + @Override + public void onSet(final SchemaDomain.SetSchema type) + { + result = createSetElement( + document, + typeToName.computeIfAbsent(type, nextName), + type.encodingType(), + type.choiceCount() + ); + } + + private Element embedOrReference(final SchemaDomain.TypeSchema type) + { + if (type.isEmbedded()) + { + return convert(type); + } + else + { + final boolean hasWritten = typeToName.containsKey(type); + if (!hasWritten) + { + topLevelTypes.appendChild(convert(type)); + } + + final String typeName = requireNonNull(typeToName.get(type)); + return createRefElement( + document, + toLowerFirstChar(typeName), + typeName + ); + } + } + + public Element convert(final SchemaDomain.TypeSchema type) + { + result = null; + type.accept(this); + return requireNonNull(result); + } + } +} From 439ee4baada8d36f451e4d8c0c10b68df2308370 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Thu, 5 Oct 2023 19:08:49 +0100 Subject: [PATCH 03/41] [Java] Fix typos. --- .../src/main/java/uk/co/real_logic/sbe/xml/EncodedDataType.java | 2 +- sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/Type.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/EncodedDataType.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/EncodedDataType.java index e7842ea47c..981bcaa8db 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/EncodedDataType.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/EncodedDataType.java @@ -89,7 +89,7 @@ public EncodedDataType(final Node node, final String givenName, final String ref if (presence() != CONSTANT) { - handleError(node, "present must be constant when valueRef is set: " + valueRef); + handleError(node, "presence must be constant when valueRef is set: " + valueRef); } } diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/Type.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/Type.java index db25c1a291..9590c3e756 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/Type.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/Type.java @@ -94,7 +94,7 @@ public Type( * @param presence of the type. * @param description of the type or null. * @param sinceVersion for the type - * @param deprecated version in which this wa.s deprecated. + * @param deprecated version in which this was deprecated. * @param semanticType of the type or null. */ public Type( From b94ef12eb404bd77b90f19a9b8028ec8f271d1b2 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Fri, 6 Oct 2023 00:43:37 +0100 Subject: [PATCH 04/41] [Java] Move property tests to their own source set. Gradle 8 complains about deprecations when using the "old style" test source set creation. Therefore, I have had to use `jvm-test-suites`, which is the Gradle 9 way of doing things. I have moved the property tests into their own source set and own test task: `propertyTest`, as they may be more expensive to run than the other unit tests. We may only wish to run them nightly in CI. I have also profiled the `propertyTest` task and removed usage of JQwik list item duplication, as it dominates the time taken during testing. The time taken is now dominated by compiling XPaths. --- build.gradle | 79 ++++++++++++++----- .../sbe/properties/ParserPropertyTest.java | 2 +- .../sbe/properties/SchemaDomain.java | 64 +++++++++++++-- .../sbe/properties/XmlSchemaWriter.java | 25 +++--- .../resources/junit-platform.properties | 3 + 5 files changed, 135 insertions(+), 38 deletions(-) rename sbe-tool/src/{test => propertyTest}/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java (97%) rename sbe-tool/src/{test => propertyTest}/java/uk/co/real_logic/sbe/properties/SchemaDomain.java (81%) rename sbe-tool/src/{test => propertyTest}/java/uk/co/real_logic/sbe/properties/XmlSchemaWriter.java (95%) create mode 100644 sbe-tool/src/propertyTest/resources/junit-platform.properties diff --git a/build.gradle b/build.gradle index c71515f561..191c4c156a 100644 --- a/build.gradle +++ b/build.gradle @@ -31,6 +31,7 @@ buildscript { plugins { id 'java-library' + id 'jvm-test-suite' id 'com.github.johnrengelman.shadow' version '8.1.1' apply false id 'com.github.ben-manes.versions' version '0.51.0' } @@ -165,6 +166,7 @@ jar.enabled = false subprojects { apply plugin: 'java-library' + apply plugin: 'jvm-test-suite' apply plugin: 'checkstyle' group = sbeGroup @@ -217,22 +219,34 @@ subprojects { } } - test { - useJUnitPlatform() + testing { + suites { + test { + useJUnitJupiter junitVersion - testLogging { - for (def level : LogLevel.values()) - { - def testLogging = get(level) - testLogging.exceptionFormat = 'full' - testLogging.events = ["FAILED", "STANDARD_OUT", "STANDARD_ERROR"] - } - } + targets { + all { + testTask.configure { + useJUnitPlatform() - javaLauncher.set(toolchainLauncher) + testLogging { + for (def level : LogLevel.values()) + { + def testLogging = get(level) + testLogging.exceptionFormat = 'full' + testLogging.events = ["FAILED", "STANDARD_OUT", "STANDARD_ERROR"] + } + } + + javaLauncher.set(toolchainLauncher) - systemProperty 'sbe.enable.ir.precedence.checks', 'true' - systemProperty 'sbe.enable.test.precedence.checks', 'true' + systemProperty 'sbe.enable.ir.precedence.checks', 'true' + systemProperty 'sbe.enable.test.precedence.checks', 'true' + } + } + } + } + } } } @@ -249,12 +263,6 @@ project(':sbe-tool') { prefer(agronaVersion) } } - testImplementation files('build/classes/java/generated') - testImplementation "org.hamcrest:hamcrest:${hamcrestVersion}" - testImplementation "org.mockito:mockito-core:${mockitoVersion}" - testImplementation "net.jqwik:jqwik:${jqwikVersion}" - testImplementation "org.junit.jupiter:junit-jupiter-params:${junitVersion}" - testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${junitVersion}" } def generatedDir = 'build/generated-src' @@ -276,6 +284,39 @@ project(':sbe-tool') { compileTestJava.dependsOn compileGeneratedJava + testing { + suites { + test { + dependencies { + implementation files('build/classes/java/generated') + implementation "org.hamcrest:hamcrest:${hamcrestVersion}" + implementation "org.mockito:mockito-core:${mockitoVersion}" + implementation "org.junit.jupiter:junit-jupiter-params:${junitVersion}" + } + } + + propertyTest(JvmTestSuite) { + useJUnitJupiter junitVersion + + dependencies { + implementation project() + implementation "net.jqwik:jqwik:${jqwikVersion}" + } + + targets { + all { + testTask.configure { + minHeapSize = '2g' + maxHeapSize = '2g' + + javaLauncher.set(toolchainLauncher) + } + } + } + } + } + } + tasks.register('generateTestCodecs', JavaExec) { dependsOn 'compileJava' mainClass.set('uk.co.real_logic.sbe.SbeTool') diff --git a/sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java similarity index 97% rename from sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java rename to sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java index 7abb2b65ca..5781777274 100644 --- a/sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java @@ -28,7 +28,7 @@ import static uk.co.real_logic.sbe.xml.XmlSchemaParser.parse; -public class ParserPropertyTest +class ParserPropertyTest { @Property void shouldParseAnyValidSchema(@ForAll("schemas") final SchemaDomain.MessageSchema schema) throws Exception diff --git a/sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/SchemaDomain.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/SchemaDomain.java similarity index 81% rename from sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/SchemaDomain.java rename to sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/SchemaDomain.java index 1605d7d781..ab2d58db15 100644 --- a/sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/SchemaDomain.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/SchemaDomain.java @@ -19,13 +19,14 @@ import net.jqwik.api.Arbitraries; import net.jqwik.api.Arbitrary; import net.jqwik.api.Combinators; +import net.jqwik.api.arbitraries.ListArbitrary; import uk.co.real_logic.sbe.PrimitiveType; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -public final class SchemaDomain +final class SchemaDomain { private static final int MAX_COMPOSITE_DEPTH = 3; private static final int MAX_GROUP_DEPTH = 3; @@ -34,6 +35,57 @@ private SchemaDomain() { } + /** + * This combinator adds duplicates to an arbitrary list. We prefer this to JQwik's built-in functionality, + * as that is inefficient and dominates test runs. + *

+ * This method works by generating a list of integers, which represent, in an alternating manner, + * the number of items to skip before selecting an item to duplicate + * and the number of items to skip before inserting the duplicate. + * + * @param maxDuplicates the maximum number of duplicates to add + * @param arbitrary the arbitrary list to duplicate items in + * @param the type of the list + * @return an arbitrary list with duplicates + */ + private static Arbitrary> withDuplicates( + final int maxDuplicates, + final ListArbitrary arbitrary) + { + return Combinators.combine( + Arbitraries.integers().list().ofMaxSize(2 * maxDuplicates), + arbitrary + ).as((positions, originalItems) -> + { + if (originalItems.isEmpty()) + { + return originalItems; + } + + final List items = new ArrayList<>(originalItems); + T itemToDupe = null; + int j = 0; + + for (final int position : positions) + { + j += position; + j %= items.size(); + j = Math.abs(j); + if (itemToDupe == null) + { + itemToDupe = items.get(j); + } + else + { + items.add(j, itemToDupe); + itemToDupe = null; + } + } + + return items; + }); + } + static final class EncodedDataTypeSchema implements TypeSchema { private final PrimitiveType primitiveType; @@ -90,11 +142,7 @@ public List fields() static Arbitrary arbitrary(final int depth) { - return TypeSchema.arbitrary(depth - 1) - .list() - .ofMinSize(1) - .ofMaxSize(3) - .injectDuplicates(0.2) + return withDuplicates(2, TypeSchema.arbitrary(depth - 1).list().ofMinSize(1).ofMaxSize(3)) .map(CompositeTypeSchema::new); } @@ -309,7 +357,7 @@ static Arbitrary arbitrary(final int depth) arbitrary(depth - 1).list().ofMaxSize(3); return Combinators.combine( - TypeSchema.arbitrary(MAX_COMPOSITE_DEPTH).list().ofMaxSize(5), + withDuplicates(2, TypeSchema.arbitrary(MAX_COMPOSITE_DEPTH).list().ofMaxSize(5)), subGroups, VarDataSchema.arbitrary().list().ofMaxSize(3) ).as(GroupSchema::new); @@ -351,7 +399,7 @@ public List varData() static Arbitrary arbitrary() { return Combinators.combine( - TypeSchema.arbitrary(MAX_COMPOSITE_DEPTH).list().ofMaxSize(10), + withDuplicates(3, TypeSchema.arbitrary(MAX_COMPOSITE_DEPTH).list().ofMaxSize(10)), GroupSchema.arbitrary(MAX_GROUP_DEPTH).list().ofMaxSize(3), VarDataSchema.arbitrary().list().ofMaxSize(3) ).as(MessageSchema::new); diff --git a/sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/XmlSchemaWriter.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/XmlSchemaWriter.java similarity index 95% rename from sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/XmlSchemaWriter.java rename to sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/XmlSchemaWriter.java index 84ff1b2881..2572bb69ca 100644 --- a/sbe-tool/src/test/java/uk/co/real_logic/sbe/properties/XmlSchemaWriter.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/XmlSchemaWriter.java @@ -22,9 +22,7 @@ import java.io.File; import java.io.StringWriter; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.function.Function; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -35,7 +33,6 @@ import javax.xml.transform.stream.StreamResult; import static java.util.Objects.requireNonNull; -import static uk.co.real_logic.sbe.generation.Generators.toLowerFirstChar; final class XmlSchemaWriter { @@ -83,7 +80,9 @@ private static void writeTo( typeToName ); + final Set visitedTypes = new HashSet<>(); appendTypes( + visitedTypes, topLevelTypes, typeSchemaConverter, schema.blockFields(), @@ -93,7 +92,7 @@ private static void writeTo( message.setAttribute("name", "TestMessage"); message.setAttribute("id", "1"); root.appendChild(message); - final MutableInteger nextMemberId = new MutableInteger(1); + final MutableInteger nextMemberId = new MutableInteger(0); appendMembers( document, typeToName, @@ -319,6 +318,7 @@ private static Element createRefElement( } private static void appendTypes( + final Set visitedTypes, final Element topLevelTypes, final TypeSchemaConverter typeSchemaConverter, final List blockFields, @@ -326,7 +326,7 @@ private static void appendTypes( { for (final SchemaDomain.TypeSchema field : blockFields) { - if (!field.isEmbedded()) + if (!field.isEmbedded() && visitedTypes.add(field)) { topLevelTypes.appendChild(typeSchemaConverter.convert(field)); } @@ -334,7 +334,7 @@ private static void appendTypes( for (final SchemaDomain.GroupSchema group : groups) { - appendTypes(topLevelTypes, typeSchemaConverter, group.blockFields(), group.groups()); + appendTypes(visitedTypes, topLevelTypes, typeSchemaConverter, group.blockFields(), group.groups()); } } @@ -370,13 +370,18 @@ public void onEncoded(final SchemaDomain.EncodedDataTypeSchema type) @Override public void onComposite(final SchemaDomain.CompositeTypeSchema type) { - final Element[] parts = type.fields().stream() + final Element[] members = type.fields().stream() .map(this::embedOrReference) .toArray(Element[]::new); + for (int i = 0; i < members.length; i++) + { + final Element member = members[i]; + member.setAttribute("name", "member" + i + "Of" + member.getAttribute("name")); + } result = createCompositeElement( document, typeToName.computeIfAbsent(type, nextName), - parts + members ); } @@ -419,7 +424,7 @@ private Element embedOrReference(final SchemaDomain.TypeSchema type) final String typeName = requireNonNull(typeToName.get(type)); return createRefElement( document, - toLowerFirstChar(typeName), + typeName, typeName ); } diff --git a/sbe-tool/src/propertyTest/resources/junit-platform.properties b/sbe-tool/src/propertyTest/resources/junit-platform.properties new file mode 100644 index 0000000000..8a3fb086c1 --- /dev/null +++ b/sbe-tool/src/propertyTest/resources/junit-platform.properties @@ -0,0 +1,3 @@ +jqwik.edgecases.default=MIXIN +jqwik.tries.default=5000 +jqwik.failures.runfirst=true \ No newline at end of file From ea7e84bacb42a4de44087ec78bb359b628a40be2 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Fri, 6 Oct 2023 09:24:01 +0100 Subject: [PATCH 05/41] [Java] Suppress javadoc warnings in propertyTest source set. --- config/checkstyle/suppressions.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/checkstyle/suppressions.xml b/config/checkstyle/suppressions.xml index 84e7e82814..2823a9547e 100644 --- a/config/checkstyle/suppressions.xml +++ b/config/checkstyle/suppressions.xml @@ -6,5 +6,6 @@ + From 526eac24d96245e4d4ca64b21e3aa9e9368d248e Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Fri, 6 Oct 2023 09:24:32 +0100 Subject: [PATCH 06/41] [Java] Split apart schema generation files. I think separate files improves the readability of the arbitrary message schema generation. --- .../sbe/properties/ParserPropertyTest.java | 9 +- .../sbe/properties/SchemaDomain.java | 408 ------------------ .../arbitraries/SbeArbitraries.java | 189 ++++++++ .../schema/CompositeTypeSchema.java | 40 ++ .../schema/EncodedDataTypeSchema.java | 52 +++ .../sbe/properties/schema/EnumTypeSchema.java | 49 +++ .../sbe/properties/schema/GroupSchema.java | 52 +++ .../sbe/properties/schema/MessageSchema.java | 52 +++ .../sbe/properties/schema/SetSchema.java | 48 +++ .../sbe/properties/schema/TypeSchema.java | 27 ++ .../properties/schema/TypeSchemaVisitor.java | 28 ++ .../sbe/properties/schema/VarDataSchema.java | 38 ++ .../{ => schema}/XmlSchemaWriter.java | 62 +-- 13 files changed, 612 insertions(+), 442 deletions(-) delete mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/SchemaDomain.java create mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java create mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/CompositeTypeSchema.java create mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java create mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EnumTypeSchema.java create mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java create mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java create mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/SetSchema.java create mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchema.java create mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchemaVisitor.java create mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java rename sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/{ => schema}/XmlSchemaWriter.java (86%) diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java index 5781777274..9f0517c5b8 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java @@ -20,6 +20,9 @@ import net.jqwik.api.ForAll; import net.jqwik.api.Property; import net.jqwik.api.Provide; +import uk.co.real_logic.sbe.properties.arbitraries.SbeArbitraries; +import uk.co.real_logic.sbe.properties.schema.MessageSchema; +import uk.co.real_logic.sbe.properties.schema.XmlSchemaWriter; import uk.co.real_logic.sbe.xml.ParserOptions; import java.io.ByteArrayInputStream; @@ -31,7 +34,7 @@ class ParserPropertyTest { @Property - void shouldParseAnyValidSchema(@ForAll("schemas") final SchemaDomain.MessageSchema schema) throws Exception + void shouldParseAnyValidSchema(@ForAll("schemas") final MessageSchema schema) throws Exception { final String xml = XmlSchemaWriter.writeString(schema); try (InputStream in = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) @@ -41,8 +44,8 @@ void shouldParseAnyValidSchema(@ForAll("schemas") final SchemaDomain.MessageSche } @Provide - Arbitrary schemas() + Arbitrary schemas() { - return SchemaDomain.MessageSchema.arbitrary(); + return SbeArbitraries.message(); } } diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/SchemaDomain.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/SchemaDomain.java deleted file mode 100644 index ab2d58db15..0000000000 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/SchemaDomain.java +++ /dev/null @@ -1,408 +0,0 @@ -/* - * Copyright 2013-2023 Real Logic Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package uk.co.real_logic.sbe.properties; - -import net.jqwik.api.Arbitraries; -import net.jqwik.api.Arbitrary; -import net.jqwik.api.Combinators; -import net.jqwik.api.arbitraries.ListArbitrary; -import uk.co.real_logic.sbe.PrimitiveType; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -final class SchemaDomain -{ - private static final int MAX_COMPOSITE_DEPTH = 3; - private static final int MAX_GROUP_DEPTH = 3; - - private SchemaDomain() - { - } - - /** - * This combinator adds duplicates to an arbitrary list. We prefer this to JQwik's built-in functionality, - * as that is inefficient and dominates test runs. - *

- * This method works by generating a list of integers, which represent, in an alternating manner, - * the number of items to skip before selecting an item to duplicate - * and the number of items to skip before inserting the duplicate. - * - * @param maxDuplicates the maximum number of duplicates to add - * @param arbitrary the arbitrary list to duplicate items in - * @param the type of the list - * @return an arbitrary list with duplicates - */ - private static Arbitrary> withDuplicates( - final int maxDuplicates, - final ListArbitrary arbitrary) - { - return Combinators.combine( - Arbitraries.integers().list().ofMaxSize(2 * maxDuplicates), - arbitrary - ).as((positions, originalItems) -> - { - if (originalItems.isEmpty()) - { - return originalItems; - } - - final List items = new ArrayList<>(originalItems); - T itemToDupe = null; - int j = 0; - - for (final int position : positions) - { - j += position; - j %= items.size(); - j = Math.abs(j); - if (itemToDupe == null) - { - itemToDupe = items.get(j); - } - else - { - items.add(j, itemToDupe); - itemToDupe = null; - } - } - - return items; - }); - } - - static final class EncodedDataTypeSchema implements TypeSchema - { - private final PrimitiveType primitiveType; - private final boolean isEmbedded; - - private EncodedDataTypeSchema( - final PrimitiveType primitiveType, - final boolean isEmbedded - ) - { - this.primitiveType = primitiveType; - this.isEmbedded = isEmbedded; - } - - public PrimitiveType primitiveType() - { - return primitiveType; - } - - @Override - public boolean isEmbedded() - { - return isEmbedded; - } - - @Override - public void accept(final TypeSchemaVisitor visitor) - { - visitor.onEncoded(this); - } - - static Arbitrary arbitrary() - { - return Combinators.combine( - Arbitraries.of(PrimitiveType.values()), - Arbitraries.of(true, false) - ).as(EncodedDataTypeSchema::new); - } - } - - static final class CompositeTypeSchema implements TypeSchema - { - private final List fields; - - private CompositeTypeSchema(final List fields) - { - this.fields = fields; - } - - public List fields() - { - return fields; - } - - static Arbitrary arbitrary(final int depth) - { - return withDuplicates(2, TypeSchema.arbitrary(depth - 1).list().ofMinSize(1).ofMaxSize(3)) - .map(CompositeTypeSchema::new); - } - - @Override - public void accept(final TypeSchemaVisitor visitor) - { - visitor.onComposite(this); - } - } - - static final class EnumTypeSchema implements TypeSchema - { - private final String encodingType; - private final List validValues; - - private EnumTypeSchema( - final String encodingType, - final List validValues) - { - this.encodingType = encodingType; - this.validValues = validValues; - } - - public String encodingType() - { - return encodingType; - } - - public List validValues() - { - return validValues; - } - - static Arbitrary arbitrary() - { - return Arbitraries.oneOf( - Arbitraries.chars().alpha() - .map(Character::toUpperCase) - .list() - .ofMaxSize(10) - .uniqueElements() - .map(values -> new EnumTypeSchema( - "char", - values.stream().map(String::valueOf).collect(Collectors.toList()) - )), - Arbitraries.integers() - .between(1, 254) - .list() - .ofMaxSize(254) - .uniqueElements() - .map(values -> new EnumTypeSchema( - "uint8", - values.stream().map(String::valueOf).collect(Collectors.toList()) - )) - ); - } - - @Override - public void accept(final TypeSchemaVisitor visitor) - { - visitor.onEnum(this); - } - } - - static final class SetSchema implements TypeSchema - { - private final String encodingType; - private final int choiceCount; - - private SetSchema( - final String encodingType, - final int choiceCount) - { - this.choiceCount = choiceCount; - this.encodingType = encodingType; - } - - public String encodingType() - { - return encodingType; - } - - public int choiceCount() - { - return choiceCount; - } - - @Override - public void accept(final TypeSchemaVisitor visitor) - { - visitor.onSet(this); - } - - static Arbitrary arbitrary() - { - return Combinators.combine( - Arbitraries.of( - "uint8", - "uint16", - "uint32", - "uint64" - ), - Arbitraries.integers().between(1, 4) - ).as(SetSchema::new); - } - } - - interface TypeSchema - { - static Arbitrary arbitrary(final int depth) - { - if (depth == 1) - { - return Arbitraries.oneOf( - EncodedDataTypeSchema.arbitrary(), - EnumTypeSchema.arbitrary(), - SetSchema.arbitrary() - ); - } - else - { - return Arbitraries.oneOf( - CompositeTypeSchema.arbitrary(depth), - EncodedDataTypeSchema.arbitrary(), - EnumTypeSchema.arbitrary(), - SetSchema.arbitrary() - ); - } - } - - default boolean isEmbedded() - { - return false; - } - - void accept(TypeSchemaVisitor visitor); - } - - interface TypeSchemaVisitor - { - void onEncoded(EncodedDataTypeSchema type); - - void onComposite(CompositeTypeSchema type); - - void onEnum(EnumTypeSchema type); - - void onSet(SetSchema type); - } - - static final class VarDataSchema - { - private final Encoding encoding; - - VarDataSchema(final Encoding encoding) - { - this.encoding = encoding; - } - - public Encoding encoding() - { - return encoding; - } - - static Arbitrary arbitrary() - { - return Arbitraries.of(Encoding.values()) - .map(VarDataSchema::new); - } - - enum Encoding - { - ASCII, - BYTES - } - } - - static final class GroupSchema - { - private final List blockFields; - private final List groups; - private final List varData; - - GroupSchema( - final List blockFields, - final List groups, - final List varData) - { - this.blockFields = blockFields; - this.groups = groups; - this.varData = varData; - } - - public List blockFields() - { - return blockFields; - } - - public List groups() - { - return groups; - } - - public List varData() - { - return varData; - } - - static Arbitrary arbitrary(final int depth) - { - final Arbitrary> subGroups = depth == 1 ? - Arbitraries.of(0).map(ignored -> new ArrayList<>()) : - arbitrary(depth - 1).list().ofMaxSize(3); - - return Combinators.combine( - withDuplicates(2, TypeSchema.arbitrary(MAX_COMPOSITE_DEPTH).list().ofMaxSize(5)), - subGroups, - VarDataSchema.arbitrary().list().ofMaxSize(3) - ).as(GroupSchema::new); - } - } - - static final class MessageSchema - { - private final List blockFields; - private final List groups; - private final List varData; - - MessageSchema( - final List blockFields, - final List groups, - final List varData - ) - { - this.blockFields = blockFields; - this.groups = groups; - this.varData = varData; - } - - public List blockFields() - { - return blockFields; - } - - public List groups() - { - return groups; - } - - public List varData() - { - return varData; - } - - static Arbitrary arbitrary() - { - return Combinators.combine( - withDuplicates(3, TypeSchema.arbitrary(MAX_COMPOSITE_DEPTH).list().ofMaxSize(10)), - GroupSchema.arbitrary(MAX_GROUP_DEPTH).list().ofMaxSize(3), - VarDataSchema.arbitrary().list().ofMaxSize(3) - ).as(MessageSchema::new); - } - } -} diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java new file mode 100644 index 0000000000..a892a27a57 --- /dev/null +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java @@ -0,0 +1,189 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties.arbitraries; + +import net.jqwik.api.Arbitraries; +import net.jqwik.api.Arbitrary; +import net.jqwik.api.Combinators; +import net.jqwik.api.arbitraries.ListArbitrary; +import uk.co.real_logic.sbe.PrimitiveType; +import uk.co.real_logic.sbe.properties.schema.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public final class SbeArbitraries +{ + private static final int MAX_COMPOSITE_DEPTH = 3; + private static final int MAX_GROUP_DEPTH = 3; + + private SbeArbitraries() + { + } + + /** + * This combinator adds duplicates to an arbitrary list. We prefer this to JQwik's built-in functionality, + * as that is inefficient and dominates test runs. + *

+ * This method works by generating a list of integers, which represent, in an alternating manner, + * the number of items to skip before selecting an item to duplicate + * and the number of items to skip before inserting the duplicate. + * + * @param maxDuplicates the maximum number of duplicates to add + * @param arbitrary the arbitrary list to duplicate items in + * @param the type of the list + * @return an arbitrary list with duplicates + */ + private static Arbitrary> withDuplicates( + final int maxDuplicates, + final ListArbitrary arbitrary) + { + return Combinators.combine( + Arbitraries.integers().list().ofMaxSize(2 * maxDuplicates), + arbitrary + ).as((positions, originalItems) -> + { + if (originalItems.isEmpty()) + { + return originalItems; + } + + final List items = new ArrayList<>(originalItems); + T itemToDupe = null; + int j = 0; + + for (final int position : positions) + { + j += position; + j %= items.size(); + j = Math.abs(j); + if (itemToDupe == null) + { + itemToDupe = items.get(j); + } + else + { + items.add(j, itemToDupe); + itemToDupe = null; + } + } + + return items; + }); + } + + private static Arbitrary encodedDataType() + { + return Combinators.combine( + Arbitraries.of(PrimitiveType.values()), + Arbitraries.of(true, false) + ).as(EncodedDataTypeSchema::new); + } + + private static Arbitrary enumType() + { + return Arbitraries.oneOf( + Arbitraries.chars().alpha() + .map(Character::toUpperCase) + .list() + .ofMaxSize(10) + .uniqueElements() + .map(values -> new EnumTypeSchema( + "char", + values.stream().map(String::valueOf).collect(Collectors.toList()) + )), + Arbitraries.integers() + .between(1, 254) + .list() + .ofMaxSize(254) + .uniqueElements() + .map(values -> new EnumTypeSchema( + "uint8", + values.stream().map(String::valueOf).collect(Collectors.toList()) + )) + ); + } + + private static Arbitrary setType() + { + return Combinators.combine( + Arbitraries.of( + "uint8", + "uint16", + "uint32", + "uint64" + ), + Arbitraries.integers().between(1, 4) + ).as(SetSchema::new); + } + + private static Arbitrary compositeType(final int depth) + { + return withDuplicates(2, type(depth - 1).list().ofMinSize(1).ofMaxSize(3)) + .map(CompositeTypeSchema::new); + } + + private static Arbitrary type(final int depth) + { + if (depth == 1) + { + return Arbitraries.oneOf( + encodedDataType(), + enumType(), + setType() + ); + } + else + { + return Arbitraries.oneOf( + compositeType(depth), + encodedDataType(), + enumType(), + setType() + ); + } + } + + private static Arbitrary group(final int depth) + { + final Arbitrary> subGroups = depth == 1 ? + Arbitraries.of(0).map(ignored -> new ArrayList<>()) : + group(depth - 1).list().ofMaxSize(3); + + return Combinators.combine( + withDuplicates(2, type(MAX_COMPOSITE_DEPTH).list().ofMaxSize(5)), + subGroups, + varData().list().ofMaxSize(3) + ).as(GroupSchema::new); + } + + private static Arbitrary varData() + { + return Arbitraries.of(VarDataSchema.Encoding.values()) + .map(VarDataSchema::new); + } + + public static Arbitrary message() + { + return Combinators.combine( + withDuplicates(3, type(MAX_COMPOSITE_DEPTH).list().ofMaxSize(10)), + group(MAX_GROUP_DEPTH).list().ofMaxSize(3), + varData().list().ofMaxSize(3) + ).as(MessageSchema::new); + } +} diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/CompositeTypeSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/CompositeTypeSchema.java new file mode 100644 index 0000000000..a6df53d778 --- /dev/null +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/CompositeTypeSchema.java @@ -0,0 +1,40 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties.schema; + +import java.util.List; + +public final class CompositeTypeSchema implements TypeSchema +{ + private final List fields; + + public CompositeTypeSchema(final List fields) + { + this.fields = fields; + } + + public List fields() + { + return fields; + } + + @Override + public void accept(final TypeSchemaVisitor visitor) + { + visitor.onComposite(this); + } +} diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java new file mode 100644 index 0000000000..b5fe76941d --- /dev/null +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java @@ -0,0 +1,52 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties.schema; + +import uk.co.real_logic.sbe.PrimitiveType; + +public final class EncodedDataTypeSchema implements TypeSchema +{ + private final PrimitiveType primitiveType; + private final boolean isEmbedded; + + public EncodedDataTypeSchema( + final PrimitiveType primitiveType, + final boolean isEmbedded + ) + { + this.primitiveType = primitiveType; + this.isEmbedded = isEmbedded; + } + + public PrimitiveType primitiveType() + { + return primitiveType; + } + + @Override + public boolean isEmbedded() + { + return isEmbedded; + } + + @Override + public void accept(final TypeSchemaVisitor visitor) + { + visitor.onEncoded(this); + } + +} diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EnumTypeSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EnumTypeSchema.java new file mode 100644 index 0000000000..fa669fe04a --- /dev/null +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EnumTypeSchema.java @@ -0,0 +1,49 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties.schema; + +import java.util.List; + +public final class EnumTypeSchema implements TypeSchema +{ + private final String encodingType; + private final List validValues; + + public EnumTypeSchema( + final String encodingType, + final List validValues) + { + this.encodingType = encodingType; + this.validValues = validValues; + } + + public String encodingType() + { + return encodingType; + } + + public List validValues() + { + return validValues; + } + + @Override + public void accept(final TypeSchemaVisitor visitor) + { + visitor.onEnum(this); + } +} diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java new file mode 100644 index 0000000000..df34e0a9b6 --- /dev/null +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java @@ -0,0 +1,52 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties.schema; + +import java.util.List; + +public final class GroupSchema +{ + private final List blockFields; + private final List groups; + private final List varData; + + public GroupSchema( + final List blockFields, + final List groups, + final List varData) + { + this.blockFields = blockFields; + this.groups = groups; + this.varData = varData; + } + + public List blockFields() + { + return blockFields; + } + + public List groups() + { + return groups; + } + + public List varData() + { + return varData; + } + +} diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java new file mode 100644 index 0000000000..c275defe3d --- /dev/null +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java @@ -0,0 +1,52 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties.schema; + +import java.util.List; + +public final class MessageSchema +{ + private final List blockFields; + private final List groups; + private final List varData; + + public MessageSchema( + final List blockFields, + final List groups, + final List varData + ) + { + this.blockFields = blockFields; + this.groups = groups; + this.varData = varData; + } + + public List blockFields() + { + return blockFields; + } + + public List groups() + { + return groups; + } + + public List varData() + { + return varData; + } +} diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/SetSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/SetSchema.java new file mode 100644 index 0000000000..95874e8381 --- /dev/null +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/SetSchema.java @@ -0,0 +1,48 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties.schema; + +public final class SetSchema implements TypeSchema +{ + private final String encodingType; + private final int choiceCount; + + public SetSchema( + final String encodingType, + final int choiceCount) + { + this.choiceCount = choiceCount; + this.encodingType = encodingType; + } + + public String encodingType() + { + return encodingType; + } + + public int choiceCount() + { + return choiceCount; + } + + @Override + public void accept(final TypeSchemaVisitor visitor) + { + visitor.onSet(this); + } + +} diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchema.java new file mode 100644 index 0000000000..311521e697 --- /dev/null +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchema.java @@ -0,0 +1,27 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties.schema; + +public interface TypeSchema +{ + default boolean isEmbedded() + { + return false; + } + + void accept(TypeSchemaVisitor visitor); +} diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchemaVisitor.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchemaVisitor.java new file mode 100644 index 0000000000..830a2e1cf7 --- /dev/null +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchemaVisitor.java @@ -0,0 +1,28 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties.schema; + +public interface TypeSchemaVisitor +{ + void onEncoded(EncodedDataTypeSchema type); + + void onComposite(CompositeTypeSchema type); + + void onEnum(EnumTypeSchema type); + + void onSet(SetSchema type); +} diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java new file mode 100644 index 0000000000..c6b39db56b --- /dev/null +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java @@ -0,0 +1,38 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties.schema; + +public final class VarDataSchema +{ + private final Encoding encoding; + + public VarDataSchema(final Encoding encoding) + { + this.encoding = encoding; + } + + public Encoding encoding() + { + return encoding; + } + + public enum Encoding + { + ASCII, + BYTES + } +} diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/XmlSchemaWriter.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/XmlSchemaWriter.java similarity index 86% rename from sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/XmlSchemaWriter.java rename to sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/XmlSchemaWriter.java index 2572bb69ca..63a10903ca 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/XmlSchemaWriter.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/XmlSchemaWriter.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package uk.co.real_logic.sbe.properties; +package uk.co.real_logic.sbe.properties.schema; import org.agrona.collections.MutableInteger; import org.w3c.dom.Document; @@ -34,13 +34,13 @@ import static java.util.Objects.requireNonNull; -final class XmlSchemaWriter +public final class XmlSchemaWriter { private XmlSchemaWriter() { } - public static String writeString(final SchemaDomain.MessageSchema schema) + public static String writeString(final MessageSchema schema) { final StringWriter writer = new StringWriter(); final StreamResult result = new StreamResult(writer); @@ -49,7 +49,7 @@ public static String writeString(final SchemaDomain.MessageSchema schema) } public static void writeFile( - final SchemaDomain.MessageSchema schema, + final MessageSchema schema, final File destination) { final StreamResult result = new StreamResult(destination); @@ -57,7 +57,7 @@ public static void writeFile( } private static void writeTo( - final SchemaDomain.MessageSchema schema, + final MessageSchema schema, final StreamResult destination) { try @@ -72,7 +72,7 @@ private static void writeTo( final Element topLevelTypes = createTypesElement(document); root.appendChild(topLevelTypes); - final HashMap typeToName = new HashMap<>(); + final HashMap typeToName = new HashMap<>(); final TypeSchemaConverter typeSchemaConverter = new TypeSchemaConverter( document, @@ -80,7 +80,7 @@ private static void writeTo( typeToName ); - final Set visitedTypes = new HashSet<>(); + final Set visitedTypes = new HashSet<>(); appendTypes( visitedTypes, topLevelTypes, @@ -127,20 +127,20 @@ private static void writeTo( private static void appendMembers( final Document document, - final HashMap typeToName, - final List blockFields, - final List groups, - final List varData, + final HashMap typeToName, + final List blockFields, + final List groups, + final List varData, final MutableInteger nextMemberId, final Element parent) { - for (final SchemaDomain.TypeSchema field : blockFields) + for (final TypeSchema field : blockFields) { final int id = nextMemberId.getAndIncrement(); - final boolean usePrimitiveName = field.isEmbedded() && field instanceof SchemaDomain.EncodedDataTypeSchema; + final boolean usePrimitiveName = field.isEmbedded() && field instanceof EncodedDataTypeSchema; final String typeName = usePrimitiveName ? - ((SchemaDomain.EncodedDataTypeSchema)field).primitiveType().primitiveName() : + ((EncodedDataTypeSchema)field).primitiveType().primitiveName() : requireNonNull(typeToName.get(field)); final Element element = document.createElement("field"); @@ -150,7 +150,7 @@ private static void appendMembers( parent.appendChild(element); } - for (final SchemaDomain.GroupSchema group : groups) + for (final GroupSchema group : groups) { final int id = nextMemberId.getAndIncrement(); @@ -168,7 +168,7 @@ private static void appendMembers( parent.appendChild(element); } - for (final SchemaDomain.VarDataSchema data : varData) + for (final VarDataSchema data : varData) { final int id = nextMemberId.getAndIncrement(); @@ -318,13 +318,13 @@ private static Element createRefElement( } private static void appendTypes( - final Set visitedTypes, + final Set visitedTypes, final Element topLevelTypes, final TypeSchemaConverter typeSchemaConverter, - final List blockFields, - final List groups) + final List blockFields, + final List groups) { - for (final SchemaDomain.TypeSchema field : blockFields) + for (final TypeSchema field : blockFields) { if (!field.isEmbedded() && visitedTypes.add(field)) { @@ -332,24 +332,24 @@ private static void appendTypes( } } - for (final SchemaDomain.GroupSchema group : groups) + for (final GroupSchema group : groups) { appendTypes(visitedTypes, topLevelTypes, typeSchemaConverter, group.blockFields(), group.groups()); } } - private static final class TypeSchemaConverter implements SchemaDomain.TypeSchemaVisitor + private static final class TypeSchemaConverter implements TypeSchemaVisitor { private final Document document; private final Element topLevelTypes; - private final Map typeToName; - private final Function nextName; + private final Map typeToName; + private final Function nextName; private Element result; private TypeSchemaConverter( final Document document, final Element topLevelTypes, - final Map typeToName) + final Map typeToName) { this.document = document; this.topLevelTypes = topLevelTypes; @@ -358,7 +358,7 @@ private TypeSchemaConverter( } @Override - public void onEncoded(final SchemaDomain.EncodedDataTypeSchema type) + public void onEncoded(final EncodedDataTypeSchema type) { result = createTypeElement( document, @@ -368,7 +368,7 @@ public void onEncoded(final SchemaDomain.EncodedDataTypeSchema type) } @Override - public void onComposite(final SchemaDomain.CompositeTypeSchema type) + public void onComposite(final CompositeTypeSchema type) { final Element[] members = type.fields().stream() .map(this::embedOrReference) @@ -386,7 +386,7 @@ public void onComposite(final SchemaDomain.CompositeTypeSchema type) } @Override - public void onEnum(final SchemaDomain.EnumTypeSchema type) + public void onEnum(final EnumTypeSchema type) { result = createEnumElement( document, @@ -397,7 +397,7 @@ public void onEnum(final SchemaDomain.EnumTypeSchema type) } @Override - public void onSet(final SchemaDomain.SetSchema type) + public void onSet(final SetSchema type) { result = createSetElement( document, @@ -407,7 +407,7 @@ public void onSet(final SchemaDomain.SetSchema type) ); } - private Element embedOrReference(final SchemaDomain.TypeSchema type) + private Element embedOrReference(final TypeSchema type) { if (type.isEmbedded()) { @@ -430,7 +430,7 @@ private Element embedOrReference(final SchemaDomain.TypeSchema type) } } - public Element convert(final SchemaDomain.TypeSchema type) + public Element convert(final TypeSchema type) { result = null; type.accept(this); From 0a2a6e43e0cdfbc60f47ff9c078cfe74148dbd2d Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Sat, 7 Oct 2023 00:06:46 +0100 Subject: [PATCH 07/41] [Java] Extend PBT to generate arbitrary encoded messages. In this commit, I've also demonstrated the usage of arbitrary encoded messages in a JsonPrinter test. In the test, I print an arbitrary value of an arbitrary schema and check that it is valid JSON. The test passes as long as we constrain the generation of strings, as JsonPrinter currently does not do any escaping. I also noted that it uses `'` rather than `"`, which AFAICS is not permitted by the JSON specification, but is permitted by our (less strict) JSON testing tool. --- build.gradle | 2 + .../sbe/properties/JsonPropertyTest.java | 56 ++ .../sbe/properties/ParserPropertyTest.java | 6 +- .../arbitraries/SbeArbitraries.java | 622 +++++++++++++++++- .../sbe/properties/schema/MessageSchema.java | 10 + ...maWriter.java => TestXmlSchemaWriter.java} | 8 +- .../resources/junit-platform.properties | 5 +- 7 files changed, 678 insertions(+), 31 deletions(-) create mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/JsonPropertyTest.java rename sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/{XmlSchemaWriter.java => TestXmlSchemaWriter.java} (98%) diff --git a/build.gradle b/build.gradle index 191c4c156a..2ccfad6010 100644 --- a/build.gradle +++ b/build.gradle @@ -59,6 +59,7 @@ def hamcrestVersion = '2.2' def mockitoVersion = '4.11.0' def junitVersion = '5.10.2' def jqwikVersion = '1.8.0' +def jsonVersion = '20230618' def jmhVersion = '1.37' def agronaVersion = '1.21.2' def agronaVersionRange = '[1.21.2,2.0[' // allow any release >= 1.21.2 and < 2.0.0 @@ -301,6 +302,7 @@ project(':sbe-tool') { dependencies { implementation project() implementation "net.jqwik:jqwik:${jqwikVersion}" + implementation "org.json:json:${jsonVersion}" } targets { diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/JsonPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/JsonPropertyTest.java new file mode 100644 index 0000000000..ab5c8d955e --- /dev/null +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/JsonPropertyTest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties; + +import net.jqwik.api.Arbitrary; +import net.jqwik.api.ForAll; +import net.jqwik.api.Property; +import net.jqwik.api.Provide; +import uk.co.real_logic.sbe.json.JsonPrinter; +import uk.co.real_logic.sbe.properties.arbitraries.SbeArbitraries; +import org.agrona.concurrent.UnsafeBuffer; +import org.json.JSONException; +import org.json.JSONObject; + +public class JsonPropertyTest +{ + @Property + void shouldGenerateValidJson( + @ForAll("encodedMessage") final SbeArbitraries.EncodedMessage encodedMessage + ) + { + final StringBuilder output = new StringBuilder(); + final JsonPrinter printer = new JsonPrinter(encodedMessage.ir()); + printer.print(output, new UnsafeBuffer(encodedMessage.buffer()), 0); + try + { + new JSONObject(output.toString()); + } + catch (final JSONException e) + { + throw new AssertionError("Invalid JSON: " + output + "\n\nSchema:\n" + encodedMessage.schema(), e); + } + } + + @Provide + Arbitrary encodedMessage() + { + final SbeArbitraries.CharGenerationMode mode = + SbeArbitraries.CharGenerationMode.JSON_PRINTER_COMPATIBLE; + return SbeArbitraries.encodedMessage(mode); + } +} diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java index 9f0517c5b8..b6ce84ef3a 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java @@ -22,7 +22,7 @@ import net.jqwik.api.Provide; import uk.co.real_logic.sbe.properties.arbitraries.SbeArbitraries; import uk.co.real_logic.sbe.properties.schema.MessageSchema; -import uk.co.real_logic.sbe.properties.schema.XmlSchemaWriter; +import uk.co.real_logic.sbe.properties.schema.TestXmlSchemaWriter; import uk.co.real_logic.sbe.xml.ParserOptions; import java.io.ByteArrayInputStream; @@ -36,7 +36,7 @@ class ParserPropertyTest @Property void shouldParseAnyValidSchema(@ForAll("schemas") final MessageSchema schema) throws Exception { - final String xml = XmlSchemaWriter.writeString(schema); + final String xml = TestXmlSchemaWriter.writeString(schema); try (InputStream in = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) { parse(in, ParserOptions.DEFAULT); @@ -46,6 +46,6 @@ void shouldParseAnyValidSchema(@ForAll("schemas") final MessageSchema schema) th @Provide Arbitrary schemas() { - return SbeArbitraries.message(); + return SbeArbitraries.messageSchema(); } } diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java index a892a27a57..eb6ef0603f 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java @@ -19,18 +19,38 @@ import net.jqwik.api.Arbitraries; import net.jqwik.api.Arbitrary; import net.jqwik.api.Combinators; +import net.jqwik.api.arbitraries.CharacterArbitrary; import net.jqwik.api.arbitraries.ListArbitrary; import uk.co.real_logic.sbe.PrimitiveType; +import uk.co.real_logic.sbe.PrimitiveValue; +import uk.co.real_logic.sbe.ir.Encoding; +import uk.co.real_logic.sbe.ir.Ir; +import uk.co.real_logic.sbe.ir.Token; import uk.co.real_logic.sbe.properties.schema.*; +import uk.co.real_logic.sbe.xml.IrGenerator; +import uk.co.real_logic.sbe.xml.ParserOptions; +import org.agrona.BitUtil; +import org.agrona.ExpandableArrayBuffer; +import org.agrona.MutableDirectBuffer; +import org.agrona.collections.MutableInteger; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.stream.Collectors; +import static uk.co.real_logic.sbe.ir.Signal.*; +import static uk.co.real_logic.sbe.xml.XmlSchemaParser.parse; + +@SuppressWarnings("EnhancedSwitchMigration") public final class SbeArbitraries { private static final int MAX_COMPOSITE_DEPTH = 3; private static final int MAX_GROUP_DEPTH = 3; + public static final int NULL_VALUE = Integer.MIN_VALUE; private SbeArbitraries() { @@ -87,7 +107,7 @@ private static Arbitrary> withDuplicates( }); } - private static Arbitrary encodedDataType() + private static Arbitrary encodedDataTypeSchema() { return Combinators.combine( Arbitraries.of(PrimitiveType.values()), @@ -95,7 +115,13 @@ private static Arbitrary encodedDataType() ).as(EncodedDataTypeSchema::new); } - private static Arbitrary enumType() + public enum CharGenerationMode + { + UNRESTRICTED, + JSON_PRINTER_COMPATIBLE + } + + private static Arbitrary enumTypeSchema() { return Arbitraries.oneOf( Arbitraries.chars().alpha() @@ -119,7 +145,7 @@ private static Arbitrary enumType() ); } - private static Arbitrary setType() + private static Arbitrary setTypeSchema() { return Combinators.combine( Arbitraries.of( @@ -132,58 +158,610 @@ private static Arbitrary setType() ).as(SetSchema::new); } - private static Arbitrary compositeType(final int depth) + private static Arbitrary compositeTypeSchema(final int depth) { - return withDuplicates(2, type(depth - 1).list().ofMinSize(1).ofMaxSize(3)) + return withDuplicates(2, typeSchema(depth - 1).list().ofMinSize(1).ofMaxSize(3)) .map(CompositeTypeSchema::new); } - private static Arbitrary type(final int depth) + private static Arbitrary typeSchema(final int depth) { if (depth == 1) { return Arbitraries.oneOf( - encodedDataType(), - enumType(), - setType() + encodedDataTypeSchema(), + enumTypeSchema(), + setTypeSchema() ); } else { return Arbitraries.oneOf( - compositeType(depth), - encodedDataType(), - enumType(), - setType() + compositeTypeSchema(depth), + encodedDataTypeSchema(), + enumTypeSchema(), + setTypeSchema() ); } } - private static Arbitrary group(final int depth) + private static Arbitrary groupSchema(final int depth) { final Arbitrary> subGroups = depth == 1 ? Arbitraries.of(0).map(ignored -> new ArrayList<>()) : - group(depth - 1).list().ofMaxSize(3); + groupSchema(depth - 1).list().ofMaxSize(3); return Combinators.combine( - withDuplicates(2, type(MAX_COMPOSITE_DEPTH).list().ofMaxSize(5)), + withDuplicates(2, typeSchema(MAX_COMPOSITE_DEPTH).list().ofMaxSize(5)), subGroups, - varData().list().ofMaxSize(3) + varDataSchema().list().ofMaxSize(3) ).as(GroupSchema::new); } - private static Arbitrary varData() + private static Arbitrary varDataSchema() { return Arbitraries.of(VarDataSchema.Encoding.values()) .map(VarDataSchema::new); } - public static Arbitrary message() + public static Arbitrary messageSchema() { return Combinators.combine( - withDuplicates(3, type(MAX_COMPOSITE_DEPTH).list().ofMaxSize(10)), - group(MAX_GROUP_DEPTH).list().ofMaxSize(3), - varData().list().ofMaxSize(3) + withDuplicates(3, typeSchema(MAX_COMPOSITE_DEPTH).list().ofMaxSize(10)), + groupSchema(MAX_GROUP_DEPTH).list().ofMaxSize(3), + varDataSchema().list().ofMaxSize(3) ).as(MessageSchema::new); } + + public interface Encoder + { + void encode(MutableDirectBuffer buffer, int offset, MutableInteger limit); + } + + private static Encoder combineEncoders(final Collection encoders) + { + return (buffer, offset, limit) -> + { + for (final Encoder encoder : encoders) + { + encoder.encode(buffer, offset, limit); + } + }; + } + + private static Arbitrary combineArbitraryEncoders(final List> encoders) + { + if (encoders.isEmpty()) + { + return Arbitraries.of(emptyEncoder()); + } + else + { + return Combinators.combine(encoders).as(SbeArbitraries::combineEncoders); + } + } + + public static CharacterArbitrary chars(final CharGenerationMode mode) + { + switch (mode) + { + case UNRESTRICTED: + return Arbitraries.chars(); + case JSON_PRINTER_COMPATIBLE: + return Arbitraries.chars().alpha(); + default: + throw new IllegalArgumentException("Unsupported mode: " + mode); + } + } + + private static Arbitrary encodedTypeEncoder( + final Encoding encoding, + final CharGenerationMode charGenerationMode) + { + final PrimitiveValue minValue = encoding.applicableMinValue(); + final PrimitiveValue maxValue = encoding.applicableMaxValue(); + + switch (encoding.primitiveType()) + { + case CHAR: + assert minValue.longValue() <= maxValue.longValue(); + return chars(charGenerationMode).map(c -> + (buffer, offset, limit) -> buffer.putChar(offset, c, encoding.byteOrder())); + + case UINT8: + case INT8: + assert (short)minValue.longValue() <= (short)maxValue.longValue(); + return Arbitraries.shorts() + .between((short)minValue.longValue(), (short)maxValue.longValue()) + .map(b -> (buffer, offset, limit) -> buffer.putByte(offset, (byte)(short)b)); + + case UINT16: + case INT16: + assert (int)minValue.longValue() <= (int)maxValue.longValue(); + return Arbitraries.integers() + .between((int)minValue.longValue(), (int)maxValue.longValue()) + .map(s -> (buffer, offset, limit) -> buffer.putShort(offset, (short)(int)s, encoding.byteOrder())); + + case UINT32: + case INT32: + assert minValue.longValue() <= maxValue.longValue(); + return Arbitraries.longs() + .between(minValue.longValue(), maxValue.longValue()) + .map(i -> (buffer, offset, limit) -> buffer.putInt(offset, (int)(long)i, encoding.byteOrder())); + + case UINT64: + return Arbitraries.longs() + .map(l -> (buffer, offset, limit) -> buffer.putLong(offset, l, encoding.byteOrder())); + + case INT64: + assert minValue.longValue() <= maxValue.longValue(); + return Arbitraries.longs() + .between(minValue.longValue(), maxValue.longValue()) + .map(l -> (buffer, offset, limit) -> buffer.putLong(offset, l, encoding.byteOrder())); + + case FLOAT: + return Arbitraries.floats() + .map(f -> (buffer, offset, limit) -> buffer.putFloat(offset, f, encoding.byteOrder())); + + case DOUBLE: + return Arbitraries.doubles() + .map(d -> (buffer, offset, limit) -> buffer.putDouble(offset, d, encoding.byteOrder())); + + default: + throw new IllegalArgumentException("Unsupported type: " + encoding.primitiveType()); + } + } + + private static Arbitrary encodedTypeEncoder( + final int offset, + final Token typeToken, + final CharGenerationMode charGenerationMode) + { + final Encoding encoding = typeToken.encoding(); + final Arbitrary arbEncoder = encodedTypeEncoder(encoding, charGenerationMode); + + if (typeToken.arrayLength() == 1) + { + return arbEncoder.map(encoder -> (buffer, bufferOffset, limit) -> + encoder.encode(buffer, bufferOffset + offset, limit)); + } + else + { + return arbEncoder.map(encoder -> (buffer, bufferOffset, limit) -> + { + for (int i = 0; i < typeToken.arrayLength(); i++) + { + encoder.encode(buffer, bufferOffset + offset + i * encoding.primitiveType().size(), limit); + } + }); + } + } + + private static Encoder emptyEncoder() + { + return (buffer, offset, limit) -> + { + }; + } + + private static Encoder integerValueEncoder(final Encoding encoding, final long value) + { + final PrimitiveType type = encoding.primitiveType(); + switch (type) + { + case CHAR: + case UINT8: + case INT8: + return (buffer, offset, limit) -> buffer.putByte(offset, (byte)value); + + case UINT16: + case INT16: + return (buffer, offset, limit) -> buffer.putShort(offset, (short)value, encoding.byteOrder()); + + case UINT32: + case INT32: + return (buffer, offset, limit) -> buffer.putInt(offset, (int)value, encoding.byteOrder()); + + case UINT64: + case INT64: + return (buffer, offset, limit) -> buffer.putLong(offset, value, encoding.byteOrder()); + + default: + throw new IllegalArgumentException("Unsupported type: " + type); + } + } + + private static Arbitrary enumEncoder( + final int offset, + final List tokens, + final Token typeToken, + final MutableInteger cursor, + final int endIdxInclusive) + { + cursor.increment(); + + final List encoders = new ArrayList<>(); + for (; cursor.get() <= endIdxInclusive; cursor.increment()) + { + final Token token = tokens.get(cursor.get()); + + if (VALID_VALUE != token.signal()) + { + throw new IllegalArgumentException("Expected VALID_VALUE token"); + } + + final Encoding encoding = token.encoding(); + final Encoder caseEncoder = integerValueEncoder(encoding, encoding.constValue().longValue()); + encoders.add(caseEncoder); + } + + if (encoders.isEmpty()) + { + final Encoder nullEncoder = integerValueEncoder( + typeToken.encoding(), + typeToken.encoding().nullValue().longValue()); + encoders.add(nullEncoder); + } + + return Arbitraries.of(encoders).map(encoder -> + (buffer, bufferOffset, limit) -> encoder.encode(buffer, bufferOffset + offset, limit)); + } + + private static Encoder choiceEncoder(final Encoding encoding) + { + final long choiceBitIdx = encoding.constValue().longValue(); + final PrimitiveType type = encoding.primitiveType(); + switch (type) + { + case UINT8: + case INT8: + return (buffer, offset, limit) -> + { + buffer.checkLimit(offset + BitUtil.SIZE_OF_BYTE); + final byte oldValue = buffer.getByte(offset); + final byte newValue = (byte)(oldValue | (1 << choiceBitIdx)); + buffer.putByte(offset, newValue); + }; + + case UINT16: + case INT16: + return (buffer, offset, limit) -> + { + buffer.checkLimit(offset + BitUtil.SIZE_OF_SHORT); + final short oldValue = buffer.getShort(offset, encoding.byteOrder()); + final short newValue = (short)(oldValue | (1 << choiceBitIdx)); + buffer.putShort(offset, newValue, encoding.byteOrder()); + }; + + case UINT32: + case INT32: + return (buffer, offset, limit) -> + { + buffer.checkLimit(offset + BitUtil.SIZE_OF_INT); + final int oldValue = buffer.getInt(offset, encoding.byteOrder()); + final int newValue = oldValue | (1 << choiceBitIdx); + buffer.putInt(offset, newValue, encoding.byteOrder()); + }; + + case UINT64: + case INT64: + return (buffer, offset, limit) -> + { + buffer.checkLimit(offset + BitUtil.SIZE_OF_LONG); + final long oldValue = buffer.getLong(offset, encoding.byteOrder()); + final long newValue = oldValue | (1L << choiceBitIdx); + buffer.putLong(offset, newValue, encoding.byteOrder()); + }; + + default: + throw new IllegalArgumentException("Unsupported type: " + type); + } + } + + private static Arbitrary bitSetEncoder( + final int offset, + final List tokens, + final MutableInteger cursor, + final int endIdxInclusive) + { + cursor.increment(); + + final List encoders = new ArrayList<>(); + for (; cursor.get() <= endIdxInclusive; cursor.increment()) + { + final Token token = tokens.get(cursor.get()); + + if (CHOICE != token.signal()) + { + throw new IllegalArgumentException("Expected CHOICE token"); + } + + final Encoding encoding = token.encoding(); + final Encoder choiceEncoder = choiceEncoder(encoding); + encoders.add(choiceEncoder); + } + + return Arbitraries.subsetOf(encoders) + .map(SbeArbitraries::combineEncoders) + .map(encoder -> (buffer, bufferOffset, limit) -> encoder.encode(buffer, bufferOffset + offset, limit)); + } + + private static Arbitrary fieldsEncoder( + final List tokens, + final MutableInteger cursor, + final int endIdxInclusive, + final boolean expectFields, + final CharGenerationMode charGenerationMode) + { + final List> encoders = new ArrayList<>(); + while (cursor.get() <= endIdxInclusive) + { + final Token memberToken = tokens.get(cursor.get()); + final int nextFieldIdx = cursor.get() + memberToken.componentTokenCount(); + + Token typeToken = memberToken; + int endFieldTokenCount = 0; + + if (BEGIN_FIELD == memberToken.signal()) + { + cursor.increment(); + typeToken = tokens.get(cursor.get()); + endFieldTokenCount = 1; + } + else if (expectFields) + { + break; + } + + final int offset = typeToken.offset(); + + if (!memberToken.isConstantEncoding()) + { + switch (typeToken.signal()) + { + case BEGIN_COMPOSITE: + cursor.increment(); + final int endCompositeTokenCount = 1; + final int lastMemberIdx = nextFieldIdx - endCompositeTokenCount - endFieldTokenCount - 1; + final Arbitrary encoder = fieldsEncoder( + tokens, cursor, lastMemberIdx, false, charGenerationMode); + final Arbitrary positionedEncoder = encoder.map(e -> + (buffer, bufferOffset, limit) -> e.encode(buffer, bufferOffset + offset, limit)); + encoders.add(positionedEncoder); + break; + + case BEGIN_ENUM: + final int endEnumTokenCount = 1; + final int lastValidValueIdx = nextFieldIdx - endFieldTokenCount - endEnumTokenCount - 1; + encoders.add(enumEncoder(offset, tokens, typeToken, cursor, lastValidValueIdx)); + break; + + case BEGIN_SET: + final int endSetTokenCount = 1; + final int lastChoiceIdx = nextFieldIdx - endFieldTokenCount - endSetTokenCount - 1; + encoders.add(bitSetEncoder(offset, tokens, cursor, lastChoiceIdx)); + break; + + case ENCODING: + encoders.add(encodedTypeEncoder(offset, typeToken, charGenerationMode)); + break; + + default: + break; + } + } + + cursor.set(nextFieldIdx); + } + + return combineArbitraryEncoders(encoders); + } + + + private static Arbitrary groupsEncoder( + final List tokens, + final MutableInteger cursor, + final int endIdxInclusive, + final CharGenerationMode charGenerationMode) + { + final List> encoders = new ArrayList<>(); + + while (cursor.get() <= endIdxInclusive) + { + final Token token = tokens.get(cursor.get()); + if (BEGIN_GROUP != token.signal()) + { + break; + } + final int nextFieldIdx = cursor.get() + token.componentTokenCount(); + + cursor.increment(); // consume BEGIN_GROUP + cursor.increment(); // consume BEGIN_COMPOSITE + final Token blockLengthToken = tokens.get(cursor.get()); + final int blockLength = token.encodedLength(); + final Encoder blockLengthEncoder = integerValueEncoder(blockLengthToken.encoding(), blockLength); + cursor.increment(); // consume ENCODED + final Token numInGroupToken = tokens.get(cursor.get()); + cursor.increment(); // consume ENCODED + cursor.increment(); // consume END_COMPOSITE + final int headerLength = blockLengthToken.encodedLength() + numInGroupToken.encodedLength(); + + + final Arbitrary groupElement = Combinators.combine( + fieldsEncoder(tokens, cursor, nextFieldIdx - 1, true, charGenerationMode), + groupsEncoder(tokens, cursor, nextFieldIdx - 1, charGenerationMode), + varDataEncoder(tokens, cursor, nextFieldIdx - 1, charGenerationMode) + ).as((fieldsEncoder, groupsEncoder, varDataEncoder) -> + (buffer, ignored, limit) -> + { + final int offset = limit.get(); + fieldsEncoder.encode(buffer, offset, null); + limit.set(offset + blockLength); + groupsEncoder.encode(buffer, NULL_VALUE, limit); + varDataEncoder.encode(buffer, NULL_VALUE, limit); + }); + + final Arbitrary repeatingGroupEncoder = groupElement.list() + .ofMaxSize(10) + .map(elements -> (buffer, ignored, limit) -> + { + final int offset = limit.get(); + limit.set(offset + headerLength); + blockLengthEncoder.encode(buffer, offset, null); + integerValueEncoder(numInGroupToken.encoding(), elements.size()) + .encode(buffer, offset + blockLengthToken.encodedLength(), null); + + for (final Encoder element : elements) + { + element.encode(buffer, NULL_VALUE, limit); + } + }); + + encoders.add(repeatingGroupEncoder); + + cursor.set(nextFieldIdx); + } + + return combineArbitraryEncoders(encoders); + } + + private static Arbitrary varDataEncoder( + final List tokens, + final MutableInteger cursor, + final int endIdxInclusive, + final CharGenerationMode charGenerationMode) + { + final List> encoders = new ArrayList<>(); + + while (cursor.get() <= endIdxInclusive) + { + final Token token = tokens.get(cursor.get()); + if (BEGIN_VAR_DATA != token.signal()) + { + break; + } + final int nextFieldIdx = cursor.get() + token.componentTokenCount(); + + cursor.increment(); // BEGIN_COMPOSITE + cursor.increment(); // ENCODED + final Token lengthToken = tokens.get(cursor.get()); + cursor.increment(); // ENCODED + final Token varDataToken = tokens.get(cursor.get()); + cursor.increment(); // END_COMPOSITE + + final String characterEncoding = varDataToken.encoding().characterEncoding(); + final Arbitrary arbitraryByte = null == characterEncoding ? + Arbitraries.bytes() : + chars(charGenerationMode).map(c -> (byte)c.charValue()); + encoders.add(arbitraryByte.list().map(bytes -> + (buffer, ignored, limit) -> + { + final int offset = limit.get(); + final int elementLength = varDataToken.encoding().primitiveType().size(); + limit.set(offset + lengthToken.encodedLength() + bytes.size() * elementLength); + integerValueEncoder(lengthToken.encoding(), bytes.size()) + .encode(buffer, offset, null); + for (int i = 0; i < bytes.size(); i++) + { + final int dataOffset = offset + lengthToken.encodedLength() + i * elementLength; + integerValueEncoder(varDataToken.encoding(), bytes.get(i)) + .encode(buffer, dataOffset, null); + } + })); + + cursor.set(nextFieldIdx); + } + + return combineArbitraryEncoders(encoders); + } + + public static Arbitrary messageValueEncoder( + final Ir ir, + final short messageId, + final CharGenerationMode charGenerationMode) + { + final List tokens = ir.getMessage(messageId); + final MutableInteger cursor = new MutableInteger(1); + + final Token token = tokens.get(0); + if (BEGIN_MESSAGE != token.signal()) + { + throw new IllegalArgumentException("Expected BEGIN_MESSAGE token"); + } + + final Arbitrary fieldsEncoder = fieldsEncoder( + tokens, cursor, tokens.size() - 1, true, charGenerationMode); + final Arbitrary groupsEncoder = groupsEncoder( + tokens, cursor, tokens.size() - 1, charGenerationMode); + final Arbitrary varDataEncoder = varDataEncoder( + tokens, cursor, tokens.size() - 1, charGenerationMode); + return Combinators.combine(fieldsEncoder, groupsEncoder, varDataEncoder) + .as((fields, groups, varData) -> (buffer, offset, limit) -> + { + final int blockLength = token.encodedLength(); + buffer.putShort(0, (short)blockLength, ir.byteOrder()); + buffer.putShort(2, messageId, ir.byteOrder()); + buffer.putShort(4, (short)ir.id(), ir.byteOrder()); + buffer.putShort(6, (short)0, ir.byteOrder()); + final int headerLength = 8; + fields.encode(buffer, offset + headerLength, null); + limit.set(offset + headerLength + blockLength); + groups.encode(buffer, NULL_VALUE, limit); + varData.encode(buffer, NULL_VALUE, limit); + }); + } + + public static final class EncodedMessage + { + private final String schema; + private final Ir ir; + private final ExpandableArrayBuffer buffer; + + private EncodedMessage(final String schema, final Ir ir, final ExpandableArrayBuffer buffer) + { + this.schema = schema; + this.ir = ir; + this.buffer = buffer; + } + + public String schema() + { + return schema; + } + + public Ir ir() + { + return ir; + } + + public ExpandableArrayBuffer buffer() + { + return buffer; + } + } + + public static Arbitrary encodedMessage(final CharGenerationMode mode) + { + return SbeArbitraries.messageSchema().flatMap(testSchema -> + { + final String xml = TestXmlSchemaWriter.writeString(testSchema); + try (InputStream in = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) + { + final uk.co.real_logic.sbe.xml.MessageSchema parsedSchema = parse(in, ParserOptions.DEFAULT); + final Ir ir = new IrGenerator().generate(parsedSchema); + return SbeArbitraries.messageValueEncoder(ir, testSchema.templateId(), mode) + .map(encoder -> + { + final ExpandableArrayBuffer buffer = new ExpandableArrayBuffer(); + final MutableInteger limit = new MutableInteger(); + encoder.encode(buffer, 0, limit); + return new EncodedMessage(xml, ir, buffer); + }); + } + catch (final Exception e) + { + throw new RuntimeException(e); + } + }).withoutEdgeCases(); + } } diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java index c275defe3d..451f61a34f 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java @@ -35,6 +35,16 @@ public MessageSchema( this.varData = varData; } + public short schemaId() + { + return 42; + } + + public short templateId() + { + return 1; + } + public List blockFields() { return blockFields; diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/XmlSchemaWriter.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java similarity index 98% rename from sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/XmlSchemaWriter.java rename to sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java index 63a10903ca..242f7c9946 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/XmlSchemaWriter.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java @@ -34,9 +34,9 @@ import static java.util.Objects.requireNonNull; -public final class XmlSchemaWriter +public final class TestXmlSchemaWriter { - private XmlSchemaWriter() + private TestXmlSchemaWriter() { } @@ -65,7 +65,7 @@ private static void writeTo( final Document document = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument(); final Element root = document.createElementNS("http://fixprotocol.io/2016/sbe", "sbe:messageSchema"); - root.setAttribute("id", "42"); + root.setAttribute("id", Short.toString(schema.schemaId())); root.setAttribute("package", "uk.co.real_logic.sbe.properties"); document.appendChild(root); @@ -90,7 +90,7 @@ private static void writeTo( final Element message = document.createElement("sbe:message"); message.setAttribute("name", "TestMessage"); - message.setAttribute("id", "1"); + message.setAttribute("id", Short.toString(schema.templateId())); root.appendChild(message); final MutableInteger nextMemberId = new MutableInteger(0); appendMembers( diff --git a/sbe-tool/src/propertyTest/resources/junit-platform.properties b/sbe-tool/src/propertyTest/resources/junit-platform.properties index 8a3fb086c1..8194cc7e80 100644 --- a/sbe-tool/src/propertyTest/resources/junit-platform.properties +++ b/sbe-tool/src/propertyTest/resources/junit-platform.properties @@ -1,3 +1,4 @@ jqwik.edgecases.default=MIXIN -jqwik.tries.default=5000 -jqwik.failures.runfirst=true \ No newline at end of file +jqwik.tries.default=500 +jqwik.failures.runfirst=true +jqwik.shrinking.bounded.seconds=90 \ No newline at end of file From 5af0274372a8a8498fa7ee2b2fdcafcca745856e Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Sat, 7 Oct 2023 03:15:25 +0100 Subject: [PATCH 08/41] [Java] Test DTOs preserve information. In this commit, I've added a round trip property test, where I show `dto.EncodeInto(...)` is the inverse of `dto.DecodeFrom(...)` and that the original encoded value is preserved through the transformation. --- .../generation/csharp/CSharpDtoGenerator.java | 7 +- .../properties/CSharpDtosPropertyTest.java | 130 ++++++++++++++++++ .../CSharpDtosPropertyTest/Program.cs | 31 +++++ .../SbePropertyTest.csproj | 14 ++ 4 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java create mode 100644 sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/Program.cs create mode 100644 sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/SbePropertyTest.csproj diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java index a0f1e21d9d..e7ceeeb155 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java @@ -266,7 +266,7 @@ private void generateFieldDecodeFrom( break; case BEGIN_COMPOSITE: - generateComplexDecodeFrom(sb, fieldToken, indent); + generateComplexDecodeFrom(sb, fieldToken, typeToken, indent); break; default: @@ -389,10 +389,12 @@ private void generatePropertyDecodeFrom( private void generateComplexDecodeFrom( final StringBuilder sb, final Token fieldToken, + final Token typeToken, final String indent) { final String propertyName = fieldToken.name(); final String formattedPropertyName = formatPropertyName(propertyName); + final String dtoClassName = formatDtoClassName(typeToken.applicableTypeName()); wrapInActingVersionCheck( sb, @@ -401,7 +403,7 @@ private void generateComplexDecodeFrom( (blkSb, blkIndent) -> { blkSb.append(blkIndent).append(formattedPropertyName).append(" = new ") - .append(formatDtoClassName(propertyName)).append("();\n") + .append(dtoClassName).append("();\n") .append(blkIndent).append(formattedPropertyName).append(".DecodeFrom(codec.") .append(formattedPropertyName).append(");\n"); }, @@ -577,7 +579,6 @@ private void generateFieldEncodeInto( final Token typeToken, final String indent) { - switch (typeToken.signal()) { case ENCODING: diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java new file mode 100644 index 0000000000..2c3cc814e1 --- /dev/null +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java @@ -0,0 +1,130 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties; + +import net.jqwik.api.Arbitrary; +import net.jqwik.api.ForAll; +import net.jqwik.api.Property; +import net.jqwik.api.Provide; +import uk.co.real_logic.sbe.generation.csharp.CSharpDtoGenerator; +import uk.co.real_logic.sbe.generation.csharp.CSharpGenerator; +import uk.co.real_logic.sbe.generation.csharp.CSharpNamespaceOutputManager; +import uk.co.real_logic.sbe.properties.arbitraries.SbeArbitraries; +import org.agrona.IoUtil; +import org.agrona.io.DirectBufferInputStream; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; + +public class CSharpDtosPropertyTest +{ + @Property + void dtoEncodeShouldBeTheInverseOfDtoDecode( + @ForAll("encodedMessage") final SbeArbitraries.EncodedMessage encodedMessage + ) throws IOException, InterruptedException + { + final Path tempDir = Files.createTempDirectory("sbe-csharp-dto-test"); + try + { + final CSharpNamespaceOutputManager outputManager = new CSharpNamespaceOutputManager( + tempDir.toString(), + "SbePropertyTest" + ); + new CSharpGenerator(encodedMessage.ir(), outputManager) + .generate(); + new CSharpDtoGenerator(encodedMessage.ir(), outputManager) + .generate(); + copyResourceToFile("/CSharpDtosPropertyTest/SbePropertyTest.csproj", tempDir); + copyResourceToFile("/CSharpDtosPropertyTest/Program.cs", tempDir); + try ( + DirectBufferInputStream inputStream = new DirectBufferInputStream(encodedMessage.buffer()); + OutputStream outputStream = Files.newOutputStream(tempDir.resolve("input.dat"))) + { + final byte[] buffer = new byte[2048]; + int read; + while ((read = inputStream.read(buffer, 0, buffer.length)) >= 0) + { + outputStream.write(buffer, 0, read); + } + } + + final Path stdout = tempDir.resolve("stdout.txt"); + final Path stderr = tempDir.resolve("stderr.txt"); + final ProcessBuilder processBuilder = new ProcessBuilder("dotnet", "run", "--", "input.dat") + .directory(tempDir.toFile()) + .redirectOutput(stdout.toFile()) + .redirectError(stderr.toFile()); + + final Process process = processBuilder.start(); + if (0 != process.waitFor()) + { + throw new AssertionError( + "Process failed with exit code: " + process.exitValue() + "\n\n" + + "STDOUT:\n" + new String(Files.readAllBytes(stdout)) + "\n\n" + + "STDERR:\n" + new String(Files.readAllBytes(stderr)) + "\n\n" + + "SCHEMA:\n" + encodedMessage.schema()); + } + + final byte[] inputBytes = encodedMessage.buffer().byteArray(); + final byte[] outputBytes = Files.readAllBytes(tempDir.resolve("output.dat")); + if (!Arrays.equals(inputBytes, outputBytes)) + { + throw new AssertionError( + "Input and output files differ\n\n" + + "SCHEMA:\n" + encodedMessage.schema()); + } + } + finally + { + IoUtil.delete(tempDir.toFile(), true); + } + } + + @Provide + Arbitrary encodedMessage() + { + final SbeArbitraries.CharGenerationMode mode = + SbeArbitraries.CharGenerationMode.JSON_PRINTER_COMPATIBLE; + return SbeArbitraries.encodedMessage(mode); + } + + private static void copyResourceToFile( + final String resourcePath, + final Path outputDir) + { + try (InputStream inputStream = CSharpDtosPropertyTest.class.getResourceAsStream(resourcePath)) + { + if (inputStream == null) + { + throw new IOException("Resource not found: " + resourcePath); + } + + final int resourceNameIndex = resourcePath.lastIndexOf('/') + 1; + final String resourceName = resourcePath.substring(resourceNameIndex); + final Path outputFilePath = outputDir.resolve(resourceName); + Files.copy(inputStream, outputFilePath); + } + catch (final IOException e) + { + throw new RuntimeException(e); + } + } +} diff --git a/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/Program.cs b/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/Program.cs new file mode 100644 index 0000000000..1efcf4bfa7 --- /dev/null +++ b/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/Program.cs @@ -0,0 +1,31 @@ +using System; +using System.IO; +using Org.SbeTool.Sbe.Dll; +using Uk.Co.Real_logic.Sbe.Properties; + +namespace SbePropertyTest { + static class Test { + static int Main(string[] args) { + if (args.Length != 1) { + Console.WriteLine("Usage: dotnet run -- $BINARY_FILE"); + return 1; + } + var binaryFile = args[0]; + var inputBytes = File.ReadAllBytes(binaryFile); + var buffer = new DirectBuffer(inputBytes); + var messageHeader = new MessageHeader(); + messageHeader.Wrap(buffer, 0, 0); + var decoder = new TestMessage(); + decoder.WrapForDecode(buffer, 8, messageHeader.BlockLength, messageHeader.Version); + var dto = new TestMessageDto(); + dto.DecodeFrom(decoder); + var outputBytes = new byte[inputBytes.Length]; + var outputBuffer = new DirectBuffer(outputBytes); + var encoder = new TestMessage(); + encoder.WrapForEncodeAndApplyHeader(outputBuffer, 0, new MessageHeader()); + dto.EncodeInto(encoder); + File.WriteAllBytes("output.dat", outputBytes); + return 0; + } + } +} diff --git a/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/SbePropertyTest.csproj b/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/SbePropertyTest.csproj new file mode 100644 index 0000000000..a1bbc69f1c --- /dev/null +++ b/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/SbePropertyTest.csproj @@ -0,0 +1,14 @@ + + + + Exe + net7.0 + + + + + /home/zach/src/real-logic/simple-binary-encoding/csharp/sbe-dll/bin/Release/netstandard2.0/SBE.dll + + + + From fdf77422434e6a2c81c1e8f426cc83451f9e3fcb Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Tue, 10 Oct 2023 11:36:17 +0100 Subject: [PATCH 09/41] [Java,C#] Make dotnet executable customisable in tests. Addresses CodeQL comment. --- .../co/real_logic/sbe/properties/CSharpDtosPropertyTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java index 2c3cc814e1..3834b86e11 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java @@ -36,6 +36,8 @@ public class CSharpDtosPropertyTest { + private static final String DOTNET_EXECUTABLE = System.getProperty("sbe.tests.dotnet.executable", "dotnet"); + @Property void dtoEncodeShouldBeTheInverseOfDtoDecode( @ForAll("encodedMessage") final SbeArbitraries.EncodedMessage encodedMessage @@ -68,7 +70,7 @@ void dtoEncodeShouldBeTheInverseOfDtoDecode( final Path stdout = tempDir.resolve("stdout.txt"); final Path stderr = tempDir.resolve("stderr.txt"); - final ProcessBuilder processBuilder = new ProcessBuilder("dotnet", "run", "--", "input.dat") + final ProcessBuilder processBuilder = new ProcessBuilder(DOTNET_EXECUTABLE, "run", "--", "input.dat") .directory(tempDir.toFile()) .redirectOutput(stdout.toFile()) .redirectError(stderr.toFile()); From 852a0fe38989d8de7f224405c7551c208a83ea68 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Tue, 10 Oct 2023 12:08:09 +0100 Subject: [PATCH 10/41] [Java] Extend schema generation to include bitsets with gaps. --- .../properties/arbitraries/SbeArbitraries.java | 15 ++++++--------- .../sbe/properties/schema/SetSchema.java | 13 +++++++------ .../properties/schema/TestXmlSchemaWriter.java | 11 +++++------ 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java index eb6ef0603f..1efd68e7c0 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java @@ -147,15 +147,12 @@ private static Arbitrary enumTypeSchema() private static Arbitrary setTypeSchema() { - return Combinators.combine( - Arbitraries.of( - "uint8", - "uint16", - "uint32", - "uint64" - ), - Arbitraries.integers().between(1, 4) - ).as(SetSchema::new); + return Arbitraries.oneOf( + Arbitraries.integers().between(0, 7).set().map(choices -> new SetSchema("uint8", choices)), + Arbitraries.integers().between(0, 15).set().map(choices -> new SetSchema("uint16", choices)), + Arbitraries.integers().between(0, 31).set().map(choices -> new SetSchema("uint32", choices)), + Arbitraries.integers().between(0, 63).set().map(choices -> new SetSchema("uint64", choices)) + ); } private static Arbitrary compositeTypeSchema(final int depth) diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/SetSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/SetSchema.java index 95874e8381..4ed41fc013 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/SetSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/SetSchema.java @@ -16,17 +16,19 @@ package uk.co.real_logic.sbe.properties.schema; +import java.util.Set; + public final class SetSchema implements TypeSchema { private final String encodingType; - private final int choiceCount; + private final Set choices; public SetSchema( final String encodingType, - final int choiceCount) + final Set choices) { - this.choiceCount = choiceCount; this.encodingType = encodingType; + this.choices = choices; } public String encodingType() @@ -34,9 +36,9 @@ public String encodingType() return encodingType; } - public int choiceCount() + public Set choices() { - return choiceCount; + return choices; } @Override @@ -44,5 +46,4 @@ public void accept(final TypeSchemaVisitor visitor) { visitor.onSet(this); } - } diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java index 242f7c9946..c7c23533bd 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java @@ -238,18 +238,17 @@ private static Element createSetElement( final Document document, final String name, final String encodingType, - final int choiceCount) + final Set choices) { final Element enumElement = document.createElement("set"); enumElement.setAttribute("name", name); enumElement.setAttribute("encodingType", encodingType); - int caseId = 0; - for (int i = 0; i < choiceCount; i++) + for (final Integer value : choices) { final Element choice = document.createElement("choice"); - choice.setAttribute("name", "option" + caseId++); - choice.setTextContent(Integer.toString(i)); + choice.setAttribute("name", "option" + value); + choice.setTextContent(value.toString()); enumElement.appendChild(choice); } @@ -403,7 +402,7 @@ public void onSet(final SetSchema type) document, typeToName.computeIfAbsent(type, nextName), type.encodingType(), - type.choiceCount() + type.choices() ); } From 92588c0433373590fd18a6d561fef1b886028d3f Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Tue, 10 Oct 2023 16:23:23 +0100 Subject: [PATCH 11/41] [Java] Use arbitrary fixed-length arrays in property tests. Previously, we were only generating non-array fields in blocks. Now, we can exercise more possibly specifications in our property-based tests. --- .../arbitraries/SbeArbitraries.java | 20 +++++++++++++------ .../schema/EncodedDataTypeSchema.java | 8 ++++++++ .../schema/TestXmlSchemaWriter.java | 20 ++++++++++++++++++- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java index 1efd68e7c0..4750da7c19 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java @@ -111,6 +111,7 @@ private static Arbitrary encodedDataTypeSchema() { return Combinators.combine( Arbitraries.of(PrimitiveType.values()), + Arbitraries.of(1, 1, 1, 2, 13), Arbitraries.of(true, false) ).as(EncodedDataTypeSchema::new); } @@ -324,13 +325,15 @@ private static Arbitrary encodedTypeEncoder( } else { - return arbEncoder.map(encoder -> (buffer, bufferOffset, limit) -> - { - for (int i = 0; i < typeToken.arrayLength(); i++) + return arbEncoder.list().ofSize(typeToken.arrayLength()) + .map(encoders -> (buffer, bufferOffset, limit) -> { - encoder.encode(buffer, bufferOffset + offset + i * encoding.primitiveType().size(), limit); - } - }); + for (int i = 0; i < typeToken.arrayLength(); i++) + { + final int elementOffset = bufferOffset + offset + i * encoding.primitiveType().size(); + encoders.get(i).encode(buffer, elementOffset, limit); + } + }); } } @@ -478,6 +481,11 @@ private static Arbitrary bitSetEncoder( encoders.add(choiceEncoder); } + if (encoders.isEmpty()) + { + return Arbitraries.of(emptyEncoder()); + } + return Arbitraries.subsetOf(encoders) .map(SbeArbitraries::combineEncoders) .map(encoder -> (buffer, bufferOffset, limit) -> encoder.encode(buffer, bufferOffset + offset, limit)); diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java index b5fe76941d..5eb092150f 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java @@ -21,14 +21,17 @@ public final class EncodedDataTypeSchema implements TypeSchema { private final PrimitiveType primitiveType; + private final int length; private final boolean isEmbedded; public EncodedDataTypeSchema( final PrimitiveType primitiveType, + final int length, final boolean isEmbedded ) { this.primitiveType = primitiveType; + this.length = length; this.isEmbedded = isEmbedded; } @@ -37,6 +40,11 @@ public PrimitiveType primitiveType() return primitiveType; } + public int length() + { + return length; + } + @Override public boolean isEmbedded() { diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java index c7c23533bd..2211c31860 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java @@ -305,6 +305,23 @@ private static Element createTypeElement( return blockLength; } + private static Element createTypeElement( + final Document document, + final String name, + final String primitiveType, + final int length) + { + final Element blockLength = document.createElement("type"); + blockLength.setAttribute("name", name); + blockLength.setAttribute("primitiveType", primitiveType); + if (length > 1) + { + blockLength.setAttribute("length", Integer.toString(length)); + } + + return blockLength; + } + private static Element createRefElement( final Document document, final String name, @@ -362,7 +379,8 @@ public void onEncoded(final EncodedDataTypeSchema type) result = createTypeElement( document, typeToName.computeIfAbsent(type, nextName), - type.primitiveType().primitiveName() + type.primitiveType().primitiveName(), + type.length() ); } From b205c4b919ed5f84bef110fad1c6be3cd8c142cb Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Wed, 11 Oct 2023 10:00:17 +0100 Subject: [PATCH 12/41] [Java] Model optional fields in property-based tests. Previously, we were only supplying required fields in the schema. Now, we allow fields and primitive types to be marked as optional. Note that they can be marked as optional in two places. Optional => null value. There are some oddities around specifying `nullValue`s: https://github.com/real-logic/simple-binary-encoding/issues/429 We currently use the default `nullValue`s for types rather than generating arbitrary ones. --- .../arbitraries/SbeArbitraries.java | 33 ++++++++++- .../schema/EncodedDataTypeSchema.java | 9 +++ .../sbe/properties/schema/FieldSchema.java | 44 +++++++++++++++ .../sbe/properties/schema/GroupSchema.java | 6 +- .../sbe/properties/schema/MessageSchema.java | 6 +- .../schema/TestXmlSchemaWriter.java | 56 ++++++++++++++----- 6 files changed, 131 insertions(+), 23 deletions(-) create mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/FieldSchema.java diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java index 4750da7c19..2a3d2bd9b3 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java @@ -112,6 +112,7 @@ private static Arbitrary encodedDataTypeSchema() return Combinators.combine( Arbitraries.of(PrimitiveType.values()), Arbitraries.of(1, 1, 1, 2, 13), + presence(), Arbitraries.of(true, false) ).as(EncodedDataTypeSchema::new); } @@ -190,12 +191,26 @@ private static Arbitrary groupSchema(final int depth) groupSchema(depth - 1).list().ofMaxSize(3); return Combinators.combine( - withDuplicates(2, typeSchema(MAX_COMPOSITE_DEPTH).list().ofMaxSize(5)), + withDuplicates( + 2, + Combinators.combine( + typeSchema(MAX_COMPOSITE_DEPTH), + presence() + ).as(FieldSchema::new).list().ofMaxSize(5) + ), subGroups, varDataSchema().list().ofMaxSize(3) ).as(GroupSchema::new); } + private static Arbitrary presence() + { + return Arbitraries.of( + Encoding.Presence.REQUIRED, + Encoding.Presence.OPTIONAL + ); + } + private static Arbitrary varDataSchema() { return Arbitraries.of(VarDataSchema.Encoding.values()) @@ -205,7 +220,14 @@ private static Arbitrary varDataSchema() public static Arbitrary messageSchema() { return Combinators.combine( - withDuplicates(3, typeSchema(MAX_COMPOSITE_DEPTH).list().ofMaxSize(10)), + + withDuplicates( + 3, + Combinators.combine( + typeSchema(MAX_COMPOSITE_DEPTH), + presence() + ).as(FieldSchema::new).list().ofMaxSize(10) + ), groupSchema(MAX_GROUP_DEPTH).list().ofMaxSize(3), varDataSchema().list().ofMaxSize(3) ).as(MessageSchema::new); @@ -752,7 +774,12 @@ public static Arbitrary encodedMessage(final CharGenerationMode final String xml = TestXmlSchemaWriter.writeString(testSchema); try (InputStream in = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) { - final uk.co.real_logic.sbe.xml.MessageSchema parsedSchema = parse(in, ParserOptions.DEFAULT); + final ParserOptions options = ParserOptions.builder() + .suppressOutput(false) + .warningsFatal(true) + .stopOnError(true) + .build(); + final uk.co.real_logic.sbe.xml.MessageSchema parsedSchema = parse(in, options); final Ir ir = new IrGenerator().generate(parsedSchema); return SbeArbitraries.messageValueEncoder(ir, testSchema.templateId(), mode) .map(encoder -> diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java index 5eb092150f..a1bbeba14e 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java @@ -17,21 +17,25 @@ package uk.co.real_logic.sbe.properties.schema; import uk.co.real_logic.sbe.PrimitiveType; +import uk.co.real_logic.sbe.ir.Encoding; public final class EncodedDataTypeSchema implements TypeSchema { private final PrimitiveType primitiveType; private final int length; + private final Encoding.Presence presence; private final boolean isEmbedded; public EncodedDataTypeSchema( final PrimitiveType primitiveType, final int length, + final Encoding.Presence presence, final boolean isEmbedded ) { this.primitiveType = primitiveType; this.length = length; + this.presence = presence; this.isEmbedded = isEmbedded; } @@ -45,6 +49,11 @@ public int length() return length; } + public Encoding.Presence presence() + { + return presence; + } + @Override public boolean isEmbedded() { diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/FieldSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/FieldSchema.java new file mode 100644 index 0000000000..406ee65cdd --- /dev/null +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/FieldSchema.java @@ -0,0 +1,44 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties.schema; + +import uk.co.real_logic.sbe.ir.Encoding; + +public final class FieldSchema +{ + private final TypeSchema type; + private final Encoding.Presence presence; + + public FieldSchema( + final TypeSchema type, + final Encoding.Presence presence + ) + { + this.type = type; + this.presence = presence; + } + + public TypeSchema type() + { + return type; + } + + public Encoding.Presence presence() + { + return presence; + } +} diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java index df34e0a9b6..53c3c81df7 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java @@ -20,12 +20,12 @@ public final class GroupSchema { - private final List blockFields; + private final List blockFields; private final List groups; private final List varData; public GroupSchema( - final List blockFields, + final List blockFields, final List groups, final List varData) { @@ -34,7 +34,7 @@ public GroupSchema( this.varData = varData; } - public List blockFields() + public List blockFields() { return blockFields; } diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java index 451f61a34f..a0decb2d6b 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java @@ -20,12 +20,12 @@ public final class MessageSchema { - private final List blockFields; + private final List blockFields; private final List groups; private final List varData; public MessageSchema( - final List blockFields, + final List blockFields, final List groups, final List varData ) @@ -45,7 +45,7 @@ public short templateId() return 1; } - public List blockFields() + public List blockFields() { return blockFields; } diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java index 2211c31860..1481126646 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java @@ -16,6 +16,7 @@ package uk.co.real_logic.sbe.properties.schema; +import uk.co.real_logic.sbe.ir.Encoding; import org.agrona.collections.MutableInteger; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -24,6 +25,7 @@ import java.io.StringWriter; import java.util.*; import java.util.function.Function; +import java.util.stream.Collectors; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.OutputKeys; @@ -85,7 +87,7 @@ private static void writeTo( visitedTypes, topLevelTypes, typeSchemaConverter, - schema.blockFields(), + schema.blockFields().stream().map(FieldSchema::type).collect(Collectors.toList()), schema.groups()); final Element message = document.createElement("sbe:message"); @@ -128,25 +130,26 @@ private static void writeTo( private static void appendMembers( final Document document, final HashMap typeToName, - final List blockFields, + final List blockFields, final List groups, final List varData, final MutableInteger nextMemberId, final Element parent) { - for (final TypeSchema field : blockFields) + for (final FieldSchema field : blockFields) { final int id = nextMemberId.getAndIncrement(); - final boolean usePrimitiveName = field.isEmbedded() && field instanceof EncodedDataTypeSchema; + final boolean usePrimitiveName = field.type().isEmbedded() && field.type() instanceof EncodedDataTypeSchema; final String typeName = usePrimitiveName ? - ((EncodedDataTypeSchema)field).primitiveType().primitiveName() : - requireNonNull(typeToName.get(field)); + ((EncodedDataTypeSchema)field.type()).primitiveType().primitiveName() : + requireNonNull(typeToName.get(field.type())); final Element element = document.createElement("field"); element.setAttribute("id", Integer.toString(id)); element.setAttribute("name", "member" + id); element.setAttribute("type", typeName); + element.setAttribute("presence", field.presence().name().toLowerCase()); parent.appendChild(element); } @@ -309,17 +312,35 @@ private static Element createTypeElement( final Document document, final String name, final String primitiveType, - final int length) + final int length, + final Encoding.Presence presence) { - final Element blockLength = document.createElement("type"); - blockLength.setAttribute("name", name); - blockLength.setAttribute("primitiveType", primitiveType); + final Element typeElement = document.createElement("type"); + typeElement.setAttribute("name", name); + typeElement.setAttribute("primitiveType", primitiveType); + if (length > 1) { - blockLength.setAttribute("length", Integer.toString(length)); + typeElement.setAttribute("length", Integer.toString(length)); } - return blockLength; + switch (presence) + { + + case REQUIRED: + typeElement.setAttribute("presence", "required"); + break; + case OPTIONAL: + typeElement.setAttribute("presence", "optional"); + break; + case CONSTANT: + typeElement.setAttribute("presence", "constant"); + break; + default: + throw new IllegalArgumentException("Unknown presence: " + presence); + } + + return typeElement; } private static Element createRefElement( @@ -350,7 +371,13 @@ private static void appendTypes( for (final GroupSchema group : groups) { - appendTypes(visitedTypes, topLevelTypes, typeSchemaConverter, group.blockFields(), group.groups()); + appendTypes( + visitedTypes, + topLevelTypes, + typeSchemaConverter, + group.blockFields().stream().map(FieldSchema::type).collect(Collectors.toList()), + group.groups() + ); } } @@ -380,7 +407,8 @@ public void onEncoded(final EncodedDataTypeSchema type) document, typeToName.computeIfAbsent(type, nextName), type.primitiveType().primitiveName(), - type.length() + type.length(), + type.presence() ); } From 58b0e927615750902cece9c62a5b9d18f1c95bbc Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Wed, 11 Oct 2023 10:17:38 +0100 Subject: [PATCH 13/41] [Java, C#] Add a GitHub workflow for slow checks. This workflow is configured to run once per day or when manually requested. At the moment, the slow checks include several property-based tests. These tests take a significant amount of time to execute; therefore, we decided it was better to keep them out of the CI build. --- .github/workflows/slow.yml | 58 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/workflows/slow.yml diff --git a/.github/workflows/slow.yml b/.github/workflows/slow.yml new file mode 100644 index 0000000000..9a22c1f415 --- /dev/null +++ b/.github/workflows/slow.yml @@ -0,0 +1,58 @@ +name: Slow checks + +on: + workflow_dispatch: + branches: + - '**' + schedule: + - cron: '0 12 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + GRADLE_OPTS: '-Dorg.gradle.daemon=false -Dorg.gradle.java.installations.auto-detect=false -Dorg.gradle.warning.mode=fail' + +permissions: + contents: read + +jobs: + property-tests: + name: Property tests + runs-on: ubuntu-22.04 + strategy: + matrix: + java: [ '21' ] + dotnet: [ '3.1.301' ] + env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Setup java + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: ${{ matrix.java }} + - name: Setup BUILD_JAVA_HOME & BUILD_JAVA_VERSION + run: | + java -Xinternalversion + echo "BUILD_JAVA_HOME=${JAVA_HOME}" >> $GITHUB_ENV + echo "BUILD_JAVA_VERSION=${{ matrix.java }}" >> $GITHUB_ENV + - name: Setup java 8 to run the Gradle script + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 8 + - name: Setup dotnet + uses: actions/setup-dotnet@v2 + with: + dotnet-version: ${{ matrix.dotnet }} + - name: Build SbeTool + run: ./gradlew + - name: Build .NET library + run: ./csharp/build.sh + - name: Run property tests + run: ./gradlew propertyTest From 4f9d51b18607ace0e4297dd6c38a8bc3300409d6 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Wed, 11 Oct 2023 19:41:37 +0100 Subject: [PATCH 14/41] [Java] Extend arbitrary varData encodings. We now explore more of the "schema space", as we generate lengths of different encodings. It surfaces an issue, reported in #955, which was fixed in another outstanding PR's commit: 3885a63d3103ac52406c660147ee929fa9aaee63. --- .../arbitraries/SbeArbitraries.java | 27 ++++-- .../schema/TestXmlSchemaWriter.java | 87 ++++++++++--------- .../sbe/properties/schema/VarDataSchema.java | 21 +++-- 3 files changed, 81 insertions(+), 54 deletions(-) diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java index 2a3d2bd9b3..cc2f5ff9ab 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java @@ -150,10 +150,18 @@ private static Arbitrary enumTypeSchema() private static Arbitrary setTypeSchema() { return Arbitraries.oneOf( - Arbitraries.integers().between(0, 7).set().map(choices -> new SetSchema("uint8", choices)), - Arbitraries.integers().between(0, 15).set().map(choices -> new SetSchema("uint16", choices)), - Arbitraries.integers().between(0, 31).set().map(choices -> new SetSchema("uint32", choices)), - Arbitraries.integers().between(0, 63).set().map(choices -> new SetSchema("uint64", choices)) + Arbitraries.integers().between(0, 7).set() + .ofMaxSize(8) + .map(choices -> new SetSchema("uint8", choices)), + Arbitraries.integers().between(0, 15).set() + .ofMaxSize(16) + .map(choices -> new SetSchema("uint16", choices)), + Arbitraries.integers().between(0, 31).set() + .ofMaxSize(32) + .map(choices -> new SetSchema("uint32", choices)), + Arbitraries.integers().between(0, 63).set() + .ofMaxSize(64) + .map(choices -> new SetSchema("uint64", choices)) ); } @@ -213,14 +221,19 @@ private static Arbitrary presence() private static Arbitrary varDataSchema() { - return Arbitraries.of(VarDataSchema.Encoding.values()) - .map(VarDataSchema::new); + return Combinators.combine( + Arbitraries.of(VarDataSchema.Encoding.values()), + Arbitraries.of( + PrimitiveType.UINT8, + PrimitiveType.UINT16, + PrimitiveType.UINT32 + ) + ).as(VarDataSchema::new); } public static Arbitrary messageSchema() { return Combinators.combine( - withDuplicates( 3, Combinators.combine( diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java index 1481126646..7ccd273017 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java @@ -20,6 +20,7 @@ import org.agrona.collections.MutableInteger; import org.w3c.dom.Document; import org.w3c.dom.Element; +import org.w3c.dom.Node; import java.io.File; import java.io.StringWriter; @@ -74,7 +75,7 @@ private static void writeTo( final Element topLevelTypes = createTypesElement(document); root.appendChild(topLevelTypes); - final HashMap typeToName = new HashMap<>(); + final HashMap typeToName = new HashMap<>(); final TypeSchemaConverter typeSchemaConverter = new TypeSchemaConverter( document, @@ -88,7 +89,8 @@ private static void writeTo( topLevelTypes, typeSchemaConverter, schema.blockFields().stream().map(FieldSchema::type).collect(Collectors.toList()), - schema.groups()); + schema.groups(), + schema.varData()); final Element message = document.createElement("sbe:message"); message.setAttribute("name", "TestMessage"); @@ -129,7 +131,7 @@ private static void writeTo( private static void appendMembers( final Document document, - final HashMap typeToName, + final HashMap typeToName, final List blockFields, final List groups, final List varData, @@ -174,21 +176,10 @@ private static void appendMembers( for (final VarDataSchema data : varData) { final int id = nextMemberId.getAndIncrement(); - final Element element = document.createElement("data"); element.setAttribute("id", Integer.toString(id)); element.setAttribute("name", "member" + id); - switch (data.encoding()) - { - case ASCII: - element.setAttribute("type", "varStringEncoding"); - break; - case BYTES: - element.setAttribute("type", "varDataEncoding"); - break; - default: - throw new IllegalStateException("Unknown encoding: " + data.encoding()); - } + element.setAttribute("type", requireNonNull(typeToName.get(data))); parent.appendChild(element); } } @@ -213,27 +204,6 @@ private static Element createTypesElement(final Document document) createTypeElement(document, "numInGroup", "uint16") )); - final Element varString = createTypeElement(document, "varData", "uint8"); - varString.setAttribute("length", "0"); - varString.setAttribute("characterEncoding", "US-ASCII"); - types.appendChild(createCompositeElement( - document, - "varStringEncoding", - createTypeElement(document, "length", "uint16"), - varString - )); - - final Element varData = createTypeElement(document, "varData", "uint8"); - final Element varDataLength = createTypeElement(document, "length", "uint32"); - varDataLength.setAttribute("maxValue", "1000000"); - varData.setAttribute("length", "0"); - types.appendChild(createCompositeElement( - document, - "varDataEncoding", - varDataLength, - varData - )); - return types; } @@ -359,7 +329,8 @@ private static void appendTypes( final Element topLevelTypes, final TypeSchemaConverter typeSchemaConverter, final List blockFields, - final List groups) + final List groups, + final List varDataFields) { for (final TypeSchema field : blockFields) { @@ -376,23 +347,29 @@ private static void appendTypes( topLevelTypes, typeSchemaConverter, group.blockFields().stream().map(FieldSchema::type).collect(Collectors.toList()), - group.groups() - ); + group.groups(), + group.varData()); + } + + for (final VarDataSchema varData : varDataFields) + { + topLevelTypes.appendChild(typeSchemaConverter.convert(varData)); } } + @SuppressWarnings("EnhancedSwitchMigration") private static final class TypeSchemaConverter implements TypeSchemaVisitor { private final Document document; private final Element topLevelTypes; - private final Map typeToName; - private final Function nextName; + private final Map typeToName; + private final Function nextName; private Element result; private TypeSchemaConverter( final Document document, final Element topLevelTypes, - final Map typeToName) + final Map typeToName) { this.document = document; this.topLevelTypes = topLevelTypes; @@ -481,5 +458,31 @@ public Element convert(final TypeSchema type) type.accept(this); return requireNonNull(result); } + + public Node convert(final VarDataSchema varData) + { + final Element lengthElement = createTypeElement(document, "length", + varData.lengthEncoding().primitiveName()); + + if (varData.lengthEncoding().size() >= 4) + { + lengthElement.setAttribute("maxValue", Integer.toString(1_000_000)); + } + + final Element varDataElement = createTypeElement(document, "varData", "uint8"); + varDataElement.setAttribute("length", "0"); + + if (varData.dataEncoding().equals(VarDataSchema.Encoding.ASCII)) + { + varDataElement.setAttribute("characterEncoding", "US-ASCII"); + } + + return createCompositeElement( + document, + typeToName.computeIfAbsent(varData, nextName), + lengthElement, + varDataElement + ); + } } } diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java index c6b39db56b..b5e98c02c7 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java @@ -16,18 +16,29 @@ package uk.co.real_logic.sbe.properties.schema; +import uk.co.real_logic.sbe.PrimitiveType; + public final class VarDataSchema { - private final Encoding encoding; + private final Encoding dataEncoding; + private final PrimitiveType lengthEncoding; + + public VarDataSchema( + final Encoding dataEncoding, + final PrimitiveType lengthEncoding) + { + this.dataEncoding = dataEncoding; + this.lengthEncoding = lengthEncoding; + } - public VarDataSchema(final Encoding encoding) + public Encoding dataEncoding() { - this.encoding = encoding; + return dataEncoding; } - public Encoding encoding() + public PrimitiveType lengthEncoding() { - return encoding; + return lengthEncoding; } public enum Encoding From 77711dd7e6285c2763794f0ee048745b250d6f4e Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Thu, 12 Oct 2023 13:27:22 +0100 Subject: [PATCH 15/41] [Java, C#] Tidy up encoded message writing in tests. Previously, we were writing all bytes in a buffer rather than just those used for encoding the message. Now, we keep track of the "limit" of the message and only use the bytes in the buffer up to the limit. --- .../sbe/properties/CSharpDtosPropertyTest.java | 9 +++++++-- .../properties/arbitraries/SbeArbitraries.java | 15 +++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java index 3834b86e11..7a4dde3f49 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java @@ -57,7 +57,11 @@ void dtoEncodeShouldBeTheInverseOfDtoDecode( copyResourceToFile("/CSharpDtosPropertyTest/SbePropertyTest.csproj", tempDir); copyResourceToFile("/CSharpDtosPropertyTest/Program.cs", tempDir); try ( - DirectBufferInputStream inputStream = new DirectBufferInputStream(encodedMessage.buffer()); + DirectBufferInputStream inputStream = new DirectBufferInputStream( + encodedMessage.buffer(), + 0, + encodedMessage.length() + ); OutputStream outputStream = Files.newOutputStream(tempDir.resolve("input.dat"))) { final byte[] buffer = new byte[2048]; @@ -85,7 +89,8 @@ void dtoEncodeShouldBeTheInverseOfDtoDecode( "SCHEMA:\n" + encodedMessage.schema()); } - final byte[] inputBytes = encodedMessage.buffer().byteArray(); + final byte[] inputBytes = new byte[encodedMessage.length()]; + encodedMessage.buffer().getBytes(0, inputBytes); final byte[] outputBytes = Files.readAllBytes(tempDir.resolve("output.dat")); if (!Arrays.equals(inputBytes, outputBytes)) { diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java index cc2f5ff9ab..2e3db4bda0 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java @@ -756,12 +756,18 @@ public static final class EncodedMessage private final String schema; private final Ir ir; private final ExpandableArrayBuffer buffer; + private final int length; - private EncodedMessage(final String schema, final Ir ir, final ExpandableArrayBuffer buffer) + private EncodedMessage( + final String schema, + final Ir ir, + final ExpandableArrayBuffer buffer, + final int length) { this.schema = schema; this.ir = ir; this.buffer = buffer; + this.length = length; } public String schema() @@ -778,6 +784,11 @@ public ExpandableArrayBuffer buffer() { return buffer; } + + public int length() + { + return length; + } } public static Arbitrary encodedMessage(final CharGenerationMode mode) @@ -800,7 +811,7 @@ public static Arbitrary encodedMessage(final CharGenerationMode final ExpandableArrayBuffer buffer = new ExpandableArrayBuffer(); final MutableInteger limit = new MutableInteger(); encoder.encode(buffer, 0, limit); - return new EncodedMessage(xml, ir, buffer); + return new EncodedMessage(xml, ir, buffer, limit.get()); }); } catch (final Exception e) From 19f69a9fae941dd5018e68c546bc7d60528a2683 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Fri, 13 Oct 2023 00:00:41 +0100 Subject: [PATCH 16/41] [C#] Upgrade to latest LTS .NET version. To support records in DTOs, we require C# 9+. This is not supported by .NET 3. Also, this old version is no longer supported by Microsoft. Therefore, I have upgraded to the latest LTS release. --- .github/workflows/codeql.yml | 2 +- .github/workflows/slow.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index db70001ed7..44a604749a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -88,7 +88,7 @@ jobs: fail-fast: false matrix: language: [ 'csharp' ] - dotnet: [ '3.1.x' ] + dotnet: [ '8.0.x' ] env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true DOTNET_CLI_TELEMETRY_OPTOUT: 1 diff --git a/.github/workflows/slow.yml b/.github/workflows/slow.yml index 9a22c1f415..a5f007f6bf 100644 --- a/.github/workflows/slow.yml +++ b/.github/workflows/slow.yml @@ -24,7 +24,7 @@ jobs: strategy: matrix: java: [ '21' ] - dotnet: [ '3.1.301' ] + dotnet: [ '8.0.x' ] env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true DOTNET_CLI_TELEMETRY_OPTOUT: 1 From 520cdb1c5eb7156b68a7789819fd4c9e2b332987 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Thu, 12 Oct 2023 23:20:21 +0100 Subject: [PATCH 17/41] [C#] Generate DTOs using C# 9+ records. The reason behind this change is to support equality and comparison of DTOs in tests by leveraging records. --- .../sbe/generation/csharp/CSharpDtoGenerator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java index e7ceeeb155..ad6c68db38 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java @@ -103,7 +103,7 @@ public void generate() throws IOException out.append(generateFileHeader(ir.applicableNamespace(), "using System.Collections.Generic;\n")); out.append(generateDocumentation(BASE_INDENT, msgToken)); - out.append(BASE_INDENT).append("public sealed partial class ").append(className).append("\n") + out.append(BASE_INDENT).append("public sealed partial record ").append(className).append("\n") .append(BASE_INDENT).append("{") .append(sb) .append(BASE_INDENT).append("}\n") @@ -136,7 +136,7 @@ private void generateGroups( sb.append("\n") .append(generateDocumentation(indent, groupToken)) - .append(indent).append("public sealed partial class ").append(groupClassName).append("\n") + .append(indent).append("public sealed partial record ").append(groupClassName).append("\n") .append(indent).append("{"); i++; @@ -1031,7 +1031,7 @@ private void generateComposite(final List tokens) throws IOException generateCompositeEncodeInto(sb, codecClassName, compositeTokens, BASE_INDENT + INDENT); generateDisplay(sb, codecClassName, "Wrap", codecClassName + ".SbeSchemaVersion", BASE_INDENT + INDENT); - out.append(BASE_INDENT).append("public sealed partial class ").append(className).append("\n") + out.append(BASE_INDENT).append("public sealed partial record ").append(className).append("\n") .append(BASE_INDENT).append("{") .append(sb) .append(BASE_INDENT).append("}\n") From c1afd033456fc7d264dad9ce22a188e877271ae4 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Tue, 17 Oct 2023 17:04:37 +0100 Subject: [PATCH 18/41] [C#] Use more-idiomatic null representations. Also, make records immutable, as it is more idiomatic to use `with` expressions. In the future, it may be preferable to declare record fields in the constructor rather than as init-only properties; however, this would be a much larger change. This is the current mapping of optional fields: - Primitives: as nullable value types - Composites: as nullable references - Enums: we already generate a `NULL_VALUE` case and we use that - Bitsets: as a flags enum where `0` is the null value - Strings: as empty string - Arrays: as empty read-only lists --- csharp/sbe-tests/DtoTests.cs | 5 +- .../generation/csharp/CSharpDtoGenerator.java | 286 ++++++++++-------- .../sbe/generation/csharp/CSharpUtil.java | 6 +- .../CSharpDtosPropertyTest/Program.cs | 3 +- .../SbePropertyTest.csproj | 2 +- 5 files changed, 169 insertions(+), 133 deletions(-) diff --git a/csharp/sbe-tests/DtoTests.cs b/csharp/sbe-tests/DtoTests.cs index 99fbcadc74..fcd0007bf7 100644 --- a/csharp/sbe-tests/DtoTests.cs +++ b/csharp/sbe-tests/DtoTests.cs @@ -16,14 +16,13 @@ public void ShouldRoundTripCar() var decoder = new Car(); decoder.WrapForDecode(inputBuffer, 0, Car.BlockLength, Car.SchemaVersion); var decoderString = decoder.ToString(); - var dto = new CarDto(); - dto.DecodeFrom(decoder); + var dto = CarDto.DecodeFrom(decoder); var outputByteArray = new byte[1024]; var outputBuffer = new DirectBuffer(outputByteArray); var encoder = new Car(); encoder.WrapForEncode(outputBuffer, 0); dto.EncodeInto(encoder); - var dtoString = dto.ToString(); + var dtoString = dto.ToSbeString(); CollectionAssert.AreEqual(inputByteArray, outputByteArray); Assert.AreEqual(decoderString, dtoString); } diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java index ad6c68db38..b6ab8638d9 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java @@ -30,7 +30,6 @@ import java.io.Writer; import java.util.ArrayList; import java.util.List; -import java.util.function.BiConsumer; import static uk.co.real_logic.sbe.generation.csharp.CSharpUtil.*; import static uk.co.real_logic.sbe.ir.GenerationUtil.collectFields; @@ -40,7 +39,6 @@ /** * DTO generator for the CSharp programming language. */ -@SuppressWarnings("CodeBlock2Expr") public class CSharpDtoGenerator implements CodeGenerator { private static final String INDENT = " "; @@ -94,13 +92,17 @@ public void generate() throws IOException collectVarData(messageBody, offset, varData); generateVarData(sb, varData, BASE_INDENT + INDENT); - generateDecodeFrom(sb, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); + generateDecodeFrom(sb, className, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); generateEncodeInto(sb, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); generateDisplay(sb, codecClassName, "WrapForEncode", null, BASE_INDENT + INDENT); try (Writer out = outputManager.createOutput(className)) { - out.append(generateFileHeader(ir.applicableNamespace(), "using System.Collections.Generic;\n")); + out.append(generateFileHeader( + ir.applicableNamespace(), + "#nullable enable\n\n", + "using System.Collections.Generic;\n", + "using System.Linq;\n")); out.append(generateDocumentation(BASE_INDENT, msgToken)); out.append(BASE_INDENT).append("public sealed partial record ").append(className).append("\n") @@ -130,9 +132,9 @@ private void generateGroups( sb.append("\n") .append(generateDocumentation(indent, groupToken)) - .append(indent).append("public List<").append(groupClassName).append("> ") + .append(indent).append("public IReadOnlyList<").append(groupClassName).append("> ") .append(formatPropertyName(groupName)) - .append(" { get; set; } = new List<").append(groupClassName).append(">();\n"); + .append(" { get; init; }\n"); sb.append("\n") .append(generateDocumentation(indent, groupToken)) @@ -155,7 +157,8 @@ private void generateGroups( i = collectVarData(tokens, i, varData); generateVarData(sb, varData, indent + INDENT); - generateDecodeFrom(sb, codecClassName, fields, groups, varData, indent + INDENT); + generateDecodeListFrom(sb, groupClassName, codecClassName, indent + INDENT); + generateDecodeFrom(sb, groupClassName, codecClassName, fields, groups, varData, indent + INDENT); generateEncodeInto(sb, codecClassName, fields, groups, varData, indent + INDENT); sb.append(indent).append("}\n"); @@ -164,23 +167,29 @@ private void generateGroups( private void generateCompositeDecodeFrom( final StringBuilder sb, + final String dtoClassName, final String codecClassName, final List tokens, final String indent) { sb.append("\n") - .append(indent).append("public void DecodeFrom(").append(codecClassName).append(" codec)\n") + .append(indent).append("public static ").append(dtoClassName).append(" DecodeFrom(") + .append(codecClassName).append(" codec)\n") .append(indent).append("{\n"); + sb.append(indent).append(INDENT).append("return new ").append(dtoClassName).append("()\n") + .append(indent).append(INDENT).append("{\n"); + for (int i = 0; i < tokens.size(); ) { final Token token = tokens.get(i); - generateFieldDecodeFrom(sb, token, token, codecClassName, indent + INDENT); + generateFieldDecodeFrom(sb, token, token, codecClassName, indent + INDENT + INDENT); i += tokens.get(i).componentTokenCount(); } + sb.append(indent).append(INDENT).append("};\n"); sb.append(indent).append("}\n"); } @@ -198,7 +207,7 @@ private void generateCompositeEncodeInto( { final Token token = tokens.get(i); - generateFieldEncodeInto(sb, token, token, indent + INDENT); + generateFieldEncodeInto(sb, codecClassName, token, token, indent + INDENT); i += tokens.get(i).componentTokenCount(); } @@ -206,8 +215,36 @@ private void generateCompositeEncodeInto( sb.append(indent).append("}\n"); } + private void generateDecodeListFrom( + final StringBuilder sb, + final String dtoClassName, + final String codecClassName, + final String indent) + { + sb.append("\n") + .append(indent).append("public static IReadOnlyList<").append(dtoClassName).append("> DecodeListFrom(") + .append(codecClassName).append(" codec)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("var ").append("list = new List<").append(dtoClassName) + .append(">(codec.Count);\n") + .append(indent).append(INDENT) + .append("while (codec.HasNext)\n") + .append(indent).append(INDENT) + .append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("var element = ").append(dtoClassName).append(".DecodeFrom(codec.Next());\n") + .append(indent).append(INDENT).append(INDENT) + .append("list.Add(element);\n") + .append(indent).append(INDENT) + .append("}\n") + .append(indent).append(INDENT) + .append("return list.AsReadOnly();\n") + .append(indent).append("}\n"); + } + private void generateDecodeFrom( final StringBuilder sb, + final String dtoClassName, final String codecClassName, final List fields, final List groups, @@ -215,12 +252,16 @@ private void generateDecodeFrom( final String indent) { sb.append("\n") - .append(indent).append("public void DecodeFrom(").append(codecClassName).append(" codec)\n") + .append(indent).append("public static ").append(dtoClassName) + .append(" DecodeFrom(").append(codecClassName).append(" codec)\n") .append(indent).append("{\n"); - generateFieldsDecodeFrom(sb, fields, codecClassName, indent + INDENT); - generateGroupsDecodeFrom(sb, groups, indent + INDENT); - generateVarDataDecodeFrom(sb, varData, indent + INDENT); + sb.append(indent).append(INDENT).append("return new ").append(dtoClassName).append("()\n") + .append(indent).append(INDENT).append("{\n"); + generateFieldsDecodeFrom(sb, fields, codecClassName, indent + INDENT + INDENT); + generateGroupsDecodeFrom(sb, groups, indent + INDENT + INDENT); + generateVarDataDecodeFrom(sb, varData, indent + INDENT + INDENT); + sb.append(indent).append(INDENT).append("};\n"); sb.append(indent).append("}\n"); } @@ -256,13 +297,13 @@ private void generateFieldDecodeFrom( break; case BEGIN_SET: - generatePropertyDecodeFrom(sb, fieldToken, "0", indent); + generatePropertyDecodeFrom(sb, codecClassName, fieldToken, "0", indent); break; case BEGIN_ENUM: final String enumName = formatClassName(typeToken.applicableTypeName()); final String nullValue = formatNamespace(ir.packageName()) + "." + enumName + ".NULL_VALUE"; - generatePropertyDecodeFrom(sb, fieldToken, nullValue, indent); + generatePropertyDecodeFrom(sb, codecClassName, fieldToken, nullValue, indent); break; case BEGIN_COMPOSITE: @@ -291,7 +332,7 @@ private void generatePrimitiveDecodeFrom( if (arrayLength == 1) { final String nullValue = codecClassName + "." + formatPropertyName(fieldToken.name()) + "NullValue"; - generatePropertyDecodeFrom(sb, fieldToken, nullValue, indent); + generatePropertyDecodeFrom(sb, codecClassName, fieldToken, nullValue, indent); } else if (arrayLength > 1) { @@ -319,44 +360,27 @@ private void generateArrayDecodeFrom( sb, fieldToken, indent, - (blkSb, blkIndent) -> - { - blkSb.append(blkIndent).append(formattedPropertyName) - .append(" = codec.Get").append(formattedPropertyName).append("();\n"); - }, - (blkSb, blkIndent) -> - { - blkSb.append(blkIndent).append(formattedPropertyName).append(" = null;\n"); - } + "codec.Get" + formattedPropertyName + "()", + "null", + null ); } else { - final String typeName = cSharpTypeName(typeToken.encoding().primitiveType()); - wrapInActingVersionCheck( sb, fieldToken, indent, - (blkSb, blkIndent) -> - { - blkSb.append(blkIndent).append(formattedPropertyName) - .append(" = new ").append(typeName).append("[") - .append(typeToken.arrayLength()).append("];\n") - .append(blkIndent).append("codec.").append(formattedPropertyName) - .append(".CopyTo(new Span<").append(typeName).append(">(") - .append(formattedPropertyName).append("));\n"); - }, - (blkSb, blkIndent) -> - { - blkSb.append(blkIndent).append(formattedPropertyName).append(" = null;\n"); - } + "codec." + formattedPropertyName + "AsSpan().ToArray()", + "null", + null ); } } private void generatePropertyDecodeFrom( final StringBuilder sb, + final String codecClassName, final Token fieldToken, final String nullValue, final String indent) @@ -374,15 +398,9 @@ private void generatePropertyDecodeFrom( sb, fieldToken, indent, - (blkSb, blkIndent) -> - { - blkSb.append(blkIndent).append(formattedPropertyName).append(" = codec.") - .append(formattedPropertyName).append(";\n"); - }, - (blkSb, blkIndent) -> - { - blkSb.append(blkIndent).append(formattedPropertyName).append(" = ").append(nullValue).append(";\n"); - } + "codec." + formattedPropertyName, + nullValue, + codecClassName + "." + formattedPropertyName + "NullValue" ); } @@ -400,17 +418,9 @@ private void generateComplexDecodeFrom( sb, fieldToken, indent, - (blkSb, blkIndent) -> - { - blkSb.append(blkIndent).append(formattedPropertyName).append(" = new ") - .append(dtoClassName).append("();\n") - .append(blkIndent).append(formattedPropertyName).append(".DecodeFrom(codec.") - .append(formattedPropertyName).append(");\n"); - }, - (blkSb, blkIndent) -> - { - blkSb.append(blkIndent).append(formattedPropertyName).append(" = null;\n"); - } + dtoClassName + ".DecodeFrom(codec." + formattedPropertyName + ")", + "null", + "null" ); } @@ -429,30 +439,13 @@ private void generateGroupsDecodeFrom( final String groupName = groupToken.name(); final String formattedPropertyName = formatPropertyName(groupName); final String groupDtoClassName = formatDtoClassName(groupName); - final String groupCodecVarName = groupName + "Codec"; - - sb.append("\n") - .append(indent) - .append(formattedPropertyName).append(" = new List<").append(groupDtoClassName).append(">();\n"); wrapInActingVersionCheck( sb, groupToken, indent, - (blkSb, blkIndent) -> - { - blkSb.append(blkIndent).append("var ").append(groupCodecVarName).append(" = codec.") - .append(formattedPropertyName).append(";\n") - .append(blkIndent).append("while (").append(groupCodecVarName).append(".HasNext)\n") - .append(blkIndent).append("{\n") - .append(blkIndent).append(INDENT) - .append("var element = new ").append(groupDtoClassName).append("();\n") - .append(blkIndent).append(INDENT) - .append("element.DecodeFrom(").append(groupCodecVarName).append(".Next());\n") - .append(blkIndent).append(INDENT) - .append(formattedPropertyName).append(".Add(element);\n") - .append(blkIndent).append("}\n"); - }, + groupDtoClassName + ".DecodeListFrom(codec." + formattedPropertyName + ")", + "new List<" + groupDtoClassName + ">(0).AsReadOnly()", null ); @@ -493,15 +486,9 @@ private void generateVarDataDecodeFrom( sb, token, indent, - (blkSb, blkIndent) -> - { - blkSb.append(blkIndent).append(formattedPropertyName) - .append(" = codec.").append(accessor).append("();\n"); - }, - (blkSb, blkIndent) -> - { - blkSb.append(blkIndent).append(formattedPropertyName).append(" = null;\n"); - } + "codec." + accessor + "()", + "null", + null ); } } @@ -511,30 +498,31 @@ private void wrapInActingVersionCheck( final StringBuilder sb, final Token token, final String indent, - final BiConsumer generatePresentBlock, - final BiConsumer generateNotPresentBlock) + final String presentExpression, + final String notPresentExpression, + final String nullCodecValueOrNull) { final String propertyName = token.name(); final String formattedPropertyName = formatPropertyName(propertyName); + sb.append(indent).append(formattedPropertyName).append(" = "); + if (token.version() > 0) { - sb.append("\n").append(indent).append("if (codec.").append(formattedPropertyName) - .append("InActingVersion())\n") - .append(indent).append("{\n"); - generatePresentBlock.accept(sb, indent + INDENT); - sb.append(indent).append("}\n"); - if (null != generateNotPresentBlock) + sb.append("codec.").append(formattedPropertyName).append("InActingVersion()"); + + if (null != nullCodecValueOrNull) { - sb.append(indent).append("else\n") - .append(indent).append("{\n"); - generateNotPresentBlock.accept(sb, indent + INDENT); - sb.append(indent).append("}\n"); + sb.append(" && codec.").append(formattedPropertyName).append(" != ").append(nullCodecValueOrNull); } + + sb.append(" ?\n"); + sb.append(indent).append(INDENT).append(presentExpression).append(" :\n") + .append(indent).append(INDENT).append(notPresentExpression).append(",\n"); } else { - generatePresentBlock.accept(sb, indent); + sb.append(presentExpression).append(",\n"); } } @@ -550,7 +538,7 @@ private void generateEncodeInto( .append(indent).append("public void EncodeInto(").append(codecClassName).append(" codec)\n") .append(indent).append("{\n"); - generateFieldsEncodeInto(sb, fields, indent + INDENT); + generateFieldsEncodeInto(sb, codecClassName, fields, indent + INDENT); generateGroupsEncodeInto(sb, groups, indent + INDENT); generateVarDataEncodeInto(sb, varData, indent + INDENT); @@ -559,6 +547,7 @@ private void generateEncodeInto( private void generateFieldsEncodeInto( final StringBuilder sb, + final String codecClassName, final List tokens, final String indent) { @@ -568,13 +557,14 @@ private void generateFieldsEncodeInto( if (signalToken.signal() == Signal.BEGIN_FIELD) { final Token encodingToken = tokens.get(i + 1); - generateFieldEncodeInto(sb, signalToken, encodingToken, indent); + generateFieldEncodeInto(sb, codecClassName, signalToken, encodingToken, indent); } } } private void generateFieldEncodeInto( final StringBuilder sb, + final String codecClassName, final Token fieldToken, final Token typeToken, final String indent) @@ -582,12 +572,12 @@ private void generateFieldEncodeInto( switch (typeToken.signal()) { case ENCODING: - generatePrimitiveEncodeInto(sb, fieldToken, typeToken, indent); + generatePrimitiveEncodeInto(sb, codecClassName, fieldToken, typeToken, indent); break; case BEGIN_SET: case BEGIN_ENUM: - generatePropertyEncodeInto(sb, fieldToken, indent); + generateEnumEncodeInto(sb, fieldToken, indent); break; case BEGIN_COMPOSITE: @@ -601,6 +591,7 @@ private void generateFieldEncodeInto( private void generatePrimitiveEncodeInto( final StringBuilder sb, + final String codecClassName, final Token fieldToken, final Token typeToken, final String indent) @@ -614,7 +605,7 @@ private void generatePrimitiveEncodeInto( if (arrayLength == 1) { - generatePropertyEncodeInto(sb, fieldToken, indent); + generatePropertyEncodeInto(sb, codecClassName, fieldToken, indent); } else if (arrayLength > 1) { @@ -638,19 +629,53 @@ private void generateArrayEncodeInto( if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) { + final String value = nullableConvertedExpression(fieldToken, formattedPropertyName, "\"\""); sb.append(indent).append("codec.Set").append(formattedPropertyName).append("(") - .append(formattedPropertyName).append(");\n"); + .append(value).append(");\n"); } else { final String typeName = cSharpTypeName(typeToken.encoding().primitiveType()); sb.append(indent).append("new Span<").append(typeName).append(">(").append(formattedPropertyName) - .append(").CopyTo(codec.").append(formattedPropertyName).append("AsSpan());\n"); + .append("?.ToArray()).CopyTo(codec.").append(formattedPropertyName).append("AsSpan());\n"); } } + private String nullableConvertedExpression( + final Token fieldToken, + final String expression, + final String nullValue) + { + return fieldToken.version() > 0 || fieldToken.isOptionalEncoding() ? + expression + " ?? " + nullValue : + expression; + } + private void generatePropertyEncodeInto( + final StringBuilder sb, + final String codecClassName, + final Token fieldToken, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + final String value = nullableConvertedExpression( + fieldToken, + formattedPropertyName, + codecClassName + "." + formattedPropertyName + "NullValue"); + + sb.append(indent).append("codec.").append(formattedPropertyName).append(" = ") + .append(value).append(";\n"); + } + + private void generateEnumEncodeInto( final StringBuilder sb, final Token fieldToken, final String indent) @@ -743,7 +768,7 @@ private void generateDisplay( final String indent) { sb.append("\n") - .append(indent).append("public override string ToString()\n") + .append(indent).append("public string ToSbeString()\n") .append(indent).append("{\n") .append(indent).append(INDENT) .append("var buffer = new DirectBuffer(new byte[128], (ignored, newSize) => new byte[newSize]);\n") @@ -800,6 +825,11 @@ private void generateFields( } } + private String optionality(final Token fieldToken, final Token typeToken) + { + return fieldToken.isOptionalEncoding() ? "?" : ""; + } + private void generateCompositeProperty( final StringBuilder sb, final String propertyName, @@ -807,11 +837,12 @@ private void generateCompositeProperty( final Token typeToken, final String indent) { - final String bitSetName = formatDtoClassName(typeToken.applicableTypeName()); + final String compositeName = formatDtoClassName(typeToken.applicableTypeName()); sb.append("\n") .append(generateDocumentation(indent, fieldToken)) - .append(indent).append("public ").append(bitSetName).append(" ").append(formatPropertyName(propertyName)) - .append(" { get; set; } = new ").append(bitSetName).append("();\n"); + .append(indent).append("public ").append(compositeName) + .append(optionality(fieldToken, typeToken)).append(" ").append(formatPropertyName(propertyName)) + .append(" { get; init; }\n"); } private void generateBitSetProperty( @@ -826,7 +857,7 @@ private void generateBitSetProperty( sb.append("\n") .append(generateDocumentation(indent, fieldToken)) .append(indent).append("public ").append(enumName).append(" ") - .append(formatPropertyName(propertyName)).append(" { get; set; }\n"); + .append(formatPropertyName(propertyName)).append(" { get; init; }\n"); } private void generateEnumProperty( @@ -856,8 +887,8 @@ private void generateEnumProperty( { sb.append("\n") .append(generateDocumentation(indent, fieldToken)) - .append(indent).append("public ").append(enumName).append(" ").append(formatPropertyName(propertyName)) - .append(" { get; set; }\n"); + .append(indent).append("public ").append(enumName).append(" ") + .append(formatPropertyName(propertyName)).append(" { get; init; }\n"); } } @@ -909,7 +940,7 @@ private void generateArrayProperty( sb.append("\n") .append(generateDocumentation(indent, fieldToken)) .append(indent).append("public string ") - .append(formatPropertyName(propertyName)).append(" { get; set; }\n"); + .append(formatPropertyName(propertyName)).append(" { get; init; }\n"); } else { @@ -917,8 +948,9 @@ private void generateArrayProperty( sb.append("\n") .append(generateDocumentation(indent, fieldToken)) - .append(indent).append("public ").append(typeName).append("[] ") - .append(formatPropertyName(propertyName)).append(" { get; set; }\n"); + .append(indent).append("public IReadOnlyList<").append(typeName).append(">") + .append(optionality(fieldToken, typeToken)).append(" ") + .append(formatPropertyName(propertyName)).append(" { get; init; }\n"); } } @@ -933,8 +965,9 @@ private void generateSingleValueProperty( sb.append("\n") .append(generateDocumentation(indent, fieldToken)) - .append(indent).append("public ").append(typeName).append(" ") - .append(formatPropertyName(propertyName)).append(" { get; set; }\n"); + .append(indent).append("public ").append(typeName) + .append(optionality(fieldToken, typeToken)).append(" ") + .append(formatPropertyName(propertyName)).append(" { get; init; }\n"); } private void generateConstPropertyMethods( @@ -984,9 +1017,11 @@ private void generateVarData( final String characterEncoding = varDataToken.encoding().characterEncoding(); final String dtoType = characterEncoding == null ? "byte[]" : "string"; + final String optionality = token.version() > 0 ? "?" : ""; + sb.append("\n") .append(indent).append("public ").append(dtoType).append(" ") - .append(formatPropertyName(propertyName)).append(" { get; set; }\n"); + .append(formatPropertyName(propertyName)).append(" { get; init; }\n"); } } } @@ -1020,14 +1055,17 @@ private void generateComposite(final List tokens) throws IOException try (Writer out = outputManager.createOutput(className)) { - out.append(generateFileHeader(ir.applicableNamespace())); + out.append(generateFileHeader(ir.applicableNamespace(), + "#nullable enable\n", + "using System.Collections.Generic;\n", + "using System.Linq;\n")); out.append(generateDocumentation(BASE_INDENT, tokens.get(0))); final StringBuilder sb = new StringBuilder(); final List compositeTokens = tokens.subList(1, tokens.size() - 1); generateCompositePropertyElements(sb, compositeTokens, BASE_INDENT + INDENT); - generateCompositeDecodeFrom(sb, codecClassName, compositeTokens, BASE_INDENT + INDENT); + generateCompositeDecodeFrom(sb, className, codecClassName, compositeTokens, BASE_INDENT + INDENT); generateCompositeEncodeInto(sb, codecClassName, compositeTokens, BASE_INDENT + INDENT); generateDisplay(sb, codecClassName, "Wrap", codecClassName + ".SbeSchemaVersion", BASE_INDENT + INDENT); diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpUtil.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpUtil.java index 124b352629..1fc1cb64b2 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpUtil.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpUtil.java @@ -85,20 +85,20 @@ static String generateLiteral(final PrimitiveType type, final String value) return literal; } - static CharSequence generateFileHeader(final String packageName, final String... imports) + static CharSequence generateFileHeader(final String packageName, final String... topLevelStatements) { return String.format( "// \n" + "// Generated SBE (Simple Binary Encoding) message codec\n" + "// \n\n" + "#pragma warning disable 1591 // disable warning on missing comments\n" + + "%1$s" + "using System;\n" + "using System.Text;\n" + - "%1$s" + "using Org.SbeTool.Sbe.Dll;\n\n" + "namespace %2$s\n" + "{\n", - String.join("", imports), + String.join("", topLevelStatements), formatNamespace(packageName)); } diff --git a/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/Program.cs b/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/Program.cs index 1efcf4bfa7..60bdc22616 100644 --- a/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/Program.cs +++ b/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/Program.cs @@ -17,8 +17,7 @@ static int Main(string[] args) { messageHeader.Wrap(buffer, 0, 0); var decoder = new TestMessage(); decoder.WrapForDecode(buffer, 8, messageHeader.BlockLength, messageHeader.Version); - var dto = new TestMessageDto(); - dto.DecodeFrom(decoder); + var dto = TestMessageDto.DecodeFrom(decoder); var outputBytes = new byte[inputBytes.Length]; var outputBuffer = new DirectBuffer(outputBytes); var encoder = new TestMessage(); diff --git a/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/SbePropertyTest.csproj b/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/SbePropertyTest.csproj index a1bbc69f1c..8b0b167d2b 100644 --- a/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/SbePropertyTest.csproj +++ b/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/SbePropertyTest.csproj @@ -2,7 +2,7 @@ Exe - net7.0 + net6.0 From 207ab1f3be2e28c9d3aa867f8dedf82191a6ebb9 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Wed, 18 Oct 2023 14:23:08 +0100 Subject: [PATCH 19/41] [IR] Change default float and double values to minimum representable values. Previously, we were using the minimum positive value as the `minValue`. --- .../src/main/java/uk/co/real_logic/sbe/PrimitiveValue.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/PrimitiveValue.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/PrimitiveValue.java index 27411749b5..0754d9a485 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/PrimitiveValue.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/PrimitiveValue.java @@ -197,7 +197,7 @@ public enum Representation /** * Maximum value representation for a single precision 32-bit floating point type. */ - public static final float MIN_VALUE_FLOAT = Float.MIN_VALUE; + public static final float MIN_VALUE_FLOAT = -Float.MAX_VALUE; /** * Maximum value representation for a single precision 32-bit floating point type. @@ -212,7 +212,7 @@ public enum Representation /** * Minimum value representation for a double precision 64-bit floating point type. */ - public static final double MIN_VALUE_DOUBLE = Double.MIN_VALUE; + public static final double MIN_VALUE_DOUBLE = -Double.MAX_VALUE; /** * Maximum value representation for a double precision 64-bit floating point type. From 4201789439cd009daa73683231485bc6dc0295de Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Wed, 18 Oct 2023 15:57:17 +0100 Subject: [PATCH 20/41] [C#] Add validation to DTOs and improve use of records. The aim of these changes are to avoid multiple representations in DTOs and to support read-only views over data. The new validation includes: - Checking null values "idiomatic null values" rather than the reserved null value, to prevent against multiple kinds of null in DTOs. - Checking primitive field values are at least `minValue` and at most `maxValue`. Note that this validation is not applied to fixed-size arrays as the specification says, "Data range attributes minValue and maxValue do not apply", under the "Fixed-length data" section. Records are now immutable. Record expressions, i.e., using `with`, are supported and will apply validation, as we have customised the `init` property accessor. I have not included support for encoding null composite values from DTOs; however, in theory, this could be added later. --- .../generation/csharp/CSharpDtoGenerator.java | 347 ++++++++++++++---- .../arbitraries/SbeArbitraries.java | 7 +- 2 files changed, 278 insertions(+), 76 deletions(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java index b6ab8638d9..d50030351a 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java @@ -79,23 +79,26 @@ public void generate() throws IOException int offset = 0; final StringBuilder sb = new StringBuilder(); + final StringBuilder ctorArgs = new StringBuilder(); final List fields = new ArrayList<>(); offset = collectFields(messageBody, offset, fields); - generateFields(sb, fields, BASE_INDENT + INDENT); + generateFields(sb, ctorArgs, codecClassName, fields, BASE_INDENT + INDENT); final List groups = new ArrayList<>(); offset = collectGroups(messageBody, offset, groups); - generateGroups(sb, codecClassName, groups, BASE_INDENT + INDENT); + generateGroups(sb, ctorArgs, className, codecClassName, groups, BASE_INDENT + INDENT); final List varData = new ArrayList<>(); collectVarData(messageBody, offset, varData); - generateVarData(sb, varData, BASE_INDENT + INDENT); + generateVarData(sb, ctorArgs, varData, BASE_INDENT + INDENT); generateDecodeFrom(sb, className, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); generateEncodeInto(sb, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); generateDisplay(sb, codecClassName, "WrapForEncode", null, BASE_INDENT + INDENT); + removeTrailingComma(ctorArgs); + try (Writer out = outputManager.createOutput(className)) { out.append(generateFileHeader( @@ -105,7 +108,9 @@ public void generate() throws IOException "using System.Linq;\n")); out.append(generateDocumentation(BASE_INDENT, msgToken)); - out.append(BASE_INDENT).append("public sealed partial record ").append(className).append("\n") + out.append(BASE_INDENT).append("public sealed partial record ").append(className).append("(\n") + .append(ctorArgs) + .append(BASE_INDENT).append(")\n") .append(BASE_INDENT).append("{") .append(sb) .append(BASE_INDENT).append("}\n") @@ -116,7 +121,9 @@ public void generate() throws IOException private void generateGroups( final StringBuilder sb, - final String parentMessageClassName, + final StringBuilder ctorArgs, + final String qualifiedParentDtoClassName, + final String qualifiedParentCodecClassName, final List tokens, final String indent) { @@ -130,38 +137,58 @@ private void generateGroups( final String groupName = groupToken.name(); final String groupClassName = formatDtoClassName(groupName); + final String formattedPropertyName = formatPropertyName(groupName); + + ctorArgs.append(indent).append("IReadOnlyList<") + .append(qualifiedParentDtoClassName).append(".").append(groupClassName).append("> ") + .append(formattedPropertyName).append(",\n"); + sb.append("\n") .append(generateDocumentation(indent, groupToken)) .append(indent).append("public IReadOnlyList<").append(groupClassName).append("> ") - .append(formatPropertyName(groupName)) - .append(" { get; init; }\n"); + .append(formattedPropertyName) + .append(" { get; init; } = ").append(formattedPropertyName).append(";\n"); - sb.append("\n") - .append(generateDocumentation(indent, groupToken)) - .append(indent).append("public sealed partial record ").append(groupClassName).append("\n") - .append(indent).append("{"); + final StringBuilder groupCtorArgs = new StringBuilder(); + final StringBuilder groupRecordBody = new StringBuilder(); i++; i += tokens.get(i).componentTokenCount(); + final String qualifiedDtoClassName = qualifiedParentDtoClassName + "." + groupClassName; + final String qualifiedCodecClassName = + qualifiedParentCodecClassName + "." + formatClassName(groupName) + "Group"; + final List fields = new ArrayList<>(); i = collectFields(tokens, i, fields); - generateFields(sb, fields, indent + INDENT); + generateFields(groupRecordBody, groupCtorArgs, qualifiedCodecClassName, fields, indent + INDENT); final List groups = new ArrayList<>(); i = collectGroups(tokens, i, groups); - final String codecClassName = parentMessageClassName + "." + formatClassName(groupName) + "Group"; - generateGroups(sb, codecClassName, groups, indent + INDENT); + generateGroups(groupRecordBody, groupCtorArgs, qualifiedDtoClassName, + qualifiedCodecClassName, groups, indent + INDENT); final List varData = new ArrayList<>(); i = collectVarData(tokens, i, varData); - generateVarData(sb, varData, indent + INDENT); + generateVarData(groupRecordBody, groupCtorArgs, varData, indent + INDENT); - generateDecodeListFrom(sb, groupClassName, codecClassName, indent + INDENT); - generateDecodeFrom(sb, groupClassName, codecClassName, fields, groups, varData, indent + INDENT); - generateEncodeInto(sb, codecClassName, fields, groups, varData, indent + INDENT); + generateDecodeListFrom( + groupRecordBody, groupClassName, qualifiedCodecClassName, indent + INDENT); + generateDecodeFrom( + groupRecordBody, groupClassName, qualifiedCodecClassName, fields, groups, varData, indent + INDENT); + generateEncodeInto( + groupRecordBody, qualifiedCodecClassName, fields, groups, varData, indent + INDENT); - sb.append(indent).append("}\n"); + removeTrailingComma(groupCtorArgs); + + sb.append("\n") + .append(generateDocumentation(indent, groupToken)) + .append(indent).append("public sealed partial record ").append(groupClassName).append("(\n") + .append(groupCtorArgs) + .append(indent).append(")\n") + .append(indent).append("{\n") + .append(groupRecordBody) + .append(indent).append("}\n"); } } @@ -177,8 +204,7 @@ private void generateCompositeDecodeFrom( .append(codecClassName).append(" codec)\n") .append(indent).append("{\n"); - sb.append(indent).append(INDENT).append("return new ").append(dtoClassName).append("()\n") - .append(indent).append(INDENT).append("{\n"); + sb.append(indent).append(INDENT).append("return new ").append(dtoClassName).append("(\n"); for (int i = 0; i < tokens.size(); ) { @@ -189,7 +215,9 @@ private void generateCompositeDecodeFrom( i += tokens.get(i).componentTokenCount(); } - sb.append(indent).append(INDENT).append("};\n"); + removeTrailingComma(sb); + + sb.append(indent).append(INDENT).append(");\n"); sb.append(indent).append("}\n"); } @@ -256,12 +284,12 @@ private void generateDecodeFrom( .append(" DecodeFrom(").append(codecClassName).append(" codec)\n") .append(indent).append("{\n"); - sb.append(indent).append(INDENT).append("return new ").append(dtoClassName).append("()\n") - .append(indent).append(INDENT).append("{\n"); + sb.append(indent).append(INDENT).append("return new ").append(dtoClassName).append("(\n"); generateFieldsDecodeFrom(sb, fields, codecClassName, indent + INDENT + INDENT); generateGroupsDecodeFrom(sb, groups, indent + INDENT + INDENT); generateVarDataDecodeFrom(sb, varData, indent + INDENT + INDENT); - sb.append(indent).append(INDENT).append("};\n"); + removeTrailingComma(sb); + sb.append(indent).append(INDENT).append(");\n"); sb.append(indent).append("}\n"); } @@ -297,13 +325,13 @@ private void generateFieldDecodeFrom( break; case BEGIN_SET: - generatePropertyDecodeFrom(sb, codecClassName, fieldToken, "0", indent); + generatePropertyDecodeFrom(sb, fieldToken, "0", null, indent); break; case BEGIN_ENUM: final String enumName = formatClassName(typeToken.applicableTypeName()); final String nullValue = formatNamespace(ir.packageName()) + "." + enumName + ".NULL_VALUE"; - generatePropertyDecodeFrom(sb, codecClassName, fieldToken, nullValue, indent); + generatePropertyDecodeFrom(sb, fieldToken, nullValue, null, indent); break; case BEGIN_COMPOSITE: @@ -331,8 +359,8 @@ private void generatePrimitiveDecodeFrom( if (arrayLength == 1) { - final String nullValue = codecClassName + "." + formatPropertyName(fieldToken.name()) + "NullValue"; - generatePropertyDecodeFrom(sb, codecClassName, fieldToken, nullValue, indent); + final String codecNullValue = codecClassName + "." + formatPropertyName(fieldToken.name()) + "NullValue"; + generatePropertyDecodeFrom(sb, fieldToken, "null", codecNullValue, indent); } else if (arrayLength > 1) { @@ -356,7 +384,7 @@ private void generateArrayDecodeFrom( if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) { - wrapInActingVersionCheck( + generateRecordPropertyAssignment( sb, fieldToken, indent, @@ -367,7 +395,7 @@ private void generateArrayDecodeFrom( } else { - wrapInActingVersionCheck( + generateRecordPropertyAssignment( sb, fieldToken, indent, @@ -380,12 +408,11 @@ private void generateArrayDecodeFrom( private void generatePropertyDecodeFrom( final StringBuilder sb, - final String codecClassName, final Token fieldToken, - final String nullValue, + final String dtoNullValue, + final String codecNullValue, final String indent) { - if (fieldToken.isConstantEncoding()) { return; @@ -394,13 +421,13 @@ private void generatePropertyDecodeFrom( final String propertyName = fieldToken.name(); final String formattedPropertyName = formatPropertyName(propertyName); - wrapInActingVersionCheck( + generateRecordPropertyAssignment( sb, fieldToken, indent, "codec." + formattedPropertyName, - nullValue, - codecClassName + "." + formattedPropertyName + "NullValue" + dtoNullValue, + codecNullValue ); } @@ -414,7 +441,7 @@ private void generateComplexDecodeFrom( final String formattedPropertyName = formatPropertyName(propertyName); final String dtoClassName = formatDtoClassName(typeToken.applicableTypeName()); - wrapInActingVersionCheck( + generateRecordPropertyAssignment( sb, fieldToken, indent, @@ -440,7 +467,7 @@ private void generateGroupsDecodeFrom( final String formattedPropertyName = formatPropertyName(groupName); final String groupDtoClassName = formatDtoClassName(groupName); - wrapInActingVersionCheck( + generateRecordPropertyAssignment( sb, groupToken, indent, @@ -482,7 +509,7 @@ private void generateVarDataDecodeFrom( "Get" + formattedPropertyName + "Bytes" : "Get" + formattedPropertyName; - wrapInActingVersionCheck( + generateRecordPropertyAssignment( sb, token, indent, @@ -494,7 +521,7 @@ private void generateVarDataDecodeFrom( } } - private void wrapInActingVersionCheck( + private void generateRecordPropertyAssignment( final StringBuilder sb, final Token token, final String indent, @@ -505,7 +532,7 @@ private void wrapInActingVersionCheck( final String propertyName = token.name(); final String formattedPropertyName = formatPropertyName(propertyName); - sb.append(indent).append(formattedPropertyName).append(" = "); + sb.append(indent).append(formattedPropertyName).append(": "); if (token.version() > 0) { @@ -699,6 +726,17 @@ private void generateComplexEncodeInto( { final String propertyName = fieldToken.name(); final String formattedPropertyName = formatPropertyName(propertyName); + + if (fieldToken.isOptionalEncoding()) + { + sb.append(indent).append("if (null == ").append(formattedPropertyName).append(")\n") + .append(indent).append("{") + .append(indent).append(INDENT).append("throw new System.InvalidOperationException(\"") + .append("Null composite value encoding (").append(formattedPropertyName) + .append(") is not supported via DTOs\");\n") + .append(indent).append("}\n\n"); + } + sb.append(indent).append(formattedPropertyName).append(".EncodeInto(codec.") .append(formattedPropertyName).append(");\n"); } @@ -789,6 +827,8 @@ private void generateDisplay( private void generateFields( final StringBuilder sb, + final StringBuilder ctorArgs, + final String codecClassName, final List tokens, final String indent) { @@ -803,19 +843,20 @@ private void generateFields( switch (encodingToken.signal()) { case ENCODING: - generatePrimitiveProperty(sb, propertyName, signalToken, encodingToken, indent); + generatePrimitiveProperty( + sb, ctorArgs, codecClassName, propertyName, signalToken, encodingToken, indent); break; case BEGIN_ENUM: - generateEnumProperty(sb, propertyName, signalToken, encodingToken, indent); + generateEnumProperty(sb, ctorArgs, propertyName, signalToken, encodingToken, indent); break; case BEGIN_SET: - generateBitSetProperty(sb, propertyName, signalToken, encodingToken, indent); + generateBitSetProperty(sb, ctorArgs, propertyName, signalToken, encodingToken, indent); break; case BEGIN_COMPOSITE: - generateCompositeProperty(sb, propertyName, signalToken, encodingToken, indent); + generateCompositeProperty(sb, ctorArgs, propertyName, signalToken, encodingToken, indent); break; default: @@ -825,28 +866,30 @@ private void generateFields( } } - private String optionality(final Token fieldToken, final Token typeToken) - { - return fieldToken.isOptionalEncoding() ? "?" : ""; - } - private void generateCompositeProperty( final StringBuilder sb, + final StringBuilder ctorArgs, final String propertyName, final Token fieldToken, final Token typeToken, final String indent) { final String compositeName = formatDtoClassName(typeToken.applicableTypeName()); + final String nullableSuffix = fieldToken.isOptionalEncoding() ? "?" : ""; + final String formattedPropertyName = formatPropertyName(propertyName); + + ctorArgs.append(indent).append(compositeName).append(" ").append(formattedPropertyName).append(",\n"); + sb.append("\n") .append(generateDocumentation(indent, fieldToken)) .append(indent).append("public ").append(compositeName) - .append(optionality(fieldToken, typeToken)).append(" ").append(formatPropertyName(propertyName)) - .append(" { get; init; }\n"); + .append(nullableSuffix).append(" ").append(formattedPropertyName) + .append(" { get; init; } = ").append(formattedPropertyName).append(";\n"); } private void generateBitSetProperty( final StringBuilder sb, + final StringBuilder ctorArgs, final String propertyName, final Token fieldToken, final Token typeToken, @@ -854,14 +897,20 @@ private void generateBitSetProperty( { final String enumName = formatClassName(typeToken.applicableTypeName()); + final String formattedPropertyName = formatPropertyName(propertyName); + + ctorArgs.append(indent).append(enumName).append(" ").append(formattedPropertyName).append(",\n"); + sb.append("\n") .append(generateDocumentation(indent, fieldToken)) .append(indent).append("public ").append(enumName).append(" ") - .append(formatPropertyName(propertyName)).append(" { get; init; }\n"); + .append(formattedPropertyName).append(" { get; init; } = ") + .append(formattedPropertyName).append(";\n"); } private void generateEnumProperty( final StringBuilder sb, + final StringBuilder ctorArgs, final String propertyName, final Token fieldToken, final Token typeToken, @@ -869,6 +918,8 @@ private void generateEnumProperty( { final String enumName = formatClassName(typeToken.applicableTypeName()); + final String formattedPropertyName = formatPropertyName(propertyName); + if (fieldToken.isConstantEncoding()) { final String constValue = fieldToken.encoding().constValue().toString(); @@ -876,7 +927,7 @@ private void generateEnumProperty( sb.append("\n") .append(generateDocumentation(indent, fieldToken)) .append(indent).append("public static ").append(enumName).append(" ") - .append(formatPropertyName(propertyName)).append("\n") + .append(formattedPropertyName).append("\n") .append(indent).append("{\n") .append(indent).append(INDENT).append("get { return ") .append(formatNamespace(ir.packageName())).append(".").append(constValue) @@ -885,15 +936,19 @@ private void generateEnumProperty( } else { + ctorArgs.append(indent).append(enumName).append(" ").append(formattedPropertyName).append(",\n"); + sb.append("\n") .append(generateDocumentation(indent, fieldToken)) .append(indent).append("public ").append(enumName).append(" ") - .append(formatPropertyName(propertyName)).append(" { get; init; }\n"); + .append(formattedPropertyName).append(" { get; init; } = ").append(formattedPropertyName).append(";\n"); } } private void generatePrimitiveProperty( final StringBuilder sb, + final StringBuilder ctorArgs, + final String codecClassName, final String propertyName, final Token fieldToken, final Token typeToken, @@ -905,12 +960,14 @@ private void generatePrimitiveProperty( } else { - generatePrimitivePropertyMethods(sb, propertyName, fieldToken, typeToken, indent); + generatePrimitivePropertyMethods(sb, ctorArgs, codecClassName, propertyName, fieldToken, typeToken, indent); } } private void generatePrimitivePropertyMethods( final StringBuilder sb, + final StringBuilder ctorArgs, + final String codecClassName, final String propertyName, final Token fieldToken, final Token typeToken, @@ -920,54 +977,162 @@ private void generatePrimitivePropertyMethods( if (arrayLength == 1) { - generateSingleValueProperty(sb, propertyName, fieldToken, typeToken, indent); + generateSingleValueProperty(sb, ctorArgs, codecClassName, propertyName, fieldToken, typeToken, indent); } else if (arrayLength > 1) { - generateArrayProperty(sb, propertyName, fieldToken, typeToken, indent); + generateArrayProperty(sb, ctorArgs, codecClassName, propertyName, fieldToken, typeToken, indent); } } private void generateArrayProperty( final StringBuilder sb, + final StringBuilder ctorArgs, + final String codecClassName, final String propertyName, final Token fieldToken, final Token typeToken, final String indent) { + final String formattedPropertyName = formatPropertyName(propertyName); + if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) { + ctorArgs.append(indent).append("string ").append(formattedPropertyName).append(",\n"); + sb.append("\n") .append(generateDocumentation(indent, fieldToken)) .append(indent).append("public string ") - .append(formatPropertyName(propertyName)).append(" { get; init; }\n"); + .append(formattedPropertyName).append(" { get; init; } = ") + .append(formattedPropertyName).append(";\n"); } else { final String typeName = cSharpTypeName(typeToken.encoding().primitiveType()); + final String fieldName = "_" + toLowerFirstChar(propertyName); + final String nullableSuffix = fieldToken.isOptionalEncoding() ? "?" : ""; + final String listTypeName = "IReadOnlyList<" + typeName + ">" + nullableSuffix; + + ctorArgs.append(indent).append(listTypeName).append(" ").append(formattedPropertyName).append(",\n"); + + sb.append("\n") + .append(indent).append("private ").append(listTypeName).append(" ").append(fieldName) + .append(" = Validate").append(formattedPropertyName).append("(").append(formattedPropertyName) + .append(");\n"); sb.append("\n") .append(generateDocumentation(indent, fieldToken)) - .append(indent).append("public IReadOnlyList<").append(typeName).append(">") - .append(optionality(fieldToken, typeToken)).append(" ") - .append(formatPropertyName(propertyName)).append(" { get; init; }\n"); + .append(indent).append("public ").append(listTypeName).append(" ") + .append(formattedPropertyName).append("\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("get => ").append(fieldName).append(";\n") + .append(indent).append(INDENT).append("init => ").append(fieldName).append(" = Validate") + .append(formattedPropertyName).append("(value);\n") + .append(indent).append("}\n"); + + sb.append("\n") + .append(indent).append("private static ").append(listTypeName).append(" Validate") + .append(formattedPropertyName).append("(").append(listTypeName).append(" value)\n") + .append(indent).append("{\n"); + + if (fieldToken.isOptionalEncoding()) + { + sb.append(indent).append(INDENT) + .append("if (value == null)\n") + .append(indent).append(INDENT) + .append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("return null;\n") + .append(indent).append(INDENT) + .append("}\n"); + } + + sb.append(indent).append(INDENT) + .append("if (value.Count > ").append(codecClassName).append(".") + .append(formattedPropertyName).append("Length)\n") + .append(indent).append(INDENT) + .append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("throw new ArgumentException(\"too many elements: \" + value.Count);\n") + .append(indent).append(INDENT) + .append("}\n") + .append(indent).append(INDENT) + .append("return value;\n") + .append(indent).append("}\n"); } } private void generateSingleValueProperty( final StringBuilder sb, + final StringBuilder ctorArgs, + final String codecClassName, final String propertyName, final Token fieldToken, final Token typeToken, final String indent) { - final String typeName = cSharpTypeName(typeToken.encoding().primitiveType()); + final String nullableSuffix = fieldToken.isOptionalEncoding() ? "?" : ""; + final String typeName = cSharpTypeName(typeToken.encoding().primitiveType()) + nullableSuffix; + final String formattedPropertyName = formatPropertyName(propertyName); + final String fieldName = "_" + toLowerFirstChar(propertyName); + + ctorArgs.append(indent).append(typeName).append(" ").append(formattedPropertyName).append(",\n"); + + sb.append("\n") + .append(indent).append("private ").append(typeName).append(" ").append(fieldName) + .append(" = Validate").append(formattedPropertyName) + .append("(").append(formattedPropertyName).append(");\n"); sb.append("\n") .append(generateDocumentation(indent, fieldToken)) - .append(indent).append("public ").append(typeName) - .append(optionality(fieldToken, typeToken)).append(" ") - .append(formatPropertyName(propertyName)).append(" { get; init; }\n"); + .append(indent).append("public ").append(typeName).append(" ") + .append(formattedPropertyName).append("\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("get => ").append(fieldName).append(";\n") + .append(indent).append(INDENT).append("init => ").append(fieldName).append(" = Validate") + .append(formattedPropertyName).append("(value);\n") + .append(indent).append("}\n"); + + sb.append("\n") + .append(indent).append("private static ").append(typeName).append(" Validate") + .append(formattedPropertyName).append("(").append(typeName).append(" value)\n") + .append(indent).append("{\n"); + + if (fieldToken.isOptionalEncoding()) + { + sb.append(indent).append(INDENT) + .append("if (value == null)\n") + .append(indent).append(INDENT) + .append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("return null;\n") + .append(indent).append(INDENT) + .append("}\n"); + + sb.append(indent).append(INDENT) + .append("if (value == ").append(codecClassName).append(".") + .append(formattedPropertyName).append("NullValue)\n") + .append(indent).append(INDENT) + .append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("throw new ArgumentException(\"null value is reserved: \" + value);\n") + .append(indent).append(INDENT) + .append("}\n"); + } + + sb.append(indent).append(INDENT) + .append("if (value < ") + .append(codecClassName).append(".").append(formattedPropertyName).append("MinValue || value > ") + .append(codecClassName).append(".").append(formattedPropertyName).append("MaxValue)\n") + .append(indent).append(INDENT) + .append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("throw new ArgumentException(\"value is out of allowed range: \" + value);\n") + .append(indent).append(INDENT) + .append("}\n") + .append(indent).append(INDENT) + .append("return value;\n") + .append(indent).append("}\n"); } private void generateConstPropertyMethods( @@ -1004,6 +1169,7 @@ private void generateConstPropertyMethods( private void generateVarData( final StringBuilder sb, + final StringBuilder ctorArgs, final List tokens, final String indent) { @@ -1017,11 +1183,14 @@ private void generateVarData( final String characterEncoding = varDataToken.encoding().characterEncoding(); final String dtoType = characterEncoding == null ? "byte[]" : "string"; - final String optionality = token.version() > 0 ? "?" : ""; + final String formattedPropertyName = formatPropertyName(propertyName); + + ctorArgs.append(indent).append(dtoType).append(" ").append(formattedPropertyName).append(",\n"); sb.append("\n") .append(indent).append("public ").append(dtoType).append(" ") - .append(formatPropertyName(propertyName)).append(" { get; init; }\n"); + .append(formattedPropertyName).append(" { get; init; } = ") + .append(formattedPropertyName).append(";\n"); } } } @@ -1062,14 +1231,19 @@ private void generateComposite(final List tokens) throws IOException out.append(generateDocumentation(BASE_INDENT, tokens.get(0))); final StringBuilder sb = new StringBuilder(); + final StringBuilder ctorArgs = new StringBuilder(); final List compositeTokens = tokens.subList(1, tokens.size() - 1); - generateCompositePropertyElements(sb, compositeTokens, BASE_INDENT + INDENT); + generateCompositePropertyElements(sb, ctorArgs, codecClassName, compositeTokens, BASE_INDENT + INDENT); generateCompositeDecodeFrom(sb, className, codecClassName, compositeTokens, BASE_INDENT + INDENT); generateCompositeEncodeInto(sb, codecClassName, compositeTokens, BASE_INDENT + INDENT); generateDisplay(sb, codecClassName, "Wrap", codecClassName + ".SbeSchemaVersion", BASE_INDENT + INDENT); - out.append(BASE_INDENT).append("public sealed partial record ").append(className).append("\n") + removeTrailingComma(ctorArgs); + + out.append(BASE_INDENT).append("public sealed partial record ").append(className).append("(\n") + .append(ctorArgs) + .append(BASE_INDENT).append(")\n") .append(BASE_INDENT).append("{") .append(sb) .append(BASE_INDENT).append("}\n") @@ -1077,8 +1251,31 @@ private void generateComposite(final List tokens) throws IOException } } + private static void removeTrailingComma(final StringBuilder ctorArgs) + { + if (ctorArgs.length() < 2) + { + return; + } + + if (ctorArgs.charAt(ctorArgs.length() - 1) != '\n') + { + return; + } + + if (ctorArgs.charAt(ctorArgs.length() - 2) != ',') + { + return; + } + + ctorArgs.setLength(ctorArgs.length() - 2); + ctorArgs.append("\n"); + } + private void generateCompositePropertyElements( final StringBuilder sb, + final StringBuilder ctorArgs, + final String codecClassName, final List tokens, final String indent) { @@ -1090,19 +1287,19 @@ private void generateCompositePropertyElements( switch (token.signal()) { case ENCODING: - generatePrimitiveProperty(sb, propertyName, token, token, indent); + generatePrimitiveProperty(sb, ctorArgs, codecClassName, propertyName, token, token, indent); break; case BEGIN_ENUM: - generateEnumProperty(sb, propertyName, token, token, indent); + generateEnumProperty(sb, ctorArgs, propertyName, token, token, indent); break; case BEGIN_SET: - generateBitSetProperty(sb, propertyName, token, token, indent); + generateBitSetProperty(sb, ctorArgs, propertyName, token, token, indent); break; case BEGIN_COMPOSITE: - generateCompositeProperty(sb, propertyName, token, token, indent); + generateCompositeProperty(sb, ctorArgs, propertyName, token, token, indent); break; default: diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java index 2e3db4bda0..b1b025d4d9 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java @@ -324,7 +324,12 @@ private static Arbitrary encodedTypeEncoder( case UINT64: return Arbitraries.longs() - .map(l -> (buffer, offset, limit) -> buffer.putLong(offset, l, encoding.byteOrder())); + .map(l -> (buffer, offset, limit) -> + { + final long nullValue = encoding.applicableNullValue().longValue(); + final long nonNullValue = l == nullValue ? minValue.longValue() : l; + buffer.putLong(offset, nonNullValue, encoding.byteOrder()); + }); case INT64: assert minValue.longValue() <= maxValue.longValue(); From bc17a46ba35ed5c8adc175d95fac952325be14d5 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Wed, 18 Oct 2023 16:57:47 +0100 Subject: [PATCH 21/41] [C#] Support DTO generation via system property. Previously, you had to pass the `CSharpDtos` FQN as the `TargetCodeGenerator` when running sbe-tool; however, that means the XML schema was parsed multiple times, as DTOs depend on flyweights, which seems wasteful. Therefore, I have introduced a system property, `sbe.csharp.generate.dtos` that also controls the generation of DTOs when targetting `CSharp`. --- .../sbe/generation/csharp/CSharp.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharp.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharp.java index 157309f319..8a88cbce56 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharp.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharp.java @@ -26,14 +26,30 @@ */ public class CSharp implements TargetCodeGenerator { + private static final boolean GENERATE_DTOS = Boolean.getBoolean("sbe.csharp.generate.dtos"); + /** * {@inheritDoc} */ public CodeGenerator newInstance(final Ir ir, final String outputDir) { - return new CSharpGenerator( + final CSharpGenerator flyweightGenerator = new CSharpGenerator( ir, TargetCodeGeneratorLoader.precedenceChecks(), new CSharpNamespaceOutputManager(outputDir, ir.applicableNamespace())); + + if (GENERATE_DTOS) + { + final CSharpDtoGenerator dtoGenerator = + new CSharpDtoGenerator(ir, new CSharpNamespaceOutputManager(outputDir, ir.applicableNamespace())); + + return () -> + { + flyweightGenerator.generate(); + dtoGenerator.generate(); + }; + } + + return flyweightGenerator; } } From 248eae90e8c67111e1888a7f7fd9dec0c90a9da2 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Thu, 19 Oct 2023 16:49:33 +0100 Subject: [PATCH 22/41] [C#] Extend tests to cover extended schemas. These tests showed some deficiencies in the DTO generation. For example, added variable-length data was not represented as nullable nor properly handled in `EncodeInto(...)`. During this work, I noticed composite "field" tokens (i.e., types) take their `token.version()` from their containing message/group field. I had to adjust some code that was using `token.version() > 0` to determine whether a field had been added, as this only works with message/group fields. --- .../generation/csharp/CSharpDtoGenerator.java | 87 ++++++++++++++----- .../properties/CSharpDtosPropertyTest.java | 11 +++ .../arbitraries/SbeArbitraries.java | 55 +++++++++--- .../sbe/properties/schema/FieldSchema.java | 11 ++- .../sbe/properties/schema/GroupSchema.java | 10 ++- .../sbe/properties/schema/MessageSchema.java | 35 +++++++- .../schema/TestXmlSchemaWriter.java | 3 + .../sbe/properties/schema/VarDataSchema.java | 10 ++- 8 files changed, 184 insertions(+), 38 deletions(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java index d50030351a..9335295663 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java @@ -93,8 +93,8 @@ public void generate() throws IOException collectVarData(messageBody, offset, varData); generateVarData(sb, ctorArgs, varData, BASE_INDENT + INDENT); - generateDecodeFrom(sb, className, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); - generateEncodeInto(sb, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); + generateMessageDecodeFrom(sb, className, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); + generateMessageEncodeInto(sb, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); generateDisplay(sb, codecClassName, "WrapForEncode", null, BASE_INDENT + INDENT); removeTrailingComma(ctorArgs); @@ -119,6 +119,12 @@ public void generate() throws IOException } } + private enum DefinitionKind + { + Composite, + Message + } + private void generateGroups( final StringBuilder sb, final StringBuilder ctorArgs, @@ -174,9 +180,9 @@ private void generateGroups( generateDecodeListFrom( groupRecordBody, groupClassName, qualifiedCodecClassName, indent + INDENT); - generateDecodeFrom( + generateMessageDecodeFrom( groupRecordBody, groupClassName, qualifiedCodecClassName, fields, groups, varData, indent + INDENT); - generateEncodeInto( + generateMessageEncodeInto( groupRecordBody, qualifiedCodecClassName, fields, groups, varData, indent + INDENT); removeTrailingComma(groupCtorArgs); @@ -210,7 +216,8 @@ private void generateCompositeDecodeFrom( { final Token token = tokens.get(i); - generateFieldDecodeFrom(sb, token, token, codecClassName, indent + INDENT + INDENT); + generateFieldDecodeFrom( + sb, DefinitionKind.Composite, token, token, codecClassName, indent + INDENT + INDENT); i += tokens.get(i).componentTokenCount(); } @@ -270,7 +277,7 @@ private void generateDecodeListFrom( .append(indent).append("}\n"); } - private void generateDecodeFrom( + private void generateMessageDecodeFrom( final StringBuilder sb, final String dtoClassName, final String codecClassName, @@ -285,7 +292,7 @@ private void generateDecodeFrom( .append(indent).append("{\n"); sb.append(indent).append(INDENT).append("return new ").append(dtoClassName).append("(\n"); - generateFieldsDecodeFrom(sb, fields, codecClassName, indent + INDENT + INDENT); + generateMessageFieldsDecodeFrom(sb, fields, codecClassName, indent + INDENT + INDENT); generateGroupsDecodeFrom(sb, groups, indent + INDENT + INDENT); generateVarDataDecodeFrom(sb, varData, indent + INDENT + INDENT); removeTrailingComma(sb); @@ -294,7 +301,7 @@ private void generateDecodeFrom( sb.append(indent).append("}\n"); } - private void generateFieldsDecodeFrom( + private void generateMessageFieldsDecodeFrom( final StringBuilder sb, final List tokens, final String codecClassName, @@ -307,35 +314,37 @@ private void generateFieldsDecodeFrom( { final Token encodingToken = tokens.get(i + 1); - generateFieldDecodeFrom(sb, signalToken, encodingToken, codecClassName, indent); + generateFieldDecodeFrom(sb, DefinitionKind.Message, signalToken, encodingToken, codecClassName, indent); } } } private void generateFieldDecodeFrom( final StringBuilder sb, + final DefinitionKind rootKind, final Token fieldToken, final Token typeToken, - final String codecClassName, final String indent) + final String codecClassName, + final String indent) { switch (typeToken.signal()) { case ENCODING: - generatePrimitiveDecodeFrom(sb, fieldToken, typeToken, codecClassName, indent); + generatePrimitiveDecodeFrom(sb, rootKind, fieldToken, typeToken, codecClassName, indent); break; case BEGIN_SET: - generatePropertyDecodeFrom(sb, fieldToken, "0", null, indent); + generatePropertyDecodeFrom(sb, rootKind, fieldToken, "0", null, indent); break; case BEGIN_ENUM: final String enumName = formatClassName(typeToken.applicableTypeName()); final String nullValue = formatNamespace(ir.packageName()) + "." + enumName + ".NULL_VALUE"; - generatePropertyDecodeFrom(sb, fieldToken, nullValue, null, indent); + generatePropertyDecodeFrom(sb, rootKind, fieldToken, nullValue, null, indent); break; case BEGIN_COMPOSITE: - generateComplexDecodeFrom(sb, fieldToken, typeToken, indent); + generateComplexDecodeFrom(sb, rootKind, fieldToken, typeToken, indent); break; default: @@ -345,6 +354,7 @@ private void generateFieldDecodeFrom( private void generatePrimitiveDecodeFrom( final StringBuilder sb, + final DefinitionKind rootKind, final Token fieldToken, final Token typeToken, final String codecClassName, @@ -360,16 +370,17 @@ private void generatePrimitiveDecodeFrom( if (arrayLength == 1) { final String codecNullValue = codecClassName + "." + formatPropertyName(fieldToken.name()) + "NullValue"; - generatePropertyDecodeFrom(sb, fieldToken, "null", codecNullValue, indent); + generatePropertyDecodeFrom(sb, rootKind, fieldToken, "null", codecNullValue, indent); } else if (arrayLength > 1) { - generateArrayDecodeFrom(sb, fieldToken, typeToken, indent); + generateArrayDecodeFrom(sb, rootKind, fieldToken, typeToken, indent); } } private void generateArrayDecodeFrom( final StringBuilder sb, + final DefinitionKind rootKind, final Token fieldToken, final Token typeToken, final String indent) @@ -386,6 +397,7 @@ private void generateArrayDecodeFrom( { generateRecordPropertyAssignment( sb, + rootKind, fieldToken, indent, "codec.Get" + formattedPropertyName + "()", @@ -397,6 +409,7 @@ private void generateArrayDecodeFrom( { generateRecordPropertyAssignment( sb, + rootKind, fieldToken, indent, "codec." + formattedPropertyName + "AsSpan().ToArray()", @@ -408,6 +421,7 @@ private void generateArrayDecodeFrom( private void generatePropertyDecodeFrom( final StringBuilder sb, + final DefinitionKind rootKind, final Token fieldToken, final String dtoNullValue, final String codecNullValue, @@ -423,6 +437,7 @@ private void generatePropertyDecodeFrom( generateRecordPropertyAssignment( sb, + rootKind, fieldToken, indent, "codec." + formattedPropertyName, @@ -433,6 +448,7 @@ private void generatePropertyDecodeFrom( private void generateComplexDecodeFrom( final StringBuilder sb, + final DefinitionKind rootKind, final Token fieldToken, final Token typeToken, final String indent) @@ -443,6 +459,7 @@ private void generateComplexDecodeFrom( generateRecordPropertyAssignment( sb, + rootKind, fieldToken, indent, dtoClassName + ".DecodeFrom(codec." + formattedPropertyName + ")", @@ -469,6 +486,7 @@ private void generateGroupsDecodeFrom( generateRecordPropertyAssignment( sb, + DefinitionKind.Message, groupToken, indent, groupDtoClassName + ".DecodeListFrom(codec." + formattedPropertyName + ")", @@ -511,6 +529,7 @@ private void generateVarDataDecodeFrom( generateRecordPropertyAssignment( sb, + DefinitionKind.Message, token, indent, "codec." + accessor + "()", @@ -523,6 +542,7 @@ private void generateVarDataDecodeFrom( private void generateRecordPropertyAssignment( final StringBuilder sb, + final DefinitionKind rootKind, final Token token, final String indent, final String presentExpression, @@ -534,8 +554,20 @@ private void generateRecordPropertyAssignment( sb.append(indent).append(formattedPropertyName).append(": "); - if (token.version() > 0) + // N.B., in the IR, the composite information is "embedded" into each message/group field. + // When we are looking at a composite "field" (i.e., a type) the `token.version()` method + // returns the "sinceVersion" of the containing field. Therefore, we cannot decide presence + // based on `token.version()` when dealing with composites. + if (rootKind.equals(DefinitionKind.Message) && token.version() > 0) { + if (token.signal() != Signal.BEGIN_VAR_DATA && !token.isOptionalEncoding()) + { + throw new IllegalStateException( + "Expected added field " + propertyName + + " (sinceVersion=" + token.version() + ") to have optional presence." + ); + } + sb.append("codec.").append(formattedPropertyName).append("InActingVersion()"); if (null != nullCodecValueOrNull) @@ -553,7 +585,7 @@ private void generateRecordPropertyAssignment( } } - private void generateEncodeInto( + private void generateMessageEncodeInto( final StringBuilder sb, final String codecClassName, final List fields, @@ -674,7 +706,7 @@ private String nullableConvertedExpression( final String expression, final String nullValue) { - return fieldToken.version() > 0 || fieldToken.isOptionalEncoding() ? + return fieldToken.isOptionalEncoding() ? expression + " ?? " + nullValue : expression; } @@ -791,9 +823,19 @@ private void generateVarDataEncodeInto( if (token.signal() == Signal.BEGIN_VAR_DATA) { final String propertyName = token.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + if (token.version() > 0) + { + sb.append(indent).append("if (null == ").append(formattedPropertyName).append(")\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("throw new System.InvalidOperationException(\"") + .append(formattedPropertyName).append(" must not be null.\");\n") + .append(indent).append("}\n\n"); + } - sb.append(indent).append("codec.Set").append(formatPropertyName(propertyName)) - .append("(").append(formatPropertyName(propertyName)).append(");\n"); + sb.append(indent).append("codec.Set").append(formattedPropertyName) + .append("(").append(formattedPropertyName).append(");\n"); } } } @@ -1181,7 +1223,8 @@ private void generateVarData( final String propertyName = token.name(); final Token varDataToken = Generators.findFirst("varData", tokens, i); final String characterEncoding = varDataToken.encoding().characterEncoding(); - final String dtoType = characterEncoding == null ? "byte[]" : "string"; + final String nullableSuffix = token.version() > 0 ? "?" : ""; + final String dtoType = (characterEncoding == null ? "byte[]" : "string") + nullableSuffix; final String formattedPropertyName = formatPropertyName(propertyName); diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java index 7a4dde3f49..8a8be0fdde 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java @@ -89,6 +89,17 @@ void dtoEncodeShouldBeTheInverseOfDtoDecode( "SCHEMA:\n" + encodedMessage.schema()); } + final byte[] errorBytes = Files.readAllBytes(stderr); + if (errorBytes.length != 0) + { + throw new AssertionError( + "Process wrote to stderr.\n\n" + + "STDOUT:\n" + new String(Files.readAllBytes(stdout)) + "\n\n" + + "STDERR:\n" + new String(errorBytes) + "\n\n" + + "SCHEMA:\n" + encodedMessage.schema() + "\n\n" + ); + } + final byte[] inputBytes = new byte[encodedMessage.length()]; encodedMessage.buffer().getBytes(0, inputBytes); final byte[] outputBytes = Files.readAllBytes(tempDir.resolve("output.dat")); diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java index b1b025d4d9..13f027fb30 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java @@ -192,6 +192,37 @@ private static Arbitrary typeSchema(final int depth) } } + private static Arbitrary addedField() + { + return Combinators.combine( + typeSchema(MAX_COMPOSITE_DEPTH), + Arbitraries.of(Encoding.Presence.OPTIONAL), + Arbitraries.of((short)1, (short)2) + ).as(FieldSchema::new); + } + + private static Arbitrary originalField() + { + return Combinators.combine( + typeSchema(MAX_COMPOSITE_DEPTH), + presence(), + Arbitraries.of((short)0) + ).as(FieldSchema::new); + } + + private static Arbitrary skewedFieldDistribution() + { + final Arbitrary originalField = originalField(); + final Arbitrary addedField = addedField(); + + return Arbitraries.oneOf( + originalField, + originalField, + originalField, + addedField + ); + } + private static Arbitrary groupSchema(final int depth) { final Arbitrary> subGroups = depth == 1 ? @@ -201,10 +232,7 @@ private static Arbitrary groupSchema(final int depth) return Combinators.combine( withDuplicates( 2, - Combinators.combine( - typeSchema(MAX_COMPOSITE_DEPTH), - presence() - ).as(FieldSchema::new).list().ofMaxSize(5) + skewedFieldDistribution().list().ofMaxSize(5) ), subGroups, varDataSchema().list().ofMaxSize(3) @@ -227,6 +255,13 @@ private static Arbitrary varDataSchema() PrimitiveType.UINT8, PrimitiveType.UINT16, PrimitiveType.UINT32 + ), + Arbitraries.of( + (short)0, + (short)0, + (short)0, + (short)1, + (short)2 ) ).as(VarDataSchema::new); } @@ -236,10 +271,7 @@ public static Arbitrary messageSchema() return Combinators.combine( withDuplicates( 3, - Combinators.combine( - typeSchema(MAX_COMPOSITE_DEPTH), - presence() - ).as(FieldSchema::new).list().ofMaxSize(10) + skewedFieldDistribution().list().ofMaxSize(10) ), groupSchema(MAX_GROUP_DEPTH).list().ofMaxSize(3), varDataSchema().list().ofMaxSize(3) @@ -747,7 +779,7 @@ public static Arbitrary messageValueEncoder( buffer.putShort(0, (short)blockLength, ir.byteOrder()); buffer.putShort(2, messageId, ir.byteOrder()); buffer.putShort(4, (short)ir.id(), ir.byteOrder()); - buffer.putShort(6, (short)0, ir.byteOrder()); + buffer.putShort(6, (short)ir.version(), ir.byteOrder()); final int headerLength = 8; fields.encode(buffer, offset + headerLength, null); limit.set(offset + headerLength + blockLength); @@ -821,7 +853,10 @@ public static Arbitrary encodedMessage(final CharGenerationMode } catch (final Exception e) { - throw new RuntimeException(e); + throw new AssertionError( + "Failed to generate encoded value for schema.\n\n" + + "SCHEMA:\n" + xml, + e); } }).withoutEdgeCases(); } diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/FieldSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/FieldSchema.java index 406ee65cdd..c1fad8abc4 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/FieldSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/FieldSchema.java @@ -22,14 +22,18 @@ public final class FieldSchema { private final TypeSchema type; private final Encoding.Presence presence; + private final short sinceVersion; public FieldSchema( final TypeSchema type, - final Encoding.Presence presence + final Encoding.Presence presence, + final short sinceVersion ) { + assert sinceVersion == 0 || presence.equals(Encoding.Presence.OPTIONAL); this.type = type; this.presence = presence; + this.sinceVersion = sinceVersion; } public TypeSchema type() @@ -41,4 +45,9 @@ public Encoding.Presence presence() { return presence; } + + public short sinceVersion() + { + return sinceVersion; + } } diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java index 53c3c81df7..47cfe343a7 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java @@ -16,7 +16,9 @@ package uk.co.real_logic.sbe.properties.schema; +import java.util.Comparator; import java.util.List; +import java.util.stream.Collectors; public final class GroupSchema { @@ -29,9 +31,13 @@ public GroupSchema( final List groups, final List varData) { - this.blockFields = blockFields; + this.blockFields = blockFields.stream() + .sorted(Comparator.comparing(FieldSchema::sinceVersion)) + .collect(Collectors.toList()); this.groups = groups; - this.varData = varData; + this.varData = varData.stream() + .sorted(Comparator.comparing(VarDataSchema::sinceVersion)) + .collect(Collectors.toList()); } public List blockFields() diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java index a0decb2d6b..2095628c39 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java @@ -16,13 +16,16 @@ package uk.co.real_logic.sbe.properties.schema; +import java.util.Comparator; import java.util.List; +import java.util.stream.Collectors; public final class MessageSchema { private final List blockFields; private final List groups; private final List varData; + private final short version; public MessageSchema( final List blockFields, @@ -30,9 +33,14 @@ public MessageSchema( final List varData ) { - this.blockFields = blockFields; + this.blockFields = blockFields.stream() + .sorted(Comparator.comparing(FieldSchema::sinceVersion)) + .collect(Collectors.toList()); this.groups = groups; - this.varData = varData; + this.varData = varData.stream() + .sorted(Comparator.comparing(VarDataSchema::sinceVersion)) + .collect(Collectors.toList()); + this.version = findMaxVersion(blockFields, groups, varData); } public short schemaId() @@ -45,6 +53,11 @@ public short templateId() return 1; } + public short version() + { + return version; + } + public List blockFields() { return blockFields; @@ -59,4 +72,22 @@ public List varData() { return varData; } + + private static short findMaxVersion( + final List fields, + final List groups, + final List varData + ) + { + final int maxFieldVersion = fields.stream() + .mapToInt(FieldSchema::sinceVersion) + .max().orElse(0); + final int maxGroupVersion = groups.stream() + .mapToInt(group -> findMaxVersion(group.blockFields(), group.groups(), group.varData())) + .max().orElse(0); + final int maxVarDataVersion = varData.stream() + .mapToInt(VarDataSchema::sinceVersion) + .max().orElse(0); + return (short)Math.max(maxFieldVersion, Math.max(maxGroupVersion, maxVarDataVersion)); + } } diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java index 7ccd273017..3c43228099 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java @@ -70,6 +70,7 @@ private static void writeTo( final Element root = document.createElementNS("http://fixprotocol.io/2016/sbe", "sbe:messageSchema"); root.setAttribute("id", Short.toString(schema.schemaId())); root.setAttribute("package", "uk.co.real_logic.sbe.properties"); + root.setAttribute("version", Short.toString(schema.version())); document.appendChild(root); final Element topLevelTypes = createTypesElement(document); @@ -152,6 +153,7 @@ private static void appendMembers( element.setAttribute("name", "member" + id); element.setAttribute("type", typeName); element.setAttribute("presence", field.presence().name().toLowerCase()); + element.setAttribute("sinceVersion", Short.toString(field.sinceVersion())); parent.appendChild(element); } @@ -180,6 +182,7 @@ private static void appendMembers( element.setAttribute("id", Integer.toString(id)); element.setAttribute("name", "member" + id); element.setAttribute("type", requireNonNull(typeToName.get(data))); + element.setAttribute("sinceVersion", Short.toString(data.sinceVersion())); parent.appendChild(element); } } diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java index b5e98c02c7..db3d5900a1 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java @@ -22,13 +22,16 @@ public final class VarDataSchema { private final Encoding dataEncoding; private final PrimitiveType lengthEncoding; + private final short sinceVersion; public VarDataSchema( final Encoding dataEncoding, - final PrimitiveType lengthEncoding) + final PrimitiveType lengthEncoding, + final short sinceVersion) { this.dataEncoding = dataEncoding; this.lengthEncoding = lengthEncoding; + this.sinceVersion = sinceVersion; } public Encoding dataEncoding() @@ -41,6 +44,11 @@ public PrimitiveType lengthEncoding() return lengthEncoding; } + public short sinceVersion() + { + return sinceVersion; + } + public enum Encoding { ASCII, From 7ee92651a2471aa7ee4fe087283397c4ac5b697d Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Thu, 9 Nov 2023 15:25:34 +0000 Subject: [PATCH 23/41] [C#] Address some of Martin's feedback re "added" var data representation. The SBE spec doesn't explicitly allow variable-length data or groups to be added to existing message schemas, i.e., only block-level fields may be added; however in practice the encoders/decoders for some languages do support the addition of var data fields. Previously, I had represented these "added" var data fields as optional strings or byte arrays, but having chatted with Martin, we think it is better to mimick the existing decoder representation, i.e., use empty strings/arrays to represent missing elements. In this commit, I've also fixed some instances where I was checking `token.version() > 0` where I should have been checking `token.version() > sinceVersionOfParentContainer`. Ideally, I would have liked to remove such checks entirely, but the C# and C++ codecs do not give sensible responses when decoding older versions in some cases, i.e., you _must_ check the presence of the field before accessing it. --- .../generation/csharp/CSharpDtoGenerator.java | 196 ++++++++++-------- .../properties/CSharpDtosPropertyTest.java | 19 +- .../arbitraries/SbeArbitraries.java | 14 +- 3 files changed, 131 insertions(+), 98 deletions(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java index 9335295663..6f25151a56 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java @@ -30,6 +30,7 @@ import java.io.Writer; import java.util.ArrayList; import java.util.List; +import java.util.function.Predicate; import static uk.co.real_logic.sbe.generation.csharp.CSharpUtil.*; import static uk.co.real_logic.sbe.ir.GenerationUtil.collectFields; @@ -43,6 +44,7 @@ public class CSharpDtoGenerator implements CodeGenerator { private static final String INDENT = " "; private static final String BASE_INDENT = INDENT; + private static final Predicate CANNOT_EXTEND = ignored -> false; private final Ir ir; private final OutputManager outputManager; @@ -93,8 +95,10 @@ public void generate() throws IOException collectVarData(messageBody, offset, varData); generateVarData(sb, ctorArgs, varData, BASE_INDENT + INDENT); - generateMessageDecodeFrom(sb, className, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); - generateMessageEncodeInto(sb, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); + generateDecodeFrom(sb, className, codecClassName, fields, groups, varData, + token -> token.version() > msgToken.version(), + BASE_INDENT + INDENT); + generateEncodeInto(sb, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); generateDisplay(sb, codecClassName, "WrapForEncode", null, BASE_INDENT + INDENT); removeTrailingComma(ctorArgs); @@ -119,12 +123,6 @@ public void generate() throws IOException } } - private enum DefinitionKind - { - Composite, - Message - } - private void generateGroups( final StringBuilder sb, final StringBuilder ctorArgs, @@ -142,9 +140,15 @@ private void generateGroups( } final String groupName = groupToken.name(); final String groupClassName = formatDtoClassName(groupName); - final String formattedPropertyName = formatPropertyName(groupName); + final Token dimToken = tokens.get(i + 1); + if (dimToken.signal() != Signal.BEGIN_COMPOSITE) + { + throw new IllegalStateException("groups must start with BEGIN_COMPOSITE: token=" + dimToken); + } + final int sinceVersion = dimToken.version(); + ctorArgs.append(indent).append("IReadOnlyList<") .append(qualifiedParentDtoClassName).append(".").append(groupClassName).append("> ") .append(formattedPropertyName).append(",\n"); @@ -167,22 +171,40 @@ private void generateGroups( final List fields = new ArrayList<>(); i = collectFields(tokens, i, fields); + generateFields(groupRecordBody, groupCtorArgs, qualifiedCodecClassName, fields, indent + INDENT); final List groups = new ArrayList<>(); i = collectGroups(tokens, i, groups); + generateGroups(groupRecordBody, groupCtorArgs, qualifiedDtoClassName, qualifiedCodecClassName, groups, indent + INDENT); final List varData = new ArrayList<>(); i = collectVarData(tokens, i, varData); + generateVarData(groupRecordBody, groupCtorArgs, varData, indent + INDENT); generateDecodeListFrom( groupRecordBody, groupClassName, qualifiedCodecClassName, indent + INDENT); - generateMessageDecodeFrom( - groupRecordBody, groupClassName, qualifiedCodecClassName, fields, groups, varData, indent + INDENT); - generateMessageEncodeInto( + + final Predicate wasAddedAfterGroup = token -> + { + final boolean addedAfterParent = token.version() > sinceVersion; + + if (addedAfterParent && token.signal() == Signal.BEGIN_VAR_DATA) + { + throw new IllegalStateException("Cannot extend var data inside a group."); + } + + return addedAfterParent; + }; + + generateDecodeFrom( + groupRecordBody, groupClassName, qualifiedCodecClassName, fields, groups, varData, + wasAddedAfterGroup, indent + INDENT); + + generateEncodeInto( groupRecordBody, qualifiedCodecClassName, fields, groups, varData, indent + INDENT); removeTrailingComma(groupCtorArgs); @@ -217,7 +239,7 @@ private void generateCompositeDecodeFrom( final Token token = tokens.get(i); generateFieldDecodeFrom( - sb, DefinitionKind.Composite, token, token, codecClassName, indent + INDENT + INDENT); + sb, CANNOT_EXTEND, token, token, codecClassName, indent + INDENT + INDENT); i += tokens.get(i).componentTokenCount(); } @@ -277,13 +299,14 @@ private void generateDecodeListFrom( .append(indent).append("}\n"); } - private void generateMessageDecodeFrom( + private void generateDecodeFrom( final StringBuilder sb, final String dtoClassName, final String codecClassName, final List fields, final List groups, final List varData, + final Predicate wasAddedAfterParent, final String indent) { sb.append("\n") @@ -292,9 +315,9 @@ private void generateMessageDecodeFrom( .append(indent).append("{\n"); sb.append(indent).append(INDENT).append("return new ").append(dtoClassName).append("(\n"); - generateMessageFieldsDecodeFrom(sb, fields, codecClassName, indent + INDENT + INDENT); + generateMessageFieldsDecodeFrom(sb, wasAddedAfterParent, fields, codecClassName, indent + INDENT + INDENT); generateGroupsDecodeFrom(sb, groups, indent + INDENT + INDENT); - generateVarDataDecodeFrom(sb, varData, indent + INDENT + INDENT); + generateVarDataDecodeFrom(sb, varData, wasAddedAfterParent, indent + INDENT + INDENT); removeTrailingComma(sb); sb.append(indent).append(INDENT).append(");\n"); @@ -303,6 +326,7 @@ private void generateMessageDecodeFrom( private void generateMessageFieldsDecodeFrom( final StringBuilder sb, + final Predicate wasAddedAfterParent, final List tokens, final String codecClassName, final String indent) @@ -314,14 +338,15 @@ private void generateMessageFieldsDecodeFrom( { final Token encodingToken = tokens.get(i + 1); - generateFieldDecodeFrom(sb, DefinitionKind.Message, signalToken, encodingToken, codecClassName, indent); + generateFieldDecodeFrom( + sb, wasAddedAfterParent, signalToken, encodingToken, codecClassName, indent); } } } private void generateFieldDecodeFrom( final StringBuilder sb, - final DefinitionKind rootKind, + final Predicate wasAddedAfterParent, final Token fieldToken, final Token typeToken, final String codecClassName, @@ -330,21 +355,21 @@ private void generateFieldDecodeFrom( switch (typeToken.signal()) { case ENCODING: - generatePrimitiveDecodeFrom(sb, rootKind, fieldToken, typeToken, codecClassName, indent); + generatePrimitiveDecodeFrom(sb, fieldToken, typeToken, wasAddedAfterParent, codecClassName, indent); break; case BEGIN_SET: - generatePropertyDecodeFrom(sb, rootKind, fieldToken, "0", null, indent); + generatePropertyDecodeFrom(sb, fieldToken, wasAddedAfterParent, "0", null, indent); break; case BEGIN_ENUM: final String enumName = formatClassName(typeToken.applicableTypeName()); final String nullValue = formatNamespace(ir.packageName()) + "." + enumName + ".NULL_VALUE"; - generatePropertyDecodeFrom(sb, rootKind, fieldToken, nullValue, null, indent); + generatePropertyDecodeFrom(sb, fieldToken, wasAddedAfterParent, nullValue, null, indent); break; case BEGIN_COMPOSITE: - generateComplexDecodeFrom(sb, rootKind, fieldToken, typeToken, indent); + generateComplexDecodeFrom(sb, fieldToken, typeToken, indent); break; default: @@ -354,9 +379,9 @@ private void generateFieldDecodeFrom( private void generatePrimitiveDecodeFrom( final StringBuilder sb, - final DefinitionKind rootKind, final Token fieldToken, final Token typeToken, + final Predicate wasAddedAfterParent, final String codecClassName, final String indent) { @@ -370,19 +395,19 @@ private void generatePrimitiveDecodeFrom( if (arrayLength == 1) { final String codecNullValue = codecClassName + "." + formatPropertyName(fieldToken.name()) + "NullValue"; - generatePropertyDecodeFrom(sb, rootKind, fieldToken, "null", codecNullValue, indent); + generatePropertyDecodeFrom(sb, fieldToken, wasAddedAfterParent, "null", codecNullValue, indent); } else if (arrayLength > 1) { - generateArrayDecodeFrom(sb, rootKind, fieldToken, typeToken, indent); + generateArrayDecodeFrom(sb, fieldToken, typeToken, wasAddedAfterParent, indent); } } private void generateArrayDecodeFrom( final StringBuilder sb, - final DefinitionKind rootKind, final Token fieldToken, final Token typeToken, + final Predicate wasAddedAfterParent, final String indent) { if (fieldToken.isConstantEncoding()) @@ -397,8 +422,8 @@ private void generateArrayDecodeFrom( { generateRecordPropertyAssignment( sb, - rootKind, fieldToken, + wasAddedAfterParent, indent, "codec.Get" + formattedPropertyName + "()", "null", @@ -409,8 +434,8 @@ private void generateArrayDecodeFrom( { generateRecordPropertyAssignment( sb, - rootKind, fieldToken, + wasAddedAfterParent, indent, "codec." + formattedPropertyName + "AsSpan().ToArray()", "null", @@ -421,8 +446,8 @@ private void generateArrayDecodeFrom( private void generatePropertyDecodeFrom( final StringBuilder sb, - final DefinitionKind rootKind, final Token fieldToken, + final Predicate wasAddedAfterParent, final String dtoNullValue, final String codecNullValue, final String indent) @@ -437,8 +462,8 @@ private void generatePropertyDecodeFrom( generateRecordPropertyAssignment( sb, - rootKind, fieldToken, + wasAddedAfterParent, indent, "codec." + formattedPropertyName, dtoNullValue, @@ -448,7 +473,6 @@ private void generatePropertyDecodeFrom( private void generateComplexDecodeFrom( final StringBuilder sb, - final DefinitionKind rootKind, final Token fieldToken, final Token typeToken, final String indent) @@ -457,15 +481,9 @@ private void generateComplexDecodeFrom( final String formattedPropertyName = formatPropertyName(propertyName); final String dtoClassName = formatDtoClassName(typeToken.applicableTypeName()); - generateRecordPropertyAssignment( - sb, - rootKind, - fieldToken, - indent, - dtoClassName + ".DecodeFrom(codec." + formattedPropertyName + ")", - "null", - "null" - ); + sb.append(indent).append(formattedPropertyName).append(": ") + .append(dtoClassName).append(".DecodeFrom(codec.") + .append(formattedPropertyName).append(")").append(",\n"); } private void generateGroupsDecodeFrom( @@ -484,10 +502,17 @@ private void generateGroupsDecodeFrom( final String formattedPropertyName = formatPropertyName(groupName); final String groupDtoClassName = formatDtoClassName(groupName); + final Token dimToken = tokens.get(i + 1); + if (dimToken.signal() != Signal.BEGIN_COMPOSITE) + { + throw new IllegalStateException("groups must start with BEGIN_COMPOSITE: token=" + dimToken); + } + final int sinceVersion = dimToken.version(); + generateRecordPropertyAssignment( sb, - DefinitionKind.Message, groupToken, + token -> token.version() > sinceVersion, indent, groupDtoClassName + ".DecodeListFrom(codec." + formattedPropertyName + ")", "new List<" + groupDtoClassName + ">(0).AsReadOnly()", @@ -511,6 +536,7 @@ private void generateGroupsDecodeFrom( private void generateVarDataDecodeFrom( final StringBuilder sb, final List tokens, + final Predicate wasAddedAfterParent, final String indent) { for (int i = 0; i < tokens.size(); i++) @@ -526,24 +552,32 @@ private void generateVarDataDecodeFrom( final String accessor = characterEncoding == null ? "Get" + formattedPropertyName + "Bytes" : "Get" + formattedPropertyName; + final String missingValue = characterEncoding == null ? + "new byte[0]" : + "\"\""; + + sb.append(indent).append(formattedPropertyName).append(": "); - generateRecordPropertyAssignment( - sb, - DefinitionKind.Message, - token, - indent, - "codec." + accessor + "()", - "null", - null - ); + if (wasAddedAfterParent.test(token)) + { + sb.append("codec.").append(formattedPropertyName).append("InActingVersion()"); + sb.append(" ?\n"); + sb.append(indent).append(INDENT).append("codec.").append(accessor).append("()") + .append(" :\n") + .append(indent).append(INDENT).append(missingValue).append(",\n"); + } + else + { + sb.append("codec.").append(accessor).append("()").append(",\n"); + } } } } private void generateRecordPropertyAssignment( final StringBuilder sb, - final DefinitionKind rootKind, final Token token, + final Predicate wasAddedAfterParent, final String indent, final String presentExpression, final String notPresentExpression, @@ -554,27 +588,29 @@ private void generateRecordPropertyAssignment( sb.append(indent).append(formattedPropertyName).append(": "); - // N.B., in the IR, the composite information is "embedded" into each message/group field. - // When we are looking at a composite "field" (i.e., a type) the `token.version()` method - // returns the "sinceVersion" of the containing field. Therefore, we cannot decide presence - // based on `token.version()` when dealing with composites. - if (rootKind.equals(DefinitionKind.Message) && token.version() > 0) - { - if (token.signal() != Signal.BEGIN_VAR_DATA && !token.isOptionalEncoding()) - { - throw new IllegalStateException( - "Expected added field " + propertyName + - " (sinceVersion=" + token.version() + ") to have optional presence." - ); - } + boolean hasPresenceCondition = false; + // Unfortunately, we need to check whether the field is in the acting version, + // as the codec may incorrectly decode data for missing fields. + if (wasAddedAfterParent.test(token)) + { sb.append("codec.").append(formattedPropertyName).append("InActingVersion()"); + hasPresenceCondition = true; + } - if (null != nullCodecValueOrNull) + if (token.isOptionalEncoding() && null != nullCodecValueOrNull) + { + if (hasPresenceCondition) { - sb.append(" && codec.").append(formattedPropertyName).append(" != ").append(nullCodecValueOrNull); + sb.append(" && "); } + sb.append("codec.").append(formattedPropertyName).append(" != ").append(nullCodecValueOrNull); + hasPresenceCondition = true; + } + + if (hasPresenceCondition) + { sb.append(" ?\n"); sb.append(indent).append(INDENT).append(presentExpression).append(" :\n") .append(indent).append(INDENT).append(notPresentExpression).append(",\n"); @@ -585,7 +621,7 @@ private void generateRecordPropertyAssignment( } } - private void generateMessageEncodeInto( + private void generateEncodeInto( final StringBuilder sb, final String codecClassName, final List fields, @@ -758,17 +794,6 @@ private void generateComplexEncodeInto( { final String propertyName = fieldToken.name(); final String formattedPropertyName = formatPropertyName(propertyName); - - if (fieldToken.isOptionalEncoding()) - { - sb.append(indent).append("if (null == ").append(formattedPropertyName).append(")\n") - .append(indent).append("{") - .append(indent).append(INDENT).append("throw new System.InvalidOperationException(\"") - .append("Null composite value encoding (").append(formattedPropertyName) - .append(") is not supported via DTOs\");\n") - .append(indent).append("}\n\n"); - } - sb.append(indent).append(formattedPropertyName).append(".EncodeInto(codec.") .append(formattedPropertyName).append(");\n"); } @@ -825,15 +850,6 @@ private void generateVarDataEncodeInto( final String propertyName = token.name(); final String formattedPropertyName = formatPropertyName(propertyName); - if (token.version() > 0) - { - sb.append(indent).append("if (null == ").append(formattedPropertyName).append(")\n") - .append(indent).append("{\n") - .append(indent).append(INDENT).append("throw new System.InvalidOperationException(\"") - .append(formattedPropertyName).append(" must not be null.\");\n") - .append(indent).append("}\n\n"); - } - sb.append(indent).append("codec.Set").append(formattedPropertyName) .append("(").append(formattedPropertyName).append(");\n"); } @@ -917,7 +933,6 @@ private void generateCompositeProperty( final String indent) { final String compositeName = formatDtoClassName(typeToken.applicableTypeName()); - final String nullableSuffix = fieldToken.isOptionalEncoding() ? "?" : ""; final String formattedPropertyName = formatPropertyName(propertyName); ctorArgs.append(indent).append(compositeName).append(" ").append(formattedPropertyName).append(",\n"); @@ -925,7 +940,7 @@ private void generateCompositeProperty( sb.append("\n") .append(generateDocumentation(indent, fieldToken)) .append(indent).append("public ").append(compositeName) - .append(nullableSuffix).append(" ").append(formattedPropertyName) + .append(" ").append(formattedPropertyName) .append(" { get; init; } = ").append(formattedPropertyName).append(";\n"); } @@ -1223,8 +1238,7 @@ private void generateVarData( final String propertyName = token.name(); final Token varDataToken = Generators.findFirst("varData", tokens, i); final String characterEncoding = varDataToken.encoding().characterEncoding(); - final String nullableSuffix = token.version() > 0 ? "?" : ""; - final String dtoType = (characterEncoding == null ? "byte[]" : "string") + nullableSuffix; + final String dtoType = characterEncoding == null ? "byte[]" : "string"; final String formattedPropertyName = formatPropertyName(propertyName); diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java index 8a8be0fdde..701ca2fcf4 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java @@ -50,10 +50,21 @@ void dtoEncodeShouldBeTheInverseOfDtoDecode( tempDir.toString(), "SbePropertyTest" ); - new CSharpGenerator(encodedMessage.ir(), outputManager) - .generate(); - new CSharpDtoGenerator(encodedMessage.ir(), outputManager) - .generate(); + + try + { + new CSharpGenerator(encodedMessage.ir(), outputManager) + .generate(); + new CSharpDtoGenerator(encodedMessage.ir(), outputManager) + .generate(); + } + catch (final Exception generationException) + { + throw new AssertionError( + "Code generation failed.\n\nSCHEMA:\n" + encodedMessage.schema(), + generationException); + } + copyResourceToFile("/CSharpDtosPropertyTest/SbePropertyTest.csproj", tempDir); copyResourceToFile("/CSharpDtosPropertyTest/Program.cs", tempDir); try ( diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java index 13f027fb30..25207cdf96 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java @@ -21,6 +21,7 @@ import net.jqwik.api.Combinators; import net.jqwik.api.arbitraries.CharacterArbitrary; import net.jqwik.api.arbitraries.ListArbitrary; +import net.jqwik.api.arbitraries.ShortArbitrary; import uk.co.real_logic.sbe.PrimitiveType; import uk.co.real_logic.sbe.PrimitiveValue; import uk.co.real_logic.sbe.ir.Encoding; @@ -235,7 +236,7 @@ private static Arbitrary groupSchema(final int depth) skewedFieldDistribution().list().ofMaxSize(5) ), subGroups, - varDataSchema().list().ofMaxSize(3) + varDataSchema(Arbitraries.of((short)0)).list().ofMaxSize(3) ).as(GroupSchema::new); } @@ -247,7 +248,7 @@ private static Arbitrary presence() ); } - private static Arbitrary varDataSchema() + private static Arbitrary varDataSchema(final Arbitrary sinceVersion) { return Combinators.combine( Arbitraries.of(VarDataSchema.Encoding.values()), @@ -256,6 +257,13 @@ private static Arbitrary varDataSchema() PrimitiveType.UINT16, PrimitiveType.UINT32 ), + sinceVersion + ).as(VarDataSchema::new); + } + + private static Arbitrary varDataSchema() + { + return varDataSchema( Arbitraries.of( (short)0, (short)0, @@ -263,7 +271,7 @@ private static Arbitrary varDataSchema() (short)1, (short)2 ) - ).as(VarDataSchema::new); + ); } public static Arbitrary messageSchema() From 1b98e1de06346eb350045fd6bdb253774e15a22d Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Fri, 10 Nov 2023 19:06:29 +0000 Subject: [PATCH 24/41] [C#] Address feedback around encode/decode methods. This commit addresses some feedback from Martin: - Makes encode and decode methods static - Improves naming: - `EncodeInto(codec)` -> `EncodeWith` - `DecodeFrom(codec)` -> `DecodeWith` - Introduces version that works directly with buffer --- csharp/sbe-tests/DtoTests.cs | 27 ++- .../generation/csharp/CSharpDtoGenerator.java | 215 ++++++++++++------ .../CSharpDtosPropertyTest/Program.cs | 4 +- 3 files changed, 163 insertions(+), 83 deletions(-) diff --git a/csharp/sbe-tests/DtoTests.cs b/csharp/sbe-tests/DtoTests.cs index fcd0007bf7..1986db5738 100644 --- a/csharp/sbe-tests/DtoTests.cs +++ b/csharp/sbe-tests/DtoTests.cs @@ -8,29 +8,42 @@ namespace Org.SbeTool.Sbe.Tests public class DtoTests { [TestMethod] - public void ShouldRoundTripCar() + public void ShouldRoundTripCar1() { var inputByteArray = new byte[1024]; var inputBuffer = new DirectBuffer(inputByteArray); - EncodeCar(inputBuffer); + EncodeCar(inputBuffer, 0); var decoder = new Car(); decoder.WrapForDecode(inputBuffer, 0, Car.BlockLength, Car.SchemaVersion); var decoderString = decoder.ToString(); - var dto = CarDto.DecodeFrom(decoder); + var dto = CarDto.DecodeWith(decoder); var outputByteArray = new byte[1024]; var outputBuffer = new DirectBuffer(outputByteArray); var encoder = new Car(); encoder.WrapForEncode(outputBuffer, 0); - dto.EncodeInto(encoder); + CarDto.EncodeWith(encoder, dto); var dtoString = dto.ToSbeString(); CollectionAssert.AreEqual(inputByteArray, outputByteArray); Assert.AreEqual(decoderString, dtoString); } + + [TestMethod] + public void ShouldRoundTripCar2() + { + var inputByteArray = new byte[1024]; + var inputBuffer = new DirectBuffer(inputByteArray); + var length = EncodeCar(inputBuffer, 0); + var dto = CarDto.DecodeFrom(inputBuffer, 0, length, Car.BlockLength, Car.SchemaVersion); + var outputByteArray = new byte[1024]; + var outputBuffer = new DirectBuffer(outputByteArray); + CarDto.EncodeInto(outputBuffer, 0, dto); + CollectionAssert.AreEqual(inputByteArray, outputByteArray); + } - private static void EncodeCar(DirectBuffer buffer) + private static int EncodeCar(DirectBuffer buffer, int offset) { var car = new Car(); - car.WrapForEncode(buffer, 0); + car.WrapForEncode(buffer, offset); car.SerialNumber = 1234; car.ModelYear = 2013; car.Available = BooleanType.T; @@ -104,6 +117,8 @@ private static void EncodeCar(DirectBuffer buffer) car.SetManufacturer("Ford"); car.SetModel("Fiesta"); car.SetActivationCode("1234"); + + return car.Limit - offset; } } } \ No newline at end of file diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java index 6f25151a56..a132e751d6 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java @@ -75,7 +75,7 @@ public void generate() throws IOException { final Token msgToken = tokens.get(0); final String codecClassName = formatClassName(msgToken.name()); - final String className = formatDtoClassName(msgToken.name()); + final String dtoClassName = formatDtoClassName(msgToken.name()); final List messageBody = tokens.subList(1, tokens.size() - 1); int offset = 0; @@ -89,21 +89,24 @@ public void generate() throws IOException final List groups = new ArrayList<>(); offset = collectGroups(messageBody, offset, groups); - generateGroups(sb, ctorArgs, className, codecClassName, groups, BASE_INDENT + INDENT); + generateGroups(sb, ctorArgs, dtoClassName, codecClassName, groups, BASE_INDENT + INDENT); final List varData = new ArrayList<>(); collectVarData(messageBody, offset, varData); generateVarData(sb, ctorArgs, varData, BASE_INDENT + INDENT); - generateDecodeFrom(sb, className, codecClassName, fields, groups, varData, + generateDecodeWith(sb, dtoClassName, codecClassName, fields, groups, varData, token -> token.version() > msgToken.version(), BASE_INDENT + INDENT); - generateEncodeInto(sb, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); + generateDecodeFrom(sb, dtoClassName, codecClassName, BASE_INDENT + INDENT); + generateEncodeWith(sb, dtoClassName, codecClassName, fields, groups, varData, + BASE_INDENT + INDENT); + generateEncodeInto(sb, dtoClassName, codecClassName, BASE_INDENT + INDENT); generateDisplay(sb, codecClassName, "WrapForEncode", null, BASE_INDENT + INDENT); removeTrailingComma(ctorArgs); - try (Writer out = outputManager.createOutput(className)) + try (Writer out = outputManager.createOutput(dtoClassName)) { out.append(generateFileHeader( ir.applicableNamespace(), @@ -112,7 +115,7 @@ public void generate() throws IOException "using System.Linq;\n")); out.append(generateDocumentation(BASE_INDENT, msgToken)); - out.append(BASE_INDENT).append("public sealed partial record ").append(className).append("(\n") + out.append(BASE_INDENT).append("public sealed partial record ").append(dtoClassName).append("(\n") .append(ctorArgs) .append(BASE_INDENT).append(")\n") .append(BASE_INDENT).append("{") @@ -185,7 +188,7 @@ private void generateGroups( generateVarData(groupRecordBody, groupCtorArgs, varData, indent + INDENT); - generateDecodeListFrom( + generateDecodeListWith( groupRecordBody, groupClassName, qualifiedCodecClassName, indent + INDENT); final Predicate wasAddedAfterGroup = token -> @@ -200,12 +203,13 @@ private void generateGroups( return addedAfterParent; }; - generateDecodeFrom( + generateDecodeWith( groupRecordBody, groupClassName, qualifiedCodecClassName, fields, groups, varData, wasAddedAfterGroup, indent + INDENT); - generateEncodeInto( - groupRecordBody, qualifiedCodecClassName, fields, groups, varData, indent + INDENT); + generateEncodeWith( + groupRecordBody, groupClassName, qualifiedCodecClassName, fields, groups, varData, + indent + INDENT); removeTrailingComma(groupCtorArgs); @@ -220,7 +224,7 @@ private void generateGroups( } } - private void generateCompositeDecodeFrom( + private void generateCompositeDecodeWith( final StringBuilder sb, final String dtoClassName, final String codecClassName, @@ -228,7 +232,7 @@ private void generateCompositeDecodeFrom( final String indent) { sb.append("\n") - .append(indent).append("public static ").append(dtoClassName).append(" DecodeFrom(") + .append(indent).append("public static ").append(dtoClassName).append(" DecodeWith(") .append(codecClassName).append(" codec)\n") .append(indent).append("{\n"); @@ -238,7 +242,7 @@ private void generateCompositeDecodeFrom( { final Token token = tokens.get(i); - generateFieldDecodeFrom( + generateFieldDecodeWith( sb, CANNOT_EXTEND, token, token, codecClassName, indent + INDENT + INDENT); i += tokens.get(i).componentTokenCount(); @@ -250,21 +254,24 @@ private void generateCompositeDecodeFrom( sb.append(indent).append("}\n"); } - private void generateCompositeEncodeInto( + private void generateCompositeEncodeWith( final StringBuilder sb, + final String dtoClassName, final String codecClassName, final List tokens, final String indent) { sb.append("\n") - .append(indent).append("public void EncodeInto(").append(codecClassName).append(" codec)\n") + .append(indent).append("public static void EncodeWith(") + .append(codecClassName).append(" codec, ") + .append(dtoClassName).append(" dto)\n") .append(indent).append("{\n"); for (int i = 0; i < tokens.size(); ) { final Token token = tokens.get(i); - generateFieldEncodeInto(sb, codecClassName, token, token, indent + INDENT); + generateFieldEncodeWith(sb, codecClassName, token, token, indent + INDENT); i += tokens.get(i).componentTokenCount(); } @@ -272,14 +279,14 @@ private void generateCompositeEncodeInto( sb.append(indent).append("}\n"); } - private void generateDecodeListFrom( + private void generateDecodeListWith( final StringBuilder sb, final String dtoClassName, final String codecClassName, final String indent) { sb.append("\n") - .append(indent).append("public static IReadOnlyList<").append(dtoClassName).append("> DecodeListFrom(") + .append(indent).append("public static IReadOnlyList<").append(dtoClassName).append("> DecodeListWith(") .append(codecClassName).append(" codec)\n") .append(indent).append("{\n") .append(indent).append(INDENT).append("var ").append("list = new List<").append(dtoClassName) @@ -289,7 +296,7 @@ private void generateDecodeListFrom( .append(indent).append(INDENT) .append("{\n") .append(indent).append(INDENT).append(INDENT) - .append("var element = ").append(dtoClassName).append(".DecodeFrom(codec.Next());\n") + .append("var element = ").append(dtoClassName).append(".DecodeWith(codec.Next());\n") .append(indent).append(INDENT).append(INDENT) .append("list.Add(element);\n") .append(indent).append(INDENT) @@ -299,7 +306,7 @@ private void generateDecodeListFrom( .append(indent).append("}\n"); } - private void generateDecodeFrom( + private void generateDecodeWith( final StringBuilder sb, final String dtoClassName, final String codecClassName, @@ -311,20 +318,40 @@ private void generateDecodeFrom( { sb.append("\n") .append(indent).append("public static ").append(dtoClassName) - .append(" DecodeFrom(").append(codecClassName).append(" codec)\n") + .append(" DecodeWith(").append(codecClassName).append(" codec)\n") .append(indent).append("{\n"); sb.append(indent).append(INDENT).append("return new ").append(dtoClassName).append("(\n"); - generateMessageFieldsDecodeFrom(sb, wasAddedAfterParent, fields, codecClassName, indent + INDENT + INDENT); - generateGroupsDecodeFrom(sb, groups, indent + INDENT + INDENT); - generateVarDataDecodeFrom(sb, varData, wasAddedAfterParent, indent + INDENT + INDENT); + generateMessageFieldsDecodeWith(sb, wasAddedAfterParent, fields, codecClassName, indent + INDENT + INDENT); + generateGroupsDecodeWith(sb, groups, indent + INDENT + INDENT); + generateVarDataDecodeWith(sb, varData, wasAddedAfterParent, indent + INDENT + INDENT); removeTrailingComma(sb); sb.append(indent).append(INDENT).append(");\n"); sb.append(indent).append("}\n"); } - private void generateMessageFieldsDecodeFrom( + private void generateDecodeFrom( + final StringBuilder sb, + final String dtoClassName, + final String codecClassName, + final String indent) + { + sb.append("\n") + .append(indent).append("public static ").append(dtoClassName) + .append(" DecodeFrom(DirectBuffer buffer, int offset, int length, ") + .append("int actingBlockLength, int actingVersion)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT) + .append("var decoder = new ").append(codecClassName).append("();\n") + .append(indent).append(INDENT) + .append("decoder.WrapForDecode(buffer, offset, actingBlockLength, actingVersion);\n") + .append(indent).append(INDENT) + .append("return DecodeWith(decoder);\n") + .append(indent).append("}\n"); + } + + private void generateMessageFieldsDecodeWith( final StringBuilder sb, final Predicate wasAddedAfterParent, final List tokens, @@ -338,13 +365,13 @@ private void generateMessageFieldsDecodeFrom( { final Token encodingToken = tokens.get(i + 1); - generateFieldDecodeFrom( + generateFieldDecodeWith( sb, wasAddedAfterParent, signalToken, encodingToken, codecClassName, indent); } } } - private void generateFieldDecodeFrom( + private void generateFieldDecodeWith( final StringBuilder sb, final Predicate wasAddedAfterParent, final Token fieldToken, @@ -355,21 +382,21 @@ private void generateFieldDecodeFrom( switch (typeToken.signal()) { case ENCODING: - generatePrimitiveDecodeFrom(sb, fieldToken, typeToken, wasAddedAfterParent, codecClassName, indent); + generatePrimitiveDecodeWith(sb, fieldToken, typeToken, wasAddedAfterParent, codecClassName, indent); break; case BEGIN_SET: - generatePropertyDecodeFrom(sb, fieldToken, wasAddedAfterParent, "0", null, indent); + generatePropertyDecodeWith(sb, fieldToken, wasAddedAfterParent, "0", null, indent); break; case BEGIN_ENUM: final String enumName = formatClassName(typeToken.applicableTypeName()); final String nullValue = formatNamespace(ir.packageName()) + "." + enumName + ".NULL_VALUE"; - generatePropertyDecodeFrom(sb, fieldToken, wasAddedAfterParent, nullValue, null, indent); + generatePropertyDecodeWith(sb, fieldToken, wasAddedAfterParent, nullValue, null, indent); break; case BEGIN_COMPOSITE: - generateComplexDecodeFrom(sb, fieldToken, typeToken, indent); + generateComplexDecodeWith(sb, fieldToken, typeToken, indent); break; default: @@ -377,7 +404,7 @@ private void generateFieldDecodeFrom( } } - private void generatePrimitiveDecodeFrom( + private void generatePrimitiveDecodeWith( final StringBuilder sb, final Token fieldToken, final Token typeToken, @@ -395,15 +422,15 @@ private void generatePrimitiveDecodeFrom( if (arrayLength == 1) { final String codecNullValue = codecClassName + "." + formatPropertyName(fieldToken.name()) + "NullValue"; - generatePropertyDecodeFrom(sb, fieldToken, wasAddedAfterParent, "null", codecNullValue, indent); + generatePropertyDecodeWith(sb, fieldToken, wasAddedAfterParent, "null", codecNullValue, indent); } else if (arrayLength > 1) { - generateArrayDecodeFrom(sb, fieldToken, typeToken, wasAddedAfterParent, indent); + generateArrayDecodeWith(sb, fieldToken, typeToken, wasAddedAfterParent, indent); } } - private void generateArrayDecodeFrom( + private void generateArrayDecodeWith( final StringBuilder sb, final Token fieldToken, final Token typeToken, @@ -444,7 +471,7 @@ private void generateArrayDecodeFrom( } } - private void generatePropertyDecodeFrom( + private void generatePropertyDecodeWith( final StringBuilder sb, final Token fieldToken, final Predicate wasAddedAfterParent, @@ -471,7 +498,7 @@ private void generatePropertyDecodeFrom( ); } - private void generateComplexDecodeFrom( + private void generateComplexDecodeWith( final StringBuilder sb, final Token fieldToken, final Token typeToken, @@ -482,11 +509,11 @@ private void generateComplexDecodeFrom( final String dtoClassName = formatDtoClassName(typeToken.applicableTypeName()); sb.append(indent).append(formattedPropertyName).append(": ") - .append(dtoClassName).append(".DecodeFrom(codec.") + .append(dtoClassName).append(".DecodeWith(codec.") .append(formattedPropertyName).append(")").append(",\n"); } - private void generateGroupsDecodeFrom( + private void generateGroupsDecodeWith( final StringBuilder sb, final List tokens, final String indent) @@ -514,7 +541,7 @@ private void generateGroupsDecodeFrom( groupToken, token -> token.version() > sinceVersion, indent, - groupDtoClassName + ".DecodeListFrom(codec." + formattedPropertyName + ")", + groupDtoClassName + ".DecodeListWith(codec." + formattedPropertyName + ")", "new List<" + groupDtoClassName + ">(0).AsReadOnly()", null ); @@ -533,7 +560,7 @@ private void generateGroupsDecodeFrom( } } - private void generateVarDataDecodeFrom( + private void generateVarDataDecodeWith( final StringBuilder sb, final List tokens, final Predicate wasAddedAfterParent, @@ -621,8 +648,9 @@ private void generateRecordPropertyAssignment( } } - private void generateEncodeInto( + private void generateEncodeWith( final StringBuilder sb, + final String dtoClassName, final String codecClassName, final List fields, final List groups, @@ -630,17 +658,49 @@ private void generateEncodeInto( final String indent) { sb.append("\n") - .append(indent).append("public void EncodeInto(").append(codecClassName).append(" codec)\n") + .append(indent).append("public static void EncodeWith(") + .append(codecClassName).append(" codec, ") + .append(dtoClassName).append(" dto)\n") .append(indent).append("{\n"); - generateFieldsEncodeInto(sb, codecClassName, fields, indent + INDENT); - generateGroupsEncodeInto(sb, groups, indent + INDENT); - generateVarDataEncodeInto(sb, varData, indent + INDENT); + generateFieldsEncodeWith(sb, codecClassName, fields, indent + INDENT); + generateGroupsEncodeWith(sb, groups, indent + INDENT); + generateVarDataEncodeWith(sb, varData, indent + INDENT); sb.append(indent).append("}\n"); } - private void generateFieldsEncodeInto( + private void generateEncodeInto( + final StringBuilder sb, + final String dtoClassName, + final String codecClassName, + final String indent) + { + sb.append("\n") + .append(indent).append("public static int EncodeInto(") + .append("DirectBuffer buffer, int offset, ") + .append(dtoClassName).append(" dto)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("var encoder = new ").append(codecClassName).append("();\n") + .append(indent).append(INDENT).append("encoder.WrapForEncode(buffer, offset);\n") + .append(indent).append(INDENT).append("EncodeWith(encoder, dto);\n") + .append(indent).append(INDENT).append("return encoder.Limit - offset;\n") + .append(indent).append("}\n"); + + sb.append("\n") + .append(indent).append("public static int EncodeWithHeaderInto(") + .append("DirectBuffer buffer, int offset, ") + .append(dtoClassName).append(" dto)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("var encoder = new ").append(codecClassName).append("();\n") + .append(indent).append(INDENT) + .append("encoder.WrapForEncodeAndApplyHeader(buffer, offset, new MessageHeader());\n") + .append(indent).append(INDENT).append("EncodeWith(encoder, dto);\n") + .append(indent).append(INDENT).append("return encoder.Limit - offset;\n") + .append(indent).append("}\n"); + } + + private void generateFieldsEncodeWith( final StringBuilder sb, final String codecClassName, final List tokens, @@ -652,12 +712,12 @@ private void generateFieldsEncodeInto( if (signalToken.signal() == Signal.BEGIN_FIELD) { final Token encodingToken = tokens.get(i + 1); - generateFieldEncodeInto(sb, codecClassName, signalToken, encodingToken, indent); + generateFieldEncodeWith(sb, codecClassName, signalToken, encodingToken, indent); } } } - private void generateFieldEncodeInto( + private void generateFieldEncodeWith( final StringBuilder sb, final String codecClassName, final Token fieldToken, @@ -667,16 +727,16 @@ private void generateFieldEncodeInto( switch (typeToken.signal()) { case ENCODING: - generatePrimitiveEncodeInto(sb, codecClassName, fieldToken, typeToken, indent); + generatePrimitiveEncodeWith(sb, codecClassName, fieldToken, typeToken, indent); break; case BEGIN_SET: case BEGIN_ENUM: - generateEnumEncodeInto(sb, fieldToken, indent); + generateEnumEncodeWith(sb, fieldToken, indent); break; case BEGIN_COMPOSITE: - generateComplexEncodeInto(sb, fieldToken, indent); + generateComplexEncodeWith(sb, fieldToken, typeToken, indent); break; default: @@ -684,7 +744,7 @@ private void generateFieldEncodeInto( } } - private void generatePrimitiveEncodeInto( + private void generatePrimitiveEncodeWith( final StringBuilder sb, final String codecClassName, final Token fieldToken, @@ -700,15 +760,15 @@ private void generatePrimitiveEncodeInto( if (arrayLength == 1) { - generatePropertyEncodeInto(sb, codecClassName, fieldToken, indent); + generatePropertyEncodeWith(sb, codecClassName, fieldToken, indent); } else if (arrayLength > 1) { - generateArrayEncodeInto(sb, fieldToken, typeToken, indent); + generateArrayEncodeWith(sb, fieldToken, typeToken, indent); } } - private void generateArrayEncodeInto( + private void generateArrayEncodeWith( final StringBuilder sb, final Token fieldToken, final Token typeToken, @@ -724,7 +784,7 @@ private void generateArrayEncodeInto( if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) { - final String value = nullableConvertedExpression(fieldToken, formattedPropertyName, "\"\""); + final String value = nullableConvertedExpression(fieldToken, "dto." + formattedPropertyName, "\"\""); sb.append(indent).append("codec.Set").append(formattedPropertyName).append("(") .append(value).append(");\n"); } @@ -732,7 +792,7 @@ private void generateArrayEncodeInto( { final String typeName = cSharpTypeName(typeToken.encoding().primitiveType()); - sb.append(indent).append("new Span<").append(typeName).append(">(").append(formattedPropertyName) + sb.append(indent).append("new Span<").append(typeName).append(">(dto.").append(formattedPropertyName) .append("?.ToArray()).CopyTo(codec.").append(formattedPropertyName).append("AsSpan());\n"); } } @@ -747,7 +807,7 @@ private String nullableConvertedExpression( expression; } - private void generatePropertyEncodeInto( + private void generatePropertyEncodeWith( final StringBuilder sb, final String codecClassName, final Token fieldToken, @@ -763,14 +823,14 @@ private void generatePropertyEncodeInto( final String value = nullableConvertedExpression( fieldToken, - formattedPropertyName, + "dto." + formattedPropertyName, codecClassName + "." + formattedPropertyName + "NullValue"); sb.append(indent).append("codec.").append(formattedPropertyName).append(" = ") .append(value).append(";\n"); } - private void generateEnumEncodeInto( + private void generateEnumEncodeWith( final StringBuilder sb, final Token fieldToken, final String indent) @@ -783,22 +843,25 @@ private void generateEnumEncodeInto( final String propertyName = fieldToken.name(); final String formattedPropertyName = formatPropertyName(propertyName); - sb.append(indent).append("codec.").append(formattedPropertyName).append(" = ") + sb.append(indent).append("codec.").append(formattedPropertyName).append(" = dto.") .append(formattedPropertyName).append(";\n"); } - private void generateComplexEncodeInto( + private void generateComplexEncodeWith( final StringBuilder sb, final Token fieldToken, + final Token typeToken, final String indent) { final String propertyName = fieldToken.name(); final String formattedPropertyName = formatPropertyName(propertyName); - sb.append(indent).append(formattedPropertyName).append(".EncodeInto(codec.") - .append(formattedPropertyName).append(");\n"); + final String dtoClassName = formatDtoClassName(typeToken.applicableTypeName()); + sb.append(indent) + .append(dtoClassName).append(".EncodeWith(codec.").append(formattedPropertyName) + .append(", dto.").append(formattedPropertyName).append(");\n"); } - private void generateGroupsEncodeInto( + private void generateGroupsEncodeWith( final StringBuilder sb, final List tokens, final String indent) @@ -813,15 +876,17 @@ private void generateGroupsEncodeInto( final String groupName = groupToken.name(); final String formattedPropertyName = formatPropertyName(groupName); final String groupCodecVarName = groupName + "Codec"; + final String groupDtoClassName = formatDtoClassName(groupName); sb.append("\n") .append(indent).append("var ").append(groupCodecVarName) .append(" = codec.").append(formattedPropertyName) - .append("Count(").append(formattedPropertyName).append(".Count);\n\n") - .append(indent).append("foreach (var group in ").append(formattedPropertyName).append(")\n") + .append("Count(dto.").append(formattedPropertyName).append(".Count);\n\n") + .append(indent).append("foreach (var group in dto.").append(formattedPropertyName).append(")\n") .append(indent).append("{\n") - .append(indent).append(INDENT).append("group.EncodeInto(").append(groupCodecVarName) - .append(".Next()").append(");\n") + .append(indent).append(INDENT).append(groupDtoClassName).append(".EncodeWith(") + .append(groupCodecVarName) + .append(".Next()").append(", group);\n") .append(indent).append("}\n\n"); i++; @@ -838,7 +903,7 @@ private void generateGroupsEncodeInto( } } - private void generateVarDataEncodeInto( + private void generateVarDataEncodeWith( final StringBuilder sb, final List tokens, final String indent) @@ -851,7 +916,7 @@ private void generateVarDataEncodeInto( final String formattedPropertyName = formatPropertyName(propertyName); sb.append(indent).append("codec.Set").append(formattedPropertyName) - .append("(").append(formattedPropertyName).append(");\n"); + .append("(dto.").append(formattedPropertyName).append(");\n"); } } } @@ -876,7 +941,7 @@ private void generateDisplay( sb.append(", ").append(actingVersion); } sb.append(");\n"); - sb.append(indent).append(INDENT).append("EncodeInto(codec);\n") + sb.append(indent).append(INDENT).append("EncodeWith(codec, this);\n") .append(indent).append(INDENT).append("StringBuilder sb = new StringBuilder();\n") .append(indent).append(INDENT).append("codec.BuildString(sb);\n") .append(indent).append(INDENT).append("return sb.ToString();\n") @@ -1292,8 +1357,8 @@ private void generateComposite(final List tokens) throws IOException final List compositeTokens = tokens.subList(1, tokens.size() - 1); generateCompositePropertyElements(sb, ctorArgs, codecClassName, compositeTokens, BASE_INDENT + INDENT); - generateCompositeDecodeFrom(sb, className, codecClassName, compositeTokens, BASE_INDENT + INDENT); - generateCompositeEncodeInto(sb, codecClassName, compositeTokens, BASE_INDENT + INDENT); + generateCompositeDecodeWith(sb, className, codecClassName, compositeTokens, BASE_INDENT + INDENT); + generateCompositeEncodeWith(sb, className, codecClassName, compositeTokens, BASE_INDENT + INDENT); generateDisplay(sb, codecClassName, "Wrap", codecClassName + ".SbeSchemaVersion", BASE_INDENT + INDENT); removeTrailingComma(ctorArgs); diff --git a/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/Program.cs b/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/Program.cs index 60bdc22616..fa73cd118d 100644 --- a/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/Program.cs +++ b/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/Program.cs @@ -17,12 +17,12 @@ static int Main(string[] args) { messageHeader.Wrap(buffer, 0, 0); var decoder = new TestMessage(); decoder.WrapForDecode(buffer, 8, messageHeader.BlockLength, messageHeader.Version); - var dto = TestMessageDto.DecodeFrom(decoder); + var dto = TestMessageDto.DecodeWith(decoder); var outputBytes = new byte[inputBytes.Length]; var outputBuffer = new DirectBuffer(outputBytes); var encoder = new TestMessage(); encoder.WrapForEncodeAndApplyHeader(outputBuffer, 0, new MessageHeader()); - dto.EncodeInto(encoder); + TestMessageDto.EncodeWith(encoder, dto); File.WriteAllBytes("output.dat", outputBytes); return 0; } From 96b70326c33c1f1756aca7e5f5746022d8aee72c Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Thu, 2 Nov 2023 11:55:35 +0000 Subject: [PATCH 25/41] [C++] Generate DTOs for non-perf-sensitive usecases. In some applications performance is not cricital. Some users would like to use SBE across their whole "estate", but don't want the "sharp edges" associated with using flyweight codecs, e.g., accidental escape. In this commit, I've added a first cut of DTO generation for C++ and a simple test based on the Car Example. The DTOs support encoding and decoding via the generated codecs using `DtoT::encode(CodecT& codec, const DtoT& dto)` and `DtoT::decode(CodecT& codec, Dto& dto)` methods. Generation can be enabled specifying the target code generator class, `uk.co.real_logic.sbe.generation.cpp.CppDtos`, or by passing a system property `-Dsbe.cpp.generate.dtos=true`. --- .../java/uk/co/real_logic/sbe/SbeTool.java | 5 + .../generation/TargetCodeGeneratorLoader.java | 21 +- .../sbe/generation/cpp/CppDtoGenerator.java | 1824 +++++++++++++++++ .../sbe/generation/cpp/CppDtos.java | 36 + .../sbe/generation/cpp/CppGenerator.java | 58 - .../sbe/generation/cpp/CppUtil.java | 58 + sbe-tool/src/test/cpp/CMakeLists.txt | 6 + sbe-tool/src/test/cpp/DtoTest.cpp | 195 ++ .../src/test/resources/dto-test-schema.xml | 110 + 9 files changed, 2250 insertions(+), 63 deletions(-) create mode 100644 sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java create mode 100644 sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtos.java create mode 100644 sbe-tool/src/test/cpp/DtoTest.cpp create mode 100644 sbe-tool/src/test/resources/dto-test-schema.xml diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/SbeTool.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/SbeTool.java index aae460dc2b..a6958a12bf 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/SbeTool.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/SbeTool.java @@ -192,6 +192,11 @@ public class SbeTool */ public static final String DECODE_UNKNOWN_ENUM_VALUES = "sbe.decode.unknown.enum.values"; + /** + * Should generate C++ DTOs. Defaults to false. + */ + public static final String GENERATE_CPP_DTOS = "sbe.cpp.generate.dtos"; + /** * Configuration option used to manage sinceVersion based transformations. When set, parsed schemas will be * transformed to discard messages and types higher than the specified version. This can be useful when needing diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/TargetCodeGeneratorLoader.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/TargetCodeGeneratorLoader.java index 54a9766449..6f872cd55e 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/TargetCodeGeneratorLoader.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/TargetCodeGeneratorLoader.java @@ -18,6 +18,7 @@ import uk.co.real_logic.sbe.generation.c.CGenerator; import uk.co.real_logic.sbe.generation.c.COutputManager; import uk.co.real_logic.sbe.generation.common.PrecedenceChecks; +import uk.co.real_logic.sbe.generation.cpp.CppDtoGenerator; import uk.co.real_logic.sbe.generation.cpp.CppGenerator; import uk.co.real_logic.sbe.generation.cpp.NamespaceOutputManager; import uk.co.real_logic.sbe.generation.golang.GolangGenerator; @@ -83,11 +84,21 @@ public CodeGenerator newInstance(final Ir ir, final String outputDir) */ public CodeGenerator newInstance(final Ir ir, final String outputDir) { - return new CppGenerator( - ir, - "true".equals(System.getProperty(DECODE_UNKNOWN_ENUM_VALUES)), - precedenceChecks(), - new NamespaceOutputManager(outputDir, ir.applicableNamespace())); + final NamespaceOutputManager outputManager = new NamespaceOutputManager( + outputDir, ir.applicableNamespace()); + final boolean decodeUnknownEnumValues = "true".equals(System.getProperty(DECODE_UNKNOWN_ENUM_VALUES)); + + final CodeGenerator codecGenerator = new CppGenerator(ir, decodeUnknownEnumValues, precedenceChecks(), + outputManager); + final CodeGenerator dtoGenerator = new CppDtoGenerator(ir, outputManager); + final CodeGenerator combinedGenerator = () -> + { + codecGenerator.generate(); + dtoGenerator.generate(); + }; + + final boolean generateDtos = "true".equals(System.getProperty(GENERATE_CPP_DTOS)); + return generateDtos ? combinedGenerator : codecGenerator; } }, diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java new file mode 100644 index 0000000000..e1e3793da8 --- /dev/null +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java @@ -0,0 +1,1824 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * Copyright (C) 2017 MarketFactory, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.generation.cpp; + +import uk.co.real_logic.sbe.PrimitiveType; +import uk.co.real_logic.sbe.generation.CodeGenerator; +import uk.co.real_logic.sbe.generation.Generators; +import uk.co.real_logic.sbe.ir.Ir; +import uk.co.real_logic.sbe.ir.Signal; +import uk.co.real_logic.sbe.ir.Token; +import org.agrona.LangUtil; +import org.agrona.Verify; +import org.agrona.generation.OutputManager; + +import java.io.IOException; +import java.io.Writer; +import java.util.*; +import java.util.stream.Collectors; + +import static uk.co.real_logic.sbe.generation.Generators.toLowerFirstChar; +import static uk.co.real_logic.sbe.generation.Generators.toUpperFirstChar; +import static uk.co.real_logic.sbe.generation.cpp.CppUtil.*; +import static uk.co.real_logic.sbe.ir.GenerationUtil.collectFields; +import static uk.co.real_logic.sbe.ir.GenerationUtil.collectGroups; +import static uk.co.real_logic.sbe.ir.GenerationUtil.collectVarData; + +/** + * DTO generator for the CSharp programming language. + */ +public class CppDtoGenerator implements CodeGenerator +{ + private static final String INDENT = " "; + private static final String BASE_INDENT = INDENT; + + private final Ir ir; + private final OutputManager outputManager; + + /** + * Create a new C# DTO {@link CodeGenerator}. + * + * @param ir for the messages and types. + * @param outputManager for generating the DTOs to. + */ + public CppDtoGenerator(final Ir ir, final OutputManager outputManager) + { + Verify.notNull(ir, "ir"); + Verify.notNull(outputManager, "outputManager"); + + this.ir = ir; + this.outputManager = outputManager; + } + + /** + * {@inheritDoc} + */ + public void generate() throws IOException + { + generateDtosForTypes(); + + for (final List tokens : ir.messages()) + { + final Token msgToken = tokens.get(0); + final String codecClassName = formatClassName(msgToken.name()); + final String className = formatDtoClassName(msgToken.name()); + + final List messageBody = tokens.subList(1, tokens.size() - 1); + int offset = 0; + + final ClassBuilder classBuilder = new ClassBuilder(className, BASE_INDENT); + + final List fields = new ArrayList<>(); + offset = collectFields(messageBody, offset, fields); + generateFields(classBuilder, codecClassName, fields, BASE_INDENT + INDENT); + + final List groups = new ArrayList<>(); + offset = collectGroups(messageBody, offset, groups); + generateGroups(classBuilder, className, codecClassName, groups, + BASE_INDENT + INDENT); + + final List varData = new ArrayList<>(); + collectVarData(messageBody, offset, varData); + generateVarData(classBuilder, varData, BASE_INDENT + INDENT); + + generateMessageDecodeFrom(classBuilder, className, codecClassName, fields, + groups, varData, BASE_INDENT + INDENT); + generateMessageEncodeInto(classBuilder, className, codecClassName, fields, groups, varData, + BASE_INDENT + INDENT); + generateDisplay(classBuilder, codecClassName, "wrapForEncode", null, BASE_INDENT + INDENT); + + try (Writer out = outputManager.createOutput(className)) + { + final List beginTypeTokensInSchema = ir.types().stream() + .map(t -> t.get(0)) + .collect(Collectors.toList()); + + final Set referencedTypes = generateTypesToIncludes(beginTypeTokensInSchema); + referencedTypes.add(codecClassName); + + out.append(generateDtoFileHeader( + ir.namespaces(), + className, + referencedTypes)); + out.append(generateDocumentation(BASE_INDENT, msgToken)); + classBuilder.appendTo(out); + out.append("} // namespace\n"); + out.append("#endif\n"); + } + } + } + + private static final class ClassBuilder + { + private final StringBuilder publicSb = new StringBuilder(); + private final StringBuilder privateSb = new StringBuilder(); + private final StringBuilder fieldSb = new StringBuilder(); + private final String className; + private final String indent; + + private ClassBuilder(final String className, final String indent) + { + this.className = className; + this.indent = indent; + } + + public StringBuilder appendPublic() + { + return publicSb; + } + + public StringBuilder appendPrivate() + { + return privateSb; + } + + public StringBuilder appendField() + { + return fieldSb; + } + + public void appendTo(final Appendable out) + { + try + { + out.append(indent).append("class ").append(className).append("\n") + .append(indent).append("{\n") + .append(indent).append("private:\n") + .append(privateSb) + .append("\n") + .append(indent).append("public:\n") + .append(publicSb) + .append("\n") + .append(indent).append("private:\n") + .append(fieldSb) + .append(indent).append("};\n"); + } + catch (final IOException exception) + { + LangUtil.rethrowUnchecked(exception); + } + } + } + + private void generateGroups( + final ClassBuilder classBuilder, + final String qualifiedParentDtoClassName, + final String qualifiedParentCodecClassName, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token groupToken = tokens.get(i); + if (groupToken.signal() != Signal.BEGIN_GROUP) + { + throw new IllegalStateException("tokens must begin with BEGIN_GROUP: token=" + groupToken); + } + final String groupName = groupToken.name(); + final String groupClassName = formatDtoClassName(groupName); + final String qualifiedDtoClassName = qualifiedParentDtoClassName + "::" + groupClassName; + + final String fieldName = "m_" + toLowerFirstChar(groupName); + final String formattedPropertyName = formatPropertyName(groupName); + + classBuilder.appendField().append(indent).append("std::vector<") + .append(qualifiedDtoClassName).append("> ") + .append(fieldName).append(";\n"); + + final ClassBuilder groupClassBuilder = new ClassBuilder(groupClassName, indent); + + i++; + i += tokens.get(i).componentTokenCount(); + + final String qualifiedCodecClassName = + qualifiedParentCodecClassName + "::" + formatClassName(groupName); + + final List fields = new ArrayList<>(); + i = collectFields(tokens, i, fields); + generateFields(groupClassBuilder, qualifiedCodecClassName, fields, indent + INDENT); + + final List groups = new ArrayList<>(); + i = collectGroups(tokens, i, groups); + generateGroups(groupClassBuilder, qualifiedDtoClassName, + qualifiedCodecClassName, groups, indent + INDENT); + + final List varData = new ArrayList<>(); + i = collectVarData(tokens, i, varData); + generateVarData(groupClassBuilder, varData, indent + INDENT); + + generateDecodeListFrom( + groupClassBuilder, groupClassName, qualifiedCodecClassName, indent + INDENT); + generateMessageDecodeFrom(groupClassBuilder, groupClassName, qualifiedCodecClassName, + fields, groups, varData, indent + INDENT); + generateMessageEncodeInto( + groupClassBuilder, groupClassName, qualifiedCodecClassName, fields, groups, varData, indent + INDENT); + + groupClassBuilder.appendTo( + classBuilder.appendPublic().append("\n").append(generateDocumentation(indent, groupToken)) + ); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, groupToken)) + .append(indent).append("[[nodiscard]] const std::vector<").append(qualifiedDtoClassName).append(">& ") + .append(formattedPropertyName).append("() const\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, groupToken)) + .append(indent).append("[[nodiscard]] std::vector<").append(qualifiedDtoClassName).append(">& ") + .append(formattedPropertyName).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, groupToken)) + .append(indent).append("void ").append(formattedPropertyName).append("(") + .append("const std::vector<").append(qualifiedDtoClassName).append(">& values)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(fieldName).append(" = values;\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, groupToken)) + .append(indent).append("void ").append(formattedPropertyName).append("(") + .append("std::vector<").append(qualifiedDtoClassName).append(">&& values)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(fieldName).append(" = std::move(values);\n") + .append(indent).append("}\n"); + } + } + + private void generateCompositeDecodeFrom( + final ClassBuilder classBuilder, + final String dtoClassName, + final String codecClassName, + final List tokens, + final String indent) + { + final StringBuilder decodeBuilder = classBuilder.appendPublic().append("\n") + .append(indent).append("static void decode(").append(codecClassName).append("& codec, ") + .append(dtoClassName).append("& dto)\n") + .append(indent).append("{\n"); + + for (int i = 0; i < tokens.size(); ) + { + final Token token = tokens.get(i); + + generateFieldDecodeFrom( + decodeBuilder, token, token, codecClassName, indent + INDENT); + + i += tokens.get(i).componentTokenCount(); + } + + decodeBuilder.append(indent).append("}\n"); + } + + private void generateCompositeEncodeInto( + final ClassBuilder classBuilder, + final String dtoClassName, + final String codecClassName, + final List tokens, + final String indent) + { + final StringBuilder encodeBuilder = classBuilder.appendPublic().append("\n") + .append(indent).append("static void encode(").append(codecClassName).append("& codec,") + .append("const ").append(dtoClassName).append("& dto)\n") + .append(indent).append("{\n"); + + for (int i = 0; i < tokens.size(); ) + { + final Token token = tokens.get(i); + + generateFieldEncodeInto(encodeBuilder, codecClassName, token, token, indent + INDENT); + + i += tokens.get(i).componentTokenCount(); + } + + encodeBuilder.append(indent).append("}\n"); + } + + private void generateDecodeListFrom( + final ClassBuilder classBuilder, + final String dtoClassName, + final String codecClassName, + final String indent) + { + classBuilder.appendPublic().append("\n") + .append(indent).append("static std::vector<").append(dtoClassName).append("> decodeMany(") + .append(codecClassName).append("& codec)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("std::vector<").append(dtoClassName) + .append("> dtos(codec.count());\n") + .append(indent).append(INDENT) + .append("for (std::size_t i = 0; i < dtos.size(); i++)\n") + .append(indent).append(INDENT) + .append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append(dtoClassName).append(" dto;\n") + .append(indent).append(INDENT).append(INDENT) + .append(dtoClassName).append("::decode(codec.next(), dto);\n") + .append(indent).append(INDENT).append(INDENT) + .append("dtos[i] = dto;\n") + .append(indent).append(INDENT) + .append("}\n") + .append(indent).append(INDENT) + .append("return dtos;\n") + .append(indent).append("}\n"); + } + + private void generateMessageDecodeFrom( + final ClassBuilder classBuilder, + final String dtoClassName, + final String codecClassName, + final List fields, + final List groups, + final List varData, + final String indent) + { + final StringBuilder decodeBuilder = classBuilder.appendPublic().append("\n") + .append(indent).append("static void decode(").append(codecClassName).append("& codec, ") + .append(dtoClassName).append("& dto)\n") + .append(indent).append("{\n"); + + generateMessageFieldsDecodeFrom(decodeBuilder, fields, codecClassName, indent + INDENT); + generateGroupsDecodeFrom(decodeBuilder, groups, indent + INDENT); + generateVarDataDecodeFrom(decodeBuilder, varData, indent + INDENT); + decodeBuilder.append(indent).append("}\n"); + } + + private void generateMessageFieldsDecodeFrom( + final StringBuilder sb, + final List tokens, + final String codecClassName, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token signalToken = tokens.get(i); + if (signalToken.signal() == Signal.BEGIN_FIELD) + { + final Token encodingToken = tokens.get(i + 1); + + generateFieldDecodeFrom(sb, signalToken, encodingToken, codecClassName, indent); + } + } + } + + private void generateFieldDecodeFrom( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String codecClassName, + final String indent) + { + switch (typeToken.signal()) + { + case ENCODING: + generatePrimitiveDecodeFrom(sb, fieldToken, typeToken, codecClassName, indent); + break; + + case BEGIN_SET: + final String bitSetName = formatDtoClassName(typeToken.applicableTypeName()); + generateBitSetDecodeFrom(sb, fieldToken, bitSetName, indent); + break; + + case BEGIN_ENUM: + generateEnumDecodeFrom(sb, fieldToken, indent); + break; + + case BEGIN_COMPOSITE: + generateCompositePropertyDecodeFrom(sb, fieldToken, typeToken, indent); + break; + + default: + break; + } + } + + private void generatePrimitiveDecodeFrom( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String codecClassName, + final String indent) + { + if (typeToken.isConstantEncoding()) + { + return; + } + + final int arrayLength = typeToken.arrayLength(); + + if (arrayLength == 1) + { + final String typeName = cppTypeName(typeToken.encoding().primitiveType()); + final String codecNullValue = codecClassName + "::" + formatPropertyName(fieldToken.name()) + "NullValue()"; + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + generateRecordPropertyAssignment( + sb, + fieldToken, + indent, + "codec." + formattedPropertyName + "()", + codecNullValue, + typeName + ); + } + else if (arrayLength > 1) + { + generateArrayDecodeFrom(sb, fieldToken, typeToken, codecClassName, indent); + } + } + + private void generateArrayDecodeFrom( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String codecClassName, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) + { + generateRecordPropertyAssignment( + sb, + fieldToken, + indent, + "std::string(codec." + formattedPropertyName + "(), " + + codecClassName + "::" + formattedPropertyName + "Length())", + null, + "std::string" + ); + } + else + { + final StringBuilder initializerList = new StringBuilder(); + initializerList.append("{ "); + final int arrayLength = typeToken.arrayLength(); + for (int i = 0; i < arrayLength; i++) + { + initializerList.append("codec.").append(formattedPropertyName).append("(").append(i).append("),"); + } + assert arrayLength > 0; + initializerList.setLength(initializerList.length() - 1); + initializerList.append(" }"); + + generateRecordPropertyAssignment( + sb, + fieldToken, + indent, + initializerList, + null, + "std::vector<" + cppTypeName(typeToken.encoding().primitiveType()) + ">" + ); + } + } + + private void generateBitSetDecodeFrom( + final StringBuilder sb, + final Token fieldToken, + final String dtoTypeName, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + if (fieldToken.isOptionalEncoding()) + { + sb.append(indent).append("if (codec.").append(formattedPropertyName).append("InActingVersion()"); + + sb.append(")\n") + .append(indent).append("{\n"); + + sb.append(indent).append(INDENT).append(dtoTypeName).append("::decode(codec.") + .append(formattedPropertyName).append("(), ") + .append("dto.").append(formattedPropertyName).append("());\n"); + + sb.append(indent).append("}\n") + .append(indent).append("else\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("dto.").append(formattedPropertyName).append("().clear();\n") + .append(indent).append("}\n"); + } + else + { + sb.append(indent).append(dtoTypeName).append("::decode(codec.") + .append(formattedPropertyName).append("(), ") + .append("dto.").append(formattedPropertyName).append("());\n"); + } + } + + private void generateEnumDecodeFrom( + final StringBuilder sb, + final Token fieldToken, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + sb.append(indent).append("dto.").append(formattedPropertyName).append("(") + .append("codec.").append(formattedPropertyName).append("());\n"); + } + + private void generateCompositePropertyDecodeFrom( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + final String dtoClassName = formatDtoClassName(typeToken.applicableTypeName()); + + sb.append(indent).append(dtoClassName).append("::decode(codec.") + .append(formattedPropertyName).append("(), ") + .append("dto.").append(formattedPropertyName).append("());\n"); + } + + private void generateGroupsDecodeFrom( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token groupToken = tokens.get(i); + if (groupToken.signal() != Signal.BEGIN_GROUP) + { + throw new IllegalStateException("tokens must begin with BEGIN_GROUP: token=" + groupToken); + } + final String groupName = groupToken.name(); + final String formattedPropertyName = formatPropertyName(groupName); + final String groupDtoClassName = formatDtoClassName(groupName); + + sb.append(indent).append("dto.").append(formattedPropertyName).append("(") + .append(groupDtoClassName).append("::decodeMany(codec.") + .append(formattedPropertyName).append("()));\n"); + + i++; + i += tokens.get(i).componentTokenCount(); + + final List fields = new ArrayList<>(); + i = collectFields(tokens, i, fields); + + final List groups = new ArrayList<>(); + i = collectGroups(tokens, i, groups); + + final List varData = new ArrayList<>(); + i = collectVarData(tokens, i, varData); + } + } + + private void generateVarDataDecodeFrom( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (int i = 0; i < tokens.size(); i++) + { + final Token token = tokens.get(i); + if (token.signal() == Signal.BEGIN_VAR_DATA) + { + final String propertyName = token.name(); + final Token varDataToken = Generators.findFirst("varData", tokens, i); + final String characterEncoding = varDataToken.encoding().characterEncoding(); + final String formattedPropertyName = formatPropertyName(propertyName); + + final boolean isOptional = token.version() > 0; + + final String dataVar = toLowerFirstChar(propertyName) + "Data"; + final String lengthVar = toLowerFirstChar(propertyName) + "Length"; + final String blockIndent = isOptional ? indent + INDENT : indent; + final StringBuilder codecValueExtraction = new StringBuilder() + .append(blockIndent).append("std::size_t ").append(lengthVar) + .append(" = codec.").append(formattedPropertyName).append("Length();\n") + .append(blockIndent).append("const char* ").append(dataVar) + .append(" = codec.").append(formattedPropertyName).append("();\n"); + + final String dtoValue; + final String nullDtoValue; + + if (characterEncoding == null) + { + dtoValue = "std::vector(" + dataVar + ", " + dataVar + " + " + lengthVar + ")"; + nullDtoValue = "std::vector()"; + } + else + { + dtoValue = "std::string(" + dataVar + ", " + lengthVar + ")"; + nullDtoValue = "\"\""; + } + + if (isOptional) + { + sb.append(indent).append("if (codec.").append(formattedPropertyName).append("InActingVersion()"); + + sb.append(")\n") + .append(indent).append("{\n"); + + sb.append(codecValueExtraction); + + sb.append(indent).append(INDENT).append("dto.").append(formattedPropertyName).append("(") + .append(dtoValue).append(");\n"); + + sb.append(indent).append("}\n") + .append(indent).append("else\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("dto.") + .append(formattedPropertyName).append("(").append(nullDtoValue).append(");\n") + .append(indent).append("}\n"); + } + else + { + sb.append(codecValueExtraction); + + sb.append(indent).append("dto.").append(formattedPropertyName).append("(") + .append(dtoValue).append(");\n"); + } + } + } + } + + private void generateRecordPropertyAssignment( + final StringBuilder sb, + final Token token, + final String indent, + final CharSequence presentExpression, + final String nullCodecValueOrNull, + final String dtoTypeName) + { + final String propertyName = token.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + if (token.isOptionalEncoding()) + { + sb.append(indent).append("if (codec.").append(formattedPropertyName).append("InActingVersion()"); + + if (null != nullCodecValueOrNull) + { + sb.append(" && codec.").append(formattedPropertyName).append("() != ").append(nullCodecValueOrNull); + } + + sb.append(")\n") + .append(indent).append("{\n"); + + sb.append(indent).append(INDENT).append("dto.").append(formattedPropertyName).append("(std::make_optional<") + .append(dtoTypeName).append(">(").append(presentExpression).append("));\n"); + + sb.append(indent).append("}\n") + .append(indent).append("else\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("dto.").append(formattedPropertyName).append("(std::nullopt);\n") + .append(indent).append("}\n"); + } + else + { + sb.append(indent).append("dto.").append(formattedPropertyName).append("(") + .append(presentExpression).append(");\n"); + } + } + + private void generateMessageEncodeInto( + final ClassBuilder classBuilder, + final String dtoClassName, + final String codecClassName, + final List fields, + final List groups, + final List varData, + final String indent) + { + final StringBuilder encodeBuilder = classBuilder.appendPublic().append("\n") + .append(indent).append("static void encode(").append(codecClassName).append("& codec, const ") + .append(dtoClassName).append("& dto)\n") + .append(indent).append("{\n"); + + generateFieldsEncodeInto(encodeBuilder, codecClassName, fields, indent + INDENT); + generateGroupsEncodeInto(encodeBuilder, groups, indent + INDENT); + generateVarDataEncodeInto(encodeBuilder, varData, indent + INDENT); + + encodeBuilder.append(indent).append("}\n"); + } + + private void generateFieldsEncodeInto( + final StringBuilder sb, + final String codecClassName, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token signalToken = tokens.get(i); + if (signalToken.signal() == Signal.BEGIN_FIELD) + { + final Token encodingToken = tokens.get(i + 1); + generateFieldEncodeInto(sb, codecClassName, signalToken, encodingToken, indent); + } + } + } + + private void generateFieldEncodeInto( + final StringBuilder sb, + final String codecClassName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + switch (typeToken.signal()) + { + case ENCODING: + generatePrimitiveEncodeInto(sb, codecClassName, fieldToken, typeToken, indent); + break; + + case BEGIN_ENUM: + generateEnumEncodeInto(sb, fieldToken, indent); + break; + + case BEGIN_SET: + case BEGIN_COMPOSITE: + generateComplexPropertyEncodeInto(sb, fieldToken, typeToken, indent); + break; + + default: + break; + } + } + + private void generatePrimitiveEncodeInto( + final StringBuilder sb, + final String codecClassName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + if (typeToken.isConstantEncoding()) + { + return; + } + + final int arrayLength = typeToken.arrayLength(); + + if (arrayLength == 1) + { + generatePrimitiveValueEncodeInto(sb, codecClassName, fieldToken, indent); + } + else if (arrayLength > 1) + { + generateArrayEncodeInto(sb, fieldToken, typeToken, indent); + } + } + + private void generateArrayEncodeInto( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) + { + final String accessor = "dto." + formattedPropertyName + "()"; + final String value = fieldToken.isOptionalEncoding() ? + accessor + ".value_or(" + "\"\"" + ")" : + accessor; + sb.append(indent).append("codec.put").append(toUpperFirstChar(propertyName)).append("(") + .append(value).append(".c_str());\n"); + } + else + { + final String typeName = cppTypeName(typeToken.encoding().primitiveType()); + final String vectorVar = toLowerFirstChar(propertyName) + "Vector"; + + final String accessor = "dto." + formattedPropertyName + "()"; + final String value = fieldToken.isOptionalEncoding() ? + accessor + ".value_or(std::vector<" + typeName + ">())" : + accessor; + + sb.append(indent).append("std::vector<").append(typeName).append("> ").append(vectorVar) + .append(" = ").append(value).append(";\n\n"); + + sb.append(indent).append("if (").append(vectorVar).append(".size() != ") + .append(typeToken.arrayLength()).append(")\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("throw std::invalid_argument(\"") + .append(propertyName) + .append(": array length != ") + .append(typeToken.arrayLength()) + .append("\");\n") + .append(indent).append("}\n\n"); + + sb.append(indent).append("for (std::uint64_t i = 0; i < ").append(typeToken.arrayLength()) + .append("; i++)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("codec.").append(formattedPropertyName).append("(i, ") + .append(vectorVar).append("[i]);\n") + .append(indent).append("}\n"); + } + } + + private void generatePrimitiveValueEncodeInto( + final StringBuilder sb, + final String codecClassName, + final Token fieldToken, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + final String nullValue = codecClassName + "::" + formattedPropertyName + "NullValue()"; + final String accessor = "dto." + formattedPropertyName + "()"; + final String value = fieldToken.isOptionalEncoding() ? + accessor + ".value_or(" + nullValue + ")" : + accessor; + + sb.append(indent).append("codec.").append(formattedPropertyName).append("(") + .append(value).append(");\n"); + } + + private void generateEnumEncodeInto( + final StringBuilder sb, + final Token fieldToken, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + sb.append(indent).append("codec.").append(formattedPropertyName).append("(dto.") + .append(formattedPropertyName).append("());\n"); + } + + private void generateComplexPropertyEncodeInto( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + final String typeName = formatDtoClassName(typeToken.applicableTypeName()); + + sb.append(indent).append(typeName).append("::encode(codec.") + .append(formattedPropertyName).append("(), dto.") + .append(formattedPropertyName).append("());\n"); + } + + private void generateGroupsEncodeInto( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token groupToken = tokens.get(i); + if (groupToken.signal() != Signal.BEGIN_GROUP) + { + throw new IllegalStateException("tokens must begin with BEGIN_GROUP: token=" + groupToken); + } + final String groupName = groupToken.name(); + final String formattedPropertyName = formatPropertyName(groupName); + final String groupCodecVarName = groupName + "Codec"; + final String groupDtoTypeName = formatDtoClassName(groupName); + + sb.append("\n") + .append(indent).append("const std::vector<").append(groupDtoTypeName).append(">& ") + .append(formattedPropertyName).append(" = dto.").append(formattedPropertyName).append("();\n\n") + .append(indent).append("auto&").append(" ").append(groupCodecVarName) + .append(" = codec.").append(formattedPropertyName) + .append("Count(").append(formattedPropertyName).append(".size());\n\n") + .append(indent).append("for (const auto& group: ").append(formattedPropertyName).append(")\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(groupDtoTypeName).append("::encode(").append(groupCodecVarName) + .append(".next(), group);\n") + .append(indent).append("}\n\n"); + + i++; + i += tokens.get(i).componentTokenCount(); + + final List fields = new ArrayList<>(); + i = collectFields(tokens, i, fields); + + final List groups = new ArrayList<>(); + i = collectGroups(tokens, i, groups); + + final List varData = new ArrayList<>(); + i = collectVarData(tokens, i, varData); + } + } + + private void generateVarDataEncodeInto( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (int i = 0; i < tokens.size(); i++) + { + final Token token = tokens.get(i); + if (token.signal() == Signal.BEGIN_VAR_DATA) + { + final String propertyName = token.name(); + final Token lengthToken = Generators.findFirst("length", tokens, i); + final String lengthTypeName = cppTypeName(lengthToken.encoding().primitiveType()); + final Token varDataToken = Generators.findFirst("varData", tokens, i); + final String characterEncoding = varDataToken.encoding().characterEncoding(); + final String formattedPropertyName = formatPropertyName(propertyName); + final String varName = toLowerFirstChar(propertyName) + "Vector"; + + sb.append(indent).append("auto& ").append(varName).append(" = dto.") + .append(formattedPropertyName).append("();\n") + .append(indent).append("codec.put").append(toUpperFirstChar(propertyName)) + .append("("); + + if (null == characterEncoding) + { + sb.append("reinterpret_cast(").append(varName).append(".data()), ") + .append("static_cast<").append(lengthTypeName).append(">(").append(varName).append(".size())"); + } + else + { + sb.append(varName); + } + + sb.append(");\n"); + } + } + } + + private void generateDisplay( + final ClassBuilder classBuilder, + final String codecClassName, + final String wrapMethod, + final String actingVersion, + final String indent) + { + final StringBuilder toStringBuilder = classBuilder.appendPublic() + .append("\n") + .append(indent).append( + "std::string string(char* tempBuffer, std::uint64_t offset, std::uint64_t length) const\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(codecClassName).append(" codec;\n") + .append(indent).append(INDENT).append("codec."); + + toStringBuilder.append(wrapMethod).append("(tempBuffer, offset"); + + if (null != actingVersion) + { + toStringBuilder.append(", ").append(actingVersion); + } + + toStringBuilder.append(", ").append("length);\n"); + + toStringBuilder.append(indent).append(INDENT).append("encode(codec, *this);\n") + .append(indent).append(INDENT).append("std::ostringstream oss;\n") + .append(indent).append(INDENT).append("oss << codec;\n") + .append(indent).append(INDENT).append("return oss.str();\n") + .append(indent).append("}\n"); + } + + private void generateFields( + final ClassBuilder classBuilder, + final String codecClassName, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token signalToken = tokens.get(i); + if (signalToken.signal() == Signal.BEGIN_FIELD) + { + final Token encodingToken = tokens.get(i + 1); + final String propertyName = signalToken.name(); + + switch (encodingToken.signal()) + { + case ENCODING: + generatePrimitiveProperty( + classBuilder, codecClassName, propertyName, signalToken, encodingToken, indent); + break; + + case BEGIN_ENUM: + generateEnumProperty(classBuilder, propertyName, signalToken, encodingToken, indent); + break; + + case BEGIN_SET: + case BEGIN_COMPOSITE: + generateComplexProperty(classBuilder, propertyName, signalToken, encodingToken, indent); + break; + + default: + break; + } + } + } + } + + private void generateComplexProperty( + final ClassBuilder classBuilder, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String typeName = formatDtoClassName(typeToken.applicableTypeName()); + final String formattedPropertyName = formatPropertyName(propertyName); + final String fieldName = "m_" + toLowerFirstChar(propertyName); + + classBuilder.appendField() + .append(indent).append(typeName).append(" ").append(fieldName).append(";\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("[[nodiscard]] const ").append(typeName).append("& ") + .append(formattedPropertyName).append("() const\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("[[nodiscard]] ").append(typeName).append("& ") + .append(formattedPropertyName).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + } + + private void generateEnumProperty( + final ClassBuilder classBuilder, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String enumName = formatClassName(typeToken.applicableTypeName()) + "::Value"; + + final String formattedPropertyName = formatPropertyName(propertyName); + + if (fieldToken.isConstantEncoding()) + { + final String constValue = fieldToken.encoding().constValue().toString(); + final String caseName = constValue.substring(constValue.indexOf(".") + 1); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("[[nodiscard]] static ").append(enumName).append(" ") + .append(formattedPropertyName).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(enumName).append("::") + .append(caseName).append(";\n") + .append(indent).append("}\n"); + } + else + { + final String fieldName = "m_" + toLowerFirstChar(propertyName); + + classBuilder.appendField() + .append(indent).append(enumName).append(" ").append(fieldName).append(";\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("[[nodiscard]] ").append(enumName).append(" ") + .append(formattedPropertyName).append("() const\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("void ").append(formattedPropertyName) + .append("(").append(enumName).append(" value)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(fieldName).append(" = value;\n") + .append(indent).append("}\n"); + } + } + + private void generatePrimitiveProperty( + final ClassBuilder classBuilder, + final String codecClassName, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + if (typeToken.isConstantEncoding()) + { + generateConstPropertyMethods(classBuilder, propertyName, fieldToken, typeToken, indent); + } + else + { + generatePrimitivePropertyMethods(classBuilder, codecClassName, propertyName, fieldToken, typeToken, indent); + } + } + + private void generatePrimitivePropertyMethods( + final ClassBuilder classBuilder, + final String codecClassName, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final int arrayLength = typeToken.arrayLength(); + + if (arrayLength == 1) + { + generateSingleValueProperty(classBuilder, codecClassName, propertyName, fieldToken, typeToken, indent); + } + else if (arrayLength > 1) + { + generateArrayProperty(classBuilder, codecClassName, propertyName, fieldToken, typeToken, indent); + } + } + + private void generateArrayProperty( + final ClassBuilder classBuilder, + final String codecClassName, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String fieldName = "m_" + toLowerFirstChar(propertyName); + final String formattedPropertyName = formatPropertyName(propertyName); + + if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) + { + classBuilder.appendField() + .append(indent).append("std::string ").append(fieldName).append(";\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("[[nodiscard]] const std::string& ") + .append(formattedPropertyName).append("() const\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("void ").append(formattedPropertyName) + .append("(const std::string& borrowedValue)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(fieldName).append(" = borrowedValue;\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("void ").append(formattedPropertyName) + .append("(std::string&& ownedValue)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(fieldName).append(" = std::move(ownedValue);\n") + .append(indent).append("}\n"); + } + else + { + final String validateMethod = "validate" + toUpperFirstChar(propertyName); + final String elementTypeName = cppTypeName(typeToken.encoding().primitiveType()); + final String vectorTypeName = "std::vector<" + elementTypeName + ">"; + final CharSequence typeName = typeWithFieldOptionality( + fieldToken, + vectorTypeName + ); + + classBuilder.appendField() + .append(indent).append(typeName).append(" ").append(fieldName).append(";\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("[[nodiscard]] ").append(typeName).append(" ") + .append(formattedPropertyName).append("() const\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("void ").append(formattedPropertyName).append("(") + .append(typeName).append("& borrowedValue").append(")\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(validateMethod).append("(borrowedValue);\n") + .append(indent).append(INDENT).append(fieldName).append(" = borrowedValue;\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("void ").append(formattedPropertyName).append("(") + .append(typeName).append("&& ownedValue").append(")\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(validateMethod).append("(ownedValue);\n") + .append(indent).append(INDENT).append(fieldName).append(" = std::move(ownedValue);\n") + .append(indent).append("}\n"); + + generateArrayValidateMethod( + classBuilder, + codecClassName, + fieldToken, + indent, + validateMethod, + typeName, + vectorTypeName, + formattedPropertyName); + } + } + + private static void generateArrayValidateMethod( + final ClassBuilder classBuilder, + final String codecClassName, + final Token fieldToken, + final String indent, + final String validateMethod, + final CharSequence typeName, + final String vectorTypeName, + final String formattedPropertyName) + { + final StringBuilder validateBuilder = classBuilder.appendPrivate().append("\n") + .append(indent).append("static void ").append(validateMethod).append("(") + .append(typeName).append(" value)\n") + .append(indent).append("{\n"); + + String value = "value"; + + if (fieldToken.isOptionalEncoding()) + { + validateBuilder.append(indent).append(INDENT) + .append("if (!value.has_value())\n") + .append(indent).append(INDENT) + .append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("return;\n") + .append(indent).append(INDENT) + .append("}\n"); + + validateBuilder.append(indent).append(INDENT) + .append(vectorTypeName).append(" actualValue = value.value();\n"); + + value = "actualValue"; + } + + validateBuilder.append(indent).append(INDENT) + .append("if (").append(value).append(".size() > ").append(codecClassName).append("::") + .append(formattedPropertyName).append("Length())\n") + .append(indent).append(INDENT) + .append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("throw std::invalid_argument(\"") + .append(formattedPropertyName) + .append(": too many elements: \" + std::to_string(") + .append(value).append(".size()));\n") + .append(indent).append(INDENT) + .append("}\n") + .append(indent).append("}\n"); + } + + private void generateSingleValueProperty( + final ClassBuilder classBuilder, + final String codecClassName, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String elementTypeName = cppTypeName(typeToken.encoding().primitiveType()); + final CharSequence typeName = typeWithFieldOptionality( + fieldToken, + elementTypeName + ); + final String formattedPropertyName = formatPropertyName(propertyName); + final String fieldName = "m_" + toLowerFirstChar(propertyName); + final String validateMethod = "validate" + toUpperFirstChar(propertyName); + + classBuilder.appendField() + .append(indent).append(typeName).append(" ").append(fieldName).append(";\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("[[nodiscard]] ").append(typeName).append(" ") + .append(formattedPropertyName).append("() const\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("void ").append(formattedPropertyName).append("(") + .append(typeName).append(" value)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(validateMethod).append("(value);\n") + .append(indent).append(INDENT).append(fieldName).append(" = value;\n") + .append(indent).append("}\n"); + + final StringBuilder validateBuilder = classBuilder.appendPrivate().append("\n") + .append(indent).append("static void ").append(validateMethod).append("(") + .append(typeName).append(" value)\n") + .append(indent).append("{\n"); + + String value = "value"; + + if (fieldToken.isOptionalEncoding()) + { + validateBuilder.append(indent).append(INDENT) + .append("if (!value.has_value())\n") + .append(indent).append(INDENT) + .append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("return;\n") + .append(indent).append(INDENT) + .append("}\n"); + + validateBuilder.append(indent).append(INDENT) + .append("if (value.value() == ").append(codecClassName).append("::") + .append(formattedPropertyName).append("NullValue())\n") + .append(indent).append(INDENT) + .append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("throw std::invalid_argument(\"") + .append(propertyName) + .append(": null value is reserved: \" + std::to_string(value.value()));\n") + .append(indent).append(INDENT) + .append("}\n"); + + validateBuilder.append(indent).append(INDENT) + .append(elementTypeName).append(" actualValue = value.value();\n"); + + value = "actualValue"; + } + + validateBuilder.append(indent).append(INDENT) + .append("if (").append(value).append(" < ") + .append(codecClassName).append("::").append(formattedPropertyName).append("MinValue() || ") + .append(value).append(" > ") + .append(codecClassName).append("::").append(formattedPropertyName).append("MaxValue())\n") + .append(indent).append(INDENT) + .append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("throw std::invalid_argument(\"") + .append(propertyName) + .append(": value is out of allowed range: \" + std::to_string(") + .append(value).append("));\n") + .append(indent).append(INDENT) + .append("}\n") + .append(indent).append("}\n"); + } + + private void generateConstPropertyMethods( + final ClassBuilder classBuilder, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) + { + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("static std::string ").append(toLowerFirstChar(propertyName)).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT) + .append("return \"").append(typeToken.encoding().constValue().toString()).append("\";\n") + .append(indent).append("}\n"); + } + else + { + final CharSequence literalValue = + generateLiteral(typeToken.encoding().primitiveType(), typeToken.encoding().constValue().toString()); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("[[nodiscard]] static ") + .append(cppTypeName(typeToken.encoding().primitiveType())) + .append(" ").append(formatPropertyName(propertyName)).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(literalValue).append(";\n") + .append(indent).append("}\n"); + } + } + + private void generateVarData( + final ClassBuilder classBuilder, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token token = tokens.get(i); + if (token.signal() == Signal.BEGIN_VAR_DATA) + { + final String propertyName = token.name(); + final Token varDataToken = Generators.findFirst("varData", tokens, i); + final String characterEncoding = varDataToken.encoding().characterEncoding(); + final String dtoType = characterEncoding == null ? "std::vector" : "std::string"; + + final String fieldName = "m_" + toLowerFirstChar(propertyName); + final String formattedPropertyName = formatPropertyName(propertyName); + + classBuilder.appendField() + .append(indent).append(dtoType).append(" ").append(fieldName).append(";\n"); + + classBuilder.appendPublic().append("\n") + .append(indent).append("[[nodiscard]] const ").append(dtoType).append("& ") + .append(formattedPropertyName).append("() const\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(indent).append("[[nodiscard]] ").append(dtoType).append("& ") + .append(formattedPropertyName).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(indent).append("void ").append(formattedPropertyName) + .append("(const ").append(dtoType).append("& borrowedValue)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(fieldName).append(" = borrowedValue;\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(indent).append("void ").append(formattedPropertyName) + .append("(").append(dtoType).append("&& ownedValue)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(fieldName).append(" = std::move(ownedValue);\n") + .append(indent).append("}\n"); + } + } + } + + private static String formatDtoClassName(final String name) + { + return formatClassName(name + "Dto"); + } + + private void generateDtosForTypes() throws IOException + { + for (final List tokens : ir.types()) + { + switch (tokens.get(0).signal()) + { + case BEGIN_COMPOSITE: + generateComposite(tokens); + break; + + case BEGIN_SET: + generateChoiceSet(tokens); + break; + + default: + break; + } + } + } + + private void generateComposite(final List tokens) throws IOException + { + final String name = tokens.get(0).applicableTypeName(); + final String className = formatDtoClassName(name); + final String codecClassName = formatClassName(name); + + try (Writer out = outputManager.createOutput(className)) + { + final List compositeTokens = tokens.subList(1, tokens.size() - 1); + final Set referencedTypes = generateTypesToIncludes(compositeTokens); + referencedTypes.add(codecClassName); + out.append(generateDtoFileHeader(ir.namespaces(), className, referencedTypes)); + out.append(generateDocumentation(BASE_INDENT, tokens.get(0))); + + final ClassBuilder classBuilder = new ClassBuilder(className, BASE_INDENT); + + generateCompositePropertyElements(classBuilder, codecClassName, compositeTokens, BASE_INDENT + INDENT); + generateCompositeDecodeFrom(classBuilder, className, codecClassName, compositeTokens, + BASE_INDENT + INDENT); + generateCompositeEncodeInto(classBuilder, className, codecClassName, compositeTokens, BASE_INDENT + INDENT); + generateDisplay(classBuilder, codecClassName, "wrap", + codecClassName + "::sbeSchemaVersion()", BASE_INDENT + INDENT); + + classBuilder.appendTo(out); + out.append("} // namespace\n"); + out.append("#endif\n"); + } + } + + private void generateChoiceSet(final List tokens) throws IOException + { + final String name = tokens.get(0).applicableTypeName(); + final String className = formatDtoClassName(name); + final String codecClassName = formatClassName(name); + + try (Writer out = outputManager.createOutput(className)) + { + final List setTokens = tokens.subList(1, tokens.size() - 1); + final Set referencedTypes = generateTypesToIncludes(setTokens); + referencedTypes.add(codecClassName); + out.append(generateDtoFileHeader(ir.namespaces(), className, referencedTypes)); + out.append(generateDocumentation(BASE_INDENT, tokens.get(0))); + + final ClassBuilder classBuilder = new ClassBuilder(className, BASE_INDENT); + + generateChoices(classBuilder, className, setTokens, BASE_INDENT + INDENT); + generateChoiceSetDecodeFrom(classBuilder, className, codecClassName, setTokens, BASE_INDENT + INDENT); + generateChoiceSetEncodeInto(classBuilder, className, codecClassName, setTokens, BASE_INDENT + INDENT); + + classBuilder.appendTo(out); + out.append("} // namespace\n"); + out.append("#endif\n"); + } + } + + private void generateChoiceSetEncodeInto( + final ClassBuilder classBuilder, + final String dtoClassName, + final String codecClassName, + final List setTokens, + final String indent) + { + final StringBuilder encodeBuilder = classBuilder.appendPublic() + .append("\n") + .append(indent).append("static void encode(\n") + .append(indent).append(INDENT).append(codecClassName).append("& codec, ") + .append("const ").append(dtoClassName).append("& dto)\n") + .append(indent).append("{\n"); + + encodeBuilder.append(indent).append(INDENT).append("codec.clear();\n"); + + for (final Token token : setTokens) + { + if (token.signal() == Signal.CHOICE) + { + final String formattedPropertyName = formatPropertyName(token.name()); + encodeBuilder.append(indent).append(INDENT).append("codec.").append(formattedPropertyName) + .append("(dto.").append(formattedPropertyName).append("());\n"); + } + } + + encodeBuilder.append(indent).append("}\n"); + } + + private void generateChoiceSetDecodeFrom( + final ClassBuilder classBuilder, + final String dtoClassName, + final String codecClassName, + final List setTokens, + final String indent) + { + final StringBuilder decodeBuilder = classBuilder.appendPublic() + .append("\n") + .append(indent).append("static void decode(\n") + .append(indent).append(INDENT).append("const ").append(codecClassName).append("& codec, ") + .append(dtoClassName).append("& dto)\n") + .append(indent).append("{\n"); + + for (final Token token : setTokens) + { + if (token.signal() == Signal.CHOICE) + { + final String formattedPropertyName = formatPropertyName(token.name()); + decodeBuilder.append(indent).append(INDENT).append("dto.").append(formattedPropertyName) + .append("(codec.").append(formattedPropertyName).append("());\n"); + } + } + + decodeBuilder.append(indent).append("}\n"); + } + + private void generateChoices( + final ClassBuilder classBuilder, + final String dtoClassName, + final List setTokens, + final String indent) + { + final List fields = new ArrayList<>(); + + for (final Token token : setTokens) + { + if (token.signal() == Signal.CHOICE) + { + final String fieldName = "m_" + toLowerFirstChar(token.name()); + final String formattedPropertyName = formatPropertyName(token.name()); + + fields.add(fieldName); + + classBuilder.appendField() + .append(indent).append("bool ").append(fieldName).append(";\n"); + + classBuilder.appendPublic() + .append("\n") + .append(indent).append("[[nodiscard]] bool ").append(formattedPropertyName).append("() const\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic() + .append("\n") + .append(indent).append(dtoClassName).append("& ") + .append(formattedPropertyName).append("(bool value)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(fieldName).append(" = value;\n") + .append(indent).append(INDENT).append("return *this;\n") + .append(indent).append("}\n"); + } + } + + final StringBuilder clearBuilder = classBuilder.appendPublic() + .append(indent).append(dtoClassName).append("& clear()\n") + .append(indent).append("{\n"); + + for (final String field : fields) + { + clearBuilder.append(indent).append(INDENT).append(field).append(" = false;\n"); + } + + clearBuilder.append(indent).append(INDENT).append("return *this;\n") + .append(indent).append("}\n"); + } + + private void generateCompositePropertyElements( + final ClassBuilder classBuilder, + final String codecClassName, + final List tokens, + final String indent) + { + for (int i = 0; i < tokens.size(); ) + { + final Token token = tokens.get(i); + final String propertyName = formatPropertyName(token.name()); + + switch (token.signal()) + { + case ENCODING: + generatePrimitiveProperty(classBuilder, codecClassName, propertyName, token, token, indent); + break; + + case BEGIN_ENUM: + generateEnumProperty(classBuilder, propertyName, token, token, indent); + break; + + case BEGIN_SET: + case BEGIN_COMPOSITE: + generateComplexProperty(classBuilder, propertyName, token, token, indent); + break; + + default: + break; + } + + i += tokens.get(i).componentTokenCount(); + } + } + + private static Set generateTypesToIncludes(final List tokens) + { + final Set typesToInclude = new HashSet<>(); + + for (final Token token : tokens) + { + switch (token.signal()) + { + case BEGIN_ENUM: + typesToInclude.add(formatClassName(token.applicableTypeName())); + break; + + case BEGIN_SET: + case BEGIN_COMPOSITE: + typesToInclude.add(formatDtoClassName(token.applicableTypeName())); + break; + + default: + break; + } + } + + return typesToInclude; + } + + private static CharSequence typeWithFieldOptionality( + final Token fieldToken, + final String typeName) + { + if (fieldToken.isOptionalEncoding()) + { + return "std::optional<" + typeName + ">"; + } + else + { + return typeName; + } + } + + private static CharSequence generateDtoFileHeader( + final CharSequence[] namespaces, + final String className, + final Collection typesToInclude) + { + final StringBuilder sb = new StringBuilder(); + + sb.append("/* Generated SBE (Simple Binary Encoding) message DTO */\n"); + + sb.append(String.format( + "#ifndef _%1$s_%2$s_CXX_H_\n" + + "#define _%1$s_%2$s_CXX_H_\n\n", + String.join("_", namespaces).toUpperCase(), + className.toUpperCase())); + + sb.append("#if __cplusplus < 201703L\n") + .append("#error DTO code requires at least C++17.\n") + .append("#endif\n\n"); + + sb.append("#include \n") + .append("#include \n") + .append("#include \n") + .append("#include \n") + .append("#include \n") + .append("#include \n") + .append("#include \n") + .append("#include \n") + .append("#include \n") + .append("#include \n"); + + if (typesToInclude != null && !typesToInclude.isEmpty()) + { + sb.append("\n"); + for (final String incName : typesToInclude) + { + sb.append("#include \"").append(incName).append(".h\"\n"); + } + } + + sb.append("\nnamespace "); + sb.append(String.join(" {\nnamespace ", namespaces)); + sb.append(" {\n\n"); + + return sb; + } + + private static String generateDocumentation(final String indent, final Token token) + { + final String description = token.description(); + if (null == description || description.isEmpty()) + { + return ""; + } + + return + indent + "/**\n" + + indent + " * " + description + "\n" + + indent + " */\n"; + } +} diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtos.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtos.java new file mode 100644 index 0000000000..dfe00b449e --- /dev/null +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtos.java @@ -0,0 +1,36 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * Copyright 2017 MarketFactory Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.generation.cpp; + +import uk.co.real_logic.sbe.generation.CodeGenerator; +import uk.co.real_logic.sbe.generation.TargetCodeGenerator; +import uk.co.real_logic.sbe.ir.Ir; + +/** + * {@link CodeGenerator} factory for CSharp DTOs. + */ +public class CppDtos implements TargetCodeGenerator +{ + /** + * {@inheritDoc} + */ + public CodeGenerator newInstance(final Ir ir, final String outputDir) + { + return new CppDtoGenerator(ir, new NamespaceOutputManager(outputDir, ir.applicableNamespace())); + } +} diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppGenerator.java index 5b5e6f0721..99f5498e38 100755 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppGenerator.java @@ -3503,64 +3503,6 @@ private CharSequence generateNullValueLiteral(final PrimitiveType primitiveType, return generateLiteral(primitiveType, encoding.applicableNullValue().toString()); } - private static CharSequence generateLiteral(final PrimitiveType type, final String value) - { - String literal = ""; - - switch (type) - { - case CHAR: - case UINT8: - case UINT16: - case INT8: - case INT16: - literal = "static_cast<" + cppTypeName(type) + ">(" + value + ")"; - break; - - case UINT32: - literal = "UINT32_C(0x" + Integer.toHexString((int)Long.parseLong(value)) + ")"; - break; - - case INT32: - final long intValue = Long.parseLong(value); - if (intValue == Integer.MIN_VALUE) - { - literal = "INT32_MIN"; - } - else - { - literal = "INT32_C(" + value + ")"; - } - break; - - case FLOAT: - literal = value.endsWith("NaN") ? "SBE_FLOAT_NAN" : value + "f"; - break; - - case INT64: - final long longValue = Long.parseLong(value); - if (longValue == Long.MIN_VALUE) - { - literal = "INT64_MIN"; - } - else - { - literal = "INT64_C(" + value + ")"; - } - break; - - case UINT64: - literal = "UINT64_C(0x" + Long.toHexString(Long.parseLong(value)) + ")"; - break; - - case DOUBLE: - literal = value.endsWith("NaN") ? "SBE_DOUBLE_NAN" : value; - break; - } - - return literal; - } - private void generateDisplay( final StringBuilder sb, final String name, diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppUtil.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppUtil.java index 76b75687d2..6ac6d8afa8 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppUtil.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppUtil.java @@ -137,4 +137,62 @@ public static String closingBraces(final int count) return sb.toString(); } + + static CharSequence generateLiteral(final PrimitiveType type, final String value) + { + String literal = ""; + + switch (type) + { + case CHAR: + case UINT8: + case UINT16: + case INT8: + case INT16: + literal = "static_cast<" + cppTypeName(type) + ">(" + value + ")"; + break; + + case UINT32: + literal = "UINT32_C(0x" + Integer.toHexString((int)Long.parseLong(value)) + ")"; + break; + + case INT32: + final long intValue = Long.parseLong(value); + if (intValue == Integer.MIN_VALUE) + { + literal = "INT32_MIN"; + } + else + { + literal = "INT32_C(" + value + ")"; + } + break; + + case FLOAT: + literal = value.endsWith("NaN") ? "SBE_FLOAT_NAN" : value + "f"; + break; + + case INT64: + final long longValue = Long.parseLong(value); + if (longValue == Long.MIN_VALUE) + { + literal = "INT64_MIN"; + } + else + { + literal = "INT64_C(" + value + ")"; + } + break; + + case UINT64: + literal = "UINT64_C(0x" + Long.toHexString(Long.parseLong(value)) + ")"; + break; + + case DOUBLE: + literal = value.endsWith("NaN") ? "SBE_DOUBLE_NAN" : value; + break; + } + + return literal; + } } diff --git a/sbe-tool/src/test/cpp/CMakeLists.txt b/sbe-tool/src/test/cpp/CMakeLists.txt index b1319e2389..92f061d2ef 100644 --- a/sbe-tool/src/test/cpp/CMakeLists.txt +++ b/sbe-tool/src/test/cpp/CMakeLists.txt @@ -39,6 +39,7 @@ set(COMPOSITE_ELEMENTS_SCHEMA ${CODEC_SCHEMA_DIR}/composite-elements-schema.xml) set(COMPOSITE_OFFSETS_SCHEMA ${CODEC_SCHEMA_DIR}/composite-offsets-schema.xml) set(MESSAGE_BLOCK_LENGTH_TEST ${CODEC_SCHEMA_DIR}/message-block-length-test.xml) set(GROUP_WITH_DATA_SCHEMA ${CODEC_SCHEMA_DIR}/group-with-data-schema.xml) +set(DTO_SCHEMA ${CODEC_SCHEMA_DIR}/dto-test-schema.xml) set(ISSUE835_SCHEMA ${CODEC_SCHEMA_DIR}/issue835.xml) set(ISSUE889_SCHEMA ${CODEC_SCHEMA_DIR}/issue889.xml) set(ACCESS_ORDER_SCHEMA ${CODEC_SCHEMA_DIR}/field-order-check-schema.xml) @@ -54,6 +55,7 @@ add_custom_command( ${COMPOSITE_OFFSETS_SCHEMA} ${MESSAGE_BLOCK_LENGTH_TEST} ${GROUP_WITH_DATA_SCHEMA} + ${DTO_SCHEMA} ${ISSUE835_SCHEMA} ${ISSUE889_SCHEMA} ${ACCESS_ORDER_SCHEMA} @@ -66,12 +68,14 @@ add_custom_command( -Dsbe.generate.precedence.checks="true" -Dsbe.precedence.checks.flag.name="SBE_ENABLE_PRECEDENCE_CHECKS_IN_TESTS" -Dsbe.cpp.disable.implicit.copying="true" + -Dsbe.cpp.generate.dtos="true" -jar ${SBE_JAR} ${CODE_GENERATION_SCHEMA} ${COMPOSITE_OFFSETS_SCHEMA} ${MESSAGE_BLOCK_LENGTH_TEST} ${GROUP_WITH_DATA_SCHEMA} ${COMPOSITE_ELEMENTS_SCHEMA} + ${DTO_SCHEMA} ${ISSUE835_SCHEMA} ${ISSUE889_SCHEMA} ${ACCESS_ORDER_SCHEMA} @@ -92,3 +96,5 @@ sbe_test(Issue835Test codecs) sbe_test(Issue889Test codecs) sbe_test(FieldAccessOrderCheckTest codecs) target_compile_definitions(FieldAccessOrderCheckTest PRIVATE SBE_ENABLE_PRECEDENCE_CHECKS_IN_TESTS) +sbe_test(DtoTest codecs) +target_compile_features(DtoTest PRIVATE cxx_std_17) diff --git a/sbe-tool/src/test/cpp/DtoTest.cpp b/sbe-tool/src/test/cpp/DtoTest.cpp new file mode 100644 index 0000000000..59f46fe149 --- /dev/null +++ b/sbe-tool/src/test/cpp/DtoTest.cpp @@ -0,0 +1,195 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if __cplusplus < 201703L +#error DTO code requires at least C++17. +#endif + +#include +#include "dto_test/ExtendedCar.h" +#include "dto_test/ExtendedCarDto.h" + +using namespace dto_test; + +static const std::size_t BUFFER_LEN = 2048; + +static const std::uint32_t SERIAL_NUMBER = 1234; +static const std::uint16_t MODEL_YEAR = 2013; +static const BooleanType::Value AVAILABLE = BooleanType::T; +static const Model::Value CODE = Model::A; +static const bool CRUISE_CONTROL = true; +static const bool SPORTS_PACK = true; +static const bool SUNROOF = false; +static const BoostType::Value BOOST_TYPE = BoostType::NITROUS; +static const std::uint8_t BOOSTER_HORSEPOWER = 200; +static const std::int32_t ADDED1 = 7; +static const std::int8_t ADDED6_1 = 11; +static const std::int8_t ADDED6_2 = 13; + +static char VEHICLE_CODE[] = { 'a', 'b', 'c', 'd', 'e', 'f' }; +static char MANUFACTURER_CODE[] = { '1', '2', '3' }; +static const char *FUEL_FIGURES_1_USAGE_DESCRIPTION = "Urban Cycle"; +static const char *FUEL_FIGURES_2_USAGE_DESCRIPTION = "Combined Cycle"; +static const char *FUEL_FIGURES_3_USAGE_DESCRIPTION = "Highway Cycle"; +static const char *MANUFACTURER = "Honda"; +static const char *MODEL = "Civic VTi"; +static const char *ACTIVATION_CODE = "deadbeef"; +static const char *ADDED5 = "feedface"; + +static const std::uint8_t PERFORMANCE_FIGURES_COUNT = 2; +static const std::uint8_t FUEL_FIGURES_COUNT = 3; +static const std::uint8_t ACCELERATION_COUNT = 3; + +static const std::uint16_t fuel1Speed = 30; +static const float fuel1Mpg = 35.9f; +static const std::int8_t fuel1Added2Element1 = 42; +static const std::int8_t fuel1Added2Element2 = 43; +static const std::int8_t fuel1Added3 = 44; +static const std::uint16_t fuel2Speed = 55; +static const float fuel2Mpg = 49.0f; +static const std::int8_t fuel2Added2Element1 = 45; +static const std::int8_t fuel2Added2Element2 = 46; +static const std::int8_t fuel2Added3 = 47; +static const std::uint16_t fuel3Speed = 75; +static const float fuel3Mpg = 40.0f; +static const std::int8_t fuel3Added2Element1 = 48; +static const std::int8_t fuel3Added2Element2 = 49; +static const std::int8_t fuel3Added3 = 50; + +static const std::uint8_t perf1Octane = 95; +static const std::uint16_t perf1aMph = 30; +static const float perf1aSeconds = 4.0f; +static const std::uint16_t perf1bMph = 60; +static const float perf1bSeconds = 7.5f; +static const std::uint16_t perf1cMph = 100; +static const float perf1cSeconds = 12.2f; + +static const std::uint8_t perf2Octane = 99; +static const std::uint16_t perf2aMph = 30; +static const float perf2aSeconds = 3.8f; +static const std::uint16_t perf2bMph = 60; +static const float perf2bSeconds = 7.1f; +static const std::uint16_t perf2cMph = 100; +static const float perf2cSeconds = 11.8f; + +static const std::uint16_t engineCapacity = 2000; +static const std::uint8_t engineNumCylinders = 4; + +class DtoTest : public testing::Test +{ + +public: + static std::uint64_t encodeCar(ExtendedCar &car) + { + car.serialNumber(SERIAL_NUMBER) + .modelYear(MODEL_YEAR) + .available(AVAILABLE) + .code(CODE) + .putVehicleCode(VEHICLE_CODE); + + car.extras().clear() + .cruiseControl(CRUISE_CONTROL) + .sportsPack(SPORTS_PACK) + .sunRoof(SUNROOF); + + car.engine() + .capacity(engineCapacity) + .numCylinders(engineNumCylinders) + .putManufacturerCode(MANUFACTURER_CODE) + .efficiency(50) + .boosterEnabled(BooleanType::Value::T) + .booster().boostType(BOOST_TYPE).horsePower(BOOSTER_HORSEPOWER); + + car.added1(ADDED1); + + car.added4(BooleanType::Value::T); + + car.added6().one(ADDED6_1).two(ADDED6_2); + + ExtendedCar::FuelFigures &fuelFigures = car.fuelFiguresCount(FUEL_FIGURES_COUNT); + + fuelFigures + .next().speed(fuel1Speed).mpg(fuel1Mpg) + .putAdded2(fuel1Added2Element1, fuel1Added2Element2) + .added3(fuel1Added3) + .putUsageDescription( + FUEL_FIGURES_1_USAGE_DESCRIPTION, static_cast(strlen(FUEL_FIGURES_1_USAGE_DESCRIPTION))); + + fuelFigures + .next().speed(fuel2Speed).mpg(fuel2Mpg) + .putAdded2(fuel2Added2Element1, fuel2Added2Element2) + .added3(fuel2Added3) + .putUsageDescription( + FUEL_FIGURES_2_USAGE_DESCRIPTION, static_cast(strlen(FUEL_FIGURES_2_USAGE_DESCRIPTION))); + + fuelFigures + .next().speed(fuel3Speed).mpg(fuel3Mpg) + .putAdded2(fuel3Added2Element1, fuel3Added2Element2) + .added3(fuel3Added3) + .putUsageDescription( + FUEL_FIGURES_3_USAGE_DESCRIPTION, static_cast(strlen(FUEL_FIGURES_3_USAGE_DESCRIPTION))); + + ExtendedCar::PerformanceFigures &perfFigs = car.performanceFiguresCount(PERFORMANCE_FIGURES_COUNT); + + perfFigs.next() + .octaneRating(perf1Octane) + .accelerationCount(ACCELERATION_COUNT) + .next().mph(perf1aMph).seconds(perf1aSeconds) + .next().mph(perf1bMph).seconds(perf1bSeconds) + .next().mph(perf1cMph).seconds(perf1cSeconds); + + perfFigs.next() + .octaneRating(perf2Octane) + .accelerationCount(ACCELERATION_COUNT) + .next().mph(perf2aMph).seconds(perf2aSeconds) + .next().mph(perf2bMph).seconds(perf2bSeconds) + .next().mph(perf2cMph).seconds(perf2cSeconds); + + car.putManufacturer(MANUFACTURER, static_cast(strlen(MANUFACTURER))) + .putModel(MODEL, static_cast(strlen(MODEL))) + .putActivationCode(ACTIVATION_CODE, static_cast(strlen(ACTIVATION_CODE))) + .putAdded5(ADDED5, static_cast(strlen(ADDED5))); + + return car.encodedLength(); + } +}; + +TEST_F(DtoTest, shouldRoundTripCar) +{ + char input[BUFFER_LEN]; + std::memset(input, 0, BUFFER_LEN); + ExtendedCar encoder1; + encoder1.wrapForEncode(input, 0, BUFFER_LEN); + const std::uint64_t encodedCarLength = encodeCar(encoder1); + ExtendedCar decoder; + decoder.wrapForDecode( + input, + 0, + ExtendedCar::sbeBlockLength(), + ExtendedCar::sbeSchemaVersion(), + encodedCarLength); + ExtendedCarDto dto; + ExtendedCarDto::decode(decoder, dto); + char output[BUFFER_LEN]; + std::memset(output, 0, BUFFER_LEN); + ExtendedCar encoder2; + encoder2.wrapForEncode(output, 0, BUFFER_LEN); + ExtendedCarDto::encode(encoder2, dto); + const std::uint64_t encodedCarLength2 = encoder2.encodedLength(); + + EXPECT_EQ(encodedCarLength, encodedCarLength2); + EXPECT_EQ(0, std::memcmp(input, output, encodedCarLength2)); +} diff --git a/sbe-tool/src/test/resources/dto-test-schema.xml b/sbe-tool/src/test/resources/dto-test-schema.xml new file mode 100644 index 0000000000..f6f832e990 --- /dev/null +++ b/sbe-tool/src/test/resources/dto-test-schema.xml @@ -0,0 +1,110 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + T + S + N + K + + + + + + + 9000 + + Petrol + + + + + + 0 + 1 + + + A + B + C + + + 0 + 1 + 2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 029fb2573432fe44fa2b9f0855614242f8511ade Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Thu, 2 Nov 2023 15:09:35 +0000 Subject: [PATCH 26/41] [C++] Only build DTOs with compilers that support C++ 17. --- .../sbe/generation/cpp/CppDtoGenerator.java | 6 +++-- sbe-tool/src/test/cpp/CMakeLists.txt | 26 +++++++++++++++++-- sbe-tool/src/test/cpp/DtoTest.cpp | 2 +- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java index e1e3793da8..60924dd8f2 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java @@ -1777,7 +1777,8 @@ private static CharSequence generateDtoFileHeader( String.join("_", namespaces).toUpperCase(), className.toUpperCase())); - sb.append("#if __cplusplus < 201703L\n") + sb.append("#if (defined(_MSVC_LANG) && _MSVC_LANG < 201703L) || ") + .append("(!defined(_MSVC_LANG) && defined(__cplusplus) && __cplusplus < 201703L)\n") .append("#error DTO code requires at least C++17.\n") .append("#endif\n\n"); @@ -1790,7 +1791,8 @@ private static CharSequence generateDtoFileHeader( .append("#include \n") .append("#include \n") .append("#include \n") - .append("#include \n"); + .append("#include \n") + .append("#include \n"); if (typesToInclude != null && !typesToInclude.isEmpty()) { diff --git a/sbe-tool/src/test/cpp/CMakeLists.txt b/sbe-tool/src/test/cpp/CMakeLists.txt index 92f061d2ef..1de7412623 100644 --- a/sbe-tool/src/test/cpp/CMakeLists.txt +++ b/sbe-tool/src/test/cpp/CMakeLists.txt @@ -96,5 +96,27 @@ sbe_test(Issue835Test codecs) sbe_test(Issue889Test codecs) sbe_test(FieldAccessOrderCheckTest codecs) target_compile_definitions(FieldAccessOrderCheckTest PRIVATE SBE_ENABLE_PRECEDENCE_CHECKS_IN_TESTS) -sbe_test(DtoTest codecs) -target_compile_features(DtoTest PRIVATE cxx_std_17) + +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # Check if the GCC version supports C++17 + if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "7.0") + sbe_test(DtoTest codecs) + target_compile_features(DtoTest PRIVATE cxx_std_17) + endif() +endif() + +if (CMAKE_CXX_COMPILER_ID STREQUAL "CLang") + # Check if CLang version supports C++17 + if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "4.0") + sbe_test(DtoTest codecs) + target_compile_features(DtoTest PRIVATE cxx_std_17) + endif() +endif() + +if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + # Check if MSVC version supports C++17 + if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "19.14") + sbe_test(DtoTest codecs) + target_compile_options(DtoTest PRIVATE /std:c++17) + endif() +endif() diff --git a/sbe-tool/src/test/cpp/DtoTest.cpp b/sbe-tool/src/test/cpp/DtoTest.cpp index 59f46fe149..3c8f675051 100644 --- a/sbe-tool/src/test/cpp/DtoTest.cpp +++ b/sbe-tool/src/test/cpp/DtoTest.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#if __cplusplus < 201703L +#if (defined(_MSVC_LANG) && _MSVC_LANG < 201703L) || (!defined(_MSVC_LANG) && defined(__cplusplus) && __cplusplus < 201703L) #error DTO code requires at least C++17. #endif From 246e6f77a0405435881666835be2c01b73ce891c Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Fri, 3 Nov 2023 09:57:00 +0000 Subject: [PATCH 27/41] [C++] Improve naming conventions in DTOs. Changes: - `encode` -> `encodeWith` - `decode` -> `decodeWith` --- .../sbe/generation/cpp/CppDtoGenerator.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java index 60924dd8f2..651a3aa676 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java @@ -274,7 +274,7 @@ private void generateCompositeDecodeFrom( final String indent) { final StringBuilder decodeBuilder = classBuilder.appendPublic().append("\n") - .append(indent).append("static void decode(").append(codecClassName).append("& codec, ") + .append(indent).append("static void decodeWith(").append(codecClassName).append("& codec, ") .append(dtoClassName).append("& dto)\n") .append(indent).append("{\n"); @@ -299,7 +299,7 @@ private void generateCompositeEncodeInto( final String indent) { final StringBuilder encodeBuilder = classBuilder.appendPublic().append("\n") - .append(indent).append("static void encode(").append(codecClassName).append("& codec,") + .append(indent).append("static void encodeWith(").append(codecClassName).append("& codec,") .append("const ").append(dtoClassName).append("& dto)\n") .append(indent).append("{\n"); @@ -322,7 +322,7 @@ private void generateDecodeListFrom( final String indent) { classBuilder.appendPublic().append("\n") - .append(indent).append("static std::vector<").append(dtoClassName).append("> decodeMany(") + .append(indent).append("static std::vector<").append(dtoClassName).append("> decodeManyWith(") .append(codecClassName).append("& codec)\n") .append(indent).append("{\n") .append(indent).append(INDENT).append("std::vector<").append(dtoClassName) @@ -334,7 +334,7 @@ private void generateDecodeListFrom( .append(indent).append(INDENT).append(INDENT) .append(dtoClassName).append(" dto;\n") .append(indent).append(INDENT).append(INDENT) - .append(dtoClassName).append("::decode(codec.next(), dto);\n") + .append(dtoClassName).append("::decodeWith(codec.next(), dto);\n") .append(indent).append(INDENT).append(INDENT) .append("dtos[i] = dto;\n") .append(indent).append(INDENT) @@ -354,7 +354,7 @@ private void generateMessageDecodeFrom( final String indent) { final StringBuilder decodeBuilder = classBuilder.appendPublic().append("\n") - .append(indent).append("static void decode(").append(codecClassName).append("& codec, ") + .append(indent).append("static void decodeWith(").append(codecClassName).append("& codec, ") .append(dtoClassName).append("& dto)\n") .append(indent).append("{\n"); @@ -526,7 +526,7 @@ private void generateBitSetDecodeFrom( sb.append(")\n") .append(indent).append("{\n"); - sb.append(indent).append(INDENT).append(dtoTypeName).append("::decode(codec.") + sb.append(indent).append(INDENT).append(dtoTypeName).append("::decodeWith(codec.") .append(formattedPropertyName).append("(), ") .append("dto.").append(formattedPropertyName).append("());\n"); @@ -538,7 +538,7 @@ private void generateBitSetDecodeFrom( } else { - sb.append(indent).append(dtoTypeName).append("::decode(codec.") + sb.append(indent).append(dtoTypeName).append("::decodeWith(codec.") .append(formattedPropertyName).append("(), ") .append("dto.").append(formattedPropertyName).append("());\n"); } @@ -571,7 +571,7 @@ private void generateCompositePropertyDecodeFrom( final String formattedPropertyName = formatPropertyName(propertyName); final String dtoClassName = formatDtoClassName(typeToken.applicableTypeName()); - sb.append(indent).append(dtoClassName).append("::decode(codec.") + sb.append(indent).append(dtoClassName).append("::decodeWith(codec.") .append(formattedPropertyName).append("(), ") .append("dto.").append(formattedPropertyName).append("());\n"); } @@ -593,7 +593,7 @@ private void generateGroupsDecodeFrom( final String groupDtoClassName = formatDtoClassName(groupName); sb.append(indent).append("dto.").append(formattedPropertyName).append("(") - .append(groupDtoClassName).append("::decodeMany(codec.") + .append(groupDtoClassName).append("::decodeManyWith(codec.") .append(formattedPropertyName).append("()));\n"); i++; @@ -729,7 +729,7 @@ private void generateMessageEncodeInto( final String indent) { final StringBuilder encodeBuilder = classBuilder.appendPublic().append("\n") - .append(indent).append("static void encode(").append(codecClassName).append("& codec, const ") + .append(indent).append("static void encodeWith(").append(codecClassName).append("& codec, const ") .append(dtoClassName).append("& dto)\n") .append(indent).append("{\n"); @@ -914,7 +914,7 @@ private void generateComplexPropertyEncodeInto( final String formattedPropertyName = formatPropertyName(propertyName); final String typeName = formatDtoClassName(typeToken.applicableTypeName()); - sb.append(indent).append(typeName).append("::encode(codec.") + sb.append(indent).append(typeName).append("::encodeWith(codec.") .append(formattedPropertyName).append("(), dto.") .append(formattedPropertyName).append("());\n"); } @@ -944,7 +944,7 @@ private void generateGroupsEncodeInto( .append("Count(").append(formattedPropertyName).append(".size());\n\n") .append(indent).append("for (const auto& group: ").append(formattedPropertyName).append(")\n") .append(indent).append("{\n") - .append(indent).append(INDENT).append(groupDtoTypeName).append("::encode(").append(groupCodecVarName) + .append(indent).append(INDENT).append(groupDtoTypeName).append("::encodeWith(").append(groupCodecVarName) .append(".next(), group);\n") .append(indent).append("}\n\n"); @@ -1024,7 +1024,7 @@ private void generateDisplay( toStringBuilder.append(", ").append("length);\n"); - toStringBuilder.append(indent).append(INDENT).append("encode(codec, *this);\n") + toStringBuilder.append(indent).append(INDENT).append("encodeWith(codec, *this);\n") .append(indent).append(INDENT).append("std::ostringstream oss;\n") .append(indent).append(INDENT).append("oss << codec;\n") .append(indent).append(INDENT).append("return oss.str();\n") @@ -1591,7 +1591,7 @@ private void generateChoiceSetEncodeInto( { final StringBuilder encodeBuilder = classBuilder.appendPublic() .append("\n") - .append(indent).append("static void encode(\n") + .append(indent).append("static void encodeWith(\n") .append(indent).append(INDENT).append(codecClassName).append("& codec, ") .append("const ").append(dtoClassName).append("& dto)\n") .append(indent).append("{\n"); @@ -1620,7 +1620,7 @@ private void generateChoiceSetDecodeFrom( { final StringBuilder decodeBuilder = classBuilder.appendPublic() .append("\n") - .append(indent).append("static void decode(\n") + .append(indent).append("static void decodeWith(\n") .append(indent).append(INDENT).append("const ").append(codecClassName).append("& codec, ") .append(dtoClassName).append("& dto)\n") .append(indent).append("{\n"); From e33c6a86affbed5a8b976483f667bb4d4e72c93f Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Fri, 3 Nov 2023 11:31:02 +0000 Subject: [PATCH 28/41] [C++] Support "to string" without specification of buffer length. As we use the generated codecs to create a string representation of our DTOs, we don't use Agrona buffers in C++, and there is no concept of resizing, it is necessary to size a temporary buffer during the construction of the string data. Previously, we were letting the user supply this value, which wasn't a very friendly API. Now, we use the `computeLength` methods on the codec to determine how big of a temporary buffer we need. Perhaps the methods will also be useful for avoiding a buffer copy when used in conjunction with Aeron. For example, a developer could use `dto.computeEncodedLength()` to initialise a buffer claim rather than copying via the `offer(...)` API. --- .../sbe/generation/cpp/CppDtoGenerator.java | 145 ++++++++++++++++-- sbe-tool/src/test/cpp/DtoTest.cpp | 16 +- 2 files changed, 143 insertions(+), 18 deletions(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java index 651a3aa676..8ee894cf08 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java @@ -100,7 +100,9 @@ public void generate() throws IOException groups, varData, BASE_INDENT + INDENT); generateMessageEncodeInto(classBuilder, className, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); - generateDisplay(classBuilder, codecClassName, "wrapForEncode", null, BASE_INDENT + INDENT); + generateComputeEncodedLength(classBuilder, codecClassName, groups, varData, BASE_INDENT + INDENT); + generateDisplay(classBuilder, className, codecClassName, "dto.computeEncodedLength()", + "wrapForEncode", null, BASE_INDENT + INDENT); try (Writer out = outputManager.createOutput(className)) { @@ -227,6 +229,8 @@ private void generateGroups( fields, groups, varData, indent + INDENT); generateMessageEncodeInto( groupClassBuilder, groupClassName, qualifiedCodecClassName, fields, groups, varData, indent + INDENT); + generateComputeEncodedLength(groupClassBuilder, qualifiedCodecClassName, groups, varData, + indent + INDENT); groupClassBuilder.appendTo( classBuilder.appendPublic().append("\n").append(generateDocumentation(indent, groupToken)) @@ -266,6 +270,103 @@ private void generateGroups( } } + private void generateComputeEncodedLength( + final ClassBuilder classBuilder, + final String qualifiedCodecClassName, + final List groupTokens, + final List varDataTokens, + final String indent) + { + final StringBuilder lengthBuilder = classBuilder.appendPublic() + .append("\n") + .append(indent).append("[[nodiscard]] std::size_t computeEncodedLength() const\n") + .append(indent).append("{\n"); + + final StringBuilder arguments = new StringBuilder(); + + for (int i = 0, size = groupTokens.size(); i < size; i++) + { + final Token groupToken = groupTokens.get(i); + if (groupToken.signal() != Signal.BEGIN_GROUP) + { + throw new IllegalStateException("tokens must begin with BEGIN_GROUP: token=" + groupToken); + } + + i++; + i += groupTokens.get(i).componentTokenCount(); + + final List fields = new ArrayList<>(); + i = collectFields(groupTokens, i, fields); + final List subGroups = new ArrayList<>(); + i = collectGroups(groupTokens, i, subGroups); + final List subVarData = new ArrayList<>(); + i = collectVarData(groupTokens, i, subVarData); + + final String groupName = groupToken.name(); + final String fieldName = "m_" + toLowerFirstChar(groupName); + + final boolean isConstLength = subGroups.isEmpty() && subVarData.isEmpty(); + final String argumentName = isConstLength ? + toLowerFirstChar(groupName) + "Count" : + toLowerFirstChar(groupName) + "Lengths"; + + if (isConstLength) + { + lengthBuilder + .append(indent).append(INDENT).append("std::size_t ").append(argumentName) + .append(" = ").append(fieldName).append(".size();\n"); + } + else + { + final String countName = argumentName + "Count"; + + lengthBuilder + .append(indent).append(INDENT).append("std::size_t ").append(countName).append(" = ") + .append(fieldName).append(".size();\n") + .append(indent).append(INDENT).append("std::vector> ") + .append(argumentName).append("(").append(countName).append(");\n") + .append(indent).append(INDENT).append("for (std::size_t i = 0; i < ").append(countName) + .append("; i++)\n") + .append(indent).append(INDENT).append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("auto& group = ").append(fieldName).append("[i];\n") + .append(indent).append(INDENT).append(INDENT) + .append(argumentName).append("[i] = group.computeEncodedLength();\n") + .append(indent).append(INDENT).append("}\n") + .append("\n"); + } + + arguments.append(argumentName).append(", "); + } + + for (int i = 0, size = varDataTokens.size(); i < size; i++) + { + final Token token = varDataTokens.get(i); + if (token.signal() == Signal.BEGIN_VAR_DATA) + { + final String propertyName = token.name(); + final Token varDataToken = Generators.findFirst("varData", varDataTokens, i); + final String fieldName = "m_" + toLowerFirstChar(propertyName); + final String argumentName = toLowerFirstChar(propertyName) + "Length"; + + lengthBuilder.append(indent).append(INDENT).append("std::size_t ").append(argumentName) + .append(" = ").append(fieldName).append(".size() * sizeof(") + .append(cppTypeName(varDataToken.encoding().primitiveType())).append(");\n"); + + arguments.append(argumentName).append(", "); + } + } + + if (arguments.length() >= 2) + { + arguments.setLength(arguments.length() - 2); + } + + lengthBuilder.append(indent).append(INDENT).append("return ").append(qualifiedCodecClassName) + .append("::computeLength(").append(arguments).append(");\n") + .append(indent).append("}\n"); + } + private void generateCompositeDecodeFrom( final ClassBuilder classBuilder, final String dtoClassName, @@ -944,8 +1045,8 @@ private void generateGroupsEncodeInto( .append("Count(").append(formattedPropertyName).append(".size());\n\n") .append(indent).append("for (const auto& group: ").append(formattedPropertyName).append(")\n") .append(indent).append("{\n") - .append(indent).append(INDENT).append(groupDtoTypeName).append("::encodeWith(").append(groupCodecVarName) - .append(".next(), group);\n") + .append(indent).append(INDENT).append(groupDtoTypeName) + .append("::encodeWith(").append(groupCodecVarName).append(".next(), group);\n") .append(indent).append("}\n\n"); i++; @@ -1002,32 +1103,44 @@ private void generateVarDataEncodeInto( private void generateDisplay( final ClassBuilder classBuilder, + final String dtoClassName, final String codecClassName, + final String lengthExpression, final String wrapMethod, final String actingVersion, final String indent) { - final StringBuilder toStringBuilder = classBuilder.appendPublic() + final StringBuilder streamBuilder = classBuilder.appendPublic() .append("\n") - .append(indent).append( - "std::string string(char* tempBuffer, std::uint64_t offset, std::uint64_t length) const\n") + .append(indent).append("friend std::ostream& operator << (std::ostream& stream, const ") + .append(dtoClassName).append("& dto)\n") .append(indent).append("{\n") .append(indent).append(INDENT).append(codecClassName).append(" codec;\n") - .append(indent).append(INDENT).append("codec."); - - toStringBuilder.append(wrapMethod).append("(tempBuffer, offset"); + .append(indent).append(INDENT).append("const std::size_t length = ") + .append(lengthExpression).append(";\n") + .append(indent).append(INDENT).append("std::vector buffer(length);\n") + .append(indent).append(INDENT).append("codec.").append(wrapMethod) + .append("(buffer.data(), 0"); if (null != actingVersion) { - toStringBuilder.append(", ").append(actingVersion); + streamBuilder.append(", ").append(actingVersion); } - toStringBuilder.append(", ").append("length);\n"); + streamBuilder.append(", ").append("length);\n"); + + streamBuilder.append(indent).append(INDENT).append("encodeWith(codec, dto);\n") + .append(indent).append(INDENT).append("stream << codec;\n") + .append(indent).append(INDENT).append("return stream;\n") + .append(indent).append("}\n"); - toStringBuilder.append(indent).append(INDENT).append("encodeWith(codec, *this);\n") - .append(indent).append(INDENT).append("std::ostringstream oss;\n") - .append(indent).append(INDENT).append("oss << codec;\n") - .append(indent).append(INDENT).append("return oss.str();\n") + classBuilder.appendPublic() + .append("\n") + .append(indent).append("[[nodiscard]] std::string string() const\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("std::ostringstream stream;\n") + .append(indent).append(INDENT).append("stream << *this;\n") + .append(indent).append(INDENT).append("return stream.str();\n") .append(indent).append("}\n"); } @@ -1547,7 +1660,7 @@ private void generateComposite(final List tokens) throws IOException generateCompositeDecodeFrom(classBuilder, className, codecClassName, compositeTokens, BASE_INDENT + INDENT); generateCompositeEncodeInto(classBuilder, className, codecClassName, compositeTokens, BASE_INDENT + INDENT); - generateDisplay(classBuilder, codecClassName, "wrap", + generateDisplay(classBuilder, className, codecClassName, codecClassName + "::encodedLength()", "wrap", codecClassName + "::sbeSchemaVersion()", BASE_INDENT + INDENT); classBuilder.appendTo(out); diff --git a/sbe-tool/src/test/cpp/DtoTest.cpp b/sbe-tool/src/test/cpp/DtoTest.cpp index 3c8f675051..2b37c6b910 100644 --- a/sbe-tool/src/test/cpp/DtoTest.cpp +++ b/sbe-tool/src/test/cpp/DtoTest.cpp @@ -174,6 +174,7 @@ TEST_F(DtoTest, shouldRoundTripCar) ExtendedCar encoder1; encoder1.wrapForEncode(input, 0, BUFFER_LEN); const std::uint64_t encodedCarLength = encodeCar(encoder1); + ExtendedCar decoder; decoder.wrapForDecode( input, @@ -182,14 +183,25 @@ TEST_F(DtoTest, shouldRoundTripCar) ExtendedCar::sbeSchemaVersion(), encodedCarLength); ExtendedCarDto dto; - ExtendedCarDto::decode(decoder, dto); + ExtendedCarDto::decodeWith(decoder, dto); + char output[BUFFER_LEN]; std::memset(output, 0, BUFFER_LEN); ExtendedCar encoder2; encoder2.wrapForEncode(output, 0, BUFFER_LEN); - ExtendedCarDto::encode(encoder2, dto); + ExtendedCarDto::encodeWith(encoder2, dto); const std::uint64_t encodedCarLength2 = encoder2.encodedLength(); + decoder.sbeRewind(); + std::ostringstream originalStringStream; + originalStringStream << decoder; + std::string originalString = originalStringStream.str(); + + std::ostringstream dtoStringStream; + dtoStringStream << dto; + std::string dtoString = dtoStringStream.str(); + EXPECT_EQ(encodedCarLength, encodedCarLength2); EXPECT_EQ(0, std::memcmp(input, output, encodedCarLength2)); + EXPECT_EQ(originalString, dtoString); } From f88d345519583d4ea84a496c71ae59bbcdc0324e Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Fri, 3 Nov 2023 15:34:09 +0000 Subject: [PATCH 29/41] [C++] Fix issues with length computation. I had incorrectly assumed that the `Flyweight::computeLength` method took _encoded lengths_ of groups etc., but actually it takes a complicated structure of group counts and variable lengths. As it was hard to build this list, I've opted for a simpler approach: do the length calculation within the generated DTO message and its groups. In this commit, I've also added some convenience methods for converting between DTOs and "byte arrays". --- .../sbe/generation/cpp/CppDtoGenerator.java | 251 +++++++++++------- sbe-tool/src/test/cpp/DtoTest.cpp | 34 ++- 2 files changed, 186 insertions(+), 99 deletions(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java index 8ee894cf08..c13a3211b1 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java @@ -96,10 +96,12 @@ public void generate() throws IOException collectVarData(messageBody, offset, varData); generateVarData(classBuilder, varData, BASE_INDENT + INDENT); - generateMessageDecodeFrom(classBuilder, className, codecClassName, fields, + generateDecodeWith(classBuilder, className, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); - generateMessageEncodeInto(classBuilder, className, codecClassName, fields, groups, varData, + generateDecodeFrom(classBuilder, className, codecClassName, BASE_INDENT + INDENT); + generateEncodeWith(classBuilder, className, codecClassName, fields, groups, varData, BASE_INDENT + INDENT); + generateEncodeInto(classBuilder, className, codecClassName, BASE_INDENT + INDENT); generateComputeEncodedLength(classBuilder, codecClassName, groups, varData, BASE_INDENT + INDENT); generateDisplay(classBuilder, className, codecClassName, "dto.computeEncodedLength()", "wrapForEncode", null, BASE_INDENT + INDENT); @@ -225,9 +227,9 @@ private void generateGroups( generateDecodeListFrom( groupClassBuilder, groupClassName, qualifiedCodecClassName, indent + INDENT); - generateMessageDecodeFrom(groupClassBuilder, groupClassName, qualifiedCodecClassName, + generateDecodeWith(groupClassBuilder, groupClassName, qualifiedCodecClassName, fields, groups, varData, indent + INDENT); - generateMessageEncodeInto( + generateEncodeWith( groupClassBuilder, groupClassName, qualifiedCodecClassName, fields, groups, varData, indent + INDENT); generateComputeEncodedLength(groupClassBuilder, qualifiedCodecClassName, groups, varData, indent + INDENT); @@ -282,7 +284,11 @@ private void generateComputeEncodedLength( .append(indent).append("[[nodiscard]] std::size_t computeEncodedLength() const\n") .append(indent).append("{\n"); - final StringBuilder arguments = new StringBuilder(); + lengthBuilder + .append(indent).append(INDENT).append("std::size_t encodedLength = 0;\n"); + + lengthBuilder.append(indent).append(INDENT).append("encodedLength += ").append(qualifiedCodecClassName) + .append("::sbeBlockLength();\n\n"); for (int i = 0, size = groupTokens.size(); i < size; i++) { @@ -304,39 +310,17 @@ private void generateComputeEncodedLength( final String groupName = groupToken.name(); final String fieldName = "m_" + toLowerFirstChar(groupName); - - final boolean isConstLength = subGroups.isEmpty() && subVarData.isEmpty(); - final String argumentName = isConstLength ? - toLowerFirstChar(groupName) + "Count" : - toLowerFirstChar(groupName) + "Lengths"; - - if (isConstLength) - { - lengthBuilder - .append(indent).append(INDENT).append("std::size_t ").append(argumentName) - .append(" = ").append(fieldName).append(".size();\n"); - } - else - { - final String countName = argumentName + "Count"; - - lengthBuilder - .append(indent).append(INDENT).append("std::size_t ").append(countName).append(" = ") - .append(fieldName).append(".size();\n") - .append(indent).append(INDENT).append("std::vector> ") - .append(argumentName).append("(").append(countName).append(");\n") - .append(indent).append(INDENT).append("for (std::size_t i = 0; i < ").append(countName) - .append("; i++)\n") - .append(indent).append(INDENT).append("{\n") - .append(indent).append(INDENT).append(INDENT) - .append("auto& group = ").append(fieldName).append("[i];\n") - .append(indent).append(INDENT).append(INDENT) - .append(argumentName).append("[i] = group.computeEncodedLength();\n") - .append(indent).append(INDENT).append("}\n") - .append("\n"); - } - - arguments.append(argumentName).append(", "); + final String groupCodecClassName = qualifiedCodecClassName + "::" + formatClassName(groupName); + + lengthBuilder + .append(indent).append(INDENT).append("encodedLength += ") + .append(groupCodecClassName).append("::sbeHeaderSize();\n\n") + .append(indent).append(INDENT).append("for (auto& group : ") + .append(fieldName).append(")\n") + .append(indent).append(INDENT).append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("encodedLength += group.computeEncodedLength();\n") + .append(indent).append(INDENT).append("}\n\n"); } for (int i = 0, size = varDataTokens.size(); i < size; i++) @@ -347,27 +331,23 @@ private void generateComputeEncodedLength( final String propertyName = token.name(); final Token varDataToken = Generators.findFirst("varData", varDataTokens, i); final String fieldName = "m_" + toLowerFirstChar(propertyName); - final String argumentName = toLowerFirstChar(propertyName) + "Length"; - lengthBuilder.append(indent).append(INDENT).append("std::size_t ").append(argumentName) - .append(" = ").append(fieldName).append(".size() * sizeof(") - .append(cppTypeName(varDataToken.encoding().primitiveType())).append(");\n"); + lengthBuilder.append(indent).append(INDENT).append("encodedLength += ") + .append(qualifiedCodecClassName).append("::") + .append(formatPropertyName(propertyName)).append("HeaderLength();\n"); - arguments.append(argumentName).append(", "); - } - } + lengthBuilder.append(indent).append(INDENT).append("encodedLength += ") + .append(fieldName).append(".size() * sizeof(") + .append(cppTypeName(varDataToken.encoding().primitiveType())).append(");\n\n"); - if (arguments.length() >= 2) - { - arguments.setLength(arguments.length() - 2); + } } - lengthBuilder.append(indent).append(INDENT).append("return ").append(qualifiedCodecClassName) - .append("::computeLength(").append(arguments).append(");\n") + lengthBuilder.append(indent).append(INDENT).append("return encodedLength;\n") .append(indent).append("}\n"); } - private void generateCompositeDecodeFrom( + private void generateCompositeDecodeWith( final ClassBuilder classBuilder, final String dtoClassName, final String codecClassName, @@ -383,7 +363,7 @@ private void generateCompositeDecodeFrom( { final Token token = tokens.get(i); - generateFieldDecodeFrom( + generateFieldDecodeWith( decodeBuilder, token, token, codecClassName, indent + INDENT); i += tokens.get(i).componentTokenCount(); @@ -392,7 +372,7 @@ private void generateCompositeDecodeFrom( decodeBuilder.append(indent).append("}\n"); } - private void generateCompositeEncodeInto( + private void generateCompositeEncodeWith( final ClassBuilder classBuilder, final String dtoClassName, final String codecClassName, @@ -408,7 +388,7 @@ private void generateCompositeEncodeInto( { final Token token = tokens.get(i); - generateFieldEncodeInto(encodeBuilder, codecClassName, token, token, indent + INDENT); + generateFieldEncodeWith(encodeBuilder, codecClassName, token, token, indent + INDENT); i += tokens.get(i).componentTokenCount(); } @@ -445,7 +425,7 @@ private void generateDecodeListFrom( .append(indent).append("}\n"); } - private void generateMessageDecodeFrom( + private void generateDecodeWith( final ClassBuilder classBuilder, final String dtoClassName, final String codecClassName, @@ -459,13 +439,35 @@ private void generateMessageDecodeFrom( .append(dtoClassName).append("& dto)\n") .append(indent).append("{\n"); - generateMessageFieldsDecodeFrom(decodeBuilder, fields, codecClassName, indent + INDENT); - generateGroupsDecodeFrom(decodeBuilder, groups, indent + INDENT); - generateVarDataDecodeFrom(decodeBuilder, varData, indent + INDENT); + generateMessageFieldsDecodeWith(decodeBuilder, fields, codecClassName, indent + INDENT); + generateGroupsDecodeWith(decodeBuilder, groups, indent + INDENT); + generateVarDataDecodeWith(decodeBuilder, varData, indent + INDENT); decodeBuilder.append(indent).append("}\n"); } - private void generateMessageFieldsDecodeFrom( + private static void generateDecodeFrom( + final ClassBuilder classBuilder, + final String dtoClassName, + final String codecClassName, + final String indent) + { + classBuilder.appendPublic() + .append("\n") + .append(indent).append("static ").append(dtoClassName).append(" decodeFrom(") + .append("char* buffer, std::uint64_t offset, ") + .append("std::uint64_t actingBlockLength, std::uint64_t actingVersion, ") + .append("std::uint64_t bufferLength)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(codecClassName).append(" codec;\n") + .append(indent).append(INDENT) + .append("codec.wrapForDecode(buffer, offset, actingBlockLength, actingVersion, bufferLength);\n") + .append(indent).append(INDENT).append(dtoClassName).append(" dto;\n") + .append(indent).append(INDENT).append(dtoClassName).append("::decodeWith(codec, dto);\n") + .append(indent).append(INDENT).append("return dto;\n") + .append(indent).append("}\n"); + } + + private void generateMessageFieldsDecodeWith( final StringBuilder sb, final List tokens, final String codecClassName, @@ -478,12 +480,12 @@ private void generateMessageFieldsDecodeFrom( { final Token encodingToken = tokens.get(i + 1); - generateFieldDecodeFrom(sb, signalToken, encodingToken, codecClassName, indent); + generateFieldDecodeWith(sb, signalToken, encodingToken, codecClassName, indent); } } } - private void generateFieldDecodeFrom( + private void generateFieldDecodeWith( final StringBuilder sb, final Token fieldToken, final Token typeToken, @@ -493,20 +495,20 @@ private void generateFieldDecodeFrom( switch (typeToken.signal()) { case ENCODING: - generatePrimitiveDecodeFrom(sb, fieldToken, typeToken, codecClassName, indent); + generatePrimitiveDecodeWith(sb, fieldToken, typeToken, codecClassName, indent); break; case BEGIN_SET: final String bitSetName = formatDtoClassName(typeToken.applicableTypeName()); - generateBitSetDecodeFrom(sb, fieldToken, bitSetName, indent); + generateBitSetDecodeWith(sb, fieldToken, bitSetName, indent); break; case BEGIN_ENUM: - generateEnumDecodeFrom(sb, fieldToken, indent); + generateEnumDecodeWith(sb, fieldToken, indent); break; case BEGIN_COMPOSITE: - generateCompositePropertyDecodeFrom(sb, fieldToken, typeToken, indent); + generateCompositePropertyDecodeWith(sb, fieldToken, typeToken, indent); break; default: @@ -514,7 +516,7 @@ private void generateFieldDecodeFrom( } } - private void generatePrimitiveDecodeFrom( + private void generatePrimitiveDecodeWith( final StringBuilder sb, final Token fieldToken, final Token typeToken, @@ -551,11 +553,11 @@ private void generatePrimitiveDecodeFrom( } else if (arrayLength > 1) { - generateArrayDecodeFrom(sb, fieldToken, typeToken, codecClassName, indent); + generateArrayDecodeWith(sb, fieldToken, typeToken, codecClassName, indent); } } - private void generateArrayDecodeFrom( + private void generateArrayDecodeWith( final StringBuilder sb, final Token fieldToken, final Token typeToken, @@ -606,7 +608,7 @@ private void generateArrayDecodeFrom( } } - private void generateBitSetDecodeFrom( + private void generateBitSetDecodeWith( final StringBuilder sb, final Token fieldToken, final String dtoTypeName, @@ -645,7 +647,7 @@ private void generateBitSetDecodeFrom( } } - private void generateEnumDecodeFrom( + private void generateEnumDecodeWith( final StringBuilder sb, final Token fieldToken, final String indent) @@ -662,7 +664,7 @@ private void generateEnumDecodeFrom( .append("codec.").append(formattedPropertyName).append("());\n"); } - private void generateCompositePropertyDecodeFrom( + private void generateCompositePropertyDecodeWith( final StringBuilder sb, final Token fieldToken, final Token typeToken, @@ -677,7 +679,7 @@ private void generateCompositePropertyDecodeFrom( .append("dto.").append(formattedPropertyName).append("());\n"); } - private void generateGroupsDecodeFrom( + private void generateGroupsDecodeWith( final StringBuilder sb, final List tokens, final String indent) @@ -711,7 +713,7 @@ private void generateGroupsDecodeFrom( } } - private void generateVarDataDecodeFrom( + private void generateVarDataDecodeWith( final StringBuilder sb, final List tokens, final String indent) @@ -820,7 +822,7 @@ private void generateRecordPropertyAssignment( } } - private void generateMessageEncodeInto( + private void generateEncodeWith( final ClassBuilder classBuilder, final String dtoClassName, final String codecClassName, @@ -834,14 +836,67 @@ private void generateMessageEncodeInto( .append(dtoClassName).append("& dto)\n") .append(indent).append("{\n"); - generateFieldsEncodeInto(encodeBuilder, codecClassName, fields, indent + INDENT); - generateGroupsEncodeInto(encodeBuilder, groups, indent + INDENT); - generateVarDataEncodeInto(encodeBuilder, varData, indent + INDENT); + generateFieldsEncodeWith(encodeBuilder, codecClassName, fields, indent + INDENT); + generateGroupsEncodeWith(encodeBuilder, groups, indent + INDENT); + generateVarDataEncodeWith(encodeBuilder, varData, indent + INDENT); encodeBuilder.append(indent).append("}\n"); } - private void generateFieldsEncodeInto( + private static void generateEncodeInto( + final ClassBuilder classBuilder, + final String dtoClassName, + final String codecClassName, + final String indent) + { + classBuilder.appendPublic() + .append("\n") + .append(indent).append("static std::size_t encodeInto(const ").append(dtoClassName).append("& dto, ") + .append("char *buffer, std::uint64_t offset, std::uint64_t bufferLength)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(codecClassName).append(" codec;\n") + .append(indent).append(INDENT).append("codec.wrapForEncode(buffer, offset, bufferLength);\n") + .append(indent).append(INDENT).append(dtoClassName).append("::encodeWith(codec, dto);\n") + .append(indent).append(INDENT).append("return codec.encodedLength();\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic() + .append("\n") + .append(indent).append("static std::size_t encodeWithHeaderInto(const ") + .append(dtoClassName).append("& dto, ") + .append("char *buffer, std::uint64_t offset, std::uint64_t bufferLength)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(codecClassName).append(" codec;\n") + .append(indent).append(INDENT).append("codec.wrapAndApplyHeader(buffer, offset, bufferLength);\n") + .append(indent).append(INDENT).append(dtoClassName).append("::encodeWith(codec, dto);\n") + .append(indent).append(INDENT).append("return codec.sbePosition() - offset;\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic() + .append("\n") + .append(indent).append("[[nodiscard]] static std::vector bytes(const ") + .append(dtoClassName).append("& dto)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("std::vector bytes(dto.computeEncodedLength());\n") + .append(indent).append(INDENT).append(dtoClassName) + .append("::encodeInto(dto, reinterpret_cast(bytes.data()), 0, bytes.size());\n") + .append(indent).append(INDENT).append("return bytes;\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic() + .append("\n") + .append(indent).append("[[nodiscard]] static std::vector bytesWithHeader(const ") + .append(dtoClassName).append("& dto)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("std::vector bytes(dto.computeEncodedLength() + ") + .append("MessageHeader::encodedLength());\n") + .append(indent).append(INDENT).append(dtoClassName) + .append("::encodeWithHeaderInto(dto, reinterpret_cast(bytes.data()), 0, bytes.size());\n") + .append(indent).append(INDENT).append("return bytes;\n") + .append(indent).append("}\n"); + } + + private void generateFieldsEncodeWith( final StringBuilder sb, final String codecClassName, final List tokens, @@ -853,12 +908,12 @@ private void generateFieldsEncodeInto( if (signalToken.signal() == Signal.BEGIN_FIELD) { final Token encodingToken = tokens.get(i + 1); - generateFieldEncodeInto(sb, codecClassName, signalToken, encodingToken, indent); + generateFieldEncodeWith(sb, codecClassName, signalToken, encodingToken, indent); } } } - private void generateFieldEncodeInto( + private void generateFieldEncodeWith( final StringBuilder sb, final String codecClassName, final Token fieldToken, @@ -868,16 +923,16 @@ private void generateFieldEncodeInto( switch (typeToken.signal()) { case ENCODING: - generatePrimitiveEncodeInto(sb, codecClassName, fieldToken, typeToken, indent); + generatePrimitiveEncodeWith(sb, codecClassName, fieldToken, typeToken, indent); break; case BEGIN_ENUM: - generateEnumEncodeInto(sb, fieldToken, indent); + generateEnumEncodeWith(sb, fieldToken, indent); break; case BEGIN_SET: case BEGIN_COMPOSITE: - generateComplexPropertyEncodeInto(sb, fieldToken, typeToken, indent); + generateComplexPropertyEncodeWith(sb, fieldToken, typeToken, indent); break; default: @@ -885,7 +940,7 @@ private void generateFieldEncodeInto( } } - private void generatePrimitiveEncodeInto( + private void generatePrimitiveEncodeWith( final StringBuilder sb, final String codecClassName, final Token fieldToken, @@ -901,15 +956,15 @@ private void generatePrimitiveEncodeInto( if (arrayLength == 1) { - generatePrimitiveValueEncodeInto(sb, codecClassName, fieldToken, indent); + generatePrimitiveValueEncodeWith(sb, codecClassName, fieldToken, indent); } else if (arrayLength > 1) { - generateArrayEncodeInto(sb, fieldToken, typeToken, indent); + generateArrayEncodeWith(sb, fieldToken, typeToken, indent); } } - private void generateArrayEncodeInto( + private void generateArrayEncodeWith( final StringBuilder sb, final Token fieldToken, final Token typeToken, @@ -964,7 +1019,7 @@ private void generateArrayEncodeInto( } } - private void generatePrimitiveValueEncodeInto( + private void generatePrimitiveValueEncodeWith( final StringBuilder sb, final String codecClassName, final Token fieldToken, @@ -988,7 +1043,7 @@ private void generatePrimitiveValueEncodeInto( .append(value).append(");\n"); } - private void generateEnumEncodeInto( + private void generateEnumEncodeWith( final StringBuilder sb, final Token fieldToken, final String indent) @@ -1005,7 +1060,7 @@ private void generateEnumEncodeInto( .append(formattedPropertyName).append("());\n"); } - private void generateComplexPropertyEncodeInto( + private void generateComplexPropertyEncodeWith( final StringBuilder sb, final Token fieldToken, final Token typeToken, @@ -1020,7 +1075,7 @@ private void generateComplexPropertyEncodeInto( .append(formattedPropertyName).append("());\n"); } - private void generateGroupsEncodeInto( + private void generateGroupsEncodeWith( final StringBuilder sb, final List tokens, final String indent) @@ -1063,7 +1118,7 @@ private void generateGroupsEncodeInto( } } - private void generateVarDataEncodeInto( + private void generateVarDataEncodeWith( final StringBuilder sb, final List tokens, final String indent) @@ -1657,9 +1712,9 @@ private void generateComposite(final List tokens) throws IOException final ClassBuilder classBuilder = new ClassBuilder(className, BASE_INDENT); generateCompositePropertyElements(classBuilder, codecClassName, compositeTokens, BASE_INDENT + INDENT); - generateCompositeDecodeFrom(classBuilder, className, codecClassName, compositeTokens, + generateCompositeDecodeWith(classBuilder, className, codecClassName, compositeTokens, BASE_INDENT + INDENT); - generateCompositeEncodeInto(classBuilder, className, codecClassName, compositeTokens, BASE_INDENT + INDENT); + generateCompositeEncodeWith(classBuilder, className, codecClassName, compositeTokens, BASE_INDENT + INDENT); generateDisplay(classBuilder, className, codecClassName, codecClassName + "::encodedLength()", "wrap", codecClassName + "::sbeSchemaVersion()", BASE_INDENT + INDENT); @@ -1686,8 +1741,8 @@ private void generateChoiceSet(final List tokens) throws IOException final ClassBuilder classBuilder = new ClassBuilder(className, BASE_INDENT); generateChoices(classBuilder, className, setTokens, BASE_INDENT + INDENT); - generateChoiceSetDecodeFrom(classBuilder, className, codecClassName, setTokens, BASE_INDENT + INDENT); - generateChoiceSetEncodeInto(classBuilder, className, codecClassName, setTokens, BASE_INDENT + INDENT); + generateChoiceSetDecodeWith(classBuilder, className, codecClassName, setTokens, BASE_INDENT + INDENT); + generateChoiceSetEncodeWith(classBuilder, className, codecClassName, setTokens, BASE_INDENT + INDENT); classBuilder.appendTo(out); out.append("} // namespace\n"); @@ -1695,7 +1750,7 @@ private void generateChoiceSet(final List tokens) throws IOException } } - private void generateChoiceSetEncodeInto( + private void generateChoiceSetEncodeWith( final ClassBuilder classBuilder, final String dtoClassName, final String codecClassName, @@ -1724,7 +1779,7 @@ private void generateChoiceSetEncodeInto( encodeBuilder.append(indent).append("}\n"); } - private void generateChoiceSetDecodeFrom( + private void generateChoiceSetDecodeWith( final ClassBuilder classBuilder, final String dtoClassName, final String codecClassName, diff --git a/sbe-tool/src/test/cpp/DtoTest.cpp b/sbe-tool/src/test/cpp/DtoTest.cpp index 2b37c6b910..f5d2d0cd58 100644 --- a/sbe-tool/src/test/cpp/DtoTest.cpp +++ b/sbe-tool/src/test/cpp/DtoTest.cpp @@ -167,7 +167,7 @@ class DtoTest : public testing::Test } }; -TEST_F(DtoTest, shouldRoundTripCar) +TEST_F(DtoTest, shouldRoundTripCar1) { char input[BUFFER_LEN]; std::memset(input, 0, BUFFER_LEN); @@ -205,3 +205,35 @@ TEST_F(DtoTest, shouldRoundTripCar) EXPECT_EQ(0, std::memcmp(input, output, encodedCarLength2)); EXPECT_EQ(originalString, dtoString); } + +TEST_F(DtoTest, shouldRoundTripCar2) +{ + char input[BUFFER_LEN]; + std::memset(input, 0, BUFFER_LEN); + ExtendedCar encoder; + encoder.wrapForEncode(input, 0, BUFFER_LEN); + const std::uint64_t encodedCarLength = encodeCar(encoder); + + ExtendedCarDto dto = ExtendedCarDto::decodeFrom( + input, + 0, + ExtendedCar::sbeBlockLength(), + ExtendedCar::sbeSchemaVersion(), + encodedCarLength); + + EXPECT_EQ(encodedCarLength, dto.computeEncodedLength()); + + std::vector output = ExtendedCarDto::bytes(dto); + + std::ostringstream originalStringStream; + originalStringStream << encoder; + std::string originalString = originalStringStream.str(); + + std::ostringstream dtoStringStream; + dtoStringStream << dto; + std::string dtoString = dtoStringStream.str(); + + EXPECT_EQ(originalString, dtoString); + EXPECT_EQ(encodedCarLength, output.size()); + EXPECT_EQ(0, std::memcmp(input, output.data(), encodedCarLength)); +} From f965d2a7e649d8b2b4646e54e8ccd032accc0057 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Thu, 9 Nov 2023 15:07:44 +0000 Subject: [PATCH 30/41] [C++] Address feedback from Todd re var data representation. It is more-idiomatic to represent variable-length data using `std::string` even when there is no character encoding specified, the the `std::string` API provides useful utilities regardless. --- .../sbe/generation/cpp/CppDtoGenerator.java | 39 ++----------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java index c13a3211b1..99aae05c3b 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java @@ -724,8 +724,6 @@ private void generateVarDataDecodeWith( if (token.signal() == Signal.BEGIN_VAR_DATA) { final String propertyName = token.name(); - final Token varDataToken = Generators.findFirst("varData", tokens, i); - final String characterEncoding = varDataToken.encoding().characterEncoding(); final String formattedPropertyName = formatPropertyName(propertyName); final boolean isOptional = token.version() > 0; @@ -739,19 +737,8 @@ private void generateVarDataDecodeWith( .append(blockIndent).append("const char* ").append(dataVar) .append(" = codec.").append(formattedPropertyName).append("();\n"); - final String dtoValue; - final String nullDtoValue; - - if (characterEncoding == null) - { - dtoValue = "std::vector(" + dataVar + ", " + dataVar + " + " + lengthVar + ")"; - nullDtoValue = "std::vector()"; - } - else - { - dtoValue = "std::string(" + dataVar + ", " + lengthVar + ")"; - nullDtoValue = "\"\""; - } + final String dtoValue = "std::string(" + dataVar + ", " + lengthVar + ")"; + final String nullDtoValue = "\"\""; if (isOptional) { @@ -1129,29 +1116,13 @@ private void generateVarDataEncodeWith( if (token.signal() == Signal.BEGIN_VAR_DATA) { final String propertyName = token.name(); - final Token lengthToken = Generators.findFirst("length", tokens, i); - final String lengthTypeName = cppTypeName(lengthToken.encoding().primitiveType()); - final Token varDataToken = Generators.findFirst("varData", tokens, i); - final String characterEncoding = varDataToken.encoding().characterEncoding(); final String formattedPropertyName = formatPropertyName(propertyName); final String varName = toLowerFirstChar(propertyName) + "Vector"; sb.append(indent).append("auto& ").append(varName).append(" = dto.") .append(formattedPropertyName).append("();\n") .append(indent).append("codec.put").append(toUpperFirstChar(propertyName)) - .append("("); - - if (null == characterEncoding) - { - sb.append("reinterpret_cast(").append(varName).append(".data()), ") - .append("static_cast<").append(lengthTypeName).append(">(").append(varName).append(".size())"); - } - else - { - sb.append(varName); - } - - sb.append(");\n"); + .append("(").append(varName).append(");\n"); } } } @@ -1629,9 +1600,7 @@ private void generateVarData( if (token.signal() == Signal.BEGIN_VAR_DATA) { final String propertyName = token.name(); - final Token varDataToken = Generators.findFirst("varData", tokens, i); - final String characterEncoding = varDataToken.encoding().characterEncoding(); - final String dtoType = characterEncoding == null ? "std::vector" : "std::string"; + final String dtoType = "std::string"; final String fieldName = "m_" + toLowerFirstChar(propertyName); final String formattedPropertyName = formatPropertyName(propertyName); From 21cde2a3d9fc82e09b28fa228c3dfda27bf3201a Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Sat, 11 Nov 2023 17:56:52 +0000 Subject: [PATCH 31/41] [C++] Add PBT for C++ DTOs. The main change in this commit is to add property-based testing around the C++ DTOs. We check the same property as the C# tests, i.e., that decoding and re-encoding via a DTO preserves the original bytes. There are some other smaller changes in this commit: 1. We now only generate arbitrary schemas where enums have at least one case, as this is required by the spec. 2. We now log details about how we encoded the `input.dat` file used in property-based tests, which can help diagnose problems in the test. --- .../sbe/generation/cpp/CppDtoGenerator.java | 33 ++- .../properties/CSharpDtosPropertyTest.java | 159 ---------- .../sbe/properties/DtosPropertyTest.java | 273 ++++++++++++++++++ .../arbitraries/SbeArbitraries.java | 267 +++++++++++++---- .../resources/CppDtosPropertyTest/main.cpp | 54 ++++ 5 files changed, 559 insertions(+), 227 deletions(-) delete mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java create mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java create mode 100644 sbe-tool/src/propertyTest/resources/CppDtosPropertyTest/main.cpp diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java index 99aae05c3b..f2371181d0 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java @@ -121,7 +121,7 @@ public void generate() throws IOException referencedTypes)); out.append(generateDocumentation(BASE_INDENT, msgToken)); classBuilder.appendTo(out); - out.append("} // namespace\n"); + out.append(CppUtil.closingBraces(ir.namespaces().length)); out.append("#endif\n"); } } @@ -1336,15 +1336,21 @@ private void generateArrayProperty( { final String fieldName = "m_" + toLowerFirstChar(propertyName); final String formattedPropertyName = formatPropertyName(propertyName); + final String validateMethod = "validate" + toUpperFirstChar(propertyName); if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) { + final CharSequence typeName = typeWithFieldOptionality( + fieldToken, + "std::string" + ); + classBuilder.appendField() - .append(indent).append("std::string ").append(fieldName).append(";\n"); + .append(indent).append(typeName).append(" ").append(fieldName).append(";\n"); classBuilder.appendPublic().append("\n") .append(generateDocumentation(indent, fieldToken)) - .append(indent).append("[[nodiscard]] const std::string& ") + .append(indent).append("[[nodiscard]] const ").append(typeName).append("& ") .append(formattedPropertyName).append("() const\n") .append(indent).append("{\n") .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") @@ -1353,22 +1359,31 @@ private void generateArrayProperty( classBuilder.appendPublic().append("\n") .append(generateDocumentation(indent, fieldToken)) .append(indent).append("void ").append(formattedPropertyName) - .append("(const std::string& borrowedValue)\n") + .append("(const ").append(typeName).append("& borrowedValue)\n") .append(indent).append("{\n") .append(indent).append(INDENT).append(fieldName).append(" = borrowedValue;\n") .append(indent).append("}\n"); classBuilder.appendPublic().append("\n") .append(generateDocumentation(indent, fieldToken)) - .append(indent).append("void ").append(formattedPropertyName) - .append("(std::string&& ownedValue)\n") + .append(indent).append("void ").append(formattedPropertyName).append("(") + .append(typeName).append("&& ownedValue)\n") .append(indent).append("{\n") .append(indent).append(INDENT).append(fieldName).append(" = std::move(ownedValue);\n") .append(indent).append("}\n"); + + generateArrayValidateMethod( + classBuilder, + codecClassName, + fieldToken, + indent, + validateMethod, + typeName, + "std::string", + formattedPropertyName); } else { - final String validateMethod = "validate" + toUpperFirstChar(propertyName); final String elementTypeName = cppTypeName(typeToken.encoding().primitiveType()); final String vectorTypeName = "std::vector<" + elementTypeName + ">"; final CharSequence typeName = typeWithFieldOptionality( @@ -1688,7 +1703,7 @@ private void generateComposite(final List tokens) throws IOException codecClassName + "::sbeSchemaVersion()", BASE_INDENT + INDENT); classBuilder.appendTo(out); - out.append("} // namespace\n"); + out.append(CppUtil.closingBraces(ir.namespaces().length)); out.append("#endif\n"); } } @@ -1714,7 +1729,7 @@ private void generateChoiceSet(final List tokens) throws IOException generateChoiceSetEncodeWith(classBuilder, className, codecClassName, setTokens, BASE_INDENT + INDENT); classBuilder.appendTo(out); - out.append("} // namespace\n"); + out.append(CppUtil.closingBraces(ir.namespaces().length)); out.append("#endif\n"); } } diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java deleted file mode 100644 index 701ca2fcf4..0000000000 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/CSharpDtosPropertyTest.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright 2013-2023 Real Logic Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package uk.co.real_logic.sbe.properties; - -import net.jqwik.api.Arbitrary; -import net.jqwik.api.ForAll; -import net.jqwik.api.Property; -import net.jqwik.api.Provide; -import uk.co.real_logic.sbe.generation.csharp.CSharpDtoGenerator; -import uk.co.real_logic.sbe.generation.csharp.CSharpGenerator; -import uk.co.real_logic.sbe.generation.csharp.CSharpNamespaceOutputManager; -import uk.co.real_logic.sbe.properties.arbitraries.SbeArbitraries; -import org.agrona.IoUtil; -import org.agrona.io.DirectBufferInputStream; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; - -public class CSharpDtosPropertyTest -{ - private static final String DOTNET_EXECUTABLE = System.getProperty("sbe.tests.dotnet.executable", "dotnet"); - - @Property - void dtoEncodeShouldBeTheInverseOfDtoDecode( - @ForAll("encodedMessage") final SbeArbitraries.EncodedMessage encodedMessage - ) throws IOException, InterruptedException - { - final Path tempDir = Files.createTempDirectory("sbe-csharp-dto-test"); - try - { - final CSharpNamespaceOutputManager outputManager = new CSharpNamespaceOutputManager( - tempDir.toString(), - "SbePropertyTest" - ); - - try - { - new CSharpGenerator(encodedMessage.ir(), outputManager) - .generate(); - new CSharpDtoGenerator(encodedMessage.ir(), outputManager) - .generate(); - } - catch (final Exception generationException) - { - throw new AssertionError( - "Code generation failed.\n\nSCHEMA:\n" + encodedMessage.schema(), - generationException); - } - - copyResourceToFile("/CSharpDtosPropertyTest/SbePropertyTest.csproj", tempDir); - copyResourceToFile("/CSharpDtosPropertyTest/Program.cs", tempDir); - try ( - DirectBufferInputStream inputStream = new DirectBufferInputStream( - encodedMessage.buffer(), - 0, - encodedMessage.length() - ); - OutputStream outputStream = Files.newOutputStream(tempDir.resolve("input.dat"))) - { - final byte[] buffer = new byte[2048]; - int read; - while ((read = inputStream.read(buffer, 0, buffer.length)) >= 0) - { - outputStream.write(buffer, 0, read); - } - } - - final Path stdout = tempDir.resolve("stdout.txt"); - final Path stderr = tempDir.resolve("stderr.txt"); - final ProcessBuilder processBuilder = new ProcessBuilder(DOTNET_EXECUTABLE, "run", "--", "input.dat") - .directory(tempDir.toFile()) - .redirectOutput(stdout.toFile()) - .redirectError(stderr.toFile()); - - final Process process = processBuilder.start(); - if (0 != process.waitFor()) - { - throw new AssertionError( - "Process failed with exit code: " + process.exitValue() + "\n\n" + - "STDOUT:\n" + new String(Files.readAllBytes(stdout)) + "\n\n" + - "STDERR:\n" + new String(Files.readAllBytes(stderr)) + "\n\n" + - "SCHEMA:\n" + encodedMessage.schema()); - } - - final byte[] errorBytes = Files.readAllBytes(stderr); - if (errorBytes.length != 0) - { - throw new AssertionError( - "Process wrote to stderr.\n\n" + - "STDOUT:\n" + new String(Files.readAllBytes(stdout)) + "\n\n" + - "STDERR:\n" + new String(errorBytes) + "\n\n" + - "SCHEMA:\n" + encodedMessage.schema() + "\n\n" - ); - } - - final byte[] inputBytes = new byte[encodedMessage.length()]; - encodedMessage.buffer().getBytes(0, inputBytes); - final byte[] outputBytes = Files.readAllBytes(tempDir.resolve("output.dat")); - if (!Arrays.equals(inputBytes, outputBytes)) - { - throw new AssertionError( - "Input and output files differ\n\n" + - "SCHEMA:\n" + encodedMessage.schema()); - } - } - finally - { - IoUtil.delete(tempDir.toFile(), true); - } - } - - @Provide - Arbitrary encodedMessage() - { - final SbeArbitraries.CharGenerationMode mode = - SbeArbitraries.CharGenerationMode.JSON_PRINTER_COMPATIBLE; - return SbeArbitraries.encodedMessage(mode); - } - - private static void copyResourceToFile( - final String resourcePath, - final Path outputDir) - { - try (InputStream inputStream = CSharpDtosPropertyTest.class.getResourceAsStream(resourcePath)) - { - if (inputStream == null) - { - throw new IOException("Resource not found: " + resourcePath); - } - - final int resourceNameIndex = resourcePath.lastIndexOf('/') + 1; - final String resourceName = resourcePath.substring(resourceNameIndex); - final Path outputFilePath = outputDir.resolve(resourceName); - Files.copy(inputStream, outputFilePath); - } - catch (final IOException e) - { - throw new RuntimeException(e); - } - } -} diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java new file mode 100644 index 0000000000..d68a1c327c --- /dev/null +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java @@ -0,0 +1,273 @@ +/* + * Copyright 2013-2023 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties; + +import net.jqwik.api.*; +import uk.co.real_logic.sbe.generation.cpp.CppDtoGenerator; +import uk.co.real_logic.sbe.generation.cpp.CppGenerator; +import uk.co.real_logic.sbe.generation.cpp.NamespaceOutputManager; +import uk.co.real_logic.sbe.generation.csharp.CSharpDtoGenerator; +import uk.co.real_logic.sbe.generation.csharp.CSharpGenerator; +import uk.co.real_logic.sbe.generation.csharp.CSharpNamespaceOutputManager; +import uk.co.real_logic.sbe.properties.arbitraries.SbeArbitraries; +import org.agrona.IoUtil; +import org.agrona.io.DirectBufferInputStream; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; + +@SuppressWarnings("ReadWriteStringCanBeUsed") +public class DtosPropertyTest +{ + private static final String DOTNET_EXECUTABLE = System.getProperty("sbe.tests.dotnet.executable", "dotnet"); + private static final String CPP_EXECUTABLE = System.getProperty("sbe.tests.cpp.executable", "g++"); + private static final boolean KEEP_DIR_ON_FAILURE = Boolean.parseBoolean( + System.getProperty("sbe.tests.keep.dir.on.failure", "true")); + + @Property + void csharpDtoEncodeShouldBeTheInverseOfDtoDecode( + @ForAll("encodedMessage") final SbeArbitraries.EncodedMessage encodedMessage + ) throws IOException, InterruptedException + { + final Path tempDir = Files.createTempDirectory("sbe-csharp-dto-test"); + boolean success = false; + + try + { + final CSharpNamespaceOutputManager outputManager = new CSharpNamespaceOutputManager( + tempDir.toString(), + "SbePropertyTest" + ); + + try + { + new CSharpGenerator(encodedMessage.ir(), outputManager) + .generate(); + new CSharpDtoGenerator(encodedMessage.ir(), outputManager) + .generate(); + } + catch (final Exception generationException) + { + throw new AssertionError( + "Code generation failed.\n\n" + + "DIR:" + tempDir + "\n\n" + + "SCHEMA:\n" + encodedMessage.schema(), + generationException); + } + + copyResourceToFile("/CSharpDtosPropertyTest/SbePropertyTest.csproj", tempDir); + copyResourceToFile("/CSharpDtosPropertyTest/Program.cs", tempDir); + + writeInputFile(encodedMessage, tempDir); + + execute(encodedMessage.schema(), tempDir, "test", + DOTNET_EXECUTABLE, "run", "--", "input.dat"); + + final byte[] inputBytes = new byte[encodedMessage.length()]; + encodedMessage.buffer().getBytes(0, inputBytes); + final byte[] outputBytes = Files.readAllBytes(tempDir.resolve("output.dat")); + if (!Arrays.equals(inputBytes, outputBytes)) + { + throw new AssertionError( + "Input and output files differ\n\n" + + "DIR:" + tempDir + "\n\n" + + "SCHEMA:\n" + encodedMessage.schema()); + } + success = true; + } + finally + { + if (!KEEP_DIR_ON_FAILURE || success) + { + IoUtil.delete(tempDir.toFile(), true); + } + else + { + Files.write( + tempDir.resolve("schema.xml"), + encodedMessage.schema().getBytes(StandardCharsets.UTF_8)); + + Files.write( + tempDir.resolve("encoding.log"), + encodedMessage.encodingLog().getBytes(StandardCharsets.UTF_8)); + } + } + } + + @Property(shrinking = ShrinkingMode.OFF) + void cppDtoEncodeShouldBeTheInverseOfDtoDecode( + @ForAll("encodedMessage") final SbeArbitraries.EncodedMessage encodedMessage + ) throws IOException, InterruptedException + { + final Path tempDir = Files.createTempDirectory("sbe-cpp-dto-test"); + boolean success = false; + + try + { + final NamespaceOutputManager outputManager = new NamespaceOutputManager( + tempDir.toString(), + "sbe_property_test" + ); + + try + { + new CppGenerator(encodedMessage.ir(), true, outputManager) + .generate(); + new CppDtoGenerator(encodedMessage.ir(), outputManager) + .generate(); + } + catch (final Exception generationException) + { + throw new AssertionError( + "Code generation failed.\n\nSCHEMA:\n" + encodedMessage.schema(), + generationException); + } + + copyResourceToFile("/CppDtosPropertyTest/main.cpp", tempDir); + + writeInputFile(encodedMessage, tempDir); + + execute(encodedMessage.schema(), tempDir, "compile", + CPP_EXECUTABLE, "--std", "c++17", "-o", "round-trip-test", "main.cpp"); + + execute(encodedMessage.schema(), tempDir, "test", + tempDir.resolve("round-trip-test").toString(), "input.dat"); + + final byte[] inputBytes = new byte[encodedMessage.length()]; + encodedMessage.buffer().getBytes(0, inputBytes); + final byte[] outputBytes = Files.readAllBytes(tempDir.resolve("output.dat")); + if (!Arrays.equals(inputBytes, outputBytes)) + { + throw new AssertionError( + "Input and output files differ\n\n" + + "SCHEMA:\n" + encodedMessage.schema()); + } + success = true; + } + finally + { + if (!KEEP_DIR_ON_FAILURE || success) + { + IoUtil.delete(tempDir.toFile(), true); + } + else + { + Files.write( + tempDir.resolve("schema.xml"), + encodedMessage.schema().getBytes(StandardCharsets.UTF_8)); + + Files.write( + tempDir.resolve("encoding.log"), + encodedMessage.encodingLog().getBytes(StandardCharsets.UTF_8)); + } + } + } + + private static void writeInputFile( + final SbeArbitraries.EncodedMessage encodedMessage, + final Path tempDir) throws IOException + { + try ( + DirectBufferInputStream inputStream = new DirectBufferInputStream( + encodedMessage.buffer(), + 0, + encodedMessage.length() + ); + OutputStream outputStream = Files.newOutputStream(tempDir.resolve("input.dat"))) + { + final byte[] buffer = new byte[2048]; + int read; + while ((read = inputStream.read(buffer, 0, buffer.length)) >= 0) + { + outputStream.write(buffer, 0, read); + } + } + } + + private static void execute( + final String schema, + final Path tempDir, + final String name, + final String... args) throws InterruptedException, IOException + { + final Path stdout = tempDir.resolve(name + "_stdout.txt"); + final Path stderr = tempDir.resolve(name + "_stderr.txt"); + final ProcessBuilder compileProcessBuilder = new ProcessBuilder(args) + .directory(tempDir.toFile()) + .redirectOutput(stdout.toFile()) + .redirectError(stderr.toFile()); + + final Process process = compileProcessBuilder.start(); + + if (0 != process.waitFor()) + { + throw new AssertionError( + "Process failed with exit code: " + process.exitValue() + "\n\n" + + "DIR:" + tempDir + "\n\n" + + "STDOUT:\n" + new String(Files.readAllBytes(stdout)) + "\n\n" + + "STDERR:\n" + new String(Files.readAllBytes(stderr)) + "\n\n" + + "SCHEMA:\n" + schema); + } + + final byte[] errorBytes = Files.readAllBytes(stderr); + if (errorBytes.length != 0) + { + throw new AssertionError( + "Process wrote to stderr.\n\n" + + "DIR:" + tempDir + "\n\n" + + "STDOUT:\n" + new String(Files.readAllBytes(stdout)) + "\n\n" + + "STDERR:\n" + new String(errorBytes) + "\n\n" + + "SCHEMA:\n" + schema + "\n\n" + ); + } + } + + @Provide + Arbitrary encodedMessage() + { + final SbeArbitraries.CharGenerationMode mode = + SbeArbitraries.CharGenerationMode.JSON_PRINTER_COMPATIBLE; + return SbeArbitraries.encodedMessage(mode); + } + + private static void copyResourceToFile( + final String resourcePath, + final Path outputDir) + { + try (InputStream inputStream = DtosPropertyTest.class.getResourceAsStream(resourcePath)) + { + if (inputStream == null) + { + throw new IOException("Resource not found: " + resourcePath); + } + + final int resourceNameIndex = resourcePath.lastIndexOf('/') + 1; + final String resourceName = resourcePath.substring(resourceNameIndex); + final Path outputFilePath = outputDir.resolve(resourceName); + Files.copy(inputStream, outputFilePath); + } + catch (final IOException e) + { + throw new RuntimeException(e); + } + } +} diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java index 25207cdf96..bfc1f9bd88 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java @@ -21,7 +21,6 @@ import net.jqwik.api.Combinators; import net.jqwik.api.arbitraries.CharacterArbitrary; import net.jqwik.api.arbitraries.ListArbitrary; -import net.jqwik.api.arbitraries.ShortArbitrary; import uk.co.real_logic.sbe.PrimitiveType; import uk.co.real_logic.sbe.PrimitiveValue; import uk.co.real_logic.sbe.ir.Encoding; @@ -38,9 +37,7 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; +import java.util.*; import java.util.stream.Collectors; import static uk.co.real_logic.sbe.ir.Signal.*; @@ -130,6 +127,7 @@ private static Arbitrary enumTypeSchema() Arbitraries.chars().alpha() .map(Character::toUpperCase) .list() + .ofMinSize(1) .ofMaxSize(10) .uniqueElements() .map(values -> new EnumTypeSchema( @@ -139,6 +137,7 @@ private static Arbitrary enumTypeSchema() Arbitraries.integers() .between(1, 254) .list() + .ofMinSize(1) .ofMaxSize(254) .uniqueElements() .map(values -> new EnumTypeSchema( @@ -286,18 +285,51 @@ public static Arbitrary messageSchema() ).as(MessageSchema::new); } - public interface Encoder + private interface Encoder { - void encode(MutableDirectBuffer buffer, int offset, MutableInteger limit); + void encode( + EncodingLogger logger, + MutableDirectBuffer buffer, + int offset, + MutableInteger limit); + } + + private static final class EncodingLogger + { + private final StringBuilder builder = new StringBuilder(); + private final Deque scope = new ArrayDeque<>(); + + public void beginScope(final String name) + { + scope.addLast(name); + } + + public StringBuilder appendLine() + { + builder.append("\n"); + scope.forEach(s -> builder.append(".").append(s)); + builder.append(": "); + return builder; + } + + public void endScope() + { + scope.removeLast(); + } + + public String toString() + { + return builder.toString(); + } } private static Encoder combineEncoders(final Collection encoders) { - return (buffer, offset, limit) -> + return (builder, buffer, offset, limit) -> { for (final Encoder encoder : encoders) { - encoder.encode(buffer, offset, limit); + encoder.encode(builder, buffer, offset, limit); } }; } @@ -339,35 +371,57 @@ private static Arbitrary encodedTypeEncoder( case CHAR: assert minValue.longValue() <= maxValue.longValue(); return chars(charGenerationMode).map(c -> - (buffer, offset, limit) -> buffer.putChar(offset, c, encoding.byteOrder())); + (builder, buffer, offset, limit) -> + { + builder.appendLine().append(c).append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_BYTE).append("]"); + buffer.putChar(offset, c, encoding.byteOrder()); + }); case UINT8: case INT8: assert (short)minValue.longValue() <= (short)maxValue.longValue(); return Arbitraries.shorts() .between((short)minValue.longValue(), (short)maxValue.longValue()) - .map(b -> (buffer, offset, limit) -> buffer.putByte(offset, (byte)(short)b)); + .map(b -> (builder, buffer, offset, limit) -> + { + builder.appendLine().append((byte)(short)b).append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_BYTE).append("]"); + buffer.putByte(offset, (byte)(short)b); + }); case UINT16: case INT16: assert (int)minValue.longValue() <= (int)maxValue.longValue(); return Arbitraries.integers() .between((int)minValue.longValue(), (int)maxValue.longValue()) - .map(s -> (buffer, offset, limit) -> buffer.putShort(offset, (short)(int)s, encoding.byteOrder())); + .map(s -> (builder, buffer, offset, limit) -> + { + builder.appendLine().append((short)(int)s).append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_SHORT).append("]"); + buffer.putShort(offset, (short)(int)s, encoding.byteOrder()); + }); case UINT32: case INT32: assert minValue.longValue() <= maxValue.longValue(); return Arbitraries.longs() .between(minValue.longValue(), maxValue.longValue()) - .map(i -> (buffer, offset, limit) -> buffer.putInt(offset, (int)(long)i, encoding.byteOrder())); + .map(i -> (builder, buffer, offset, limit) -> + { + builder.appendLine().append((int)(long)i).append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_INT).append("]"); + buffer.putInt(offset, (int)(long)i, encoding.byteOrder()); + }); case UINT64: return Arbitraries.longs() - .map(l -> (buffer, offset, limit) -> + .map(l -> (builder, buffer, offset, limit) -> { final long nullValue = encoding.applicableNullValue().longValue(); final long nonNullValue = l == nullValue ? minValue.longValue() : l; + builder.appendLine().append(nonNullValue).append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_LONG).append("]"); buffer.putLong(offset, nonNullValue, encoding.byteOrder()); }); @@ -375,15 +429,30 @@ private static Arbitrary encodedTypeEncoder( assert minValue.longValue() <= maxValue.longValue(); return Arbitraries.longs() .between(minValue.longValue(), maxValue.longValue()) - .map(l -> (buffer, offset, limit) -> buffer.putLong(offset, l, encoding.byteOrder())); + .map(l -> (builder, buffer, offset, limit) -> + { + builder.appendLine().append(l).append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_LONG).append("]"); + buffer.putLong(offset, l, encoding.byteOrder()); + }); case FLOAT: return Arbitraries.floats() - .map(f -> (buffer, offset, limit) -> buffer.putFloat(offset, f, encoding.byteOrder())); + .map(f -> (builder, buffer, offset, limit) -> + { + builder.appendLine().append(f).append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_FLOAT).append("]"); + buffer.putFloat(offset, f, encoding.byteOrder()); + }); case DOUBLE: return Arbitraries.doubles() - .map(d -> (buffer, offset, limit) -> buffer.putDouble(offset, d, encoding.byteOrder())); + .map(d -> (builder, buffer, offset, limit) -> + { + builder.appendLine().append(d).append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_DOUBLE).append("]"); + buffer.putDouble(offset, d, encoding.byteOrder()); + }); default: throw new IllegalArgumentException("Unsupported type: " + encoding.primitiveType()); @@ -400,18 +469,20 @@ private static Arbitrary encodedTypeEncoder( if (typeToken.arrayLength() == 1) { - return arbEncoder.map(encoder -> (buffer, bufferOffset, limit) -> - encoder.encode(buffer, bufferOffset + offset, limit)); + return arbEncoder.map(encoder -> (builder, buffer, bufferOffset, limit) -> + encoder.encode(builder, buffer, bufferOffset + offset, limit)); } else { return arbEncoder.list().ofSize(typeToken.arrayLength()) - .map(encoders -> (buffer, bufferOffset, limit) -> + .map(encoders -> (builder, buffer, bufferOffset, limit) -> { for (int i = 0; i < typeToken.arrayLength(); i++) { + builder.beginScope("[" + i + "]"); final int elementOffset = bufferOffset + offset + i * encoding.primitiveType().size(); - encoders.get(i).encode(buffer, elementOffset, limit); + encoders.get(i).encode(builder, buffer, elementOffset, limit); + builder.endScope(); } }); } @@ -419,7 +490,7 @@ private static Arbitrary encodedTypeEncoder( private static Encoder emptyEncoder() { - return (buffer, offset, limit) -> + return (builder, buffer, offset, limit) -> { }; } @@ -432,19 +503,43 @@ private static Encoder integerValueEncoder(final Encoding encoding, final long v case CHAR: case UINT8: case INT8: - return (buffer, offset, limit) -> buffer.putByte(offset, (byte)value); + return (builder, buffer, offset, limit) -> + { + builder.appendLine().append((byte)value).append("[").append(value).append("]") + .append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_BYTE).append("]"); + buffer.putByte(offset, (byte)value); + }; case UINT16: case INT16: - return (buffer, offset, limit) -> buffer.putShort(offset, (short)value, encoding.byteOrder()); + return (builder, buffer, offset, limit) -> + { + builder.appendLine().append((short)value).append("[").append(value).append("]") + .append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_SHORT).append("]"); + buffer.putShort(offset, (short)value, encoding.byteOrder()); + }; case UINT32: case INT32: - return (buffer, offset, limit) -> buffer.putInt(offset, (int)value, encoding.byteOrder()); + return (builder, buffer, offset, limit) -> + { + builder.appendLine().append((int)value).append("[").append(value).append("]") + .append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_INT).append("]"); + buffer.putInt(offset, (int)value, encoding.byteOrder()); + }; case UINT64: case INT64: - return (buffer, offset, limit) -> buffer.putLong(offset, value, encoding.byteOrder()); + return (builder, buffer, offset, limit) -> + { + builder.appendLine().append(value).append("[").append(value).append("]") + .append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_LONG).append("]"); + buffer.putLong(offset, value, encoding.byteOrder()); + }; default: throw new IllegalArgumentException("Unsupported type: " + type); @@ -484,7 +579,8 @@ private static Arbitrary enumEncoder( } return Arbitraries.of(encoders).map(encoder -> - (buffer, bufferOffset, limit) -> encoder.encode(buffer, bufferOffset + offset, limit)); + (builder, buffer, bufferOffset, limit) -> + encoder.encode(builder, buffer, bufferOffset + offset, limit)); } private static Encoder choiceEncoder(final Encoding encoding) @@ -495,42 +591,54 @@ private static Encoder choiceEncoder(final Encoding encoding) { case UINT8: case INT8: - return (buffer, offset, limit) -> + return (builder, buffer, offset, limit) -> { buffer.checkLimit(offset + BitUtil.SIZE_OF_BYTE); final byte oldValue = buffer.getByte(offset); final byte newValue = (byte)(oldValue | (1 << choiceBitIdx)); buffer.putByte(offset, newValue); + builder.appendLine().append("oldValue: ").append(oldValue); + builder.appendLine().append(newValue).append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_BYTE).append("]"); }; case UINT16: case INT16: - return (buffer, offset, limit) -> + return (builder, buffer, offset, limit) -> { buffer.checkLimit(offset + BitUtil.SIZE_OF_SHORT); final short oldValue = buffer.getShort(offset, encoding.byteOrder()); final short newValue = (short)(oldValue | (1 << choiceBitIdx)); buffer.putShort(offset, newValue, encoding.byteOrder()); + builder.appendLine().append("oldValue: ").append(oldValue); + builder.appendLine().append(newValue).append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_SHORT).append("]"); }; case UINT32: case INT32: - return (buffer, offset, limit) -> + return (builder, buffer, offset, limit) -> { buffer.checkLimit(offset + BitUtil.SIZE_OF_INT); final int oldValue = buffer.getInt(offset, encoding.byteOrder()); final int newValue = oldValue | (1 << choiceBitIdx); buffer.putInt(offset, newValue, encoding.byteOrder()); + builder.appendLine().append("oldValue: ").append(oldValue); + builder.appendLine().append(newValue).append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_INT).append("]"); }; case UINT64: case INT64: - return (buffer, offset, limit) -> + return (builder, buffer, offset, limit) -> { buffer.checkLimit(offset + BitUtil.SIZE_OF_LONG); final long oldValue = buffer.getLong(offset, encoding.byteOrder()); final long newValue = oldValue | (1L << choiceBitIdx); buffer.putLong(offset, newValue, encoding.byteOrder()); + builder.appendLine().append("oldValue: ").append(oldValue); + builder.appendLine().append(newValue).append(" @ ").append(offset) + .append("[").append(BitUtil.SIZE_OF_LONG).append("]"); }; default: @@ -568,7 +676,8 @@ private static Arbitrary bitSetEncoder( return Arbitraries.subsetOf(encoders) .map(SbeArbitraries::combineEncoders) - .map(encoder -> (buffer, bufferOffset, limit) -> encoder.encode(buffer, bufferOffset + offset, limit)); + .map(encoder -> (builder, buffer, bufferOffset, limit) -> + encoder.encode(builder, buffer, bufferOffset + offset, limit)); } private static Arbitrary fieldsEncoder( @@ -602,6 +711,7 @@ else if (expectFields) if (!memberToken.isConstantEncoding()) { + Arbitrary fieldEncoder = null; switch (typeToken.signal()) { case BEGIN_COMPOSITE: @@ -610,30 +720,40 @@ else if (expectFields) final int lastMemberIdx = nextFieldIdx - endCompositeTokenCount - endFieldTokenCount - 1; final Arbitrary encoder = fieldsEncoder( tokens, cursor, lastMemberIdx, false, charGenerationMode); - final Arbitrary positionedEncoder = encoder.map(e -> - (buffer, bufferOffset, limit) -> e.encode(buffer, bufferOffset + offset, limit)); - encoders.add(positionedEncoder); + fieldEncoder = encoder.map(e -> + (builder, buffer, bufferOffset, limit) -> + e.encode(builder, buffer, bufferOffset + offset, limit)); break; case BEGIN_ENUM: final int endEnumTokenCount = 1; final int lastValidValueIdx = nextFieldIdx - endFieldTokenCount - endEnumTokenCount - 1; - encoders.add(enumEncoder(offset, tokens, typeToken, cursor, lastValidValueIdx)); + fieldEncoder = enumEncoder(offset, tokens, typeToken, cursor, lastValidValueIdx); break; case BEGIN_SET: final int endSetTokenCount = 1; final int lastChoiceIdx = nextFieldIdx - endFieldTokenCount - endSetTokenCount - 1; - encoders.add(bitSetEncoder(offset, tokens, cursor, lastChoiceIdx)); + fieldEncoder = bitSetEncoder(offset, tokens, cursor, lastChoiceIdx); break; case ENCODING: - encoders.add(encodedTypeEncoder(offset, typeToken, charGenerationMode)); + fieldEncoder = encodedTypeEncoder(offset, typeToken, charGenerationMode); break; default: break; } + + if (fieldEncoder != null) + { + encoders.add(fieldEncoder.map(encoder -> (builder, buffer, off, limit) -> + { + builder.beginScope(memberToken.name()); + encoder.encode(builder, buffer, off, limit); + builder.endScope(); + })); + } } cursor.set(nextFieldIdx); @@ -677,29 +797,39 @@ private static Arbitrary groupsEncoder( groupsEncoder(tokens, cursor, nextFieldIdx - 1, charGenerationMode), varDataEncoder(tokens, cursor, nextFieldIdx - 1, charGenerationMode) ).as((fieldsEncoder, groupsEncoder, varDataEncoder) -> - (buffer, ignored, limit) -> + (builder, buffer, ignored, limit) -> { final int offset = limit.get(); - fieldsEncoder.encode(buffer, offset, null); + fieldsEncoder.encode(builder, buffer, offset, null); limit.set(offset + blockLength); - groupsEncoder.encode(buffer, NULL_VALUE, limit); - varDataEncoder.encode(buffer, NULL_VALUE, limit); + builder.appendLine().append("limit: ").append(offset).append(" -> ").append(limit.get()); + groupsEncoder.encode(builder, buffer, NULL_VALUE, limit); + varDataEncoder.encode(builder, buffer, NULL_VALUE, limit); }); final Arbitrary repeatingGroupEncoder = groupElement.list() .ofMaxSize(10) - .map(elements -> (buffer, ignored, limit) -> + .map(elements -> (builder, buffer, ignored, limit) -> { final int offset = limit.get(); limit.set(offset + headerLength); - blockLengthEncoder.encode(buffer, offset, null); + builder.beginScope(token.name()); + builder.appendLine().append("limit: ").append(offset).append(" -> ").append(limit.get()); + builder.beginScope("blockLength"); + blockLengthEncoder.encode(builder, buffer, offset, null); + builder.endScope(); + builder.beginScope("numInGroup"); integerValueEncoder(numInGroupToken.encoding(), elements.size()) - .encode(buffer, offset + blockLengthToken.encodedLength(), null); - - for (final Encoder element : elements) + .encode(builder, buffer, offset + blockLengthToken.encodedLength(), null); + builder.endScope(); + for (int i = 0; i < elements.size(); i++) { - element.encode(buffer, NULL_VALUE, limit); + final Encoder element = elements.get(i); + builder.beginScope("[" + i + "]"); + element.encode(builder, buffer, NULL_VALUE, limit); + builder.endScope(); } + builder.endScope(); }); encoders.add(repeatingGroupEncoder); @@ -738,20 +868,28 @@ private static Arbitrary varDataEncoder( final Arbitrary arbitraryByte = null == characterEncoding ? Arbitraries.bytes() : chars(charGenerationMode).map(c -> (byte)c.charValue()); - encoders.add(arbitraryByte.list().map(bytes -> - (buffer, ignored, limit) -> + encoders.add(arbitraryByte.list() + .ofMaxSize((int)Math.min(lengthToken.encoding().applicableMaxValue().longValue(), 260L)) + .map(bytes -> (builder, buffer, ignored, limit) -> { final int offset = limit.get(); final int elementLength = varDataToken.encoding().primitiveType().size(); limit.set(offset + lengthToken.encodedLength() + bytes.size() * elementLength); + builder.beginScope(token.name()); + builder.appendLine().append("limit: ").append(offset).append(" -> ").append(limit.get()); + builder.beginScope("length"); integerValueEncoder(lengthToken.encoding(), bytes.size()) - .encode(buffer, offset, null); + .encode(builder, buffer, offset, null); + builder.endScope(); for (int i = 0; i < bytes.size(); i++) { final int dataOffset = offset + lengthToken.encodedLength() + i * elementLength; + builder.beginScope("[" + i + "]"); integerValueEncoder(varDataToken.encoding(), bytes.get(i)) - .encode(buffer, dataOffset, null); + .encode(builder, buffer, dataOffset, null); + builder.endScope(); } + builder.endScope(); })); cursor.set(nextFieldIdx); @@ -760,7 +898,7 @@ private static Arbitrary varDataEncoder( return combineArbitraryEncoders(encoders); } - public static Arbitrary messageValueEncoder( + private static Arbitrary messageValueEncoder( final Ir ir, final short messageId, final CharGenerationMode charGenerationMode) @@ -781,7 +919,7 @@ public static Arbitrary messageValueEncoder( final Arbitrary varDataEncoder = varDataEncoder( tokens, cursor, tokens.size() - 1, charGenerationMode); return Combinators.combine(fieldsEncoder, groupsEncoder, varDataEncoder) - .as((fields, groups, varData) -> (buffer, offset, limit) -> + .as((fields, groups, varData) -> (builder, buffer, offset, limit) -> { final int blockLength = token.encodedLength(); buffer.putShort(0, (short)blockLength, ir.byteOrder()); @@ -789,10 +927,12 @@ public static Arbitrary messageValueEncoder( buffer.putShort(4, (short)ir.id(), ir.byteOrder()); buffer.putShort(6, (short)ir.version(), ir.byteOrder()); final int headerLength = 8; - fields.encode(buffer, offset + headerLength, null); + fields.encode(builder, buffer, offset + headerLength, null); + final int oldLimit = limit.get(); limit.set(offset + headerLength + blockLength); - groups.encode(buffer, NULL_VALUE, limit); - varData.encode(buffer, NULL_VALUE, limit); + builder.appendLine().append("limit: ").append(oldLimit).append(" -> ").append(limit.get()); + groups.encode(builder, buffer, NULL_VALUE, limit); + varData.encode(builder, buffer, NULL_VALUE, limit); }); } @@ -802,17 +942,20 @@ public static final class EncodedMessage private final Ir ir; private final ExpandableArrayBuffer buffer; private final int length; + private final String encodingLog; private EncodedMessage( final String schema, final Ir ir, final ExpandableArrayBuffer buffer, - final int length) + final int length, + final String encodingLog) { this.schema = schema; this.ir = ir; this.buffer = buffer; this.length = length; + this.encodingLog = encodingLog; } public String schema() @@ -834,6 +977,11 @@ public int length() { return length; } + + public String encodingLog() + { + return encodingLog; + } } public static Arbitrary encodedMessage(final CharGenerationMode mode) @@ -853,10 +1001,11 @@ public static Arbitrary encodedMessage(final CharGenerationMode return SbeArbitraries.messageValueEncoder(ir, testSchema.templateId(), mode) .map(encoder -> { + final EncodingLogger logger = new EncodingLogger(); final ExpandableArrayBuffer buffer = new ExpandableArrayBuffer(); final MutableInteger limit = new MutableInteger(); - encoder.encode(buffer, 0, limit); - return new EncodedMessage(xml, ir, buffer, limit.get()); + encoder.encode(logger, buffer, 0, limit); + return new EncodedMessage(xml, ir, buffer, limit.get(), logger.toString()); }); } catch (final Exception e) diff --git a/sbe-tool/src/propertyTest/resources/CppDtosPropertyTest/main.cpp b/sbe-tool/src/propertyTest/resources/CppDtosPropertyTest/main.cpp new file mode 100644 index 0000000000..5d366adf6d --- /dev/null +++ b/sbe-tool/src/propertyTest/resources/CppDtosPropertyTest/main.cpp @@ -0,0 +1,54 @@ +#include +#include +#include +#include +#include "sbe_property_test/MessageHeader.h" +#include "sbe_property_test/TestMessage.h" +#include "sbe_property_test/TestMessageDto.h" + +using namespace uk::co::real_logic::sbe::properties; + +int main(int argc, char* argv[]) { + if (argc != 2) { + std::cout << "Usage: " << argv[0] << " $BINARY_FILE" << std::endl; + return 1; + } + + std::string binaryFile = argv[1]; + + std::cout << "Reading binary file: " << binaryFile << std::endl; + std::ifstream file(binaryFile, std::ios::binary); + std::vector inputBytes((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + + char* buffer = inputBytes.data(); + std::size_t bufferLength = inputBytes.size(); + + MessageHeader messageHeader(buffer, bufferLength); + + TestMessage decoder; + decoder.wrapForDecode( + buffer, + MessageHeader::encodedLength(), + messageHeader.blockLength(), + messageHeader.version(), + bufferLength); + + std::cout << "Decoding binary into DTO" << std::endl; + TestMessageDto dto; + TestMessageDto::decodeWith(decoder, dto); + std::vector outputBytes(inputBytes.size()); + char* outputBuffer = outputBytes.data(); + + TestMessage encoder; + encoder.wrapAndApplyHeader(outputBuffer, 0, bufferLength); + TestMessageDto::encodeWith(encoder, dto); + + std::cout << "Writing binary file: output.dat" << std::endl; + std::ofstream outputFile("output.dat", std::ios::binary); + outputFile.write(outputBuffer, outputBytes.size()); + + std::cout << "Done" << std::endl; + + return 0; +} From a0fc52546fe42059102280e39a15140cfce3acc1 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Mon, 13 Nov 2023 00:40:01 +0000 Subject: [PATCH 32/41] [Java, C++, C#] Ensure buffer is large enough to contain message. Previously, when generating arbitrary encoded values the generator would not necessarily extend the buffer to the size necessary to hold the message, e.g., if optional fields were left unset at the end of the message block. We now call `checkLimit` to "reserve" space for the full block length at the message and group level. This fixes an exception seen in the slow tests: ``` java.lang.IndexOutOfBoundsException: index=0 length=129 capacity=128 at org.agrona.AbstractMutableDirectBuffer.boundsCheck0(AbstractMutableDirectBuffer.java:1719) at org.agrona.AbstractMutableDirectBuffer.getBytes(AbstractMutableDirectBuffer.java:464) at org.agrona.io.DirectBufferInputStream.read(DirectBufferInputStream.java:175) at uk.co.real_logic.sbe.properties.DtosPropertyTest.writeInputFile(DtosPropertyTest.java:199) at uk.co.real_logic.sbe.properties.DtosPropertyTest.cppDtoEncodeShouldBeTheInverseOfDtoDecode(DtosPropertyTest.java:147) at java.lang.reflect.Method.invoke(Method.java:498) at net.jqwik.engine.execution.CheckedPropertyFactory.lambda$createRawFunction$1(CheckedPropertyFactory.java:84) at net.jqwik.engine.execution.CheckedPropertyFactory.lambda$createRawFunction$2(CheckedPropertyFactory.java:91) at net.jqwik.engine.properties.CheckedFunction.execute(CheckedFunction.java:17) at net.jqwik.api.lifecycle.AroundTryHook.lambda$static$0(AroundTryHook.java:57) at net.jqwik.engine.execution.lifecycle.HookSupport.lambda$wrap$2(HookSupport.java:48) at net.jqwik.engine.hooks.lifecycle.TryLifecycleMethodsHook.aroundTry(TryLifecycleMethodsHook.java:57) at net.jqwik.engine.execution.lifecycle.HookSupport.lambda$wrap$3(HookSupport.java:53) at net.jqwik.engine.execution.lifecycle.HookSupport.lambda$wrap$2(HookSupport.java:48) at net.jqwik.engine.hooks.lifecycle.BeforeTryMembersHook.aroundTry(BeforeTryMembersHook.java:69) at net.jqwik.engine.execution.lifecycle.HookSupport.lambda$wrap$3(HookSupport.java:53) at net.jqwik.engine.execution.CheckedPropertyFactory.lambda$createTryExecutor$0(CheckedPropertyFactory.java:60) at net.jqwik.engine.execution.lifecycle.AroundTryLifecycle.execute(AroundTryLifecycle.java:23) at net.jqwik.engine.properties.GenericProperty.testPredicate(GenericProperty.java:166) at net.jqwik.engine.properties.GenericProperty.check(GenericProperty.java:68) at net.jqwik.engine.execution.CheckedProperty.check(CheckedProperty.java:67) at net.jqwik.engine.execution.PropertyMethodExecutor.executeProperty(PropertyMethodExecutor.java:90) at net.jqwik.engine.execution.PropertyMethodExecutor.executeMethod(PropertyMethodExecutor.java:69) at net.jqwik.engine.execution.PropertyMethodExecutor.lambda$execute$0(PropertyMethodExecutor.java:49) at net.jqwik.api.lifecycle.AroundPropertyHook.lambda$static$0(AroundPropertyHook.java:46) at net.jqwik.engine.execution.lifecycle.HookSupport.lambda$wrap$0(HookSupport.java:26) at net.jqwik.api.lifecycle.PropertyExecutor.executeAndFinally(PropertyExecutor.java:39) at net.jqwik.engine.hooks.lifecycle.PropertyLifecycleMethodsHook.aroundProperty(PropertyLifecycleMethodsHook.java:56) at net.jqwik.engine.execution.lifecycle.HookSupport.lambda$wrap$1(HookSupport.java:31) at net.jqwik.engine.execution.lifecycle.HookSupport.lambda$wrap$0(HookSupport.java:26) at net.jqwik.engine.hooks.statistics.StatisticsHook.aroundProperty(StatisticsHook.java:37) at net.jqwik.engine.execution.lifecycle.HookSupport.lambda$wrap$1(HookSupport.java:31) at net.jqwik.engine.execution.lifecycle.HookSupport.lambda$wrap$0(HookSupport.java:26) at net.jqwik.engine.hooks.lifecycle.AutoCloseableHook.aroundProperty(AutoCloseableHook.java:13) at net.jqwik.engine.execution.lifecycle.HookSupport.lambda$wrap$1(HookSupport.java:31) at net.jqwik.engine.execution.PropertyMethodExecutor.execute(PropertyMethodExecutor.java:47) at net.jqwik.engine.execution.PropertyTaskCreator.executeTestMethod(PropertyTaskCreator.java:166) at net.jqwik.engine.execution.PropertyTaskCreator.lambda$createTask$1(PropertyTaskCreator.java:51) at net.jqwik.engine.execution.lifecycle.CurrentDomainContext.runWithContext(CurrentDomainContext.java:28) at net.jqwik.engine.execution.PropertyTaskCreator.lambda$createTask$2(PropertyTaskCreator.java:50) at net.jqwik.engine.execution.pipeline.ExecutionTask$1.lambda$execute$0(ExecutionTask.java:31) at net.jqwik.engine.execution.lifecycle.CurrentTestDescriptor.runWithDescriptor(CurrentTestDescriptor.java:17) at net.jqwik.engine.execution.pipeline.ExecutionTask$1.execute(ExecutionTask.java:31) at net.jqwik.engine.execution.pipeline.ExecutionPipeline.runToTermination(ExecutionPipeline.java:82) at net.jqwik.engine.execution.JqwikExecutor.execute(JqwikExecutor.java:46) at net.jqwik.engine.JqwikTestEngine.executeTests(JqwikTestEngine.java:70) at net.jqwik.engine.JqwikTestEngine.execute(JqwikTestEngine.java:53) ``` --- .../real_logic/sbe/properties/arbitraries/SbeArbitraries.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java index bfc1f9bd88..66bfeebcf0 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java @@ -801,6 +801,7 @@ private static Arbitrary groupsEncoder( { final int offset = limit.get(); fieldsEncoder.encode(builder, buffer, offset, null); + buffer.checkLimit(offset + blockLength); limit.set(offset + blockLength); builder.appendLine().append("limit: ").append(offset).append(" -> ").append(limit.get()); groupsEncoder.encode(builder, buffer, NULL_VALUE, limit); @@ -929,6 +930,7 @@ private static Arbitrary messageValueEncoder( final int headerLength = 8; fields.encode(builder, buffer, offset + headerLength, null); final int oldLimit = limit.get(); + buffer.checkLimit(offset + headerLength + blockLength); limit.set(offset + headerLength + blockLength); builder.appendLine().append("limit: ").append(oldLimit).append(" -> ").append(limit.get()); groups.encode(builder, buffer, NULL_VALUE, limit); From 2195d6e9754c56780009d8721bd3e69bb95ee3c3 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Sat, 11 Nov 2023 18:41:10 +0000 Subject: [PATCH 33/41] [CI] Fix dependency conflict in JQwik. Recently JUnit was upgraded. Rebasing revealed a dependency conflict. --- .github/workflows/slow.yml | 2 -- build.gradle | 10 ++++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/slow.yml b/.github/workflows/slow.yml index a5f007f6bf..ba932f8a9e 100644 --- a/.github/workflows/slow.yml +++ b/.github/workflows/slow.yml @@ -50,8 +50,6 @@ jobs: uses: actions/setup-dotnet@v2 with: dotnet-version: ${{ matrix.dotnet }} - - name: Build SbeTool - run: ./gradlew - name: Build .NET library run: ./csharp/build.sh - name: Run property tests diff --git a/build.gradle b/build.gradle index 2ccfad6010..2bb3c39f22 100644 --- a/build.gradle +++ b/build.gradle @@ -58,7 +58,7 @@ def checkstyleVersion = '9.3' def hamcrestVersion = '2.2' def mockitoVersion = '4.11.0' def junitVersion = '5.10.2' -def jqwikVersion = '1.8.0' +def jqwikVersion = '1.8.1' def jsonVersion = '20230618' def jmhVersion = '1.37' def agronaVersion = '1.21.2' @@ -297,11 +297,17 @@ project(':sbe-tool') { } propertyTest(JvmTestSuite) { + // We should be able to use _only_ the JQwik engine, but this issue is outstanding: + // https://github.com/gradle/gradle/issues/21299 useJUnitJupiter junitVersion dependencies { implementation project() - implementation "net.jqwik:jqwik:${jqwikVersion}" + implementation("net.jqwik:jqwik:${jqwikVersion}") { + // Exclude JUnit 5 dependencies that are already provided due to useJUnitJupiter + exclude group: 'org.junit.platform', module: 'junit-platform-commons' + exclude group: 'org.junit.platform', module: 'junit-platform-engine' + } implementation "org.json:json:${jsonVersion}" } From b9ed08b7e29b06e5f85280f43a77bb9c7c9aceee Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Mon, 13 Nov 2023 10:01:22 +0000 Subject: [PATCH 34/41] [CI] Upload slow test artifacts upon failure. --- .github/workflows/slow.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/slow.yml b/.github/workflows/slow.yml index ba932f8a9e..9b790d276e 100644 --- a/.github/workflows/slow.yml +++ b/.github/workflows/slow.yml @@ -54,3 +54,9 @@ jobs: run: ./csharp/build.sh - name: Run property tests run: ./gradlew propertyTest + - name: Upload test results + uses: actions/upload-artifact@v3 + if: success() || failure() + with: + name: property-tests + path: sbe-tool/build/reports/tests/propertyTest From b27bb68c83c5aef1da5f8ca4800d04bfde7716c6 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Mon, 13 Nov 2023 12:45:25 +0000 Subject: [PATCH 35/41] [C#] Provide path to SBE.dll for C# property tests in CI. --- build.gradle | 3 +++ .../co/real_logic/sbe/properties/DtosPropertyTest.java | 10 +++++++--- .../CSharpDtosPropertyTest/SbePropertyTest.csproj | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 2bb3c39f22..c21ddf3eba 100644 --- a/build.gradle +++ b/build.gradle @@ -311,6 +311,7 @@ project(':sbe-tool') { implementation "org.json:json:${jsonVersion}" } + targets { all { testTask.configure { @@ -318,6 +319,8 @@ project(':sbe-tool') { maxHeapSize = '2g' javaLauncher.set(toolchainLauncher) + + systemProperty 'sbe.dll', "${rootProject.projectDir}/csharp/sbe-dll/bin/Release/netstandard2.0/SBE.dll" } } } diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java index d68a1c327c..8adc57679d 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java @@ -39,6 +39,8 @@ public class DtosPropertyTest { private static final String DOTNET_EXECUTABLE = System.getProperty("sbe.tests.dotnet.executable", "dotnet"); + private static final String SBE_DLL = + System.getProperty("sbe.dll", "csharp/sbe-dll/bin/Release/netstandard2.0/SBE.dll"); private static final String CPP_EXECUTABLE = System.getProperty("sbe.tests.cpp.executable", "g++"); private static final boolean KEEP_DIR_ON_FAILURE = Boolean.parseBoolean( System.getProperty("sbe.tests.keep.dir.on.failure", "true")); @@ -80,7 +82,9 @@ void csharpDtoEncodeShouldBeTheInverseOfDtoDecode( writeInputFile(encodedMessage, tempDir); execute(encodedMessage.schema(), tempDir, "test", - DOTNET_EXECUTABLE, "run", "--", "input.dat"); + DOTNET_EXECUTABLE, "run", + "--property:SBE_DLL=" + SBE_DLL, + "--", "input.dat"); final byte[] inputBytes = new byte[encodedMessage.length()]; encodedMessage.buffer().getBytes(0, inputBytes); @@ -211,12 +215,12 @@ private static void execute( { final Path stdout = tempDir.resolve(name + "_stdout.txt"); final Path stderr = tempDir.resolve(name + "_stderr.txt"); - final ProcessBuilder compileProcessBuilder = new ProcessBuilder(args) + final ProcessBuilder processBuilder = new ProcessBuilder(args) .directory(tempDir.toFile()) .redirectOutput(stdout.toFile()) .redirectError(stderr.toFile()); - final Process process = compileProcessBuilder.start(); + final Process process = processBuilder.start(); if (0 != process.waitFor()) { diff --git a/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/SbePropertyTest.csproj b/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/SbePropertyTest.csproj index 8b0b167d2b..5da3f76095 100644 --- a/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/SbePropertyTest.csproj +++ b/sbe-tool/src/propertyTest/resources/CSharpDtosPropertyTest/SbePropertyTest.csproj @@ -7,7 +7,7 @@ - /home/zach/src/real-logic/simple-binary-encoding/csharp/sbe-dll/bin/Release/netstandard2.0/SBE.dll + $(SBE_DLL) From 1e3cf8fea5afc3c8adcc08f9b55bb144b106ec9f Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Wed, 1 May 2024 16:19:14 +0100 Subject: [PATCH 36/41] [Java] Adjust copyright banners. --- .../uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java | 2 +- .../main/java/uk/co/real_logic/sbe/generation/cpp/CppDtos.java | 2 +- .../co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java | 2 +- .../java/uk/co/real_logic/sbe/generation/csharp/CSharpDtos.java | 2 +- .../java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java | 2 +- .../java/uk/co/real_logic/sbe/properties/JsonPropertyTest.java | 2 +- .../uk/co/real_logic/sbe/properties/ParserPropertyTest.java | 2 +- .../real_logic/sbe/properties/arbitraries/SbeArbitraries.java | 2 +- .../real_logic/sbe/properties/schema/CompositeTypeSchema.java | 2 +- .../real_logic/sbe/properties/schema/EncodedDataTypeSchema.java | 2 +- .../uk/co/real_logic/sbe/properties/schema/EnumTypeSchema.java | 2 +- .../uk/co/real_logic/sbe/properties/schema/FieldSchema.java | 2 +- .../uk/co/real_logic/sbe/properties/schema/GroupSchema.java | 2 +- .../uk/co/real_logic/sbe/properties/schema/MessageSchema.java | 2 +- .../java/uk/co/real_logic/sbe/properties/schema/SetSchema.java | 2 +- .../real_logic/sbe/properties/schema/TestXmlSchemaWriter.java | 2 +- .../java/uk/co/real_logic/sbe/properties/schema/TypeSchema.java | 2 +- .../co/real_logic/sbe/properties/schema/TypeSchemaVisitor.java | 2 +- .../uk/co/real_logic/sbe/properties/schema/VarDataSchema.java | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java index f2371181d0..168f6df665 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtoGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * Copyright (C) 2017 MarketFactory, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtos.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtos.java index dfe00b449e..344e48d213 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtos.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtos.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * Copyright 2017 MarketFactory Inc * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java index a132e751d6..e023b8fd95 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtoGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * Copyright (C) 2017 MarketFactory, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtos.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtos.java index 79859d6764..4fc0cc513f 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtos.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/csharp/CSharpDtos.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * Copyright 2017 MarketFactory Inc * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java index 8adc57679d..f71e416309 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/JsonPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/JsonPropertyTest.java index ab5c8d955e..a9b8cd0987 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/JsonPropertyTest.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/JsonPropertyTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java index b6ce84ef3a..1b59dca624 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/ParserPropertyTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java index 66bfeebcf0..a47a93eabf 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/CompositeTypeSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/CompositeTypeSchema.java index a6df53d778..06123ffc9f 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/CompositeTypeSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/CompositeTypeSchema.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java index a1bbeba14e..7abc254e0b 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EnumTypeSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EnumTypeSchema.java index fa669fe04a..fa28ef0a3a 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EnumTypeSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EnumTypeSchema.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/FieldSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/FieldSchema.java index c1fad8abc4..9745b042a1 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/FieldSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/FieldSchema.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java index 47cfe343a7..2aa3dcd3c3 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java index 2095628c39..d992d6559f 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/SetSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/SetSchema.java index 4ed41fc013..5acbc9da40 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/SetSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/SetSchema.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java index 3c43228099..b7e3d6f8c1 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchema.java index 311521e697..ca236a0673 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchema.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchemaVisitor.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchemaVisitor.java index 830a2e1cf7..2651febfc7 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchemaVisitor.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TypeSchemaVisitor.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java index db3d5900a1..1bfd64621d 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/VarDataSchema.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 Real Logic Limited. + * Copyright 2013-2024 Real Logic Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 328cc427f23629e7bfdaf9caae1f1b540aa9b430 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Mon, 13 May 2024 13:40:51 +0100 Subject: [PATCH 37/41] [Java] Add support for DTO generation. This commit adds support for generating SBE DTOs in Java. Unlike the C# and C++ support, there isn't a natural candidate for the idiomatic representation of optional fields, e.g., `Optional` is only meant for use as a return type. Therefore, the mapping between Java types and the encoding is simpler than in these other languages. In a future commit, I will extend the property-based tests to cover Java DTOs. --- build.gradle | 16 + .../java/uk/co/real_logic/sbe/SbeTool.java | 7 +- .../generation/TargetCodeGeneratorLoader.java | 20 +- .../sbe/generation/cpp/CppDtos.java | 2 +- .../sbe/generation/java/JavaDtoGenerator.java | 1794 +++++++++++++++++ .../sbe/generation/java/JavaDtos.java | 35 + .../sbe/generation/java/JavaGenerator.java | 18 +- .../sbe/generation/java/JavaUtil.java | 11 + .../sbe/generation/java/DtoTest.java | 154 ++ .../resources/example-extension-schema.xml | 103 + 10 files changed, 2141 insertions(+), 19 deletions(-) create mode 100644 sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtoGenerator.java create mode 100644 sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtos.java create mode 100644 sbe-tool/src/test/java/uk/co/real_logic/sbe/generation/java/DtoTest.java create mode 100644 sbe-tool/src/test/resources/example-extension-schema.xml diff --git a/build.gradle b/build.gradle index c21ddf3eba..b661f8b406 100644 --- a/build.gradle +++ b/build.gradle @@ -280,6 +280,7 @@ project(':sbe-tool') { compileGeneratedJava { dependsOn 'generateTestCodecs' + dependsOn 'generateTestDtos' classpath += sourceSets.main.runtimeClasspath } @@ -344,6 +345,21 @@ project(':sbe-tool') { 'src/test/resources/field-order-check-schema.xml'] } + tasks.register('generateTestDtos', JavaExec) { + dependsOn 'compileJava' + mainClass.set('uk.co.real_logic.sbe.SbeTool') + classpath = sourceSets.main.runtimeClasspath + systemProperties( + 'sbe.output.dir': generatedDir, + 'sbe.target.language': 'java', + 'sbe.validation.stop.on.error': 'true', + 'sbe.validation.xsd': validationXsdPath, + 'sbe.generate.precedence.checks': 'true', + 'sbe.java.precedence.checks.property.name': 'sbe.enable.test.precedence.checks', + 'sbe.java.generate.dtos': 'true') + args = ['src/test/resources/example-extension-schema.xml'] + } + jar { manifest.attributes( 'Specification-Title': 'Simple Binary Encoding', diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/SbeTool.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/SbeTool.java index a6958a12bf..d65a6a4601 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/SbeTool.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/SbeTool.java @@ -195,7 +195,12 @@ public class SbeTool /** * Should generate C++ DTOs. Defaults to false. */ - public static final String GENERATE_CPP_DTOS = "sbe.cpp.generate.dtos"; + public static final String CPP_GENERATE_DTOS = "sbe.cpp.generate.dtos"; + + /** + * Should generate Java DTOs. Defaults to false. + */ + public static final String JAVA_GENERATE_DTOS = "sbe.java.generate.dtos"; /** * Configuration option used to manage sinceVersion based transformations. When set, parsed schemas will be diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/TargetCodeGeneratorLoader.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/TargetCodeGeneratorLoader.java index 6f872cd55e..2b2383dfd2 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/TargetCodeGeneratorLoader.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/TargetCodeGeneratorLoader.java @@ -23,6 +23,7 @@ import uk.co.real_logic.sbe.generation.cpp.NamespaceOutputManager; import uk.co.real_logic.sbe.generation.golang.GolangGenerator; import uk.co.real_logic.sbe.generation.golang.GolangOutputManager; +import uk.co.real_logic.sbe.generation.java.JavaDtoGenerator; import uk.co.real_logic.sbe.generation.java.JavaGenerator; import uk.co.real_logic.sbe.generation.java.JavaOutputManager; import uk.co.real_logic.sbe.generation.rust.RustGenerator; @@ -47,7 +48,9 @@ public enum TargetCodeGeneratorLoader implements TargetCodeGenerator */ public CodeGenerator newInstance(final Ir ir, final String outputDir) { - return new JavaGenerator( + final JavaOutputManager outputManager = new JavaOutputManager(outputDir, ir.applicableNamespace()); + + final JavaGenerator codecGenerator = new JavaGenerator( ir, System.getProperty(JAVA_ENCODING_BUFFER_TYPE, JAVA_DEFAULT_ENCODING_BUFFER_TYPE), System.getProperty(JAVA_DECODING_BUFFER_TYPE, JAVA_DEFAULT_DECODING_BUFFER_TYPE), @@ -56,7 +59,18 @@ public CodeGenerator newInstance(final Ir ir, final String outputDir) "true".equals(System.getProperty(DECODE_UNKNOWN_ENUM_VALUES)), "true".equals(System.getProperty(TYPES_PACKAGE_OVERRIDE)), precedenceChecks(), - new JavaOutputManager(outputDir, ir.applicableNamespace())); + outputManager); + + final JavaDtoGenerator dtoGenerator = new JavaDtoGenerator(ir, outputManager); + + final CodeGenerator combinedGenerator = () -> + { + codecGenerator.generate(); + dtoGenerator.generate(); + }; + + final boolean generateDtos = "true".equals(System.getProperty(JAVA_GENERATE_DTOS)); + return generateDtos ? combinedGenerator : codecGenerator; } }, @@ -97,7 +111,7 @@ public CodeGenerator newInstance(final Ir ir, final String outputDir) dtoGenerator.generate(); }; - final boolean generateDtos = "true".equals(System.getProperty(GENERATE_CPP_DTOS)); + final boolean generateDtos = "true".equals(System.getProperty(CPP_GENERATE_DTOS)); return generateDtos ? combinedGenerator : codecGenerator; } }, diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtos.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtos.java index 344e48d213..09dbfb052f 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtos.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/cpp/CppDtos.java @@ -22,7 +22,7 @@ import uk.co.real_logic.sbe.ir.Ir; /** - * {@link CodeGenerator} factory for CSharp DTOs. + * {@link CodeGenerator} factory for C++ DTOs. */ public class CppDtos implements TargetCodeGenerator { diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtoGenerator.java new file mode 100644 index 0000000000..b5fd05e665 --- /dev/null +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtoGenerator.java @@ -0,0 +1,1794 @@ +/* + * Copyright 2013-2024 Real Logic Limited. + * Copyright (C) 2017 MarketFactory, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.generation.java; + +import uk.co.real_logic.sbe.PrimitiveType; +import uk.co.real_logic.sbe.generation.CodeGenerator; +import uk.co.real_logic.sbe.generation.Generators; +import uk.co.real_logic.sbe.ir.Ir; +import uk.co.real_logic.sbe.ir.Signal; +import uk.co.real_logic.sbe.ir.Token; +import org.agrona.LangUtil; +import org.agrona.Verify; +import org.agrona.generation.OutputManager; + +import java.io.IOException; +import java.io.Writer; +import java.util.ArrayList; +import java.util.List; + +import static uk.co.real_logic.sbe.generation.Generators.toLowerFirstChar; +import static uk.co.real_logic.sbe.generation.Generators.toUpperFirstChar; +import static uk.co.real_logic.sbe.generation.java.JavaUtil.*; +import static uk.co.real_logic.sbe.ir.GenerationUtil.collectFields; +import static uk.co.real_logic.sbe.ir.GenerationUtil.collectGroups; +import static uk.co.real_logic.sbe.ir.GenerationUtil.collectVarData; + +/** + * DTO generator for the Java programming language. + */ +public class JavaDtoGenerator implements CodeGenerator +{ + private static final String INDENT = " "; + private static final String BASE_INDENT = ""; + + private final Ir ir; + private final OutputManager outputManager; + + /** + * Create a new C# DTO {@link CodeGenerator}. + * + * @param ir for the messages and types. + * @param outputManager for generating the DTOs to. + */ + public JavaDtoGenerator(final Ir ir, final OutputManager outputManager) + { + Verify.notNull(ir, "ir"); + Verify.notNull(outputManager, "outputManager"); + + this.ir = ir; + this.outputManager = outputManager; + } + + /** + * {@inheritDoc} + */ + public void generate() throws IOException + { + generateDtosForTypes(); + + for (final List tokens : ir.messages()) + { + final Token msgToken = tokens.get(0); + final String encoderClassName = encoderName(msgToken.name()); + final String decoderClassName = decoderName(msgToken.name()); + final String dtoClassName = formatDtoClassName(msgToken.name()); + + final List messageBody = tokens.subList(1, tokens.size() - 1); + int offset = 0; + + final ClassBuilder classBuilder = new ClassBuilder(dtoClassName, BASE_INDENT, "public final"); + + final List fields = new ArrayList<>(); + offset = collectFields(messageBody, offset, fields); + generateFields(classBuilder, decoderClassName, fields, BASE_INDENT + INDENT); + + final List groups = new ArrayList<>(); + offset = collectGroups(messageBody, offset, groups); + generateGroups(classBuilder, dtoClassName, encoderClassName, decoderClassName, groups, + BASE_INDENT + INDENT); + + final List varData = new ArrayList<>(); + collectVarData(messageBody, offset, varData); + generateVarData(classBuilder, varData, BASE_INDENT + INDENT); + + generateDecodeWith(classBuilder, dtoClassName, decoderClassName, fields, + groups, varData, BASE_INDENT + INDENT); + generateDecodeFrom(classBuilder, dtoClassName, decoderClassName, BASE_INDENT + INDENT); + generateEncodeWith(classBuilder, dtoClassName, encoderClassName, fields, groups, varData, + BASE_INDENT + INDENT); + generateEncodeWithOverloads(classBuilder, dtoClassName, encoderClassName, BASE_INDENT + INDENT); + generateComputeEncodedLength(classBuilder, decoderClassName, + decoderClassName + ".BLOCK_LENGTH", + groups, varData, BASE_INDENT + INDENT); + generateDisplay(classBuilder, encoderClassName, "computeEncodedLength()", + BASE_INDENT + INDENT); + + try (Writer out = outputManager.createOutput(dtoClassName)) + { + out.append(generateDtoFileHeader(ir.applicableNamespace())); + out.append("import org.agrona.DirectBuffer;\n"); + out.append("import org.agrona.MutableDirectBuffer;\n"); + out.append("import org.agrona.concurrent.UnsafeBuffer;\n\n"); + out.append("import java.util.ArrayList;\n"); + out.append("import java.util.List;\n\n"); + out.append(generateDocumentation(BASE_INDENT, msgToken)); + classBuilder.appendTo(out); + } + } + } + + private static final class ClassBuilder + { + private final StringBuilder fieldSb = new StringBuilder(); + private final StringBuilder privateSb = new StringBuilder(); + private final StringBuilder publicSb = new StringBuilder(); + private final String className; + private final String indent; + private final String modifiers; + + private ClassBuilder( + final String className, + final String indent, + final String modifiers) + { + this.className = className; + this.indent = indent; + this.modifiers = modifiers.length() == 0 ? modifiers : modifiers + " "; + } + + public StringBuilder appendField() + { + return fieldSb; + } + + public StringBuilder appendPrivate() + { + return privateSb; + } + + public StringBuilder appendPublic() + { + return publicSb; + } + + public void appendTo(final Appendable out) + { + try + { + out.append(indent).append(modifiers).append("class ").append(className).append("\n") + .append(indent).append("{\n") + .append(fieldSb) + .append("\n") + .append(privateSb) + .append("\n") + .append(publicSb) + .append("\n") + .append(indent).append("}\n"); + } + catch (final IOException exception) + { + LangUtil.rethrowUnchecked(exception); + } + } + } + + private void generateGroups( + final ClassBuilder classBuilder, + final String qualifiedParentDtoClassName, + final String qualifiedParentEncoderClassName, + final String qualifiedParentDecoderClassName, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token groupToken = tokens.get(i); + if (groupToken.signal() != Signal.BEGIN_GROUP) + { + throw new IllegalStateException("tokens must begin with BEGIN_GROUP: token=" + groupToken); + } + final String groupName = groupToken.name(); + final String groupClassName = formatDtoClassName(groupName); + final String qualifiedDtoClassName = qualifiedParentDtoClassName + "." + groupClassName; + + final String fieldName = formatFieldName(groupName); + final String formattedPropertyName = formatPropertyName(groupName); + + classBuilder.appendField().append(indent).append("private List<") + .append(qualifiedDtoClassName).append("> ") + .append(fieldName).append(" = new ArrayList<>();\n"); + + final ClassBuilder groupClassBuilder = new ClassBuilder(groupClassName, indent, "public static final"); + + i++; + i += tokens.get(i).componentTokenCount(); + + final String qualifiedEncoderClassName = + qualifiedParentEncoderClassName + "." + encoderName(groupName); + final String qualifiedDecoderClassName = + qualifiedParentDecoderClassName + "." + decoderName(groupName); + + final List fields = new ArrayList<>(); + i = collectFields(tokens, i, fields); + generateFields(groupClassBuilder, qualifiedDecoderClassName, fields, indent + INDENT); + + final List groups = new ArrayList<>(); + i = collectGroups(tokens, i, groups); + generateGroups(groupClassBuilder, qualifiedDtoClassName, + qualifiedEncoderClassName, qualifiedDecoderClassName, groups, indent + INDENT); + + final List varData = new ArrayList<>(); + i = collectVarData(tokens, i, varData); + generateVarData(groupClassBuilder, varData, indent + INDENT); + + generateDecodeListWith( + groupClassBuilder, groupClassName, qualifiedDecoderClassName, indent + INDENT); + generateDecodeWith(groupClassBuilder, groupClassName, qualifiedDecoderClassName, + fields, groups, varData, indent + INDENT); + generateEncodeWith( + groupClassBuilder, groupClassName, qualifiedEncoderClassName, fields, groups, varData, indent + INDENT); + generateComputeEncodedLength(groupClassBuilder, qualifiedDecoderClassName, + qualifiedDecoderClassName + ".sbeBlockLength()", + groups, varData, indent + INDENT); + + groupClassBuilder.appendTo( + classBuilder.appendPublic().append("\n").append(generateDocumentation(indent, groupToken)) + ); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, groupToken)) + .append(indent).append("public List<").append(qualifiedDtoClassName).append("> ") + .append(formattedPropertyName).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, groupToken)) + .append(indent).append("public void ").append(formattedPropertyName).append("(") + .append("List<").append(qualifiedDtoClassName).append("> value)") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(fieldName).append(" = value;\n") + .append(indent).append("}\n"); + } + } + + private void generateComputeEncodedLength( + final ClassBuilder classBuilder, + final String qualifiedDecoderClassName, + final String blockLengthExpression, + final List groupTokens, + final List varDataTokens, + final String indent) + { + final StringBuilder lengthBuilder = classBuilder.appendPublic() + .append("\n") + .append(indent).append("public int computeEncodedLength()\n") + .append(indent).append("{\n"); + + lengthBuilder + .append(indent).append(INDENT).append("int encodedLength = 0;\n"); + + lengthBuilder.append(indent).append(INDENT).append("encodedLength += ").append(blockLengthExpression) + .append(";\n\n"); + + for (int i = 0, size = groupTokens.size(); i < size; i++) + { + final Token groupToken = groupTokens.get(i); + if (groupToken.signal() != Signal.BEGIN_GROUP) + { + throw new IllegalStateException("tokens must begin with BEGIN_GROUP: token=" + groupToken); + } + + i++; + i += groupTokens.get(i).componentTokenCount(); + + final List fields = new ArrayList<>(); + i = collectFields(groupTokens, i, fields); + final List subGroups = new ArrayList<>(); + i = collectGroups(groupTokens, i, subGroups); + final List subVarData = new ArrayList<>(); + i = collectVarData(groupTokens, i, subVarData); + + final String groupName = groupToken.name(); + final String fieldName = formatFieldName(groupName); + final String groupDecoderClassName = qualifiedDecoderClassName + "." + decoderName(groupName); + final String groupDtoClassName = formatDtoClassName(groupName); + + lengthBuilder + .append(indent).append(INDENT).append("encodedLength += ") + .append(groupDecoderClassName).append(".sbeHeaderSize();\n\n") + .append(indent).append(INDENT).append("for (").append(groupDtoClassName).append(" group : ") + .append(fieldName).append(")\n") + .append(indent).append(INDENT).append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("encodedLength += group.computeEncodedLength();\n") + .append(indent).append(INDENT).append("}\n\n"); + } + + for (int i = 0, size = varDataTokens.size(); i < size; i++) + { + final Token token = varDataTokens.get(i); + if (token.signal() == Signal.BEGIN_VAR_DATA) + { + final String propertyName = token.name(); + final Token varDataToken = Generators.findFirst("varData", varDataTokens, i); + final String fieldName = formatFieldName(propertyName); + + lengthBuilder.append(indent).append(INDENT).append("encodedLength += ") + .append(qualifiedDecoderClassName).append(".") + .append(formatPropertyName(propertyName)).append("HeaderLength();\n"); + + lengthBuilder.append(indent).append(INDENT).append("encodedLength += ") + .append(fieldName).append(".length()"); + + final int elementByteLength = varDataToken.encoding().primitiveType().size(); + if (elementByteLength != 1) + { + lengthBuilder.append(" * ").append(elementByteLength); + } + + lengthBuilder.append(";\n\n"); + } + } + + lengthBuilder.append(indent).append(INDENT).append("return encodedLength;\n") + .append(indent).append("}\n"); + } + + private void generateCompositeDecodeWith( + final ClassBuilder classBuilder, + final String dtoClassName, + final String decoderClassName, + final List tokens, + final String indent) + { + final StringBuilder decodeBuilder = classBuilder.appendPublic().append("\n") + .append(indent).append("public static void decodeWith(").append(decoderClassName).append(" decoder, ") + .append(dtoClassName).append(" dto)\n") + .append(indent).append("{\n"); + + for (int i = 0; i < tokens.size(); ) + { + final Token token = tokens.get(i); + + generateFieldDecodeWith( + decodeBuilder, token, token, decoderClassName, indent + INDENT); + + i += tokens.get(i).componentTokenCount(); + } + + decodeBuilder.append(indent).append("}\n"); + } + + private void generateCompositeEncodeWith( + final ClassBuilder classBuilder, + final String dtoClassName, + final String encoderClassName, + final List tokens, + final String indent) + { + final StringBuilder encodeBuilder = classBuilder.appendPublic().append("\n") + .append(indent).append("public static void encodeWith(").append(encoderClassName).append(" encoder, ") + .append(dtoClassName).append(" dto)\n") + .append(indent).append("{\n"); + + for (int i = 0; i < tokens.size(); ) + { + final Token token = tokens.get(i); + + generateFieldEncodeWith(encodeBuilder, token, token, indent + INDENT); + + i += tokens.get(i).componentTokenCount(); + } + + encodeBuilder.append(indent).append("}\n"); + } + + private void generateDecodeListWith( + final ClassBuilder classBuilder, + final String dtoClassName, + final String decoderClassName, + final String indent) + { + classBuilder.appendPublic().append("\n") + .append(indent).append("static List<").append(dtoClassName).append("> decodeManyWith(") + .append(decoderClassName).append(" decoder)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("List<").append(dtoClassName) + .append("> dtos = new ArrayList<>(decoder.count());\n") + .append(indent).append(INDENT) + .append("while (decoder.hasNext())\n") + .append(indent).append(INDENT) + .append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append(dtoClassName).append(" dto = new ").append(dtoClassName).append("();\n") + .append(indent).append(INDENT).append(INDENT) + .append(dtoClassName).append(".decodeWith(decoder.next(), dto);\n") + .append(indent).append(INDENT).append(INDENT) + .append("dtos.add(dto);\n") + .append(indent).append(INDENT) + .append("}\n") + .append(indent).append(INDENT) + .append("return dtos;\n") + .append(indent).append("}\n"); + } + + private void generateDecodeWith( + final ClassBuilder classBuilder, + final String dtoClassName, + final String decoderClassName, + final List fields, + final List groups, + final List varData, + final String indent) + { + final StringBuilder decodeBuilder = classBuilder.appendPublic().append("\n") + .append(indent).append("public static void decodeWith(").append(decoderClassName).append(" decoder, ") + .append(dtoClassName).append(" dto)\n") + .append(indent).append("{\n"); + + generateMessageFieldsDecodeWith(decodeBuilder, fields, decoderClassName, indent + INDENT); + generateGroupsDecodeWith(decodeBuilder, groups, indent + INDENT); + generateVarDataDecodeWith(decodeBuilder, varData, indent + INDENT); + decodeBuilder.append(indent).append("}\n"); + } + + private static void generateDecodeFrom( + final ClassBuilder classBuilder, + final String dtoClassName, + final String decoderClassName, + final String indent) + { + classBuilder.appendPublic() + .append("\n") + .append(indent).append("public static ").append(dtoClassName).append(" decodeFrom(") + .append("DirectBuffer buffer, int offset, ") + .append("short actingBlockLength, short actingVersion)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(decoderClassName).append(" decoder = new ") + .append(decoderClassName).append("();\n") + .append(indent).append(INDENT) + .append("decoder.wrap(buffer, offset, actingBlockLength, actingVersion);\n") + .append(indent).append(INDENT).append(dtoClassName).append(" dto = new ") + .append(dtoClassName).append("();\n") + .append(indent).append(INDENT).append("decodeWith(decoder, dto);\n") + .append(indent).append(INDENT).append("return dto;\n") + .append(indent).append("}\n"); + } + + private void generateMessageFieldsDecodeWith( + final StringBuilder sb, + final List tokens, + final String decoderClassName, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token signalToken = tokens.get(i); + if (signalToken.signal() == Signal.BEGIN_FIELD) + { + final Token encodingToken = tokens.get(i + 1); + + generateFieldDecodeWith(sb, signalToken, encodingToken, decoderClassName, indent); + } + } + } + + private void generateFieldDecodeWith( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String decoderClassName, + final String indent) + { + switch (typeToken.signal()) + { + case ENCODING: + generatePrimitiveDecodeWith(sb, fieldToken, typeToken, decoderClassName, indent); + break; + + case BEGIN_SET: + final String bitSetName = formatDtoClassName(typeToken.applicableTypeName()); + generateBitSetDecodeWith(sb, fieldToken, bitSetName, indent); + break; + + case BEGIN_ENUM: + generateEnumDecodeWith(sb, fieldToken, indent); + break; + + case BEGIN_COMPOSITE: + generateCompositePropertyDecodeWith(sb, fieldToken, typeToken, indent); + break; + + default: + break; + } + } + + private void generatePrimitiveDecodeWith( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String decoderClassName, + final String indent) + { + if (typeToken.isConstantEncoding()) + { + return; + } + + final int arrayLength = typeToken.arrayLength(); + + if (arrayLength == 1) + { + final String decoderNullValue = + decoderClassName + "." + formatPropertyName(fieldToken.name()) + "NullValue()"; + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + generateRecordPropertyAssignment( + sb, + fieldToken, + indent, + "decoder.actingVersion() >= " + decoderClassName + "." + formattedPropertyName + "SinceVersion()", + "decoder." + formattedPropertyName + "()", + decoderNullValue + ); + } + else if (arrayLength > 1) + { + generateArrayDecodeWith(sb, decoderClassName, fieldToken, typeToken, indent); + } + } + + private void generateArrayDecodeWith( + final StringBuilder sb, + final String decoderClassName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) + { + generateRecordPropertyAssignment( + sb, + fieldToken, + indent, + "decoder.actingVersion() >= " + decoderClassName + "." + formattedPropertyName + "SinceVersion()", + "decoder." + formattedPropertyName + "()", + "\"\"" + ); + } + else + { + final StringBuilder initializerList = new StringBuilder(); + final String elementType = javaTypeName(typeToken.encoding().primitiveType()); + initializerList.append("new ").append(elementType).append("[] { "); + final int arrayLength = typeToken.arrayLength(); + for (int i = 0; i < arrayLength; i++) + { + initializerList.append("decoder.").append(formattedPropertyName).append("(").append(i).append("),"); + } + assert arrayLength > 0; + initializerList.setLength(initializerList.length() - 1); + initializerList.append(" }"); + + generateRecordPropertyAssignment( + sb, + fieldToken, + indent, + "decoder.actingVersion() >= " + decoderClassName + "." + formattedPropertyName + "SinceVersion()", + initializerList, + null + ); + } + } + + private void generateBitSetDecodeWith( + final StringBuilder sb, + final Token fieldToken, + final String dtoTypeName, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + if (fieldToken.isOptionalEncoding()) + { + sb.append(indent).append("if (decoder.").append(formattedPropertyName).append("InActingVersion()"); + + sb.append(")\n") + .append(indent).append("{\n"); + + sb.append(indent).append(INDENT).append(dtoTypeName).append(".decodeWith(decoder.") + .append(formattedPropertyName).append("(), ") + .append("dto.").append(formattedPropertyName).append("());\n"); + + sb.append(indent).append("}\n") + .append(indent).append("else\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("dto.").append(formattedPropertyName).append("().clear();\n") + .append(indent).append("}\n"); + } + else + { + sb.append(indent).append(dtoTypeName).append(".decodeWith(decoder.") + .append(formattedPropertyName).append("(), ") + .append("dto.").append(formattedPropertyName).append("());\n"); + } + } + + private void generateEnumDecodeWith( + final StringBuilder sb, + final Token fieldToken, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + sb.append(indent).append("dto.").append(formattedPropertyName).append("(") + .append("decoder.").append(formattedPropertyName).append("());\n"); + } + + private void generateCompositePropertyDecodeWith( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + final String dtoClassName = formatDtoClassName(typeToken.applicableTypeName()); + + sb.append(indent).append(dtoClassName).append(".decodeWith(decoder.") + .append(formattedPropertyName).append("(), ") + .append("dto.").append(formattedPropertyName).append("());\n"); + } + + private void generateGroupsDecodeWith( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token groupToken = tokens.get(i); + if (groupToken.signal() != Signal.BEGIN_GROUP) + { + throw new IllegalStateException("tokens must begin with BEGIN_GROUP: token=" + groupToken); + } + final String groupName = groupToken.name(); + final String formattedPropertyName = formatPropertyName(groupName); + final String groupDtoClassName = formatDtoClassName(groupName); + + sb.append(indent).append("dto.").append(formattedPropertyName).append("(") + .append(groupDtoClassName).append(".decodeManyWith(decoder.") + .append(formattedPropertyName).append("()));\n"); + + i++; + i += tokens.get(i).componentTokenCount(); + + final List fields = new ArrayList<>(); + i = collectFields(tokens, i, fields); + + final List groups = new ArrayList<>(); + i = collectGroups(tokens, i, groups); + + final List varData = new ArrayList<>(); + i = collectVarData(tokens, i, varData); + } + } + + private void generateVarDataDecodeWith( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (int i = 0; i < tokens.size(); i++) + { + final Token token = tokens.get(i); + if (token.signal() == Signal.BEGIN_VAR_DATA) + { + final String propertyName = token.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + final Token varDataToken = Generators.findFirst("varData", tokens, i); + final String characterEncoding = varDataToken.encoding().characterEncoding(); + + final boolean isOptional = token.version() > 0; + final String blockIndent = isOptional ? indent + INDENT : indent; + + final String dataVar = toLowerFirstChar(propertyName) + "Data"; + + final StringBuilder decoderValueExtraction = new StringBuilder(); + + if (characterEncoding == null) + { + decoderValueExtraction.append(blockIndent).append("byte[] ").append(dataVar) + .append(" = new byte[decoder.").append(formattedPropertyName).append("Length()];\n") + .append(blockIndent).append("decoder.get").append(formattedPropertyName) + .append("(").append(dataVar).append(", 0, decoder.").append(formattedPropertyName) + .append("Length());\n"); + } + else + { + decoderValueExtraction.append(blockIndent).append("String ").append(dataVar) + .append(" = decoder.").append(formattedPropertyName).append("();\n"); + } + + if (isOptional) + { + sb.append(indent).append("if (decoder.").append(formattedPropertyName).append("InActingVersion()"); + + sb.append(")\n") + .append(indent).append("{\n"); + + sb.append(decoderValueExtraction); + + sb.append(indent).append(INDENT).append("dto.").append(formattedPropertyName).append("(") + .append(dataVar).append(");\n"); + + final String nullDtoValue = "\"\""; + + sb.append(indent).append("}\n") + .append(indent).append("else\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("dto.") + .append(formattedPropertyName).append("(").append(nullDtoValue).append(");\n") + .append(indent).append("}\n"); + } + else + { + sb.append(decoderValueExtraction); + + sb.append(indent).append("dto.").append(formattedPropertyName).append("(") + .append(dataVar).append(");\n"); + } + } + } + } + + private void generateRecordPropertyAssignment( + final StringBuilder sb, + final Token token, + final String indent, + final String presenceExpression, + final CharSequence getExpression, + final String nullDecoderValueOrNull) + { + final String propertyName = token.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + if (token.isOptionalEncoding()) + { + + sb.append(indent).append("if (").append(presenceExpression).append(")\n") + .append(indent).append("{\n"); + + sb.append(indent).append(INDENT).append("dto.").append(formattedPropertyName).append("(") + .append(getExpression).append(");\n"); + + final String nullValue = nullDecoderValueOrNull == null ? "null" : nullDecoderValueOrNull; + + sb.append(indent).append("}\n") + .append(indent).append("else\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("dto.").append(formattedPropertyName).append("(") + .append(nullValue).append(");\n") + .append(indent).append("}\n"); + } + else + { + sb.append(indent).append("dto.").append(formattedPropertyName).append("(") + .append(getExpression).append(");\n"); + } + } + + private void generateEncodeWith( + final ClassBuilder classBuilder, + final String dtoClassName, + final String encoderClassName, + final List fields, + final List groups, + final List varData, + final String indent) + { + final StringBuilder encodeBuilder = classBuilder.appendPublic().append("\n") + .append(indent).append("public static void encodeWith(").append(encoderClassName).append(" encoder, ") + .append(dtoClassName).append(" dto)\n") + .append(indent).append("{\n"); + + generateFieldsEncodeWith(encodeBuilder, fields, indent + INDENT); + generateGroupsEncodeWith(encodeBuilder, encoderClassName, groups, indent + INDENT); + generateVarDataEncodeWith(encodeBuilder, varData, indent + INDENT); + + encodeBuilder.append(indent).append("}\n"); + } + + private static void generateEncodeWithOverloads( + final ClassBuilder classBuilder, + final String dtoClassName, + final String encoderClassName, + final String indent) + { + classBuilder.appendPublic() + .append("\n") + .append(indent).append("public static int encodeWith(").append(dtoClassName).append(" dto, ") + .append("MutableDirectBuffer buffer, int offset)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(encoderClassName).append(" encoder = new ") + .append(encoderClassName).append("();\n") + .append(indent).append(INDENT).append("encoder.wrap(buffer, offset);\n") + .append(indent).append(INDENT).append("encodeWith(encoder, dto);\n") + .append(indent).append(INDENT).append("return encoder.encodedLength();\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic() + .append("\n") + .append(indent).append("public static int encodeWithHeaderWith(") + .append(dtoClassName).append(" dto, ") + .append("MutableDirectBuffer buffer, int offset)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(encoderClassName).append(" encoder = new ") + .append(encoderClassName).append("();\n") + .append(indent).append(INDENT) + .append("encoder.wrapAndApplyHeader(buffer, offset, new MessageHeaderEncoder());\n") + .append(indent).append(INDENT).append("encodeWith(encoder, dto);\n") + .append(indent).append(INDENT).append("return encoder.limit() - offset;\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic() + .append("\n") + .append(indent).append("public static byte[] bytes(") + .append(dtoClassName).append(" dto)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("byte[] bytes = new byte[dto.computeEncodedLength()];\n") + .append(indent).append(INDENT).append("encodeWith(dto, new UnsafeBuffer(bytes), 0);\n") + .append(indent).append(INDENT).append("return bytes;\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic() + .append("\n") + .append(indent).append("public static byte[] bytesWithHeader(") + .append(dtoClassName).append(" dto)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("byte[] bytes = new byte[dto.computeEncodedLength() + ") + .append("MessageHeaderEncoder.ENCODED_LENGTH];\n") + .append(indent).append(INDENT).append("encodeWithHeaderWith(dto, new UnsafeBuffer(bytes), 0);\n") + .append(indent).append(INDENT).append("return bytes;\n") + .append(indent).append("}\n"); + } + + private void generateFieldsEncodeWith( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token signalToken = tokens.get(i); + if (signalToken.signal() == Signal.BEGIN_FIELD) + { + final Token encodingToken = tokens.get(i + 1); + generateFieldEncodeWith(sb, signalToken, encodingToken, indent); + } + } + } + + private void generateFieldEncodeWith( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String indent) + { + switch (typeToken.signal()) + { + case ENCODING: + generatePrimitiveEncodeWith(sb, fieldToken, typeToken, indent); + break; + + case BEGIN_ENUM: + generateEnumEncodeWith(sb, fieldToken, indent); + break; + + case BEGIN_SET: + case BEGIN_COMPOSITE: + generateComplexPropertyEncodeWith(sb, fieldToken, typeToken, indent); + break; + + default: + break; + } + } + + private void generatePrimitiveEncodeWith( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String indent) + { + if (typeToken.isConstantEncoding()) + { + return; + } + + final int arrayLength = typeToken.arrayLength(); + + if (arrayLength == 1) + { + generatePrimitiveValueEncodeWith(sb, fieldToken, indent); + } + else if (arrayLength > 1) + { + generateArrayEncodeWith(sb, fieldToken, typeToken, indent); + } + } + + private void generateArrayEncodeWith( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) + { + sb.append(indent).append("encoder.").append(toLowerFirstChar(propertyName)).append("(") + .append("dto.").append(formattedPropertyName).append("());\n"); + } + else if (typeToken.encoding().primitiveType() == PrimitiveType.UINT8) + { + sb.append(indent).append("encoder.put").append(toUpperFirstChar(propertyName)).append("(") + .append("dto.").append(formattedPropertyName).append("());\n"); + } + else + { + final String javaTypeName = javaTypeName(typeToken.encoding().primitiveType()); + sb.append(indent).append(javaTypeName).append("[] ").append(formattedPropertyName).append(" = ") + .append("dto.").append(formattedPropertyName).append("();\n") + .append(indent).append("for (int i = 0; i < ").append(formattedPropertyName).append(".length; i++)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("encoder.").append(formattedPropertyName).append("(") + .append("i, ").append(formattedPropertyName).append("[i]);\n") + .append(indent).append("}\n"); + } + } + + private void generatePrimitiveValueEncodeWith( + final StringBuilder sb, + final Token fieldToken, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + final String accessor = "dto." + formattedPropertyName + "()"; + + sb.append(indent).append("encoder.").append(formattedPropertyName).append("(") + .append(accessor).append(");\n"); + } + + private void generateEnumEncodeWith( + final StringBuilder sb, + final Token fieldToken, + final String indent) + { + if (fieldToken.isConstantEncoding()) + { + return; + } + + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + + sb.append(indent).append("encoder.").append(formattedPropertyName).append("(dto.") + .append(formattedPropertyName).append("());\n"); + } + + private void generateComplexPropertyEncodeWith( + final StringBuilder sb, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String propertyName = fieldToken.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + final String typeName = formatDtoClassName(typeToken.applicableTypeName()); + + sb.append(indent).append(typeName).append(".encodeWith(encoder.") + .append(formattedPropertyName).append("(), dto.") + .append(formattedPropertyName).append("());\n"); + } + + private void generateGroupsEncodeWith( + final StringBuilder sb, + final String parentEncoderClassName, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token groupToken = tokens.get(i); + if (groupToken.signal() != Signal.BEGIN_GROUP) + { + throw new IllegalStateException("tokens must begin with BEGIN_GROUP: token=" + groupToken); + } + final String groupName = groupToken.name(); + final String formattedPropertyName = formatPropertyName(groupName); + final String groupEncoderVarName = groupName + "Encoder"; + final String groupDtoTypeName = formatDtoClassName(groupName); + final String groupEncoderTypeName = parentEncoderClassName + "." + encoderName(groupName); + + sb.append("\n") + .append(indent).append("List<").append(groupDtoTypeName).append("> ") + .append(formattedPropertyName).append(" = dto.").append(formattedPropertyName).append("();\n\n") + .append(indent).append(groupEncoderTypeName).append(" ").append(groupEncoderVarName) + .append(" = encoder.").append(formattedPropertyName) + .append("Count(").append(formattedPropertyName).append(".size());\n\n") + .append(indent).append("for (").append(groupDtoTypeName).append(" group : ") + .append(formattedPropertyName).append(")\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(groupDtoTypeName) + .append(".encodeWith(").append(groupEncoderVarName).append(".next(), group);\n") + .append(indent).append("}\n\n"); + + i++; + i += tokens.get(i).componentTokenCount(); + + final List fields = new ArrayList<>(); + i = collectFields(tokens, i, fields); + + final List groups = new ArrayList<>(); + i = collectGroups(tokens, i, groups); + + final List varData = new ArrayList<>(); + i = collectVarData(tokens, i, varData); + } + } + + private void generateVarDataEncodeWith( + final StringBuilder sb, + final List tokens, + final String indent) + { + for (int i = 0; i < tokens.size(); i++) + { + final Token token = tokens.get(i); + if (token.signal() == Signal.BEGIN_VAR_DATA) + { + final String propertyName = token.name(); + final String formattedPropertyName = formatPropertyName(propertyName); + final Token varDataToken = Generators.findFirst("varData", tokens, i); + final String characterEncoding = varDataToken.encoding().characterEncoding(); + + if (characterEncoding == null) + { + sb.append(indent).append("encoder.put").append(toUpperFirstChar(propertyName)).append("(") + .append("dto.").append(formattedPropertyName).append("());\n"); + } + else + { + sb.append(indent).append("encoder.").append(formattedPropertyName).append("(") + .append("dto.").append(formattedPropertyName).append("());\n"); + } + } + } + } + + private void generateDisplay( + final ClassBuilder classBuilder, + final String encoderClassName, + final String lengthExpression, + final String indent) + { + final StringBuilder sb = classBuilder.appendPublic(); + + sb.append("\n") + .append(indent).append("public String toString()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT) + .append("MutableDirectBuffer buffer = new UnsafeBuffer(new byte[").append(lengthExpression).append("]);\n") + .append(indent).append(INDENT).append(encoderClassName).append(" encoder = new ") + .append(encoderClassName).append("();\n") + .append(indent).append(INDENT).append("encoder."); + + sb.append("wrap").append("(buffer, 0);\n"); + + sb.append(indent).append(INDENT).append("encodeWith(encoder, this);\n") + .append(indent).append(INDENT).append("StringBuilder sb = new StringBuilder();\n") + .append(indent).append(INDENT).append("encoder.appendTo(sb);\n") + .append(indent).append(INDENT).append("return sb.toString();\n") + .append(indent).append("}\n"); + } + + private void generateFields( + final ClassBuilder classBuilder, + final String decoderClassName, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token signalToken = tokens.get(i); + if (signalToken.signal() == Signal.BEGIN_FIELD) + { + final Token encodingToken = tokens.get(i + 1); + final String propertyName = signalToken.name(); + + switch (encodingToken.signal()) + { + case ENCODING: + generatePrimitiveProperty( + classBuilder, decoderClassName, propertyName, signalToken, encodingToken, indent); + break; + + case BEGIN_ENUM: + generateEnumProperty(classBuilder, propertyName, signalToken, encodingToken, indent); + break; + + case BEGIN_SET: + case BEGIN_COMPOSITE: + generateComplexProperty(classBuilder, propertyName, signalToken, encodingToken, indent); + break; + + default: + break; + } + } + } + } + + private void generateComplexProperty( + final ClassBuilder classBuilder, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String typeName = formatDtoClassName(typeToken.applicableTypeName()); + final String formattedPropertyName = formatPropertyName(propertyName); + final String fieldName = formatFieldName(propertyName); + + classBuilder.appendField() + .append(indent).append("private ").append(typeName).append(" ").append(fieldName) + .append(" = new ").append(typeName).append("();\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public ").append(typeName).append(" ") + .append(formattedPropertyName).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public void ") + .append(formattedPropertyName).append("(").append(typeName).append(" value)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(fieldName).append(" = value;\n") + .append(indent).append("}\n"); + } + + private void generateEnumProperty( + final ClassBuilder classBuilder, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String enumName = formatClassName(typeToken.applicableTypeName()); + + final String formattedPropertyName = formatPropertyName(propertyName); + + if (fieldToken.isConstantEncoding()) + { + final String constValue = fieldToken.encoding().constValue().toString(); + final String caseName = constValue.substring(constValue.indexOf(".") + 1); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public static ").append(enumName).append(" ") + .append(formattedPropertyName).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(enumName).append(".") + .append(caseName).append(";\n") + .append(indent).append("}\n"); + } + else + { + final String fieldName = formatFieldName(propertyName); + + classBuilder.appendField() + .append(indent).append("private ").append(enumName).append(" ").append(fieldName).append(";\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public ").append(enumName).append(" ") + .append(formattedPropertyName).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public void ").append(formattedPropertyName) + .append("(").append(enumName).append(" value)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(fieldName).append(" = value;\n") + .append(indent).append("}\n"); + } + } + + private void generatePrimitiveProperty( + final ClassBuilder classBuilder, + final String decoderClassName, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + if (typeToken.isConstantEncoding()) + { + generateConstPropertyMethods(classBuilder, propertyName, fieldToken, typeToken, indent); + } + else + { + generatePrimitivePropertyMethods( + classBuilder, decoderClassName, propertyName, fieldToken, typeToken, indent); + } + } + + private void generatePrimitivePropertyMethods( + final ClassBuilder classBuilder, + final String decoderClassName, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final int arrayLength = typeToken.arrayLength(); + + if (arrayLength == 1) + { + generateSingleValueProperty(classBuilder, decoderClassName, propertyName, fieldToken, typeToken, indent); + } + else if (arrayLength > 1) + { + generateArrayProperty(classBuilder, decoderClassName, propertyName, fieldToken, typeToken, indent); + } + } + + private void generateArrayProperty( + final ClassBuilder classBuilder, + final String decoderClassName, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String formattedPropertyName = formatPropertyName(propertyName); + final String fieldName = formatFieldName(propertyName); + final String validateMethod = "validate" + toUpperFirstChar(propertyName); + + if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) + { + final CharSequence typeName = "String"; + + classBuilder.appendField() + .append(indent).append("private ").append(typeName).append(" ").append(fieldName).append(";\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public ").append(typeName).append(" ") + .append(formattedPropertyName).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("void ").append(formattedPropertyName) + .append("(").append(typeName).append(" value)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(validateMethod).append("(value);\n") + .append(indent).append(INDENT).append(fieldName).append(" = value;\n") + .append(indent).append("}\n"); + + generateArrayValidateMethod( + classBuilder, + decoderClassName, + indent, + validateMethod, + typeName, + ".length()", + formattedPropertyName); + } + else + { + final String elementTypeName = javaTypeName(typeToken.encoding().primitiveType()); + final String typeName = elementTypeName + "[]"; + + classBuilder.appendField() + .append(indent).append("private ").append(typeName).append(" ").append(fieldName).append(";\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public ").append(typeName).append(" ") + .append(formattedPropertyName).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public void ").append(formattedPropertyName).append("(") + .append(typeName).append(" value").append(")\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(validateMethod).append("(value);\n") + .append(indent).append(INDENT).append(fieldName).append(" = value;\n") + .append(indent).append("}\n"); + + generateArrayValidateMethod( + classBuilder, + decoderClassName, + indent, + validateMethod, + typeName, + ".length", + formattedPropertyName); + } + } + + private static void generateArrayValidateMethod( + final ClassBuilder classBuilder, + final String decoderClassName, + final String indent, + final String validateMethod, + final CharSequence typeName, + final String lengthAccessor, + final String formattedPropertyName) + { + final StringBuilder validateBuilder = classBuilder.appendPrivate().append("\n") + .append(indent).append("private static void ").append(validateMethod).append("(") + .append(typeName).append(" value)\n") + .append(indent).append("{\n"); + + validateBuilder.append(indent).append(INDENT) + .append("if (value").append(lengthAccessor).append(" > ").append(decoderClassName).append(".") + .append(formattedPropertyName).append("Length())\n") + .append(indent).append(INDENT) + .append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("throw new IllegalArgumentException(\"") + .append(formattedPropertyName) + .append(": too many elements: \" + ") + .append("value").append(lengthAccessor).append(");\n") + .append(indent).append(INDENT) + .append("}\n") + .append(indent).append("}\n"); + } + + private void generateSingleValueProperty( + final ClassBuilder classBuilder, + final String decoderClassName, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + final String typeName = javaTypeName(typeToken.encoding().primitiveType()); + final String formattedPropertyName = formatPropertyName(propertyName); + final String fieldName = formatFieldName(propertyName); + final String validateMethod = "validate" + toUpperFirstChar(propertyName); + + final boolean representedWithinJavaType = typeToken.encoding().primitiveType() != PrimitiveType.UINT64; + + final StringBuilder validationCall = new StringBuilder(); + + if (representedWithinJavaType) + { + final StringBuilder validateBuilder = classBuilder.appendPrivate().append("\n") + .append(indent).append("private static void ").append(validateMethod).append("(") + .append(typeName).append(" value)\n") + .append(indent).append("{\n"); + + validateBuilder.append(indent).append(INDENT) + .append("if (value < ") + .append(decoderClassName).append(".").append(formattedPropertyName).append("MinValue() || ") + .append("value").append(" > ") + .append(decoderClassName).append(".").append(formattedPropertyName).append("MaxValue())\n") + .append(indent).append(INDENT) + .append("{\n") + .append(indent).append(INDENT).append(INDENT) + .append("throw new IllegalArgumentException(\"") + .append(propertyName) + .append(": value is out of allowed range: \" + ") + .append("value").append(");\n") + .append(indent).append(INDENT) + .append("}\n") + .append(indent).append("}\n"); + + validationCall.append(indent).append(INDENT).append(validateMethod).append("(value);\n"); + } + + classBuilder.appendField() + .append(indent).append("private ").append(typeName).append(" ").append(fieldName).append(";\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public ").append(typeName).append(" ") + .append(formattedPropertyName).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public void ").append(formattedPropertyName).append("(") + .append(typeName).append(" value)\n") + .append(indent).append("{\n") + .append(validationCall) + .append(indent).append(INDENT).append(fieldName).append(" = value;\n") + .append(indent).append("}\n"); + } + + private void generateConstPropertyMethods( + final ClassBuilder classBuilder, + final String propertyName, + final Token fieldToken, + final Token typeToken, + final String indent) + { + if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) + { + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public static String ").append(toLowerFirstChar(propertyName)).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT) + .append("return \"").append(typeToken.encoding().constValue().toString()).append("\";\n") + .append(indent).append("}\n"); + } + else + { + final CharSequence literalValue = + generateLiteral(typeToken.encoding().primitiveType(), typeToken.encoding().constValue().toString()); + + classBuilder.appendPublic().append("\n") + .append(generateDocumentation(indent, fieldToken)) + .append(indent).append("public static ") + .append(javaTypeName(typeToken.encoding().primitiveType())) + .append(" ").append(formatPropertyName(propertyName)).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(literalValue).append(";\n") + .append(indent).append("}\n"); + } + } + + private void generateVarData( + final ClassBuilder classBuilder, + final List tokens, + final String indent) + { + for (int i = 0, size = tokens.size(); i < size; i++) + { + final Token token = tokens.get(i); + if (token.signal() == Signal.BEGIN_VAR_DATA) + { + final String propertyName = token.name(); + final Token varDataToken = Generators.findFirst("varData", tokens, i); + final String characterEncoding = varDataToken.encoding().characterEncoding(); + final String dtoType = characterEncoding == null ? "byte[]" : "String"; + + final String fieldName = formatFieldName(propertyName); + final String formattedPropertyName = formatPropertyName(propertyName); + + classBuilder.appendField() + .append(indent).append("private ").append(dtoType).append(" ").append(fieldName).append(";\n"); + + classBuilder.appendPublic().append("\n") + .append(indent).append("public ").append(dtoType).append(" ") + .append(formattedPropertyName).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic().append("\n") + .append(indent).append("public void ").append(formattedPropertyName) + .append("(").append(dtoType).append(" value)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(fieldName).append(" = value;\n") + .append(indent).append("}\n"); + } + } + } + + private static String formatDtoClassName(final String name) + { + return formatClassName(name + "Dto"); + } + + private void generateDtosForTypes() throws IOException + { + for (final List tokens : ir.types()) + { + switch (tokens.get(0).signal()) + { + case BEGIN_COMPOSITE: + generateComposite(tokens); + break; + + case BEGIN_SET: + generateChoiceSet(tokens); + break; + + default: + break; + } + } + } + + private void generateComposite(final List tokens) throws IOException + { + final String name = tokens.get(0).applicableTypeName(); + final String className = formatDtoClassName(name); + final String encoderClassName = encoderName(name); + final String decoderClassName = decoderName(name); + + try (Writer out = outputManager.createOutput(className)) + { + final List compositeTokens = tokens.subList(1, tokens.size() - 1); + out.append(generateDtoFileHeader(ir.applicableNamespace())); + out.append("import org.agrona.DirectBuffer;\n"); + out.append("import org.agrona.MutableDirectBuffer;\n"); + out.append("import org.agrona.concurrent.UnsafeBuffer;\n\n"); + out.append(generateDocumentation(BASE_INDENT, tokens.get(0))); + + final ClassBuilder classBuilder = new ClassBuilder(className, BASE_INDENT, "public final"); + + generateCompositePropertyElements(classBuilder, decoderClassName, compositeTokens, + BASE_INDENT + INDENT); + generateCompositeDecodeWith(classBuilder, className, decoderClassName, compositeTokens, + BASE_INDENT + INDENT); + generateCompositeEncodeWith(classBuilder, className, encoderClassName, compositeTokens, + BASE_INDENT + INDENT); + generateDisplay(classBuilder, encoderClassName, encoderClassName + ".ENCODED_LENGTH", + BASE_INDENT + INDENT); + + classBuilder.appendTo(out); + } + } + + private void generateChoiceSet(final List tokens) throws IOException + { + final String name = tokens.get(0).applicableTypeName(); + final String className = formatDtoClassName(name); + final String encoderClassName = encoderName(name); + final String decoderClassName = decoderName(name); + + try (Writer out = outputManager.createOutput(className)) + { + final List setTokens = tokens.subList(1, tokens.size() - 1); + out.append(generateDtoFileHeader(ir.applicableNamespace())); + out.append(generateDocumentation(BASE_INDENT, tokens.get(0))); + + final ClassBuilder classBuilder = new ClassBuilder(className, BASE_INDENT, "public final"); + + generateChoices(classBuilder, className, setTokens, BASE_INDENT + INDENT); + generateChoiceSetDecodeWith(classBuilder, className, decoderClassName, setTokens, BASE_INDENT + INDENT); + generateChoiceSetEncodeWith(classBuilder, className, encoderClassName, setTokens, BASE_INDENT + INDENT); + + classBuilder.appendTo(out); + } + } + + private void generateChoiceSetEncodeWith( + final ClassBuilder classBuilder, + final String dtoClassName, + final String encoderClassName, + final List setTokens, + final String indent) + { + final StringBuilder encodeBuilder = classBuilder.appendPublic() + .append("\n") + .append(indent).append("public static void encodeWith(\n") + .append(indent).append(INDENT).append(encoderClassName).append(" encoder, ") + .append(dtoClassName).append(" dto)\n") + .append(indent).append("{\n"); + + encodeBuilder.append(indent).append(INDENT).append("encoder.clear();\n"); + + for (final Token token : setTokens) + { + if (token.signal() == Signal.CHOICE) + { + final String formattedPropertyName = formatPropertyName(token.name()); + encodeBuilder.append(indent).append(INDENT).append("encoder.").append(formattedPropertyName) + .append("(dto.").append(formattedPropertyName).append("());\n"); + } + } + + encodeBuilder.append(indent).append("}\n"); + } + + private void generateChoiceSetDecodeWith( + final ClassBuilder classBuilder, + final String dtoClassName, + final String decoderClassName, + final List setTokens, + final String indent) + { + final StringBuilder decodeBuilder = classBuilder.appendPublic() + .append("\n") + .append(indent).append("public static void decodeWith(\n") + .append(indent).append(INDENT).append(decoderClassName).append(" decoder, ") + .append(dtoClassName).append(" dto)\n") + .append(indent).append("{\n"); + + for (final Token token : setTokens) + { + if (token.signal() == Signal.CHOICE) + { + final String formattedPropertyName = formatPropertyName(token.name()); + decodeBuilder.append(indent).append(INDENT).append("dto.").append(formattedPropertyName) + .append("(decoder.").append(formattedPropertyName).append("());\n"); + } + } + + decodeBuilder.append(indent).append("}\n"); + } + + private void generateChoices( + final ClassBuilder classBuilder, + final String dtoClassName, + final List setTokens, + final String indent) + { + final List fields = new ArrayList<>(); + + for (final Token token : setTokens) + { + if (token.signal() == Signal.CHOICE) + { + final String fieldName = formatFieldName(token.name()); + final String formattedPropertyName = formatPropertyName(token.name()); + + fields.add(fieldName); + + classBuilder.appendField() + .append(indent).append("boolean ").append(fieldName).append(";\n"); + + classBuilder.appendPublic() + .append("\n") + .append(indent).append("public boolean ").append(formattedPropertyName).append("()\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append("return ").append(fieldName).append(";\n") + .append(indent).append("}\n"); + + classBuilder.appendPublic() + .append("\n") + .append(indent).append(dtoClassName).append(" ") + .append(formattedPropertyName).append("(boolean value)\n") + .append(indent).append("{\n") + .append(indent).append(INDENT).append(fieldName).append(" = value;\n") + .append(indent).append(INDENT).append("return this;\n") + .append(indent).append("}\n"); + } + } + + final StringBuilder clearBuilder = classBuilder.appendPublic() + .append(indent).append(dtoClassName).append(" clear()\n") + .append(indent).append("{\n"); + + for (final String field : fields) + { + clearBuilder.append(indent).append(INDENT).append(field).append(" = false;\n"); + } + + clearBuilder.append(indent).append(INDENT).append("return this;\n") + .append(indent).append("}\n"); + } + + private void generateCompositePropertyElements( + final ClassBuilder classBuilder, + final String decoderClassName, + final List tokens, + final String indent) + { + for (int i = 0; i < tokens.size(); ) + { + final Token token = tokens.get(i); + final String propertyName = formatPropertyName(token.name()); + + switch (token.signal()) + { + case ENCODING: + generatePrimitiveProperty(classBuilder, decoderClassName, propertyName, token, token, indent); + break; + + case BEGIN_ENUM: + generateEnumProperty(classBuilder, propertyName, token, token, indent); + break; + + case BEGIN_SET: + case BEGIN_COMPOSITE: + generateComplexProperty(classBuilder, propertyName, token, token, indent); + break; + + default: + break; + } + + i += tokens.get(i).componentTokenCount(); + } + } + + private static CharSequence generateDtoFileHeader(final String packageName) + { + final StringBuilder sb = new StringBuilder(); + + sb.append("/* Generated SBE (Simple Binary Encoding) message DTO */\n"); + sb.append("package ").append(packageName).append(";\n\n"); + + return sb; + } + + private static String generateDocumentation(final String indent, final Token token) + { + final String description = token.description(); + if (null == description || description.isEmpty()) + { + return ""; + } + + return + indent + "/**\n" + + indent + " * " + description + "\n" + + indent + " */\n"; + } + + private static String formatFieldName(final String propertyName) + { + return formatPropertyName(propertyName); + } +} diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtos.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtos.java new file mode 100644 index 0000000000..45eeccff93 --- /dev/null +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtos.java @@ -0,0 +1,35 @@ +/* + * Copyright 2013-2024 Real Logic Limited. + * Copyright (C) 2017 MarketFactory, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.generation.java; + +import uk.co.real_logic.sbe.generation.CodeGenerator; +import uk.co.real_logic.sbe.generation.TargetCodeGenerator; +import uk.co.real_logic.sbe.ir.Ir; + +/** + * {@link CodeGenerator} factory for Java DTOs. + */ +public class JavaDtos implements TargetCodeGenerator +{ + /** + * {@inheritDoc} + */ + public CodeGenerator newInstance(final Ir ir, final String outputDir) + { + return new JavaDtoGenerator(ir, new JavaOutputManager(outputDir, ir.applicableNamespace())); + } +} diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java index 1a5e17bb20..01cd0cf121 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java @@ -303,7 +303,7 @@ private void generateEncoder( if (shouldGenerateGroupOrderAnnotation) { - generateAnnotations(BASE_INDENT, className, groups, out, this::encoderName); + generateAnnotations(BASE_INDENT, className, groups, out, JavaUtil::encoderName); } out.append(generateDeclaration(className, implementsString, msgToken)); @@ -816,7 +816,7 @@ private void generateDecoder( if (shouldGenerateGroupOrderAnnotation) { - generateAnnotations(BASE_INDENT, className, groups, out, this::decoderName); + generateAnnotations(BASE_INDENT, className, groups, out, JavaUtil::decoderName); } out.append(generateDeclaration(className, implementsString, msgToken)); @@ -872,7 +872,7 @@ private void generateDecoderGroups( if (shouldGenerateGroupOrderAnnotation) { - generateAnnotations(indent + INDENT, groupName, groups, sb, this::decoderName); + generateAnnotations(indent + INDENT, groupName, groups, sb, JavaUtil::decoderName); } generateGroupDecoderClassHeader(sb, groupName, outerClassName, fieldPrecedenceModel, groupToken, tokens, groups, index, indent + INDENT); @@ -926,7 +926,7 @@ private void generateEncoderGroups( if (shouldGenerateGroupOrderAnnotation) { - generateAnnotations(indent + INDENT, groupClassName, groups, sb, this::encoderName); + generateAnnotations(indent + INDENT, groupClassName, groups, sb, JavaUtil::encoderName); } generateGroupEncoderClassHeader( sb, groupName, outerClassName, fieldPrecedenceModel, groupToken, @@ -4701,16 +4701,6 @@ private static String validateBufferImplementation( } } - private String encoderName(final String className) - { - return formatClassName(className) + "Encoder"; - } - - private String decoderName(final String className) - { - return formatClassName(className) + "Decoder"; - } - private String implementsInterface(final String interfaceName) { return shouldGenerateInterfaces ? " implements " + interfaceName : ""; diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaUtil.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaUtil.java index 5bdc6e38b0..2431542d30 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaUtil.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaUtil.java @@ -464,6 +464,16 @@ public static void generateGroupEncodePropertyJavadoc( .append(indent).append(" */"); } + static String encoderName(final String className) + { + return formatClassName(className) + "Encoder"; + } + + static String decoderName(final String className) + { + return formatClassName(className) + "Decoder"; + } + private static void escapeJavadoc(final Appendable out, final String doc) throws IOException { for (int i = 0, length = doc.length(); i < length; i++) @@ -511,4 +521,5 @@ private static void escapeJavadoc(final StringBuilder sb, final String doc) } } } + } diff --git a/sbe-tool/src/test/java/uk/co/real_logic/sbe/generation/java/DtoTest.java b/sbe-tool/src/test/java/uk/co/real_logic/sbe/generation/java/DtoTest.java new file mode 100644 index 0000000000..a28b0ac374 --- /dev/null +++ b/sbe-tool/src/test/java/uk/co/real_logic/sbe/generation/java/DtoTest.java @@ -0,0 +1,154 @@ +/* + * Copyright 2013-2024 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.generation.java; + +import extension.*; +import org.agrona.ExpandableArrayBuffer; +import org.agrona.MutableDirectBuffer; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +public class DtoTest +{ + @Test + void shouldRoundTripCar1() + { + final ExpandableArrayBuffer inputBuffer = new ExpandableArrayBuffer(); + encodeCar(inputBuffer, 0); + + final CarDecoder decoder = new CarDecoder(); + decoder.wrap(inputBuffer, 0, CarDecoder.BLOCK_LENGTH, CarDecoder.SCHEMA_VERSION); + final String decoderString = decoder.toString(); + + final CarDto dto = new CarDto(); + CarDto.decodeWith(decoder, dto); + + final ExpandableArrayBuffer outputBuffer = new ExpandableArrayBuffer(); + final CarEncoder encoder = new CarEncoder(); + encoder.wrap(outputBuffer, 0); + CarDto.encodeWith(encoder, dto); + + final String dtoString = dto.toString(); + + assertThat(outputBuffer.byteArray(), equalTo(inputBuffer.byteArray())); + assertThat(dtoString, equalTo(decoderString)); + } + + @Test + void shouldRoundTripCar2() + { + final ExpandableArrayBuffer inputBuffer = new ExpandableArrayBuffer(); + final int inputLength = encodeCar(inputBuffer, 0); + + final CarDto dto = + CarDto.decodeFrom(inputBuffer, 0, (short)CarDecoder.BLOCK_LENGTH, (short)CarDecoder.SCHEMA_VERSION); + + final ExpandableArrayBuffer outputBuffer = new ExpandableArrayBuffer(); + final int outputLength = CarDto.encodeWith(dto, outputBuffer, 0); + + assertThat(outputLength, equalTo(inputLength)); + assertThat(outputBuffer.byteArray(), equalTo(inputBuffer.byteArray())); + } + + private static int encodeCar(final MutableDirectBuffer buffer, final int offset) + { + final CarEncoder car = new CarEncoder(); + car.wrap(buffer, offset); + car.serialNumber(1234); + car.modelYear(2013); + car.available(BooleanType.T); + car.code(Model.A); + car.vehicleCode("ABCDEF"); + + for (int i = 0, size = CarEncoder.someNumbersLength(); i < size; i++) + { + car.someNumbers(i, i); + } + + car.extras().cruiseControl(true).sportsPack(true); + + car.cupHolderCount((short)119); + + car.engine().capacity(2000) + .numCylinders((short)4) + .manufacturerCode("ABC") + .efficiency((byte)35) + .boosterEnabled(BooleanType.T) + .booster().boostType(BoostType.NITROUS) + .horsePower((short)200); + + final CarEncoder.FuelFiguresEncoder fuelFigures = car.fuelFiguresCount(3); + fuelFigures.next() + .speed(30) + .mpg(35.9f) + .usageDescription("this is a description"); + + fuelFigures.next() + .speed(55) + .mpg(49.0f) + .usageDescription("this is a description"); + + fuelFigures.next() + .speed(75) + .mpg(40.0f) + .usageDescription("this is a description"); + + final CarEncoder.PerformanceFiguresEncoder perfFigures = car.performanceFiguresCount(2); + + perfFigures.next() + .octaneRating((short)95); + + CarEncoder.PerformanceFiguresEncoder.AccelerationEncoder acceleration = perfFigures.accelerationCount(3); + + acceleration.next() + .mph(30) + .seconds(4.0f); + + acceleration.next() + .mph(60) + .seconds(7.5f); + + acceleration.next() + .mph(100) + .seconds(12.2f); + + perfFigures.next() + .octaneRating((short)99); + + acceleration = perfFigures.accelerationCount(3); + + acceleration.next() + .mph(30) + .seconds(3.8f); + + acceleration.next() + .mph(60) + .seconds(7.1f); + + acceleration.next() + .mph(100) + .seconds(11.8f); + + car.manufacturer("Ford"); + car.model("Fiesta"); + car.activationCode("1234"); + + return car.limit(); + } +} diff --git a/sbe-tool/src/test/resources/example-extension-schema.xml b/sbe-tool/src/test/resources/example-extension-schema.xml new file mode 100644 index 0000000000..3770489b9f --- /dev/null +++ b/sbe-tool/src/test/resources/example-extension-schema.xml @@ -0,0 +1,103 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + T + S + N + K + + + + + + + 9000 + + Petrol + + + + + + 0 + 1 + + + A + B + C + + + 0 + 1 + 2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 41f4b0e27932ef5dfca502b52fe24ff5e86f4f08 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Tue, 14 May 2024 11:57:24 +0100 Subject: [PATCH 38/41] [Java] Extend property-based tests to exercise Java DTOs. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The added test (non-exhaustively) checks the property: ``` ∀ msg ∈ MessageSchemas, ∀ encoding ∈ EncodingsOf(msg), encoding = dtoEncode(dtoDecode(encoding)) ``` The added test shows some compilation failures, which need to be resolved before enabling it. --- .../sbe/generation/java/JavaGenerator.java | 1 + .../sbe/properties/DtosPropertyTest.java | 112 +++++++++- .../utils/InMemoryOutputManager.java | 207 ++++++++++++++++++ 3 files changed, 319 insertions(+), 1 deletion(-) create mode 100644 sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/utils/InMemoryOutputManager.java diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java index 01cd0cf121..2d939309b9 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java @@ -160,6 +160,7 @@ public JavaGenerator( { Verify.notNull(ir, "ir"); Verify.notNull(outputManager, "outputManager"); + Verify.notNull(precedenceChecks, "precedenceChecks"); this.ir = ir; this.shouldSupportTypesPackageNames = shouldSupportTypesPackageNames; diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java index f71e416309..fed7d4c94d 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java @@ -13,28 +13,42 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package uk.co.real_logic.sbe.properties; import net.jqwik.api.*; +import uk.co.real_logic.sbe.generation.common.PrecedenceChecks; import uk.co.real_logic.sbe.generation.cpp.CppDtoGenerator; import uk.co.real_logic.sbe.generation.cpp.CppGenerator; import uk.co.real_logic.sbe.generation.cpp.NamespaceOutputManager; import uk.co.real_logic.sbe.generation.csharp.CSharpDtoGenerator; import uk.co.real_logic.sbe.generation.csharp.CSharpGenerator; import uk.co.real_logic.sbe.generation.csharp.CSharpNamespaceOutputManager; +import uk.co.real_logic.sbe.generation.java.JavaDtoGenerator; +import uk.co.real_logic.sbe.generation.java.JavaGenerator; +import uk.co.real_logic.sbe.ir.generated.MessageHeaderDecoder; import uk.co.real_logic.sbe.properties.arbitraries.SbeArbitraries; +import uk.co.real_logic.sbe.properties.utils.InMemoryOutputManager; +import org.agrona.DirectBuffer; +import org.agrona.ExpandableArrayBuffer; import org.agrona.IoUtil; +import org.agrona.MutableDirectBuffer; +import org.agrona.concurrent.UnsafeBuffer; import org.agrona.io.DirectBufferInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URLClassLoader; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; +import static uk.co.real_logic.sbe.SbeTool.JAVA_DEFAULT_DECODING_BUFFER_TYPE; +import static uk.co.real_logic.sbe.SbeTool.JAVA_DEFAULT_ENCODING_BUFFER_TYPE; + @SuppressWarnings("ReadWriteStringCanBeUsed") public class DtosPropertyTest { @@ -44,6 +58,101 @@ public class DtosPropertyTest private static final String CPP_EXECUTABLE = System.getProperty("sbe.tests.cpp.executable", "g++"); private static final boolean KEEP_DIR_ON_FAILURE = Boolean.parseBoolean( System.getProperty("sbe.tests.keep.dir.on.failure", "true")); + private final ExpandableArrayBuffer outputBuffer = new ExpandableArrayBuffer(); + +// @Test +// void oneRun() throws IOException, ClassNotFoundException, InvocationTargetException, NoSuchMethodException, +// IllegalAccessException +// { +// javaDtoEncodeShouldBeTheInverseOfDtoDecode(encodedMessage().sample()); +// } + + @Property + @Disabled + void javaDtoEncodeShouldBeTheInverseOfDtoDecode( + @ForAll("encodedMessage") final SbeArbitraries.EncodedMessage encodedMessage + ) throws IOException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException, + IllegalAccessException + { + final String packageName = encodedMessage.ir().applicableNamespace(); + final InMemoryOutputManager outputManager = new InMemoryOutputManager(packageName); + + boolean success = false; + + try + { + try + { + new JavaGenerator( + encodedMessage.ir(), + JAVA_DEFAULT_ENCODING_BUFFER_TYPE, + JAVA_DEFAULT_DECODING_BUFFER_TYPE, + false, + false, + false, + false, + PrecedenceChecks.newInstance(new PrecedenceChecks.Context()), + outputManager) + .generate(); + + new JavaDtoGenerator(encodedMessage.ir(), outputManager) + .generate(); + } + catch (final Exception generationException) + { + throw new AssertionError( + "Code generation failed.\n\n" + + "SCHEMA:\n" + encodedMessage.schema(), + generationException); + } + + try (URLClassLoader generatedClassLoader = outputManager.compileGeneratedSources()) + { + final Class dtoClass = + generatedClassLoader.loadClass(packageName + ".TestMessageDto"); + + final Method decodeFrom = + dtoClass.getMethod("decodeFrom", DirectBuffer.class, int.class, short.class, short.class); + + final Method encodeWith = + dtoClass.getMethod("encodeWithHeaderWith", dtoClass, MutableDirectBuffer.class, int.class); + + final int inputLength = encodedMessage.length(); + final ExpandableArrayBuffer inputBuffer = encodedMessage.buffer(); + final MessageHeaderDecoder header = new MessageHeaderDecoder().wrap(inputBuffer, 0); + final short blockLength = (short)header.blockLength(); + final short actingVersion = (short)header.version(); + final Object dto = decodeFrom.invoke(null, + encodedMessage.buffer(), MessageHeaderDecoder.ENCODED_LENGTH, blockLength, actingVersion); + outputBuffer.setMemory(0, outputBuffer.capacity(), (byte)0); + final int outputLength = (int)encodeWith.invoke(null, dto, outputBuffer, 0); + if (!areEqual(inputBuffer, inputLength, outputBuffer, outputLength)) + { + throw new AssertionError( + "Input and output differ\n\n" + + "SCHEMA:\n" + encodedMessage.schema()); + } + + success = true; + } + } + finally + { + if (!success) + { + outputManager.dumpSources(); + } + } + } + + private boolean areEqual( + final ExpandableArrayBuffer inputBuffer, + final int inputLength, + final ExpandableArrayBuffer outputBuffer, + final int outputLength) + { + return new UnsafeBuffer(inputBuffer, 0, inputLength).equals(new UnsafeBuffer(outputBuffer, 0, outputLength)); + } @Property void csharpDtoEncodeShouldBeTheInverseOfDtoDecode( @@ -274,4 +383,5 @@ private static void copyResourceToFile( throw new RuntimeException(e); } } + } diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/utils/InMemoryOutputManager.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/utils/InMemoryOutputManager.java new file mode 100644 index 0000000000..a34669593e --- /dev/null +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/utils/InMemoryOutputManager.java @@ -0,0 +1,207 @@ +/* + * Copyright 2013-2024 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.co.real_logic.sbe.properties.utils; + +import org.agrona.generation.DynamicPackageOutputManager; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.net.URI; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.*; +import javax.tools.*; + +/** + * An implementation of {@link DynamicPackageOutputManager} that stores generated source code in memory and compiles it + * on demand. + */ +public class InMemoryOutputManager implements DynamicPackageOutputManager +{ + private final String packageName; + private final Map sourceFiles = new HashMap<>(); + private String packageNameOverride; + + public InMemoryOutputManager(final String packageName) + { + this.packageName = packageName; + } + + @Override + public Writer createOutput(final String name) + { + return new InMemoryWriter(name); + } + + @Override + public void setPackageName(final String packageName) + { + packageNameOverride = packageName; + } + + /** + * Compile the generated sources and return a {@link URLClassLoader} that can be used to load the generated classes. + * + * @return a {@link URLClassLoader} that can be used to load the generated classes + */ + public URLClassLoader compileGeneratedSources() + { + final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + final StandardJavaFileManager standardFileManager = compiler.getStandardFileManager(null, null, null); + final InMemoryFileManager fileManager = new InMemoryFileManager(standardFileManager); + final JavaCompiler.CompilationTask task = compiler.getTask( + null, + fileManager, + null, + null, + null, + sourceFiles.values() + ); + + if (!task.call()) + { + throw new IllegalStateException("Compilation failed"); + } + + final GeneratedCodeLoader classLoader = new GeneratedCodeLoader(getClass().getClassLoader()); + classLoader.defineClasses(fileManager); + return classLoader; + } + + public void dumpSources() + { + sourceFiles.forEach((qualifiedName, file) -> + { + System.out.println( + System.lineSeparator() + "Source file: " + qualifiedName + + System.lineSeparator() + file.sourceCode + + System.lineSeparator()); + }); + } + + class InMemoryWriter extends StringWriter + { + private final String name; + + InMemoryWriter(final String name) + { + this.name = name; + } + + @Override + public void close() throws IOException + { + super.close(); + final String actingPackageName = packageNameOverride == null ? packageName : packageNameOverride; + packageNameOverride = null; + + final String qualifiedName = actingPackageName + "." + name; + final InMemoryJavaFileObject sourceFile = + new InMemoryJavaFileObject(qualifiedName, getBuffer().toString()); + + final InMemoryJavaFileObject existingFile = sourceFiles.putIfAbsent(qualifiedName, sourceFile); + + if (existingFile != null && !Objects.equals(existingFile.sourceCode, sourceFile.sourceCode)) + { + throw new IllegalStateException("Duplicate (but different) class: " + qualifiedName); + } + } + } + + static class InMemoryFileManager extends ForwardingJavaFileManager + { + private final List outputFiles = new ArrayList<>(); + + InMemoryFileManager(final StandardJavaFileManager fileManager) + { + super(fileManager); + } + + @Override + public JavaFileObject getJavaFileForOutput( + final Location location, + final String className, + final JavaFileObject.Kind kind, + final FileObject sibling) + { + final InMemoryJavaFileObject outputFile = new InMemoryJavaFileObject(className, kind); + outputFiles.add(outputFile); + return outputFile; + } + + public Collection outputFiles() + { + return outputFiles; + } + } + + static class InMemoryJavaFileObject extends SimpleJavaFileObject + { + private final String sourceCode; + private final ByteArrayOutputStream outputStream; + + InMemoryJavaFileObject(final String className, final String sourceCode) + { + super(URI.create("string:///" + className.replace('.', '/') + Kind.SOURCE.extension), Kind.SOURCE); + this.sourceCode = sourceCode; + this.outputStream = new ByteArrayOutputStream(); + } + + InMemoryJavaFileObject(final String className, final Kind kind) + { + super(URI.create("mem:///" + className.replace('.', '/') + kind.extension), kind); + this.sourceCode = null; + this.outputStream = new ByteArrayOutputStream(); + } + + @Override + public CharSequence getCharContent(final boolean ignoreEncodingErrors) + { + return sourceCode; + } + + @Override + public ByteArrayOutputStream openOutputStream() + { + return outputStream; + } + + public byte[] getClassBytes() + { + return outputStream.toByteArray(); + } + } + + static class GeneratedCodeLoader extends URLClassLoader + { + GeneratedCodeLoader(final ClassLoader parent) + { + super(new URL[0], parent); + } + + void defineClasses(final InMemoryFileManager fileManager) + { + fileManager.outputFiles().forEach(file -> + { + final byte[] classBytes = file.getClassBytes(); + super.defineClass(file.getName(), classBytes, 0, classBytes.length); + }); + } + } +} From 75df4629c72ea06c1fd73d420709fa3868183f44 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Wed, 15 May 2024 11:45:10 +0100 Subject: [PATCH 39/41] [Java] Avoid checking parentMessage.actingVersion inside composite. Most not present conditions were already elided via `generateFieldNotPresentCondition(inComposite=true, ...)` but the condition within the wrap method was not. Here's an example of a failing schema found with the DTO property-based tests: ``` ``` And generated code (within a composite without a `parentMessage`): ``` public void wrapMember0OfType0(final DirectBuffer wrapBuffer) { if (parentMessage.actingVersion < 1) { wrapBuffer.wrap(buffer, offset, 0); return; } wrapBuffer.wrap(buffer, offset + 0, 2); } ``` --- .../real_logic/sbe/generation/java/JavaGenerator.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java index 2d939309b9..0a47fbfe56 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java @@ -1665,7 +1665,7 @@ private void generateVarDataWrapDecoder( indent + " }\n", propertyName, readOnlyBuffer, - generateWrapFieldNotPresentCondition(token.version(), indent), + generateWrapFieldNotPresentCondition(false, token.version(), indent), accessOrderListenerCall, sizeOfLengthField, PrimitiveType.UINT32 == lengthType ? "(int)" : "", @@ -2671,9 +2671,12 @@ private CharSequence generatePrimitivePropertyEncode( generatePut(encoding.primitiveType(), "offset + " + offset, "value", byteOrderStr)); } - private CharSequence generateWrapFieldNotPresentCondition(final int sinceVersion, final String indent) + private CharSequence generateWrapFieldNotPresentCondition( + final boolean inComposite, + final int sinceVersion, + final String indent) { - if (0 == sinceVersion) + if (inComposite || 0 == sinceVersion) { return ""; } @@ -2925,7 +2928,7 @@ else if (encoding.primitiveType() == PrimitiveType.UINT8) indent + " }\n", Generators.toUpperFirstChar(propertyName), readOnlyBuffer, - generateWrapFieldNotPresentCondition(propertyToken.version(), indent), + generateWrapFieldNotPresentCondition(inComposite, propertyToken.version(), indent), accessOrderListenerCall, offset, fieldLength); From 60bd11986c5b6186c5a9e6c5bb122ec2676b424c Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Wed, 15 May 2024 11:54:55 +0100 Subject: [PATCH 40/41] [Java] Fix remaining DTO issues uncovered with PBT. The main issue was around determining when a field might not be present. We were checking `token.version() > 0`, which isn't accurate in composites or groups. In this commit, I've also started to use the `Footnotes` API in JQWIK, which allows useful debug information to be captured _only_ for failing attempts. It then prints out the shrunken footnotes upon a failure. --- .../sbe/generation/java/JavaDtoGenerator.java | 131 ++++++++------ .../sbe/generation/java/JavaGenerator.java | 4 + .../sbe/properties/DtosPropertyTest.java | 161 ++++++++++-------- .../utils/InMemoryOutputManager.java | 14 +- .../resources/example-extension-schema.xml | 4 +- 5 files changed, 184 insertions(+), 130 deletions(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtoGenerator.java index b5fd05e665..0056ba4703 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtoGenerator.java @@ -30,6 +30,7 @@ import java.io.Writer; import java.util.ArrayList; import java.util.List; +import java.util.function.Predicate; import static uk.co.real_logic.sbe.generation.Generators.toLowerFirstChar; import static uk.co.real_logic.sbe.generation.Generators.toUpperFirstChar; @@ -43,6 +44,7 @@ */ public class JavaDtoGenerator implements CodeGenerator { + private static final Predicate ALWAYS_FALSE_PREDICATE = ignored -> false; private static final String INDENT = " "; private static final String BASE_INDENT = ""; @@ -97,7 +99,7 @@ public void generate() throws IOException generateVarData(classBuilder, varData, BASE_INDENT + INDENT); generateDecodeWith(classBuilder, dtoClassName, decoderClassName, fields, - groups, varData, BASE_INDENT + INDENT); + groups, varData, BASE_INDENT + INDENT, fieldToken -> fieldToken.version() > msgToken.version()); generateDecodeFrom(classBuilder, dtoClassName, decoderClassName, BASE_INDENT + INDENT); generateEncodeWith(classBuilder, dtoClassName, encoderClassName, fields, groups, varData, BASE_INDENT + INDENT); @@ -196,6 +198,13 @@ private void generateGroups( final String groupClassName = formatDtoClassName(groupName); final String qualifiedDtoClassName = qualifiedParentDtoClassName + "." + groupClassName; + final Token dimToken = tokens.get(i + 1); + if (dimToken.signal() != Signal.BEGIN_COMPOSITE) + { + throw new IllegalStateException("groups must start with BEGIN_COMPOSITE: token=" + dimToken); + } + final int sinceVersion = dimToken.version(); + final String fieldName = formatFieldName(groupName); final String formattedPropertyName = formatPropertyName(groupName); @@ -226,10 +235,22 @@ private void generateGroups( i = collectVarData(tokens, i, varData); generateVarData(groupClassBuilder, varData, indent + INDENT); + final Predicate wasAddedAfterGroup = token -> + { + final boolean addedAfterParent = token.version() > sinceVersion; + + if (addedAfterParent && token.signal() == Signal.BEGIN_VAR_DATA) + { + throw new IllegalStateException("Cannot extend var data inside a group."); + } + + return addedAfterParent; + }; + generateDecodeListWith( groupClassBuilder, groupClassName, qualifiedDecoderClassName, indent + INDENT); generateDecodeWith(groupClassBuilder, groupClassName, qualifiedDecoderClassName, - fields, groups, varData, indent + INDENT); + fields, groups, varData, indent + INDENT, wasAddedAfterGroup); generateEncodeWith( groupClassBuilder, groupClassName, qualifiedEncoderClassName, fields, groups, varData, indent + INDENT); generateComputeEncodedLength(groupClassBuilder, qualifiedDecoderClassName, @@ -324,8 +345,10 @@ private void generateComputeEncodedLength( .append(qualifiedDecoderClassName).append(".") .append(formatPropertyName(propertyName)).append("HeaderLength();\n"); + final String characterEncoding = varDataToken.encoding().characterEncoding(); + final String lengthAccessor = characterEncoding == null ? ".length" : ".length()"; lengthBuilder.append(indent).append(INDENT).append("encodedLength += ") - .append(fieldName).append(".length()"); + .append(fieldName).append(lengthAccessor); final int elementByteLength = varDataToken.encoding().primitiveType().size(); if (elementByteLength != 1) @@ -358,7 +381,7 @@ private void generateCompositeDecodeWith( final Token token = tokens.get(i); generateFieldDecodeWith( - decodeBuilder, token, token, decoderClassName, indent + INDENT); + decodeBuilder, token, token, decoderClassName, indent + INDENT, ALWAYS_FALSE_PREDICATE); i += tokens.get(i).componentTokenCount(); } @@ -426,16 +449,17 @@ private void generateDecodeWith( final List fields, final List groups, final List varData, - final String indent) + final String indent, + final Predicate wasAddedAfterParent) { final StringBuilder decodeBuilder = classBuilder.appendPublic().append("\n") .append(indent).append("public static void decodeWith(").append(decoderClassName).append(" decoder, ") .append(dtoClassName).append(" dto)\n") .append(indent).append("{\n"); - generateMessageFieldsDecodeWith(decodeBuilder, fields, decoderClassName, indent + INDENT); + generateMessageFieldsDecodeWith(decodeBuilder, fields, decoderClassName, indent + INDENT, wasAddedAfterParent); generateGroupsDecodeWith(decodeBuilder, groups, indent + INDENT); - generateVarDataDecodeWith(decodeBuilder, varData, indent + INDENT); + generateVarDataDecodeWith(decodeBuilder, decoderClassName, varData, indent + INDENT, wasAddedAfterParent); decodeBuilder.append(indent).append("}\n"); } @@ -466,7 +490,8 @@ private void generateMessageFieldsDecodeWith( final StringBuilder sb, final List tokens, final String decoderClassName, - final String indent) + final String indent, + final Predicate wasAddedAfterParent) { for (int i = 0, size = tokens.size(); i < size; i++) { @@ -475,7 +500,7 @@ private void generateMessageFieldsDecodeWith( { final Token encodingToken = tokens.get(i + 1); - generateFieldDecodeWith(sb, signalToken, encodingToken, decoderClassName, indent); + generateFieldDecodeWith(sb, signalToken, encodingToken, decoderClassName, indent, wasAddedAfterParent); } } } @@ -485,17 +510,18 @@ private void generateFieldDecodeWith( final Token fieldToken, final Token typeToken, final String decoderClassName, - final String indent) + final String indent, + final Predicate wasAddedAfterParent) { switch (typeToken.signal()) { case ENCODING: - generatePrimitiveDecodeWith(sb, fieldToken, typeToken, decoderClassName, indent); + generatePrimitiveDecodeWith(sb, fieldToken, typeToken, decoderClassName, indent, wasAddedAfterParent); break; case BEGIN_SET: final String bitSetName = formatDtoClassName(typeToken.applicableTypeName()); - generateBitSetDecodeWith(sb, fieldToken, bitSetName, indent); + generateBitSetDecodeWith(sb, decoderClassName, fieldToken, bitSetName, indent, wasAddedAfterParent); break; case BEGIN_ENUM: @@ -516,7 +542,8 @@ private void generatePrimitiveDecodeWith( final Token fieldToken, final Token typeToken, final String decoderClassName, - final String indent) + final String indent, + final Predicate wasAddedAfterParent) { if (typeToken.isConstantEncoding()) { @@ -543,12 +570,13 @@ private void generatePrimitiveDecodeWith( indent, "decoder.actingVersion() >= " + decoderClassName + "." + formattedPropertyName + "SinceVersion()", "decoder." + formattedPropertyName + "()", - decoderNullValue + decoderNullValue, + wasAddedAfterParent ); } else if (arrayLength > 1) { - generateArrayDecodeWith(sb, decoderClassName, fieldToken, typeToken, indent); + generateArrayDecodeWith(sb, decoderClassName, fieldToken, typeToken, indent, wasAddedAfterParent); } } @@ -557,7 +585,8 @@ private void generateArrayDecodeWith( final String decoderClassName, final Token fieldToken, final Token typeToken, - final String indent) + final String indent, + final Predicate wasAddedAfterParent) { if (fieldToken.isConstantEncoding()) { @@ -566,8 +595,9 @@ private void generateArrayDecodeWith( final String propertyName = fieldToken.name(); final String formattedPropertyName = formatPropertyName(propertyName); + final PrimitiveType primitiveType = typeToken.encoding().primitiveType(); - if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) + if (primitiveType == PrimitiveType.CHAR) { generateRecordPropertyAssignment( sb, @@ -575,13 +605,14 @@ private void generateArrayDecodeWith( indent, "decoder.actingVersion() >= " + decoderClassName + "." + formattedPropertyName + "SinceVersion()", "decoder." + formattedPropertyName + "()", - "\"\"" + "\"\"", + wasAddedAfterParent ); } else { final StringBuilder initializerList = new StringBuilder(); - final String elementType = javaTypeName(typeToken.encoding().primitiveType()); + final String elementType = javaTypeName(primitiveType); initializerList.append("new ").append(elementType).append("[] { "); final int arrayLength = typeToken.arrayLength(); for (int i = 0; i < arrayLength; i++) @@ -598,16 +629,19 @@ private void generateArrayDecodeWith( indent, "decoder.actingVersion() >= " + decoderClassName + "." + formattedPropertyName + "SinceVersion()", initializerList, - null + "new " + elementType + "[0]", + wasAddedAfterParent ); } } private void generateBitSetDecodeWith( final StringBuilder sb, + final String decoderClassName, final Token fieldToken, final String dtoTypeName, - final String indent) + final String indent, + final Predicate wasAddedAfterParent) { if (fieldToken.isConstantEncoding()) { @@ -617,11 +651,11 @@ private void generateBitSetDecodeWith( final String propertyName = fieldToken.name(); final String formattedPropertyName = formatPropertyName(propertyName); - if (fieldToken.isOptionalEncoding()) + if (wasAddedAfterParent.test(fieldToken)) { - sb.append(indent).append("if (decoder.").append(formattedPropertyName).append("InActingVersion()"); - - sb.append(")\n") + sb.append(indent).append("if (decoder.actingVersion() >= ") + .append(decoderClassName).append(".") + .append(formattedPropertyName).append("SinceVersion())\n") .append(indent).append("{\n"); sb.append(indent).append(INDENT).append(dtoTypeName).append(".decodeWith(decoder.") @@ -710,8 +744,10 @@ private void generateGroupsDecodeWith( private void generateVarDataDecodeWith( final StringBuilder sb, + final String decoderClassName, final List tokens, - final String indent) + final String indent, + final Predicate wasAddedAfterParent) { for (int i = 0; i < tokens.size(); i++) { @@ -723,7 +759,7 @@ private void generateVarDataDecodeWith( final Token varDataToken = Generators.findFirst("varData", tokens, i); final String characterEncoding = varDataToken.encoding().characterEncoding(); - final boolean isOptional = token.version() > 0; + final boolean isOptional = wasAddedAfterParent.test(token); final String blockIndent = isOptional ? indent + INDENT : indent; final String dataVar = toLowerFirstChar(propertyName) + "Data"; @@ -734,7 +770,7 @@ private void generateVarDataDecodeWith( { decoderValueExtraction.append(blockIndent).append("byte[] ").append(dataVar) .append(" = new byte[decoder.").append(formattedPropertyName).append("Length()];\n") - .append(blockIndent).append("decoder.get").append(formattedPropertyName) + .append(blockIndent).append("decoder.get").append(toUpperFirstChar(formattedPropertyName)) .append("(").append(dataVar).append(", 0, decoder.").append(formattedPropertyName) .append("Length());\n"); } @@ -746,9 +782,9 @@ private void generateVarDataDecodeWith( if (isOptional) { - sb.append(indent).append("if (decoder.").append(formattedPropertyName).append("InActingVersion()"); - - sb.append(")\n") + sb.append(indent).append("if (decoder.actingVersion() >= ") + .append(decoderClassName).append(".") + .append(formattedPropertyName).append("SinceVersion())\n") .append(indent).append("{\n"); sb.append(decoderValueExtraction); @@ -756,7 +792,7 @@ private void generateVarDataDecodeWith( sb.append(indent).append(INDENT).append("dto.").append(formattedPropertyName).append("(") .append(dataVar).append(");\n"); - final String nullDtoValue = "\"\""; + final String nullDtoValue = characterEncoding == null ? "new byte[0]" : "\"\""; sb.append(indent).append("}\n") .append(indent).append("else\n") @@ -782,27 +818,25 @@ private void generateRecordPropertyAssignment( final String indent, final String presenceExpression, final CharSequence getExpression, - final String nullDecoderValueOrNull) + final String nullDecoderValue, + final Predicate wasAddedAfterParent) { final String propertyName = token.name(); final String formattedPropertyName = formatPropertyName(propertyName); - if (token.isOptionalEncoding()) + if (wasAddedAfterParent.test(token)) { - sb.append(indent).append("if (").append(presenceExpression).append(")\n") .append(indent).append("{\n"); sb.append(indent).append(INDENT).append("dto.").append(formattedPropertyName).append("(") .append(getExpression).append(");\n"); - final String nullValue = nullDecoderValueOrNull == null ? "null" : nullDecoderValueOrNull; - sb.append(indent).append("}\n") .append(indent).append("else\n") .append(indent).append("{\n") .append(indent).append(INDENT).append("dto.").append(formattedPropertyName).append("(") - .append(nullValue).append(");\n") + .append(nullDecoderValue).append(");\n") .append(indent).append("}\n"); } else @@ -966,19 +1000,16 @@ private void generateArrayEncodeWith( final String propertyName = fieldToken.name(); final String formattedPropertyName = formatPropertyName(propertyName); - if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) + final PrimitiveType primitiveType = typeToken.encoding().primitiveType(); + + if (primitiveType == PrimitiveType.CHAR) { sb.append(indent).append("encoder.").append(toLowerFirstChar(propertyName)).append("(") .append("dto.").append(formattedPropertyName).append("());\n"); } - else if (typeToken.encoding().primitiveType() == PrimitiveType.UINT8) - { - sb.append(indent).append("encoder.put").append(toUpperFirstChar(propertyName)).append("(") - .append("dto.").append(formattedPropertyName).append("());\n"); - } else { - final String javaTypeName = javaTypeName(typeToken.encoding().primitiveType()); + final String javaTypeName = javaTypeName(primitiveType); sb.append(indent).append(javaTypeName).append("[] ").append(formattedPropertyName).append(" = ") .append("dto.").append(formattedPropertyName).append("();\n") .append(indent).append("for (int i = 0; i < ").append(formattedPropertyName).append(".length; i++)\n") @@ -1103,7 +1134,9 @@ private void generateVarDataEncodeWith( if (characterEncoding == null) { sb.append(indent).append("encoder.put").append(toUpperFirstChar(propertyName)).append("(") - .append("dto.").append(formattedPropertyName).append("());\n"); + .append("dto.").append(formattedPropertyName).append("(),") + .append("0,") + .append("dto.").append(formattedPropertyName).append("().length);\n"); } else { @@ -1310,7 +1343,9 @@ private void generateArrayProperty( final String fieldName = formatFieldName(propertyName); final String validateMethod = "validate" + toUpperFirstChar(propertyName); - if (typeToken.encoding().primitiveType() == PrimitiveType.CHAR) + final PrimitiveType primitiveType = typeToken.encoding().primitiveType(); + + if (primitiveType == PrimitiveType.CHAR) { final CharSequence typeName = "String"; @@ -1345,7 +1380,7 @@ private void generateArrayProperty( } else { - final String elementTypeName = javaTypeName(typeToken.encoding().primitiveType()); + final String elementTypeName = javaTypeName(primitiveType); final String typeName = elementTypeName + "[]"; classBuilder.appendField() diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java index 0a47fbfe56..7d0d4ba97b 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java @@ -1034,6 +1034,10 @@ private void generateGroupDecoderClassHeader( .append(indent).append(" {\n") .append(indent).append(" return blockLength;\n") .append(indent).append(" }\n\n") + .append(indent).append(" public int actingVersion()\n") + .append(indent).append(" {\n") + .append(indent).append(" return parentMessage.actingVersion;\n") + .append(indent).append(" }\n\n") .append(indent).append(" public int count()\n") .append(indent).append(" {\n") .append(indent).append(" return count;\n") diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java index fed7d4c94d..b21ba27674 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java @@ -16,6 +16,8 @@ package uk.co.real_logic.sbe.properties; import net.jqwik.api.*; +import net.jqwik.api.footnotes.EnableFootnotes; +import net.jqwik.api.footnotes.Footnotes; import uk.co.real_logic.sbe.generation.common.PrecedenceChecks; import uk.co.real_logic.sbe.generation.cpp.CppDtoGenerator; import uk.co.real_logic.sbe.generation.cpp.CppGenerator; @@ -28,10 +30,7 @@ import uk.co.real_logic.sbe.ir.generated.MessageHeaderDecoder; import uk.co.real_logic.sbe.properties.arbitraries.SbeArbitraries; import uk.co.real_logic.sbe.properties.utils.InMemoryOutputManager; -import org.agrona.DirectBuffer; -import org.agrona.ExpandableArrayBuffer; -import org.agrona.IoUtil; -import org.agrona.MutableDirectBuffer; +import org.agrona.*; import org.agrona.concurrent.UnsafeBuffer; import org.agrona.io.DirectBufferInputStream; @@ -45,40 +44,33 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; +import java.util.Base64; +import java.util.stream.Stream; +import static org.junit.jupiter.api.Assertions.fail; import static uk.co.real_logic.sbe.SbeTool.JAVA_DEFAULT_DECODING_BUFFER_TYPE; import static uk.co.real_logic.sbe.SbeTool.JAVA_DEFAULT_ENCODING_BUFFER_TYPE; @SuppressWarnings("ReadWriteStringCanBeUsed") +@EnableFootnotes public class DtosPropertyTest { private static final String DOTNET_EXECUTABLE = System.getProperty("sbe.tests.dotnet.executable", "dotnet"); private static final String SBE_DLL = System.getProperty("sbe.dll", "csharp/sbe-dll/bin/Release/netstandard2.0/SBE.dll"); private static final String CPP_EXECUTABLE = System.getProperty("sbe.tests.cpp.executable", "g++"); - private static final boolean KEEP_DIR_ON_FAILURE = Boolean.parseBoolean( - System.getProperty("sbe.tests.keep.dir.on.failure", "true")); private final ExpandableArrayBuffer outputBuffer = new ExpandableArrayBuffer(); -// @Test -// void oneRun() throws IOException, ClassNotFoundException, InvocationTargetException, NoSuchMethodException, -// IllegalAccessException -// { -// javaDtoEncodeShouldBeTheInverseOfDtoDecode(encodedMessage().sample()); -// } - @Property - @Disabled void javaDtoEncodeShouldBeTheInverseOfDtoDecode( - @ForAll("encodedMessage") final SbeArbitraries.EncodedMessage encodedMessage + @ForAll("encodedMessage") final SbeArbitraries.EncodedMessage encodedMessage, + final Footnotes footnotes ) throws IOException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException, IllegalAccessException { final String packageName = encodedMessage.ir().applicableNamespace(); final InMemoryOutputManager outputManager = new InMemoryOutputManager(packageName); - boolean success = false; - try { try @@ -100,10 +92,7 @@ void javaDtoEncodeShouldBeTheInverseOfDtoDecode( } catch (final Exception generationException) { - throw new AssertionError( - "Code generation failed.\n\n" + - "SCHEMA:\n" + encodedMessage.schema(), - generationException); + fail("Code generation failed.", generationException); } try (URLClassLoader generatedClassLoader = outputManager.compileGeneratedSources()) @@ -128,39 +117,29 @@ void javaDtoEncodeShouldBeTheInverseOfDtoDecode( final int outputLength = (int)encodeWith.invoke(null, dto, outputBuffer, 0); if (!areEqual(inputBuffer, inputLength, outputBuffer, outputLength)) { - throw new AssertionError( - "Input and output differ\n\n" + - "SCHEMA:\n" + encodedMessage.schema()); + fail("Input and output differ"); } - - success = true; } } - finally + catch (final Throwable throwable) { - if (!success) - { - outputManager.dumpSources(); - } - } - } + addInputFootnotes(footnotes, encodedMessage); - private boolean areEqual( - final ExpandableArrayBuffer inputBuffer, - final int inputLength, - final ExpandableArrayBuffer outputBuffer, - final int outputLength) - { - return new UnsafeBuffer(inputBuffer, 0, inputLength).equals(new UnsafeBuffer(outputBuffer, 0, outputLength)); + final StringBuilder generatedSources = new StringBuilder(); + outputManager.dumpSources(generatedSources); + footnotes.addFootnote(generatedSources.toString()); + + throw throwable; + } } @Property void csharpDtoEncodeShouldBeTheInverseOfDtoDecode( - @ForAll("encodedMessage") final SbeArbitraries.EncodedMessage encodedMessage + @ForAll("encodedMessage") final SbeArbitraries.EncodedMessage encodedMessage, + final Footnotes footnotes ) throws IOException, InterruptedException { final Path tempDir = Files.createTempDirectory("sbe-csharp-dto-test"); - boolean success = false; try { @@ -205,34 +184,27 @@ void csharpDtoEncodeShouldBeTheInverseOfDtoDecode( "DIR:" + tempDir + "\n\n" + "SCHEMA:\n" + encodedMessage.schema()); } - success = true; } - finally + catch (final Throwable throwable) { - if (!KEEP_DIR_ON_FAILURE || success) - { - IoUtil.delete(tempDir.toFile(), true); - } - else - { - Files.write( - tempDir.resolve("schema.xml"), - encodedMessage.schema().getBytes(StandardCharsets.UTF_8)); + addInputFootnotes(footnotes, encodedMessage); + addGeneratedSourcesFootnotes(footnotes, tempDir, ".cs"); - Files.write( - tempDir.resolve("encoding.log"), - encodedMessage.encodingLog().getBytes(StandardCharsets.UTF_8)); - } + throw throwable; + } + finally + { + IoUtil.delete(tempDir.toFile(), true); } } @Property(shrinking = ShrinkingMode.OFF) void cppDtoEncodeShouldBeTheInverseOfDtoDecode( - @ForAll("encodedMessage") final SbeArbitraries.EncodedMessage encodedMessage + @ForAll("encodedMessage") final SbeArbitraries.EncodedMessage encodedMessage, + final Footnotes footnotes ) throws IOException, InterruptedException { final Path tempDir = Files.createTempDirectory("sbe-cpp-dto-test"); - boolean success = false; try { @@ -274,24 +246,17 @@ void cppDtoEncodeShouldBeTheInverseOfDtoDecode( "Input and output files differ\n\n" + "SCHEMA:\n" + encodedMessage.schema()); } - success = true; } - finally + catch (final Throwable throwable) { - if (!KEEP_DIR_ON_FAILURE || success) - { - IoUtil.delete(tempDir.toFile(), true); - } - else - { - Files.write( - tempDir.resolve("schema.xml"), - encodedMessage.schema().getBytes(StandardCharsets.UTF_8)); + addInputFootnotes(footnotes, encodedMessage); + addGeneratedSourcesFootnotes(footnotes, tempDir, ".cpp"); - Files.write( - tempDir.resolve("encoding.log"), - encodedMessage.encodingLog().getBytes(StandardCharsets.UTF_8)); - } + throw throwable; + } + finally + { + IoUtil.delete(tempDir.toFile(), true); } } @@ -384,4 +349,52 @@ private static void copyResourceToFile( } } + private boolean areEqual( + final ExpandableArrayBuffer inputBuffer, + final int inputLength, + final ExpandableArrayBuffer outputBuffer, + final int outputLength) + { + return new UnsafeBuffer(inputBuffer, 0, inputLength).equals(new UnsafeBuffer(outputBuffer, 0, outputLength)); + } + + private void addGeneratedSourcesFootnotes( + final Footnotes footnotes, + final Path directory, + final String suffix) + { + try (Stream contents = Files.walk(directory)) + { + contents + .filter(path -> path.toString().endsWith(suffix)) + .forEach(path -> + { + try + { + footnotes.addFootnote(System.lineSeparator() + "File: " + path + + System.lineSeparator() + + new String(Files.readAllBytes(path), StandardCharsets.UTF_8)); + } + catch (final IOException exn) + { + LangUtil.rethrowUnchecked(exn); + } + }); + } + catch (final IOException exn) + { + LangUtil.rethrowUnchecked(exn); + } + } + + public void addInputFootnotes(final Footnotes footnotes, final SbeArbitraries.EncodedMessage encodedMessage) + { + final byte[] messageBytes = new byte[encodedMessage.length()]; + encodedMessage.buffer().getBytes(0, messageBytes); + final byte[] base64EncodedMessageBytes = Base64.getEncoder().encode(messageBytes); + + footnotes.addFootnote("Schema:" + System.lineSeparator() + encodedMessage.schema()); + footnotes.addFootnote("Input Message:" + System.lineSeparator() + + new String(base64EncodedMessageBytes, StandardCharsets.UTF_8)); + } } diff --git a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/utils/InMemoryOutputManager.java b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/utils/InMemoryOutputManager.java index a34669593e..a17f74914e 100644 --- a/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/utils/InMemoryOutputManager.java +++ b/sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/utils/InMemoryOutputManager.java @@ -84,14 +84,16 @@ public URLClassLoader compileGeneratedSources() return classLoader; } - public void dumpSources() + public void dumpSources(final StringBuilder builder) { + builder.append(System.lineSeparator()).append("Generated sources file count: ").append(sourceFiles.size()) + .append(System.lineSeparator()); + sourceFiles.forEach((qualifiedName, file) -> { - System.out.println( - System.lineSeparator() + "Source file: " + qualifiedName + - System.lineSeparator() + file.sourceCode + - System.lineSeparator()); + builder.append(System.lineSeparator()).append("Source file: ").append(qualifiedName) + .append(System.lineSeparator()).append(file.sourceCode) + .append(System.lineSeparator()); }); } @@ -200,7 +202,7 @@ void defineClasses(final InMemoryFileManager fileManager) fileManager.outputFiles().forEach(file -> { final byte[] classBytes = file.getClassBytes(); - super.defineClass(file.getName(), classBytes, 0, classBytes.length); + super.defineClass(null, classBytes, 0, classBytes.length); }); } } diff --git a/sbe-tool/src/test/resources/example-extension-schema.xml b/sbe-tool/src/test/resources/example-extension-schema.xml index 3770489b9f..ab89ba12cc 100644 --- a/sbe-tool/src/test/resources/example-extension-schema.xml +++ b/sbe-tool/src/test/resources/example-extension-schema.xml @@ -2,7 +2,7 @@ @@ -86,7 +86,7 @@ - + From 6e0234b0c9dbd1d585407530c6b453a89fd72ea8 Mon Sep 17 00:00:00 2001 From: Zach Bray Date: Wed, 15 May 2024 12:02:41 +0100 Subject: [PATCH 41/41] [Java] Tidy up spacing. --- .../co/real_logic/sbe/generation/java/JavaDtoGenerator.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtoGenerator.java b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtoGenerator.java index 0056ba4703..4b8ceb876d 100644 --- a/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtoGenerator.java +++ b/sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaDtoGenerator.java @@ -165,11 +165,8 @@ public void appendTo(final Appendable out) out.append(indent).append(modifiers).append("class ").append(className).append("\n") .append(indent).append("{\n") .append(fieldSb) - .append("\n") .append(privateSb) - .append("\n") .append(publicSb) - .append("\n") .append(indent).append("}\n"); } catch (final IOException exception) @@ -420,7 +417,7 @@ private void generateDecodeListWith( final String indent) { classBuilder.appendPublic().append("\n") - .append(indent).append("static List<").append(dtoClassName).append("> decodeManyWith(") + .append(indent).append("public static List<").append(dtoClassName).append("> decodeManyWith(") .append(decoderClassName).append(" decoder)\n") .append(indent).append("{\n") .append(indent).append(INDENT).append("List<").append(dtoClassName)