Parsing and serializing CQL annotations in ELM XML and JSON (Kotlin f…

…eature branch) (#1493)

* XML and JSON serialization for mixed content in ELM

* Fix detekt checks

Src/java/cql-to-elm/build.gradle.kts

@@ -9,10 +9,6 @@ dependencies {
 
     implementation("org.jetbrains.kotlinx:kotlinx-io-core-jvm:0.6.0")
 
-    // Temporary until we can get rid of the dependency on wrapping
-    // the CQL annotations in a JAXBElement for narrative generation
-    implementation("jakarta.xml.bind:jakarta.xml.bind-api:4.0.1")
-
     testImplementation(project(":elm-xmlutil"))
     testImplementation(project(":model-xmlutil"))
     testImplementation(project(":quick"))

Src/java/cql-to-elm/src/main/kotlin/org/cqframework/cql/cql2elm/preprocessor/CqlPreprocessorElmCommonVisitor.kt

@@ -2,10 +2,7 @@
 
 package org.cqframework.cql.cql2elm.preprocessor
 
-import jakarta.xml.bind.JAXBElement
-import java.io.Serializable
 import java.util.*
-import javax.xml.namespace.QName
 import org.antlr.v4.kotlinruntime.ParserRuleContext
 import org.antlr.v4.kotlinruntime.TokenStream
 import org.antlr.v4.kotlinruntime.misc.Interval
@@ -763,32 +760,32 @@ abstract class CqlPreprocessorElmCommonVisitor(
             return Pair.of(header.substring(startFrom).trim { it <= ' ' }, header.length)
         }
 
-        fun wrapNarrative(narrative: Narrative): Serializable {
+        fun wrapNarrative(narrative: Narrative): Any {
             @Suppress("ForbiddenComment")
             /*
-            TODO: Should be able to collapse narrative if the span doesn't have an attribute
-            That's what this code is doing, but it doesn't work and I don't have time to debug it
-            if (narrative.getR() == null) {
-                StringBuilder content = new StringBuilder();
-                boolean onlyStrings = true;
-                for (Serializable s : narrative.getContent()) {
-                    if (s instanceof String) {
-                        content.append((String)s);
+            This code collapses the narrative if the span doesn't have an attribute.
+            It does work but creates a different (simplified) ELM.
+
+            if (narrative.r == null) {
+                val content = StringBuilder()
+                var onlyStrings = true
+                for (s in narrative.content) {
+                    if (s is String) {
+                        content.append(s)
                     }
                     else {
-                        onlyStrings = false;
+                        onlyStrings = false
                     }
                 }
                 if (onlyStrings) {
-                    return content.toString();
+                    return content.toString()
                 }
             }
+
+            return narrative
             */
-            return JAXBElement(
-                QName("urn:hl7-org:cql-annotations:r1", "s"),
-                Narrative::class.java,
-                narrative
-            )
+
+            return narrative
         }
 
         fun isValidIdentifier(tagName: String): Boolean {

Src/java/cql-to-elm/src/test/java/org/cqframework/cql/cql2elm/CommentTests.java

@@ -4,7 +4,6 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.*;
 
-import jakarta.xml.bind.JAXBElement;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
@@ -42,11 +41,10 @@ void comments() throws IOException {
         Annotation a = (Annotation) def.getAnnotation().get(0);
         assertThat(a.getS().getContent(), notNullValue());
         assertThat(a.getS().getContent().size(), is(2));
-        assertThat(a.getS().getContent().get(0), instanceOf(JAXBElement.class));
-        JAXBElement e = (JAXBElement) a.getS().getContent().get(0);
+        var e = a.getS().getContent().get(0);
         assertThat(e, notNullValue());
-        assertThat(e.getValue(), instanceOf(Narrative.class));
-        Narrative n = (Narrative) e.getValue();
+        assertThat(e, instanceOf(Narrative.class));
+        Narrative n = (Narrative) e;
         assertThat(n.getContent(), notNullValue());
         assertThat(n.getContent().size(), is(4));
         assertThat(n.getContent().get(0), instanceOf(String.class));
@@ -63,11 +61,10 @@ void comments() throws IOException {
         a = (Annotation) def.getAnnotation().get(0);
         assertThat(a.getS().getContent(), notNullValue());
         assertThat(a.getS().getContent().size(), is(2));
-        assertThat(a.getS().getContent().get(0), instanceOf(JAXBElement.class));
-        e = (JAXBElement) a.getS().getContent().get(0);
+        e = a.getS().getContent().get(0);
         assertThat(e, notNullValue());
-        assertThat(e.getValue(), instanceOf(Narrative.class));
-        n = (Narrative) e.getValue();
+        assertThat(e, instanceOf(Narrative.class));
+        n = (Narrative) e;
         assertThat(n.getContent(), notNullValue());
         assertThat(n.getContent().size(), is(4));
         assertThat(n.getContent().get(0), instanceOf(String.class));

Src/java/elm-fhir/src/main/java/org/cqframework/cql/elm/requirements/fhir/DataRequirementsProcessor.java

@@ -1,8 +1,6 @@
 package org.cqframework.cql.elm.requirements.fhir;
 
 import ca.uhn.fhir.context.FhirVersionEnum;
-import jakarta.xml.bind.JAXBElement;
-import java.io.Serializable;
 import java.time.ZonedDateTime;
 import java.util.*;
 import java.util.List;
@@ -450,19 +448,12 @@ private String toNarrativeText(org.hl7.cql_annotations.r1.Annotation a) {
     }
 
     private void addNarrativeText(StringBuilder sb, org.hl7.cql_annotations.r1.Narrative n) {
-        for (Serializable s : n.getContent()) {
+        for (var s : n.getContent()) {
             if (s instanceof org.hl7.cql_annotations.r1.Narrative) {
                 addNarrativeText(sb, (org.hl7.cql_annotations.r1.Narrative) s);
             } else if (s instanceof String) {
                 sb.append((String) s);
             }
-            // TODO: THIS IS WRONG... SHOULDN'T NEED TO KNOW ABOUT JAXB TO ACCOMPLISH THIS
-            else if (s instanceof JAXBElement<?>) {
-                JAXBElement<?> j = (JAXBElement<?>) s;
-                if (j.getValue() instanceof org.hl7.cql_annotations.r1.Narrative) {
-                    addNarrativeText(sb, (org.hl7.cql_annotations.r1.Narrative) j.getValue());
-                }
-            }
         }
     }
 

Src/java/elm-test/src/test/java/org/cqframework/cql/elm/ElmDeserializeTests.java

@@ -12,9 +12,10 @@
 import org.cqframework.cql.cql2elm.CqlCompilerOptions;
 import org.cqframework.cql.cql2elm.CqlTranslator;
 import org.cqframework.cql.cql2elm.LibraryBuilder;
+import org.hl7.cql_annotations.r1.Annotation;
 import org.hl7.cql_annotations.r1.CqlToElmInfo;
+import org.hl7.cql_annotations.r1.Narrative;
 import org.hl7.elm.r1.*;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 class ElmDeserializeTests {
@@ -30,7 +31,6 @@ void elmTests() {
     }
 
     @Test
-    @Disabled("TODO: Re-enable once XmlUtil-based ELM JSON deserialization is implemented for annotations")
     void jsonANCFHIRDummyLibraryLoad() {
         try {
             final Library library = deserializeJsonLibrary("ElmDeserialize/ANCFHIRDummy.json");
@@ -55,8 +55,19 @@ void jsonANCFHIRDummyLibraryLoad() {
             assertTrue(
                     ((SingletonFrom) library.getStatements().getDef().get(0).getExpression()).getOperand()
                             instanceof Retrieve);
-            assertNotNull(library.getStatements().getDef().get(1));
-            assertTrue(library.getStatements().getDef().get(1).getExpression() instanceof Retrieve);
+            var observationsStatement = library.getStatements().getDef().get(1);
+            assertNotNull(observationsStatement);
+            assertTrue(observationsStatement.getExpression() instanceof Retrieve);
+
+            assertTrue(observationsStatement.getAnnotation().get(0) instanceof Annotation);
+            var annotation = (Annotation) observationsStatement.getAnnotation().get(0);
+            assertNotNull(annotation.getS());
+            var narrative = annotation.getS();
+            assertTrue(narrative.getContent().get(1) instanceof Narrative);
+            var nestedNarrative = (Narrative) narrative.getContent().get(1);
+            assertTrue(nestedNarrative.getContent().get(0) instanceof Narrative);
+            nestedNarrative = (Narrative) nestedNarrative.getContent().get(0);
+            assertEquals("[", nestedNarrative.getContent().get(0));
 
             verifySigLevels(library, LibraryBuilder.SignatureLevel.All);
         } catch (IOException e) {
@@ -100,7 +111,6 @@ void jsonAdultOutpatientEncountersFHIR4LibraryLoad() {
     }
 
     @Test
-    @Disabled("Invalid XML value at position: 85:29: Index -1 out of bounds for length 2")
     void xmlLibraryLoad() {
         try {
             final Library library =
@@ -130,11 +140,20 @@ void xmlLibraryLoad() {
             assertTrue(
                     ((SingletonFrom) library.getStatements().getDef().get(0).getExpression()).getOperand()
                             instanceof Retrieve);
-            assertEquals(
-                    "Qualifying Encounters",
-                    library.getStatements().getDef().get(1).getName());
-            assertNotNull(library.getStatements().getDef().get(1));
-            assertTrue(library.getStatements().getDef().get(1).getExpression() instanceof Query);
+            var qualifyingEncountersStatement = library.getStatements().getDef().get(1);
+            assertEquals("Qualifying Encounters", qualifyingEncountersStatement.getName());
+            assertNotNull(qualifyingEncountersStatement);
+            assertTrue(qualifyingEncountersStatement.getExpression() instanceof Query);
+            assertTrue(qualifyingEncountersStatement.getAnnotation().get(0) instanceof Annotation);
+            var annotation =
+                    (Annotation) qualifyingEncountersStatement.getAnnotation().get(0);
+            assertNotNull(annotation.getS());
+            var narrative = annotation.getS();
+            assertEquals("\n               ", narrative.getContent().get(0));
+            assertTrue(narrative.getContent().get(3) instanceof Narrative);
+            var nestedNarrative = (Narrative) narrative.getContent().get(3);
+            assertEquals("\n                  ", nestedNarrative.getContent().get(0));
+            assertTrue(nestedNarrative.getContent().get(1) instanceof Narrative);
 
             verifySigLevels(library, LibraryBuilder.SignatureLevel.Overloads);
         } catch (IOException e) {
@@ -144,7 +163,6 @@ void xmlLibraryLoad() {
     }
 
     @Test
-    @Disabled("TODO: Re-enable once XmlUtil-based ELM JSON deserialization is implemented for annotations")
     void jsonTerminologyLibraryLoad() {
         try {
             final Library library = deserializeJsonLibrary("ElmDeserialize/ANCFHIRTerminologyDummy.json");
@@ -200,7 +218,6 @@ private void testElmDeserialization(String directoryName) throws URISyntaxExcept
     }
 
     @Test
-    @Disabled("Invalid XML value at position: 59:29: Index -1 out of bounds for length 2")
     void regressionTestJsonSerializer() throws URISyntaxException {
         // This test validates that the ELM library deserialized from the Json matches the ELM library deserialized from
         // Xml

Src/java/elm-xmlutil/src/main/java/org/cqframework/cql/elm/serializing/xmlutil/ElmJsonLibraryCommon.kt

@@ -3,11 +3,13 @@ package org.cqframework.cql.elm.serializing.xmlutil
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.modules.plus
 import kotlinx.serialization.modules.serializersModuleOf
+import org.cqframework.cql.elm.serializing.NarrativeJsonSerializer
 import org.hl7.elm_modelinfo.r1.serializing.BigDecimalJsonSerializer
 
 val json = Json {
     serializersModule =
         serializersModuleOf(BigDecimalJsonSerializer) +
+            serializersModuleOf(NarrativeJsonSerializer) +
             org.hl7.elm.r1.serializersModule +
             org.hl7.cql_annotations.r1.serializersModule
     explicitNulls = false

Src/java/elm-xmlutil/src/main/java/org/cqframework/cql/elm/serializing/xmlutil/ElmXmlLibraryCommon.kt

@@ -1,20 +1,61 @@
 package org.cqframework.cql.elm.serializing.xmlutil
 
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.builtins.serializer
+import kotlinx.serialization.modules.SerializersModule
 import kotlinx.serialization.modules.plus
+import kotlinx.serialization.modules.polymorphic
 import kotlinx.serialization.modules.serializersModuleOf
 import nl.adaptivity.xmlutil.QName
+import nl.adaptivity.xmlutil.XMLConstants
+import nl.adaptivity.xmlutil.serialization.DefaultXmlSerializationPolicy
 import nl.adaptivity.xmlutil.serialization.XML
+import nl.adaptivity.xmlutil.serialization.structure.SafeParentInfo
 import org.hl7.elm_modelinfo.r1.serializing.BigDecimalXmlSerializer
 
+val builder =
+    DefaultXmlSerializationPolicy.Builder().apply {
+        // Use xsi:type for handling polymorphism
+        typeDiscriminatorName = QName(XMLConstants.XSI_NS_URI, "type", XMLConstants.XSI_PREFIX)
+    }
+
+@OptIn(ExperimentalSerializationApi::class)
+val customPolicy =
+    object : DefaultXmlSerializationPolicy(builder) {
+        override fun isTransparentPolymorphic(
+            serializerParent: SafeParentInfo,
+            tagParent: SafeParentInfo
+        ): Boolean {
+            // Switch on transparent polymorphic mode for mixed content
+            if (
+                serializerParent.elementSerialDescriptor.serialName ==
+                    "kotlinx.serialization.Polymorphic<Any>"
+            ) {
+                return true
+            }
+            return super.isTransparentPolymorphic(serializerParent, tagParent)
+        }
+    }
+
+// Mixed content can include text and Narrative elements
+val mixedContentSerializersModule = SerializersModule {
+    polymorphic(Any::class) {
+        polymorphic(Any::class, String::class, String.serializer())
+        polymorphic(
+            Any::class,
+            org.hl7.cql_annotations.r1.Narrative::class,
+            org.hl7.cql_annotations.r1.Narrative.serializer()
+        )
+    }
+}
+
 val xml =
     XML(
         serializersModuleOf(BigDecimalXmlSerializer) +
+            mixedContentSerializersModule +
             org.hl7.elm.r1.serializersModule +
             org.hl7.cql_annotations.r1.serializersModule
     ) {
+        policy = customPolicy
         xmlDeclMode = nl.adaptivity.xmlutil.XmlDeclMode.Charset
-        defaultPolicy {
-            typeDiscriminatorName =
-                QName("http://www.w3.org/2001/XMLSchema-instance", "type", "xsi")
-        }
     }