From 78ecd3af4ae2c690198c5e9620b1abd1210c0089 Mon Sep 17 00:00:00 2001 From: Brian Harrington Date: Mon, 9 Dec 2024 06:24:19 -0600 Subject: [PATCH] json: enabled lenient UTF encoding When using smile enable lenient UTF encoding. In some cases log strings can have invalid UTF characters that work for encoding as JSON, but fail surrogate checks with smile. The lenient encoding allows these to go through, but replaces the invalid characters with `\uFFFD`. --- .../src/main/scala/com/netflix/atlas/json/Json.scala | 2 ++ .../test/scala/com/netflix/atlas/json/JsonSuite.scala | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/atlas-json/src/main/scala/com/netflix/atlas/json/Json.scala b/atlas-json/src/main/scala/com/netflix/atlas/json/Json.scala index 416a1b608..6b166df5e 100644 --- a/atlas-json/src/main/scala/com/netflix/atlas/json/Json.scala +++ b/atlas-json/src/main/scala/com/netflix/atlas/json/Json.scala @@ -26,6 +26,7 @@ import com.fasterxml.jackson.core.json.JsonWriteFeature import com.fasterxml.jackson.databind.* import com.fasterxml.jackson.databind.module.SimpleModule import com.fasterxml.jackson.dataformat.smile.SmileFactory +import com.fasterxml.jackson.dataformat.smile.SmileGenerator import com.fasterxml.jackson.datatype.jdk8.Jdk8Module import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.module.scala.DefaultScalaModule @@ -75,6 +76,7 @@ object Json { .builder() .enable(StreamReadFeature.AUTO_CLOSE_SOURCE) .enable(StreamWriteFeature.AUTO_CLOSE_TARGET) + .enable(SmileGenerator.Feature.LENIENT_UTF_ENCODING) .build() private val jsonMapper = newMapper(jsonFactory) diff --git a/atlas-json/src/test/scala/com/netflix/atlas/json/JsonSuite.scala b/atlas-json/src/test/scala/com/netflix/atlas/json/JsonSuite.scala index 81d81efbc..870fd6d2d 100644 --- a/atlas-json/src/test/scala/com/netflix/atlas/json/JsonSuite.scala +++ b/atlas-json/src/test/scala/com/netflix/atlas/json/JsonSuite.scala @@ -24,6 +24,7 @@ import com.fasterxml.jackson.core.JsonParseException import com.fasterxml.jackson.core.JsonToken import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.SerializationFeature +import com.fasterxml.jackson.databind.node.ObjectNode import munit.FunSuite import java.time.Instant @@ -448,6 +449,15 @@ class JsonSuite extends FunSuite { assertEquals(json, """{"name":"name","values":[]}""") } + test("unmatched surrogate pairs") { + val json = "{\"test\": 1, \"test\uD83D\": 2}" + val jsonNode = Json.decode[ObjectNode](json) + val smile = Json.smileEncode(jsonNode) + val smileNode = Json.smileDecode[ObjectNode](smile) + val expected = Json.decode[ObjectNode](json.replace('\uD83D', '\uFFFD')) + assertEquals(smileNode, expected) + } + test("decode from JsonData") { def parse(json: String): List[JsonSuiteSimple] = { val node = Json.decode[JsonNode](json)