From c40b7a66036749b706a6c058afae4f46d366a121 Mon Sep 17 00:00:00 2001 From: Andriy Plokhotnyuk Date: Sun, 2 Feb 2025 08:47:49 +0100 Subject: [PATCH] More efficient decoding of `BigInt` and `java.math.BigInteger` values (#1275) --- .../scala/zio/json/internal/SafeNumbers.scala | 10 +--- .../zio/json/internal/UnsafeNumbers.scala | 38 ++++++++----- .../scala/zio/json/internal/SafeNumbers.scala | 10 +--- .../zio/json/internal/UnsafeNumbers.scala | 54 ++++++++++--------- .../scala/zio/json/internal/SafeNumbers.scala | 10 +--- .../zio/json/internal/UnsafeNumbers.scala | 38 ++++++++----- .../src/main/scala/zio/json/JsonDecoder.scala | 2 +- .../main/scala/zio/json/internal/lexer.scala | 27 ++-------- 8 files changed, 89 insertions(+), 100 deletions(-) diff --git a/zio-json/js/src/main/scala/zio/json/internal/SafeNumbers.scala b/zio-json/js/src/main/scala/zio/json/internal/SafeNumbers.scala index 75b3fe5c..0c971822 100644 --- a/zio-json/js/src/main/scala/zio/json/internal/SafeNumbers.scala +++ b/zio-json/js/src/main/scala/zio/json/internal/SafeNumbers.scala @@ -56,10 +56,7 @@ object SafeNumbers { try LongSome(UnsafeNumbers.long(num)) catch { case _: UnexpectedEnd | UnsafeNumber => LongNone } - def bigInteger( - num: String, - max_bits: Int = 128 - ): Option[java.math.BigInteger] = + def bigInteger(num: String, max_bits: Int = 128): Option[java.math.BigInteger] = try Some(UnsafeNumbers.bigInteger(num, max_bits)) catch { case _: UnexpectedEnd | UnsafeNumber => None } @@ -71,10 +68,7 @@ object SafeNumbers { try DoubleSome(UnsafeNumbers.double(num, max_bits)) catch { case _: UnexpectedEnd | UnsafeNumber => DoubleNone } - def bigDecimal( - num: String, - max_bits: Int = 128 - ): Option[java.math.BigDecimal] = + def bigDecimal(num: String, max_bits: Int = 128): Option[java.math.BigDecimal] = try Some(UnsafeNumbers.bigDecimal(num, max_bits)) catch { case _: UnexpectedEnd | UnsafeNumber => None } diff --git a/zio-json/js/src/main/scala/zio/json/internal/UnsafeNumbers.scala b/zio-json/js/src/main/scala/zio/json/internal/UnsafeNumbers.scala index 24e25633..4c8e7fd2 100644 --- a/zio-json/js/src/main/scala/zio/json/internal/UnsafeNumbers.scala +++ b/zio-json/js/src/main/scala/zio/json/internal/UnsafeNumbers.scala @@ -117,28 +117,38 @@ object UnsafeNumbers { val negate = current == '-' if (negate) current = in.readChar().toInt if (current < '0' || current > '9') throw UnsafeNumber - var bigM10: java.math.BigInteger = null - var m10 = (current - '0').toLong + var loM10 = (current - '0').toLong + var loDigits = 1 + var hiM10: java.math.BigDecimal = null while ({ current = in.read() '0' <= current && current <= '9' }) { - if (m10 < 922337203685477580L) { - if (m10 <= 0) m10 = (current - '0').toLong - else m10 = (m10 << 3) + (m10 << 1) + (current - '0') - } else { - if (bigM10 eq null) bigM10 = java.math.BigInteger.valueOf(m10) - bigM10 = bigM10.multiply(java.math.BigInteger.TEN).add(bigIntegers(current - '0')) - if (bigM10.bitLength >= max_bits) throw UnsafeNumber + loM10 = (loM10 << 3) + (loM10 << 1) + (current - '0') + loDigits += 1 + if (loM10 >= 100000000000000000L) { + if (negate) loM10 = -loM10 + val bd = java.math.BigDecimal.valueOf(loM10) + if (hiM10 eq null) hiM10 = bd + else { + hiM10 = hiM10.scaleByPowerOfTen(loDigits).add(bd) + if (hiM10.unscaledValue.bitLength >= max_bits) throw UnsafeNumber + } + loM10 = 0 + loDigits = 0 } } if (consume && current != -1) throw UnsafeNumber - if (bigM10 eq null) { - if (negate) m10 = -m10 - return java.math.BigInteger.valueOf(m10) + if (hiM10 eq null) { + if (negate) loM10 = -loM10 + return java.math.BigInteger.valueOf(loM10) } - if (negate) bigM10 = bigM10.negate - bigM10 + if (loDigits != 0) { + if (negate) loM10 = -loM10 + hiM10 = hiM10.scaleByPowerOfTen(loDigits).add(java.math.BigDecimal.valueOf(loM10)) + if (hiM10.unscaledValue.bitLength >= max_bits) throw UnsafeNumber + } + hiM10.unscaledValue } def bigDecimal(num: String, max_bits: Int): java.math.BigDecimal = diff --git a/zio-json/jvm/src/main/scala/zio/json/internal/SafeNumbers.scala b/zio-json/jvm/src/main/scala/zio/json/internal/SafeNumbers.scala index 23022573..4a6489a3 100644 --- a/zio-json/jvm/src/main/scala/zio/json/internal/SafeNumbers.scala +++ b/zio-json/jvm/src/main/scala/zio/json/internal/SafeNumbers.scala @@ -56,10 +56,7 @@ object SafeNumbers { try LongSome(UnsafeNumbers.long(num)) catch { case _: UnexpectedEnd | UnsafeNumber => LongNone } - def bigInteger( - num: String, - max_bits: Int = 128 - ): Option[java.math.BigInteger] = + def bigInteger(num: String, max_bits: Int = 128): Option[java.math.BigInteger] = try Some(UnsafeNumbers.bigInteger(num, max_bits)) catch { case _: UnexpectedEnd | UnsafeNumber => None } @@ -71,10 +68,7 @@ object SafeNumbers { try DoubleSome(UnsafeNumbers.double(num, max_bits)) catch { case _: UnexpectedEnd | UnsafeNumber => DoubleNone } - def bigDecimal( - num: String, - max_bits: Int = 128 - ): Option[java.math.BigDecimal] = + def bigDecimal(num: String, max_bits: Int = 128): Option[java.math.BigDecimal] = try Some(UnsafeNumbers.bigDecimal(num, max_bits)) catch { case _: UnexpectedEnd | UnsafeNumber => None } diff --git a/zio-json/jvm/src/main/scala/zio/json/internal/UnsafeNumbers.scala b/zio-json/jvm/src/main/scala/zio/json/internal/UnsafeNumbers.scala index f67b17a9..4d9b3437 100644 --- a/zio-json/jvm/src/main/scala/zio/json/internal/UnsafeNumbers.scala +++ b/zio-json/jvm/src/main/scala/zio/json/internal/UnsafeNumbers.scala @@ -117,28 +117,38 @@ object UnsafeNumbers { val negate = current == '-' if (negate) current = in.readChar().toInt if (current < '0' || current > '9') throw UnsafeNumber - var bigM10: java.math.BigInteger = null - var m10 = (current - '0').toLong + var loM10 = (current - '0').toLong + var loDigits = 1 + var hiM10: java.math.BigDecimal = null while ({ current = in.read() '0' <= current && current <= '9' }) { - if (m10 < 922337203685477580L) { - if (m10 <= 0) m10 = (current - '0').toLong - else m10 = m10 * 10 + (current - '0') - } else { - if (bigM10 eq null) bigM10 = java.math.BigInteger.valueOf(m10) - bigM10 = bigM10.multiply(java.math.BigInteger.TEN).add(bigIntegers(current - '0')) - if (bigM10.bitLength >= max_bits) throw UnsafeNumber + loM10 = loM10 * 10 + (current - '0') + loDigits += 1 + if (loM10 >= 100000000000000000L) { + if (negate) loM10 = -loM10 + val bd = java.math.BigDecimal.valueOf(loM10) + if (hiM10 eq null) hiM10 = bd + else { + hiM10 = hiM10.scaleByPowerOfTen(loDigits).add(bd) + if (hiM10.unscaledValue.bitLength >= max_bits) throw UnsafeNumber + } + loM10 = 0 + loDigits = 0 } } if (consume && current != -1) throw UnsafeNumber - if (bigM10 eq null) { - if (negate) m10 = -m10 - return java.math.BigInteger.valueOf(m10) + if (hiM10 eq null) { + if (negate) loM10 = -loM10 + return java.math.BigInteger.valueOf(loM10) } - if (negate) bigM10 = bigM10.negate - bigM10 + if (loDigits != 0) { + if (negate) loM10 = -loM10 + hiM10 = hiM10.scaleByPowerOfTen(loDigits).add(java.math.BigDecimal.valueOf(loM10)) + if (hiM10.unscaledValue.bitLength >= max_bits) throw UnsafeNumber + } + hiM10.unscaledValue } def bigDecimal(num: String, max_bits: Int): java.math.BigDecimal = @@ -326,11 +336,8 @@ object UnsafeNumbers { else if (e10 >= 39) Float.PositiveInfinity else { var shift = java.lang.Long.numberOfLeadingZeros(m10) - var m2 = unsignedMultiplyHigh( - pow10Mantissas(e10 + 343), - m10 << shift - ) // FIXME: Use Math.unsignedMultiplyHigh after dropping of JDK 17 support - var e2 = (e10 * 108853 >> 15) - shift + 1 // (e10 * Math.log(10) / Math.log(2)).toInt - shift + 1 + var m2 = unsignedMultiplyHigh(pow10Mantissas(e10 + 343), m10 << shift) + var e2 = (e10 * 108853 >> 15) - shift + 1 // (e10 * Math.log(10) / Math.log(2)).toInt - shift + 1 shift = java.lang.Long.numberOfLeadingZeros(m2) m2 <<= shift e2 -= shift @@ -469,11 +476,8 @@ object UnsafeNumbers { else if (e10 >= 310) Double.PositiveInfinity else { var shift = java.lang.Long.numberOfLeadingZeros(m10) - var m2 = unsignedMultiplyHigh( - pow10Mantissas(e10 + 343), - m10 << shift - ) // FIXME: Use Math.unsignedMultiplyHigh after dropping of JDK 17 support - var e2 = (e10 * 108853 >> 15) - shift + 1 // (e10 * Math.log(10) / Math.log(2)).toInt - shift + 1 + var m2 = unsignedMultiplyHigh(pow10Mantissas(e10 + 343), m10 << shift) + var e2 = (e10 * 108853 >> 15) - shift + 1 // (e10 * Math.log(10) / Math.log(2)).toInt - shift + 1 shift = java.lang.Long.numberOfLeadingZeros(m2) m2 <<= shift e2 -= shift @@ -510,7 +514,7 @@ object UnsafeNumbers { } @inline private[this] def unsignedMultiplyHigh(x: Long, y: Long): Long = - Math.multiplyHigh(x, y) + x + y // Use implementation that works only when both params are negative + Math.multiplyHigh(x, y) + x + y // FIXME: Use Math.unsignedMultiplyHigh after dropping of JDK 17 support private[this] final val bigIntegers: Array[java.math.BigInteger] = (0L to 9L).map(java.math.BigInteger.valueOf).toArray diff --git a/zio-json/native/src/main/scala/zio/json/internal/SafeNumbers.scala b/zio-json/native/src/main/scala/zio/json/internal/SafeNumbers.scala index 16836932..fcecf03a 100644 --- a/zio-json/native/src/main/scala/zio/json/internal/SafeNumbers.scala +++ b/zio-json/native/src/main/scala/zio/json/internal/SafeNumbers.scala @@ -56,10 +56,7 @@ object SafeNumbers { try LongSome(UnsafeNumbers.long(num)) catch { case _: UnexpectedEnd | UnsafeNumber => LongNone } - def bigInteger( - num: String, - max_bits: Int = 128 - ): Option[java.math.BigInteger] = + def bigInteger(num: String, max_bits: Int = 128): Option[java.math.BigInteger] = try Some(UnsafeNumbers.bigInteger(num, max_bits)) catch { case _: UnexpectedEnd | UnsafeNumber => None } @@ -71,10 +68,7 @@ object SafeNumbers { try DoubleSome(UnsafeNumbers.double(num, max_bits)) catch { case _: UnexpectedEnd | UnsafeNumber => DoubleNone } - def bigDecimal( - num: String, - max_bits: Int = 128 - ): Option[java.math.BigDecimal] = + def bigDecimal(num: String, max_bits: Int = 128): Option[java.math.BigDecimal] = try Some(UnsafeNumbers.bigDecimal(num, max_bits)) catch { case _: UnexpectedEnd | UnsafeNumber => None } diff --git a/zio-json/native/src/main/scala/zio/json/internal/UnsafeNumbers.scala b/zio-json/native/src/main/scala/zio/json/internal/UnsafeNumbers.scala index 056e2980..74250d59 100644 --- a/zio-json/native/src/main/scala/zio/json/internal/UnsafeNumbers.scala +++ b/zio-json/native/src/main/scala/zio/json/internal/UnsafeNumbers.scala @@ -117,28 +117,38 @@ object UnsafeNumbers { val negate = current == '-' if (negate) current = in.readChar().toInt if (current < '0' || current > '9') throw UnsafeNumber - var bigM10: java.math.BigInteger = null - var m10 = (current - '0').toLong + var loM10 = (current - '0').toLong + var loDigits = 1 + var hiM10: java.math.BigDecimal = null while ({ current = in.read() '0' <= current && current <= '9' }) { - if (m10 < 922337203685477580L) { - if (m10 <= 0) m10 = (current - '0').toLong - else m10 = m10 * 10 + (current - '0') - } else { - if (bigM10 eq null) bigM10 = java.math.BigInteger.valueOf(m10) - bigM10 = bigM10.multiply(java.math.BigInteger.TEN).add(bigIntegers(current - '0')) - if (bigM10.bitLength >= max_bits) throw UnsafeNumber + loM10 = loM10 * 10 + (current - '0') + loDigits += 1 + if (loM10 >= 100000000000000000L) { + if (negate) loM10 = -loM10 + val bd = java.math.BigDecimal.valueOf(loM10) + if (hiM10 eq null) hiM10 = bd + else { + hiM10 = hiM10.scaleByPowerOfTen(loDigits).add(bd) + if (hiM10.unscaledValue.bitLength >= max_bits) throw UnsafeNumber + } + loM10 = 0 + loDigits = 0 } } if (consume && current != -1) throw UnsafeNumber - if (bigM10 eq null) { - if (negate) m10 = -m10 - return java.math.BigInteger.valueOf(m10) + if (hiM10 eq null) { + if (negate) loM10 = -loM10 + return java.math.BigInteger.valueOf(loM10) } - if (negate) bigM10 = bigM10.negate - bigM10 + if (loDigits != 0) { + if (negate) loM10 = -loM10 + hiM10 = hiM10.scaleByPowerOfTen(loDigits).add(java.math.BigDecimal.valueOf(loM10)) + if (hiM10.unscaledValue.bitLength >= max_bits) throw UnsafeNumber + } + hiM10.unscaledValue } def bigDecimal(num: String, max_bits: Int): java.math.BigDecimal = diff --git a/zio-json/shared/src/main/scala/zio/json/JsonDecoder.scala b/zio-json/shared/src/main/scala/zio/json/JsonDecoder.scala index 43272c23..f03f30bf 100644 --- a/zio-json/shared/src/main/scala/zio/json/JsonDecoder.scala +++ b/zio-json/shared/src/main/scala/zio/json/JsonDecoder.scala @@ -325,7 +325,7 @@ object JsonDecoder extends GeneratedTupleDecoders with DecoderLowPriority1 with implicit val int: JsonDecoder[Int] = number(Lexer.int, _.intValueExact()) implicit val long: JsonDecoder[Long] = number(Lexer.long, _.longValueExact()) implicit val bigInteger: JsonDecoder[java.math.BigInteger] = number(Lexer.bigInteger, _.toBigIntegerExact) - implicit val scalaBigInt: JsonDecoder[BigInt] = bigInteger.map(x => x) + implicit val scalaBigInt: JsonDecoder[BigInt] = number(Lexer.bigInteger, _.toBigIntegerExact) implicit val float: JsonDecoder[Float] = number(Lexer.float, _.floatValue()) implicit val double: JsonDecoder[Double] = number(Lexer.double, _.doubleValue()) implicit val bigDecimal: JsonDecoder[java.math.BigDecimal] = number(Lexer.bigDecimal, identity) diff --git a/zio-json/shared/src/main/scala/zio/json/internal/lexer.scala b/zio-json/shared/src/main/scala/zio/json/internal/lexer.scala index b411a052..6f93bfc8 100644 --- a/zio-json/shared/src/main/scala/zio/json/internal/lexer.scala +++ b/zio-json/shared/src/main/scala/zio/json/internal/lexer.scala @@ -78,21 +78,13 @@ object Lexer { // messages) by only checking for what we expect to see (Jon Pretty's idea). // // returns the index of the matched field, or -1 - def field( - trace: List[JsonError], - in: OneCharReader, - matrix: StringMatrix - ): Int = { + def field(trace: List[JsonError], in: OneCharReader, matrix: StringMatrix): Int = { val f = enumeration(trace, in, matrix) char(trace, in, ':') f } - def enumeration( - trace: List[JsonError], - in: OneCharReader, - matrix: StringMatrix - ): Int = { + def enumeration(trace: List[JsonError], in: OneCharReader, matrix: StringMatrix): Int = { var c = in.nextNonWhitespace() if (c != '"') error("'\"'", c, trace) var bs = matrix.initial @@ -181,10 +173,7 @@ object Lexer { } // useful for embedded documents, e.g. CSV contained inside JSON - def streamingString( - trace: List[JsonError], - in: OneCharReader - ): java.io.Reader = { + def streamingString(trace: List[JsonError], in: OneCharReader): java.io.Reader = { char(trace, in, '"') new OneCharReader { def close(): Unit = in.close() @@ -346,10 +335,7 @@ object Lexer { case UnsafeNumbers.UnsafeNumber => error("expected a Long", trace) } - def bigInteger( - trace: List[JsonError], - in: RetractReader - ): java.math.BigInteger = + def bigInteger(trace: List[JsonError], in: RetractReader): java.math.BigInteger = try { val i = UnsafeNumbers.bigInteger_(in, false, NumberMaxBits) in.retract() @@ -376,10 +362,7 @@ object Lexer { case UnsafeNumbers.UnsafeNumber => error("expected a Double", trace) } - def bigDecimal( - trace: List[JsonError], - in: RetractReader - ): java.math.BigDecimal = + def bigDecimal(trace: List[JsonError], in: RetractReader): java.math.BigDecimal = try { val i = UnsafeNumbers.bigDecimal_(in, false, NumberMaxBits) in.retract()