Integer overflow in src/idct/scalar.rs

[personnumber3377]

Hi!

I fuzzed your library, and I found an integer overflow in your library. Here is a crashing file:

and now here is my stacktrace:

thread 'main' panicked at /home/oof/.cargo/registry/src/index.crates.io-6f17d22bba15001f/zune-jpeg-0.3.17/src/idct/scalar.rs:126:22:
attempt to add with overflow
stack backtrace:
   0:     0x56398e665f65 - std::backtrace_rs::backtrace::libunwind::trace::h23054e327d0d4b55
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/../../backtrace/src/backtrace/libunwind.rs:116:5
   1:     0x56398e665f65 - std::backtrace_rs::backtrace::trace_unsynchronized::h0cc587407d7f7f64
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0x56398e665f65 - std::sys_common::backtrace::_print_fmt::h4feeb59774730d6b
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/sys_common/backtrace.rs:68:5
   3:     0x56398e665f65 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::hd736fd5964392270
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/sys_common/backtrace.rs:44:22
   4:     0x56398e6831bb - core::fmt::rt::Argument::fmt::h105051d8ea1ade1e
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/fmt/rt.rs:165:63
   5:     0x56398e6831bb - core::fmt::write::hc6043626647b98ea
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/fmt/mod.rs:1168:21
   6:     0x56398e6640bf - std::io::Write::write_fmt::h0d24b3e0473045db
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/io/mod.rs:1835:15
   7:     0x56398e665d3e - std::sys_common::backtrace::_print::h62df6fc36dcebfc8
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/sys_common/backtrace.rs:47:5
   8:     0x56398e665d3e - std::sys_common::backtrace::print::h45eb8174d25a1e76
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/sys_common/backtrace.rs:34:9
   9:     0x56398e6670d9 - std::panicking::default_hook::{{closure}}::haf3f0170eb4f3b53
  10:     0x56398e666e7a - std::panicking::default_hook::hb5d3b27aa9f6dcda
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:298:9
  11:     0x56398e667573 - std::panicking::rust_panic_with_hook::h6b49d59f86ee588c
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:795:13
  12:     0x56398e66741b - std::panicking::begin_panic_handler::{{closure}}::hd4c2f7ed79b82b70
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:656:13
  13:     0x56398e666429 - std::sys_common::backtrace::__rust_end_short_backtrace::h2946d6d32d7ea1ad
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/sys_common/backtrace.rs:171:18
  14:     0x56398e667187 - rust_begin_unwind
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:652:5
  15:     0x56398e5f70c3 - core::panicking::panic_fmt::ha02418e5cd774672
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/panicking.rs:72:14
  16:     0x56398e5f7767 - core::panicking::panic_const::panic_const_add_overflow::h343c6c3f46bad3f5
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/panicking.rs:179:21
  17:     0x56398e62574e - zune_jpeg::idct::scalar::idct_int::h3e0733d3dc18e01a
                               at /home/oof/.cargo/registry/src/index.crates.io-6f17d22bba15001f/zune-jpeg-0.3.17/src/idct/scalar.rs:126:22
  18:     0x56398e5fe804 - zune_jpeg::mcu::<impl zune_jpeg::decoder::JpegDecoder>::decode_mcu_width::h04b32da8921e97df
                               at /home/oof/.cargo/registry/src/index.crates.io-6f17d22bba15001f/zune-jpeg-0.3.17/src/mcu.rs:253:29
  19:     0x56398e5fd581 - zune_jpeg::mcu::<impl zune_jpeg::decoder::JpegDecoder>::decode_mcu_ycbcr_baseline::h5c7d330aaf3f3d95
                               at /home/oof/.cargo/registry/src/index.crates.io-6f17d22bba15001f/zune-jpeg-0.3.17/src/mcu.rs:184:13
  20:     0x56398e5fbf04 - zune_jpeg::decoder::JpegDecoder::decode_into::h514045e9d418e569
                               at /home/oof/.cargo/registry/src/index.crates.io-6f17d22bba15001f/zune-jpeg-0.3.17/src/decoder.rs:758:13
  21:     0x56398e5f94c6 - zune_jpeg::decoder::JpegDecoder::decode::hb020c337c556e450
                               at /home/oof/.cargo/registry/src/index.crates.io-6f17d22bba15001f/zune-jpeg-0.3.17/src/decoder.rs:197:9
  22:     0x56398e5f7f93 - image_stuff::main::hcfe26f0a01771933
                               at /home/oof/imagebug/example/image_stuff/src/main.rs:11:22
  23:     0x56398e5f862b - core::ops::function::FnOnce::call_once::h22411a4f40c3fc33
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/ops/function.rs:250:5
  24:     0x56398e5f8a8e - std::sys_common::backtrace::__rust_begin_short_backtrace::he35a3884dfc990d0
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/sys_common/backtrace.rs:155:18
  25:     0x56398e5f8b91 - std::rt::lang_start::{{closure}}::h54e9ce293f217484
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:159:18
  26:     0x56398e66187d - core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once::hf57beef1b8c334ce
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/ops/function.rs:284:13
  27:     0x56398e66187d - std::panicking::try::do_call::h65791b6ab5d9b39f
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:559:40
  28:     0x56398e66187d - std::panicking::try::h5a3dd25e8a379a23
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:523:19
  29:     0x56398e66187d - std::panic::catch_unwind::he2ce8403bab77de2
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panic.rs:149:14
  30:     0x56398e66187d - std::rt::lang_start_internal::{{closure}}::h0b55d0da19178545
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:141:48
  31:     0x56398e66187d - std::panicking::try::do_call::h33cbeb674c7644e0
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:559:40
  32:     0x56398e66187d - std::panicking::try::h530c58b4c9daadba
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:523:19
  33:     0x56398e66187d - std::panic::catch_unwind::h92e02677901e11e4
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panic.rs:149:14
  34:     0x56398e66187d - std::rt::lang_start_internal::hcee5ed89fc25829a
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:141:20
  35:     0x56398e5f8b6a - std::rt::lang_start::h7e196705de507415
                               at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:158:17
  36:     0x56398e5f811e - main
  37:     0x7f4835daad90 - __libc_start_call_main
                               at ./csu/../sysdeps/nptl/libc_start_call_main.h:58:16
  38:     0x7f4835daae40 - __libc_start_main_impl
                               at ./csu/../csu/libc-start.c:392:3
  39:     0x56398e5f7955 - _start
  40:                0x0 - <unknown>

The offending line is here:

let t3 = p1 + p2 * 3135;

in this function:

pub fn idct_int(in_vector: &mut [i32; 64], out_vector: &mut [i16], stride: usize) {
    // Temporary variables.

    let mut pos = 0;

    let mut i = 0;
    // Don't check for zeroes inside loop, lift it and check outside
    // we want to accelerate the case with 63 0 ac coeff
    if &in_vector[1..] == &[0_i32; 63] {
        // okay then if you work, yay, let's write you really quick
        let coeff = [(((in_vector[0] >> 3) + 128) as i16).clamp(0, 255); 8];

        macro_rules! store {
            ($index:tt) => {
                // position of the MCU
                let mcu_stride: &mut [i16; 8] = out_vector
                    .get_mut($index..$index + 8)
                    .unwrap()
                    .try_into()
                    .unwrap();
                // copy coefficients
                mcu_stride.copy_from_slice(&coeff);
                // increment index
                $index += stride;
            };
        }
        // write to four positions
        store!(pos);
        store!(pos);
        store!(pos);
        store!(pos);

        store!(pos);
        store!(pos);
        store!(pos);
        store!(pos);
    } else {
        // because the compiler fails to see that it can be auto_vectorised so i'll
        // leave it here check out [idct_int_slow, and idct_int_1D to get what i mean ] https://godbolt.org/z/8hqW9z9j9
        for ptr in 0..8 {
            let p2 = in_vector[ptr + 16];
            let p3 = in_vector[ptr + 48];

            let p1 = (p2 + p3).wrapping_mul(2217);

            let t2 = p1 + p3 * -7567;
            let t3 = p1 + p2 * 3135;

            let p2 = in_vector[ptr];
            let p3 = in_vector[32 + ptr];
            let t0 = fsh(p2 + p3);
            let t1 = fsh(p2 - p3);

            let x0 = t0 + t3 + 512;
            let x3 = t0 - t3 + 512;
            let x1 = t1 + t2 + 512;
            let x2 = t1 - t2 + 512;

            // odd part
            let mut t0 = in_vector[ptr + 56];
            let mut t1 = in_vector[ptr + 40];
            let mut t2 = in_vector[ptr + 24];
            let mut t3 = in_vector[ptr + 8];

            let p3 = t0 + t2;
            let p4 = t1 + t3;
            let p1 = t0 + t3;
            let p2 = t1 + t2;
            let p5 = (p3 + p4) * 4816;

            t0 *= 1223;
            t1 *= 8410;
            t2 *= 12586;
            t3 *= 6149;

            let p1 = p5 + p1 * -3685;
            let p2 = p5 + p2 * -10497;
            let p3 = p3 * -8034;
            let p4 = p4 * -1597;

            t3 += p1 + p4;
            t2 += p2 + p3;
            t1 += p2 + p4;
            t0 += p1 + p3;

            // constants scaled things up by 1<<12; let's bring them back
            // down, but keep 2 extra bits of precision
            in_vector[ptr] = (x0 + t3) >> 10;
            in_vector[ptr + 8] = (x1 + t2) >> 10;
            in_vector[ptr + 16] = (x2 + t1) >> 10;
            in_vector[ptr + 24] = (x3 + t0) >> 10;
            in_vector[ptr + 32] = (x3 - t0) >> 10;
            in_vector[ptr + 40] = (x2 - t1) >> 10;
            in_vector[ptr + 48] = (x1 - t2) >> 10;
            in_vector[ptr + 56] = (x0 - t3) >> 10;
        }

        // This is vectorised in architectures supporting SSE 4.1
        while i < 64 {
            // We won't try to short circuit here because it rarely works

            // Even part
            let p2 = in_vector[i + 2];
            let p3 = in_vector[i + 6];

            let p1 = (p2 + p3) * 2217;
            let t2 = p1 + p3 * -7567;
            let t3 = p1 + p2 * 3135;

            let p2 = in_vector[i];
            let p3 = in_vector[i + 4];

            let t0 = fsh(p2 + p3);
            let t1 = fsh(p2 - p3);
            // constants scaled things up by 1<<12, plus we had 1<<2 from first
            // loop, plus horizontal and vertical each scale by sqrt(8) so together
            // we've got an extra 1<<3, so 1<<17 total we need to remove.
            // so we want to round that, which means adding 0.5 * 1<<17,
            // aka 65536. Also, we'll end up with -128 to 127 that we want
            // to encode as 0..255 by adding 128, so we'll add that before the shift
            let x0 = t0 + t3 + SCALE_BITS;
            let x3 = t0 - t3 + SCALE_BITS;
            let x1 = t1 + t2 + SCALE_BITS;
            let x2 = t1 - t2 + SCALE_BITS;
            // odd part
            let mut t0 = in_vector[i + 7];
            let mut t1 = in_vector[i + 5];
            let mut t2 = in_vector[i + 3];
            let mut t3 = in_vector[i + 1];

            let p3 = t0 + t2;
            let p4 = t1 + t3;
            let p1 = t0 + t3;
            let p2 = t1 + t2;
            let p5 = (p3 + p4) * f2f(1.175875602);

            t0 = t0.wrapping_mul(1223);
            t1 = t1.wrapping_mul(8410);
            t2 = t2.wrapping_mul(12586);
            t3 = t3.wrapping_mul(6149);

            let p1 = p5 + p1 * -3685;
            let p2 = p5 + p2 * -10497;
            let p3 = p3 * -8034;
            let p4 = p4 * -1597;

            t3 += p1 + p4;
            t2 += p2 + p3;
            t1 += p2 + p4;
            t0 += p1 + p3;

            let out: &mut [i16; 8] = out_vector
                .get_mut(pos..pos + 8)
                .unwrap()
                .try_into()
                .unwrap();

            out[0] = clamp((x0 + t3) >> 17);
            out[1] = clamp((x1 + t2) >> 17);
            out[2] = clamp((x2 + t1) >> 17);
            out[3] = clamp((x3 + t0) >> 17);
            out[4] = clamp((x3 - t0) >> 17);
            out[5] = clamp((x2 - t1) >> 17);
            out[6] = clamp((x1 - t2) >> 17);
            out[7] = clamp((x0 - t3) >> 17);

            i += 8;

            pos += stride;
        }
    }
}

this was found by fuzzing.