aom: Increase scaling in linsolve_wiener

From 22bcd3de8f4fc43c4e87ef9dedfb40594fb1569b Mon Sep 17 00:00:00 2001
From: Debargha Mukherjee <[EMAIL REDACTED]>
Date: Thu, 23 May 2024 01:24:40 +0000
Subject: [PATCH] Increase scaling in linsolve_wiener

Reduces likelihood of overflows and re-enables previously
disabled overflow tests.

Bug: b:330639949, oss-fuzz:68195

Change-Id: I2c3dc9bd5783836d65b3c7a4452061b27cf4c82c
---
 av1/encoder/pickrst.c | 2 +-
 test/wiener_test.cc   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/av1/encoder/pickrst.c b/av1/encoder/pickrst.c
index 7c30e3a9d..0b0ca1c8e 100644
--- a/av1/encoder/pickrst.c
+++ b/av1/encoder/pickrst.c
@@ -1179,7 +1179,7 @@ static int linsolve_wiener(int n, int64_t *A, int stride, int64_t *b,
       if (abs_akj > max_abs_akj) max_abs_akj = abs_akj;
     }
     const int scale_threshold = 1 << 22;
-    const int scaler_A = max_abs_akj < scale_threshold ? 1 : (1 << 5);
+    const int scaler_A = max_abs_akj < scale_threshold ? 1 : (1 << 6);
     const int scaler_c = max_abs_akj < scale_threshold ? 1 : (1 << 7);
     const int scaler = scaler_c * scaler_A;
 
diff --git a/test/wiener_test.cc b/test/wiener_test.cc
index d018d8021..4508af227 100644
--- a/test/wiener_test.cc
+++ b/test/wiener_test.cc
@@ -1721,7 +1721,7 @@ TEST(SearchWienerTest, 8bitSignedIntegerOverflowInUpdateBSepSym) {
 
 // A test that reproduces crbug.com/oss-fuzz/68195: signed integer overflow in
 // linsolve_wiener().
-TEST(SearchWienerTest, DISABLED_8bitSignedIntegerOverflowInLinsolveWiener) {
+TEST(SearchWienerTest, 8bitSignedIntegerOverflowInLinsolveWiener) {
   constexpr int kWidth = 4;
   constexpr int kHeight = 3;
   constexpr unsigned char kBuffer[kWidth * kHeight] = {
@@ -1889,7 +1889,7 @@ TEST(SearchWienerTest, 10bitSignedIntegerOverflowInLinsolveWiener) {
 
 // A test that reproduces b/330639949: signed integer overflow in
 // linsolve_wiener().
-TEST(SearchWienerTest, DISABLED_12bitSignedIntegerOverflowInLinsolveWiener) {
+TEST(SearchWienerTest, 12bitSignedIntegerOverflowInLinsolveWiener) {
   constexpr int kWidth = 173;
   constexpr int kHeight = 3;
   // Since the image format is YUV 4:2:0, aom_img_wrap() expects the buffer is