aom: Fix integer overflows caused by max target bitrate

From 52ba0aed29cccb1560d9dad41eefeede0ef28565 Mon Sep 17 00:00:00 2001
From: Wan-Teh Chang <[EMAIL REDACTED]>
Date: Tue, 25 Jun 2024 14:28:21 -0700
Subject: [PATCH] Fix integer overflows caused by max target bitrate

Fix signed integer overflows caused by setting the targer bitrate to the
maximum value (2000000).

Tested: Build libaom with -DSANITIZE=undefined and the run
./aomenc husky.yuv -o AV1_husky_2000000_10000000_10000000.webm --good \
  --cpu-used=2 -v -t 0 -w 352 -h 288 --fps=10000000/10000000 \
  --target-bitrate=2000000 --limit=150 --test-decode=fatal --passes=2 \
  --lag-in-frames=35 --min-q=0 --max-q=63 --min-gf-interval=4 \
  --max-gf-interval=32 --arnr-maxframes=7 --arnr-strength=5 \
  --kf-max-dist=9999 --aq-mode=0 --undershoot-pct=100 \
  --overshoot-pct=100 --bias-pct=50

Fix the signed integer overflows strictly in the order they are reported
by UBSan, because a later overflow with a negative operand may have been
caused by an earlier overflow.

Bug: 349440066
Change-Id: Ideee322cc38f251461f67676258bb4bc4ab867f1
---
 av1/encoder/pass2_strategy.c | 12 ++++++++++--
 av1/encoder/ratectrl.c       |  8 ++++----
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/av1/encoder/pass2_strategy.c b/av1/encoder/pass2_strategy.c
index 619b2de81..a04890171 100644
--- a/av1/encoder/pass2_strategy.c
+++ b/av1/encoder/pass2_strategy.c
@@ -946,7 +946,10 @@ static void allocate_gf_group_bits(GF_GROUP *gf_group,
       case ARF_UPDATE:
       case INTNL_ARF_UPDATE:
         arf_extra_bits = layer_extra_bits[gf_group->layer_depth[idx]];
-        gf_group->bit_allocation[idx] = base_frame_bits + arf_extra_bits;
+        gf_group->bit_allocation[idx] =
+            (base_frame_bits > INT_MAX - arf_extra_bits)
+                ? INT_MAX
+                : (base_frame_bits + arf_extra_bits);
         break;
       case INTNL_OVERLAY_UPDATE:
       case OVERLAY_UPDATE: gf_group->bit_allocation[idx] = 0; break;
@@ -4136,7 +4139,12 @@ void av1_twopass_postencode_update(AV1_COMP *cpi) {
   }
 
   // Target vs actual bits for this arf group.
-  twopass->rolling_arf_group_target_bits += rc->base_frame_target;
+  if (twopass->rolling_arf_group_target_bits >
+      INT_MAX - rc->base_frame_target) {
+    twopass->rolling_arf_group_target_bits = INT_MAX;
+  } else {
+    twopass->rolling_arf_group_target_bits += rc->base_frame_target;
+  }
   twopass->rolling_arf_group_actual_bits += rc->projected_frame_size;
 
   // Calculate the pct rc error.
diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c
index a34ce7875..cd73772b4 100644
--- a/av1/encoder/ratectrl.c
+++ b/av1/encoder/ratectrl.c
@@ -2229,8 +2229,8 @@ void av1_rc_compute_frame_size_bounds(const AV1_COMP *cpi, int frame_target,
     const int tolerance = (int)AOMMAX(
         100, ((int64_t)cpi->sf.hl_sf.recode_tolerance * frame_target) / 100);
     *frame_under_shoot_limit = AOMMAX(frame_target - tolerance, 0);
-    *frame_over_shoot_limit =
-        AOMMIN(frame_target + tolerance, cpi->rc.max_frame_bandwidth);
+    *frame_over_shoot_limit = (int)AOMMIN((int64_t)frame_target + tolerance,
+                                          cpi->rc.max_frame_bandwidth);
   }
 }
 
@@ -2355,9 +2355,9 @@ void av1_rc_postencode_update(AV1_COMP *cpi, uint64_t bytes_used) {
                                                      cm->width, cm->height));
   if (current_frame->frame_type != KEY_FRAME) {
     p_rc->rolling_target_bits = (int)ROUND_POWER_OF_TWO_64(
-        p_rc->rolling_target_bits * 3 + rc->this_frame_target, 2);
+        (int64_t)p_rc->rolling_target_bits * 3 + rc->this_frame_target, 2);
     p_rc->rolling_actual_bits = (int)ROUND_POWER_OF_TWO_64(
-        p_rc->rolling_actual_bits * 3 + rc->projected_frame_size, 2);
+        (int64_t)p_rc->rolling_actual_bits * 3 + rc->projected_frame_size, 2);
   }
 
   // Actual bits spent