aom: Fix further overflow issue in VBR.

From 1c1f80b4ba7e28fbd42fed6a15a02e982973ed5d Mon Sep 17 00:00:00 2001
From: Paul Wilkins <[EMAIL REDACTED]>
Date: Thu, 23 May 2024 17:56:57 +0100
Subject: [PATCH] Fix further overflow issue in VBR.

This patch fixes some additional cases where under extreme conditions
some of the VBR adjustment variables can wrap.

As this happens on a per frame level the extra saturation checks should
not be an issue for performance.

Change-Id: If67a39ed92a7c0358f4006b946c6a2b4225db545
---
 av1/encoder/pass2_strategy.c | 19 +++++++++++-------
 av1/encoder/ratectrl.c       | 37 ++++++++++++++++++++----------------
 2 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/av1/encoder/pass2_strategy.c b/av1/encoder/pass2_strategy.c
index 8618212f6..6b63afc39 100644
--- a/av1/encoder/pass2_strategy.c
+++ b/av1/encoder/pass2_strategy.c
@@ -3401,14 +3401,18 @@ static int get_section_target_bandwidth(AV1_COMP *cpi) {
   CurrentFrame *const current_frame = &cm->current_frame;
   RATE_CONTROL *const rc = &cpi->rc;
   TWO_PASS *const twopass = &cpi->ppi->twopass;
-  int section_target_bandwidth;
+  int64_t section_target_bandwidth;
   const int frames_left = (int)(twopass->stats_buf_ctx->total_stats->count -
                                 current_frame->frame_number);
   if (cpi->ppi->lap_enabled)
-    section_target_bandwidth = (int)rc->avg_frame_bandwidth;
-  else
-    section_target_bandwidth = (int)(twopass->bits_left / frames_left);
-  return section_target_bandwidth;
+    section_target_bandwidth = rc->avg_frame_bandwidth;
+  else {
+    section_target_bandwidth = twopass->bits_left / frames_left;
+    section_target_bandwidth = (section_target_bandwidth < INT_MAX)
+                                   ? section_target_bandwidth
+                                   : INT_MAX;
+  }
+  return (int)section_target_bandwidth;
 }
 
 static INLINE void set_twopass_params_based_on_fp_stats(
@@ -4267,8 +4271,9 @@ void av1_twopass_postencode_update(AV1_COMP *cpi) {
       if (rc->projected_frame_size < fast_extra_thresh) {
         p_rc->vbr_bits_off_target_fast +=
             fast_extra_thresh - rc->projected_frame_size;
-        p_rc->vbr_bits_off_target_fast = AOMMIN(p_rc->vbr_bits_off_target_fast,
-                                                (4 * rc->avg_frame_bandwidth));
+        p_rc->vbr_bits_off_target_fast =
+            AOMMIN(p_rc->vbr_bits_off_target_fast,
+                   (4 * (int64_t)rc->avg_frame_bandwidth));
       }
     }
 
diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c
index 86dbdcaf7..148334aa8 100644
--- a/av1/encoder/ratectrl.c
+++ b/av1/encoder/ratectrl.c
@@ -2595,6 +2595,8 @@ static void vbr_rate_correction(AV1_COMP *cpi, int *this_frame_target) {
 #else
   int64_t vbr_bits_off_target = p_rc->vbr_bits_off_target;
 #endif
+  int64_t frame_target = *this_frame_target;
+
   const int stats_count =
       cpi->ppi->twopass.stats_buf_ctx->total_stats != NULL
           ? (int)cpi->ppi->twopass.stats_buf_ctx->total_stats->count
@@ -2603,13 +2605,13 @@ static void vbr_rate_correction(AV1_COMP *cpi, int *this_frame_target) {
       16, (int)(stats_count - (int)cpi->common.current_frame.frame_number));
   assert(VBR_PCT_ADJUSTMENT_LIMIT <= 100);
   if (frame_window > 0) {
-    const int max_delta = (int)AOMMIN(
-        abs((int)(vbr_bits_off_target / frame_window)),
-        ((int64_t)(*this_frame_target) * VBR_PCT_ADJUSTMENT_LIMIT) / 100);
+    const int64_t max_delta =
+        AOMMIN(llabs((vbr_bits_off_target / frame_window)),
+               (frame_target * VBR_PCT_ADJUSTMENT_LIMIT) / 100);
 
     // vbr_bits_off_target > 0 means we have extra bits to spend
     // vbr_bits_off_target < 0 we are currently overshooting
-    *this_frame_target += (vbr_bits_off_target >= 0) ? max_delta : -max_delta;
+    frame_target += (vbr_bits_off_target >= 0) ? max_delta : -max_delta;
   }
 
 #if CONFIG_FPMT_TEST
@@ -2626,32 +2628,35 @@ static void vbr_rate_correction(AV1_COMP *cpi, int *this_frame_target) {
       p_rc->vbr_bits_off_target_fast &&
 #endif
       !rc->is_src_frame_alt_ref) {
-    int one_frame_bits = AOMMAX(rc->avg_frame_bandwidth, *this_frame_target);
-    int fast_extra_bits;
+    int64_t one_frame_bits = AOMMAX(rc->avg_frame_bandwidth, frame_target);
+    int64_t fast_extra_bits;
 #if CONFIG_FPMT_TEST
-    fast_extra_bits = (int)AOMMIN(vbr_bits_off_target_fast, one_frame_bits);
+    fast_extra_bits = AOMMIN(vbr_bits_off_target_fast, one_frame_bits);
     fast_extra_bits =
-        (int)AOMMIN(fast_extra_bits,
-                    AOMMAX(one_frame_bits / 8, vbr_bits_off_target_fast / 8));
+        AOMMIN(fast_extra_bits,
+               AOMMAX(one_frame_bits / 8, vbr_bits_off_target_fast / 8));
 #else
+    fast_extra_bits = AOMMIN(p_rc->vbr_bits_off_target_fast, one_frame_bits);
     fast_extra_bits =
-        (int)AOMMIN(p_rc->vbr_bits_off_target_fast, one_frame_bits);
-    fast_extra_bits = (int)AOMMIN(
-        fast_extra_bits,
-        AOMMAX(one_frame_bits / 8, p_rc->vbr_bits_off_target_fast / 8));
+        AOMMIN(fast_extra_bits,
+               AOMMAX(one_frame_bits / 8, p_rc->vbr_bits_off_target_fast / 8));
 #endif
+    fast_extra_bits = (fast_extra_bits < INT_MAX) ? fast_extra_bits : INT_MAX;
     if (fast_extra_bits > 0) {
-      // Update this_frame_target only if additional bits are available from
+      // Update frame_target only if additional bits are available from
       // local undershoot.
-      *this_frame_target += (int)fast_extra_bits;
+      frame_target += fast_extra_bits;
     }
     // Store the fast_extra_bits of the frame and reduce it from
     // vbr_bits_off_target_fast during postencode stage.
-    rc->frame_level_fast_extra_bits = fast_extra_bits;
+    rc->frame_level_fast_extra_bits = (int)fast_extra_bits;
     // Retaining the condition to udpate during postencode stage since
     // fast_extra_bits are calculated based on vbr_bits_off_target_fast.
     cpi->do_update_vbr_bits_off_target_fast = 1;
   }
+
+  // Clamp the target for the frame to the maximum allowed for one frame.
+  *this_frame_target = (int)((frame_target < INT_MAX) ? frame_target : INT_MAX);
 }
 
 void av1_set_target_rate(AV1_COMP *cpi, int width, int height) {