aom: Fix high target data rate overflow.

From 699cefee092b8e2f35f580f44c5be709c569722f Mon Sep 17 00:00:00 2001
From: Paul Wilkins <[EMAIL REDACTED]>
Date: Wed, 15 May 2024 12:28:25 +0100
Subject: [PATCH] Fix high target data rate overflow.

These change fixes issues that can occur if the user specifies a very
high target data rate or rate per frame.

Fixes some issue with overflow of int variables used to hold bitrate
values (rate per second, rate per frame etc).

This patch also imposes a new maximum for the passed in target bitrate.
This value is passed in in kbits (so multiplied by 1000
internally).

Change-Id: Ia3b60bf1110d85cdce161492561bdda1cff61c63
---
 aom/aom_encoder.h            |  1 +
 av1/av1_cx_iface.c           |  3 ++-
 av1/encoder/encoder.h        |  2 +-
 av1/encoder/pass2_strategy.c | 16 ++++++++++------
 av1/encoder/ratectrl.c       |  9 +++++----
 5 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/aom/aom_encoder.h b/aom/aom_encoder.h
index 9bdadd693..15cf21b04 100644
--- a/aom/aom_encoder.h
+++ b/aom/aom_encoder.h
@@ -637,6 +637,7 @@ typedef struct aom_codec_enc_cfg {
   /*!\brief Target data rate
    *
    * Target bitrate to use for this stream, in kilobits per second.
+   * Max allowed value is 2000000
    */
   unsigned int rc_target_bitrate;
 
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index c09a02c81..c26f2aafb 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -674,6 +674,7 @@ static aom_codec_err_t validate_config(aom_codec_alg_priv_t *ctx,
   RANGE_CHECK(cfg, g_timebase.num, 1, cfg->g_timebase.den);
   RANGE_CHECK_HI(cfg, g_profile, MAX_PROFILES - 1);
 
+  RANGE_CHECK_HI(cfg, rc_target_bitrate, 2000000);
   RANGE_CHECK_HI(cfg, rc_max_quantizer, 63);
   RANGE_CHECK_HI(cfg, rc_min_quantizer, cfg->rc_max_quantizer);
   RANGE_CHECK_BOOL(extra_cfg, lossless);
@@ -3331,7 +3332,7 @@ static aom_codec_err_t encoder_encode(aom_codec_alg_priv_t *ctx,
       if (ppi->cpi->oxcf.pass != 1) {
         ppi->total_time_compress_data += cpi->time_compress_data;
         ppi->total_recode_hits += cpi->frame_recode_hits;
-        ppi->total_bytes += cpi->bytes;
+        ppi->total_bytes += (uint64_t)cpi->bytes;
         for (int i = 0; i < MAX_MODES; i++) {
           ppi->total_mode_chosen_counts[i] += cpi->mode_chosen_counts[i];
         }
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index a919bd906..b0fc5cd78 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -2793,7 +2793,7 @@ typedef struct AV1_PRIMARY {
   double total_blockiness;
   double worst_blockiness;
 
-  int total_bytes;
+  uint64_t total_bytes;
   double summed_quality;
   double summed_weights;
   double summed_quality_hbd;
diff --git a/av1/encoder/pass2_strategy.c b/av1/encoder/pass2_strategy.c
index bd8620c2b..8618212f6 100644
--- a/av1/encoder/pass2_strategy.c
+++ b/av1/encoder/pass2_strategy.c
@@ -268,9 +268,12 @@ static double calc_correction_factor(double err_per_mb, int q) {
 
 // Similar to find_qindex_by_rate() function in ratectrl.c, but includes
 // calculation of a correction_factor.
-static int find_qindex_by_rate_with_correction(
-    int desired_bits_per_mb, aom_bit_depth_t bit_depth, double error_per_mb,
-    double group_weight_factor, int best_qindex, int worst_qindex) {
+static int find_qindex_by_rate_with_correction(uint64_t desired_bits_per_mb,
+                                               aom_bit_depth_t bit_depth,
+                                               double error_per_mb,
+                                               double group_weight_factor,
+                                               int best_qindex,
+                                               int worst_qindex) {
   assert(best_qindex <= worst_qindex);
   int low = best_qindex;
   int high = worst_qindex;
@@ -279,7 +282,8 @@ static int find_qindex_by_rate_with_correction(
     const int mid = (low + high) >> 1;
     const double q_factor = calc_correction_factor(error_per_mb, mid);
     const double q = av1_convert_qindex_to_q(mid, bit_depth);
-    const int mid_bits_per_mb = (int)((q_factor * group_weight_factor) / q);
+    const uint64_t mid_bits_per_mb =
+        (uint64_t)((q_factor * group_weight_factor) / q);
 
     if (mid_bits_per_mb > desired_bits_per_mb) {
       low = mid + 1;
@@ -328,8 +332,8 @@ static int get_twopass_worst_quality(AV1_COMP *cpi, const double av_frame_err,
                             : cpi->common.mi_params.MBs;
     const int active_mbs = AOMMAX(1, num_mbs - (int)(num_mbs * inactive_zone));
     const double av_err_per_mb = av_frame_err / (1.0 - inactive_zone);
-    const int target_norm_bits_per_mb =
-        (int)((uint64_t)av_target_bandwidth << BPER_MB_NORMBITS) / active_mbs;
+    const uint64_t target_norm_bits_per_mb =
+        ((uint64_t)av_target_bandwidth << BPER_MB_NORMBITS) / active_mbs;
     int rate_err_tol = AOMMIN(rc_cfg->under_shoot_pct, rc_cfg->over_shoot_pct);
 
     // Update bpm correction factor based on previous GOP rate error.
diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c
index 9daeb45c8..a32e0b5ba 100644
--- a/av1/encoder/ratectrl.c
+++ b/av1/encoder/ratectrl.c
@@ -2550,7 +2550,7 @@ void av1_rc_set_gf_interval_range(const AV1_COMP *const cpi,
 void av1_rc_update_framerate(AV1_COMP *cpi, int width, int height) {
   const AV1EncoderConfig *const oxcf = &cpi->oxcf;
   RATE_CONTROL *const rc = &cpi->rc;
-  int vbr_max_bits;
+  int64_t vbr_max_bits;
   const int MBs = av1_get_MBs(width, height);
 
   rc->avg_frame_bandwidth =
@@ -2569,10 +2569,11 @@ void av1_rc_update_framerate(AV1_COMP *cpi, int width, int height) {
   // be acheived because of a user specificed max q (e.g. when the user
   // specifies lossless encode.
   vbr_max_bits =
-      (int)(((int64_t)rc->avg_frame_bandwidth * oxcf->rc_cfg.vbrmax_section) /
-            100);
+      ((int64_t)rc->avg_frame_bandwidth * oxcf->rc_cfg.vbrmax_section) / 100;
+  vbr_max_bits = (vbr_max_bits < INT_MAX) ? vbr_max_bits : INT_MAX;
+
   rc->max_frame_bandwidth =
-      AOMMAX(AOMMAX((MBs * MAX_MB_RATE), MAXRATE_1080P), vbr_max_bits);
+      AOMMAX(AOMMAX((MBs * MAX_MB_RATE), MAXRATE_1080P), (int)vbr_max_bits);
 
   av1_rc_set_gf_interval_range(cpi, rc);
 }