From 52ba0aed29cccb1560d9dad41eefeede0ef28565 Mon Sep 17 00:00:00 2001
From: Wan-Teh Chang <[EMAIL REDACTED]>
Date: Tue, 25 Jun 2024 14:28:21 -0700
Subject: [PATCH] Fix integer overflows caused by max target bitrate
Fix signed integer overflows caused by setting the targer bitrate to the
maximum value (2000000).
Tested: Build libaom with -DSANITIZE=undefined and the run
./aomenc husky.yuv -o AV1_husky_2000000_10000000_10000000.webm --good \
--cpu-used=2 -v -t 0 -w 352 -h 288 --fps=10000000/10000000 \
--target-bitrate=2000000 --limit=150 --test-decode=fatal --passes=2 \
--lag-in-frames=35 --min-q=0 --max-q=63 --min-gf-interval=4 \
--max-gf-interval=32 --arnr-maxframes=7 --arnr-strength=5 \
--kf-max-dist=9999 --aq-mode=0 --undershoot-pct=100 \
--overshoot-pct=100 --bias-pct=50
Fix the signed integer overflows strictly in the order they are reported
by UBSan, because a later overflow with a negative operand may have been
caused by an earlier overflow.
Bug: 349440066
Change-Id: Ideee322cc38f251461f67676258bb4bc4ab867f1
---
av1/encoder/pass2_strategy.c | 12 ++++++++++--
av1/encoder/ratectrl.c | 8 ++++----
2 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/av1/encoder/pass2_strategy.c b/av1/encoder/pass2_strategy.c
index 619b2de81..a04890171 100644
--- a/av1/encoder/pass2_strategy.c
+++ b/av1/encoder/pass2_strategy.c
@@ -946,7 +946,10 @@ static void allocate_gf_group_bits(GF_GROUP *gf_group,
case ARF_UPDATE:
case INTNL_ARF_UPDATE:
arf_extra_bits = layer_extra_bits[gf_group->layer_depth[idx]];
- gf_group->bit_allocation[idx] = base_frame_bits + arf_extra_bits;
+ gf_group->bit_allocation[idx] =
+ (base_frame_bits > INT_MAX - arf_extra_bits)
+ ? INT_MAX
+ : (base_frame_bits + arf_extra_bits);
break;
case INTNL_OVERLAY_UPDATE:
case OVERLAY_UPDATE: gf_group->bit_allocation[idx] = 0; break;
@@ -4136,7 +4139,12 @@ void av1_twopass_postencode_update(AV1_COMP *cpi) {
}
// Target vs actual bits for this arf group.
- twopass->rolling_arf_group_target_bits += rc->base_frame_target;
+ if (twopass->rolling_arf_group_target_bits >
+ INT_MAX - rc->base_frame_target) {
+ twopass->rolling_arf_group_target_bits = INT_MAX;
+ } else {
+ twopass->rolling_arf_group_target_bits += rc->base_frame_target;
+ }
twopass->rolling_arf_group_actual_bits += rc->projected_frame_size;
// Calculate the pct rc error.
diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c
index a34ce7875..cd73772b4 100644
--- a/av1/encoder/ratectrl.c
+++ b/av1/encoder/ratectrl.c
@@ -2229,8 +2229,8 @@ void av1_rc_compute_frame_size_bounds(const AV1_COMP *cpi, int frame_target,
const int tolerance = (int)AOMMAX(
100, ((int64_t)cpi->sf.hl_sf.recode_tolerance * frame_target) / 100);
*frame_under_shoot_limit = AOMMAX(frame_target - tolerance, 0);
- *frame_over_shoot_limit =
- AOMMIN(frame_target + tolerance, cpi->rc.max_frame_bandwidth);
+ *frame_over_shoot_limit = (int)AOMMIN((int64_t)frame_target + tolerance,
+ cpi->rc.max_frame_bandwidth);
}
}
@@ -2355,9 +2355,9 @@ void av1_rc_postencode_update(AV1_COMP *cpi, uint64_t bytes_used) {
cm->width, cm->height));
if (current_frame->frame_type != KEY_FRAME) {
p_rc->rolling_target_bits = (int)ROUND_POWER_OF_TWO_64(
- p_rc->rolling_target_bits * 3 + rc->this_frame_target, 2);
+ (int64_t)p_rc->rolling_target_bits * 3 + rc->this_frame_target, 2);
p_rc->rolling_actual_bits = (int)ROUND_POWER_OF_TWO_64(
- p_rc->rolling_actual_bits * 3 + rc->projected_frame_size, 2);
+ (int64_t)p_rc->rolling_actual_bits * 3 + rc->projected_frame_size, 2);
}
// Actual bits spent