aom: Multiply by stride in ptrdiff_t to avoid overflow

From 1b14d99e1275ccbe0f2d2b7473ff0f470a831d0b Mon Sep 17 00:00:00 2001
From: Wan-Teh Chang <[EMAIL REDACTED]>
Date: Wed, 21 Feb 2024 17:05:53 -0800
Subject: [PATCH] Multiply by stride in ptrdiff_t to avoid overflow

Change-Id: I626c0b191060ab1e8e951171d53792a646bad57f
---
 av1/common/cdef.c        |  8 +++++---
 av1/common/restoration.c | 19 ++++++++++++-------
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/av1/common/cdef.c b/av1/common/cdef.c
index 7a35b90e9..855654a9f 100644
--- a/av1/common/cdef.c
+++ b/av1/common/cdef.c
@@ -11,6 +11,7 @@
 
 #include <assert.h>
 #include <math.h>
+#include <stddef.h>
 #include <string.h>
 
 #include "config/aom_scale_rtcd.h"
@@ -93,7 +94,7 @@ void av1_cdef_copy_sb8_16_lowbd(uint16_t *const dst, int dstride,
                                 const uint8_t *src, int src_voffset,
                                 int src_hoffset, int sstride, int vsize,
                                 int hsize) {
-  const uint8_t *base = &src[src_voffset * sstride + src_hoffset];
+  const uint8_t *base = &src[src_voffset * (ptrdiff_t)sstride + src_hoffset];
   cdef_copy_rect8_8bit_to_16bit(dst, dstride, base, sstride, hsize, vsize);
 }
 
@@ -102,7 +103,7 @@ void av1_cdef_copy_sb8_16_highbd(uint16_t *const dst, int dstride,
                                  int src_hoffset, int sstride, int vsize,
                                  int hsize) {
   const uint16_t *base =
-      &CONVERT_TO_SHORTPTR(src)[src_voffset * sstride + src_hoffset];
+      &CONVERT_TO_SHORTPTR(src)[src_voffset * (ptrdiff_t)sstride + src_hoffset];
   cdef_copy_rect8_16bit_to_16bit(dst, dstride, base, sstride, hsize, vsize);
 }
 
@@ -248,7 +249,8 @@ static void cdef_prepare_fb(const AV1_COMMON *const cm, CdefBlockInfo *fb_info,
 
 static INLINE void cdef_filter_fb(CdefBlockInfo *const fb_info, int plane,
                                   uint8_t use_highbitdepth) {
-  int offset = fb_info->dst_stride * fb_info->roffset + fb_info->coffset;
+  ptrdiff_t offset =
+      (ptrdiff_t)fb_info->dst_stride * fb_info->roffset + fb_info->coffset;
   if (use_highbitdepth) {
     av1_cdef_filter_fb(
         NULL, CONVERT_TO_SHORTPTR(fb_info->dst + offset), fb_info->dst_stride,
diff --git a/av1/common/restoration.c b/av1/common/restoration.c
index e96e2f89f..d262feab8 100644
--- a/av1/common/restoration.c
+++ b/av1/common/restoration.c
@@ -11,6 +11,7 @@
  */
 
 #include <math.h>
+#include <stddef.h>
 
 #include "config/aom_config.h"
 #include "config/aom_dsp_rtcd.h"
@@ -116,8 +117,9 @@ void av1_loop_restoration_precal(void) {
 #endif
 }
 
-static void extend_frame_lowbd(uint8_t *data, int width, int height, int stride,
-                               int border_horz, int border_vert) {
+static void extend_frame_lowbd(uint8_t *data, int width, int height,
+                               ptrdiff_t stride, int border_horz,
+                               int border_vert) {
   uint8_t *data_p;
   int i;
   for (i = 0; i < height; ++i) {
@@ -137,7 +139,8 @@ static void extend_frame_lowbd(uint8_t *data, int width, int height, int stride,
 
 #if CONFIG_AV1_HIGHBITDEPTH
 static void extend_frame_highbd(uint16_t *data, int width, int height,
-                                int stride, int border_horz, int border_vert) {
+                                ptrdiff_t stride, int border_horz,
+                                int border_vert) {
   uint16_t *data_p;
   int i, j;
   for (i = 0; i < height; ++i) {
@@ -989,8 +992,10 @@ void av1_loop_restoration_filter_unit(
 
   int unit_h = limits->v_end - limits->v_start;
   int unit_w = limits->h_end - limits->h_start;
-  uint8_t *data8_tl = data8 + limits->v_start * stride + limits->h_start;
-  uint8_t *dst8_tl = dst8 + limits->v_start * dst_stride + limits->h_start;
+  uint8_t *data8_tl =
+      data8 + limits->v_start * (ptrdiff_t)stride + limits->h_start;
+  uint8_t *dst8_tl =
+      dst8 + limits->v_start * (ptrdiff_t)dst_stride + limits->h_start;
 
   if (unit_rtype == RESTORE_NONE) {
     copy_rest_unit(unit_w, unit_h, data8_tl, stride, dst8_tl, dst_stride,
@@ -1351,7 +1356,7 @@ static void save_deblock_boundary_lines(
   const int is_uv = plane > 0;
   const uint8_t *src_buf = REAL_PTR(use_highbd, frame->buffers[plane]);
   const int src_stride = frame->strides[is_uv] << use_highbd;
-  const uint8_t *src_rows = src_buf + row * src_stride;
+  const uint8_t *src_rows = src_buf + row * (ptrdiff_t)src_stride;
 
   uint8_t *bdry_buf = is_above ? boundaries->stripe_boundary_above
                                : boundaries->stripe_boundary_below;
@@ -1406,7 +1411,7 @@ static void save_cdef_boundary_lines(const YV12_BUFFER_CONFIG *frame,
   const int is_uv = plane > 0;
   const uint8_t *src_buf = REAL_PTR(use_highbd, frame->buffers[plane]);
   const int src_stride = frame->strides[is_uv] << use_highbd;
-  const uint8_t *src_rows = src_buf + row * src_stride;
+  const uint8_t *src_rows = src_buf + row * (ptrdiff_t)src_stride;
 
   uint8_t *bdry_buf = is_above ? boundaries->stripe_boundary_above
                                : boundaries->stripe_boundary_below;