From 688012dca2c39033aa2dc7bcea9796787cfd1b44 Mon Sep 17 00:00:00 2001
From: Su_Laus <[EMAIL REDACTED]>
Date: Sat, 4 Feb 2023 23:24:21 +0100
Subject: [PATCH] tiffcrop correctly update buffersize after rotateImage()
fix#520 -- enlarge buffsize and check integer overflow within rotateImage().
---
tools/tiffcrop.c | 36 +++++++++++++++++++++++++++++++++---
1 file changed, 33 insertions(+), 3 deletions(-)
diff --git a/tools/tiffcrop.c b/tools/tiffcrop.c
index f8b66188..ca23529b 100644
--- a/tools/tiffcrop.c
+++ b/tools/tiffcrop.c
@@ -9560,7 +9560,8 @@ static int rotateImage(uint16_t rotation, struct image_data *image,
uint32_t bytes_per_pixel, bytes_per_sample;
uint32_t row, rowsize, src_offset, dst_offset;
uint32_t i, col, width, length;
- uint32_t colsize, buffsize, col_offset, pix_offset;
+ uint32_t colsize, col_offset, pix_offset;
+ tmsize_t buffsize;
unsigned char *ibuff;
unsigned char *src;
unsigned char *dst;
@@ -9573,12 +9574,40 @@ static int rotateImage(uint16_t rotation, struct image_data *image,
spp = image->spp;
bps = image->bps;
+ if ((spp != 0 && bps != 0 &&
+ width > (uint32_t)((UINT32_MAX - 7) / spp / bps)) ||
+ (spp != 0 && bps != 0 &&
+ length > (uint32_t)((UINT32_MAX - 7) / spp / bps)))
+ {
+ TIFFError("rotateImage", "Integer overflow detected.");
+ return (-1);
+ }
rowsize = ((bps * spp * width) + 7) / 8;
colsize = ((bps * spp * length) + 7) / 8;
if ((colsize * width) > (rowsize * length))
- buffsize = (colsize + 1) * width;
+ {
+ if (((tmsize_t)colsize + 1) != 0 &&
+ (tmsize_t)width > ((TIFF_TMSIZE_T_MAX - NUM_BUFF_OVERSIZE_BYTES) /
+ ((tmsize_t)colsize + 1)))
+ {
+ TIFFError("rotateImage",
+ "Integer overflow when calculating buffer size.");
+ return (-1);
+ }
+ buffsize = ((tmsize_t)colsize + 1) * width;
+ }
else
+ {
+ if (((tmsize_t)rowsize + 1) != 0 &&
+ (tmsize_t)length > ((TIFF_TMSIZE_T_MAX - NUM_BUFF_OVERSIZE_BYTES) /
+ ((tmsize_t)rowsize + 1)))
+ {
+ TIFFError("rotateImage",
+ "Integer overflow when calculating buffer size.");
+ return (-1);
+ }
buffsize = (rowsize + 1) * length;
+ }
bytes_per_sample = (bps + 7) / 8;
bytes_per_pixel = ((bps * spp) + 7) / 8;
@@ -9607,7 +9636,8 @@ static int rotateImage(uint16_t rotation, struct image_data *image,
(unsigned char *)limitMalloc(buffsize + NUM_BUFF_OVERSIZE_BYTES)))
{
TIFFError("rotateImage",
- "Unable to allocate rotation buffer of %1u bytes",
+ "Unable to allocate rotation buffer of %" TIFF_SSIZE_FORMAT
+ " bytes ",
buffsize + NUM_BUFF_OVERSIZE_BYTES);
return (-1);
}