libtiff: Merge branch 'tiffcrop_fix_CoverityScan_tmsize_issue' into 'master'

From bdd7eba876c006dc3767b88f9c7e148dcbffd20d Mon Sep 17 00:00:00 2001
From: Su Laus <[EMAIL REDACTED]>
Date: Tue, 8 Nov 2022 14:03:05 +0000
Subject: [PATCH] tiffcrop: should fix some Coverity Scan issues
 OVERFLOW_BEFORE_WIDEN

---
 tools/tiffcrop.c | 48 ++++++++++++++++++++++++------------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/tools/tiffcrop.c b/tools/tiffcrop.c
index 90b75786..da28a750 100644
--- a/tools/tiffcrop.c
+++ b/tools/tiffcrop.c
@@ -860,16 +860,16 @@ static int readContigTilesIntoBuffer (TIFF* in, uint8_t* buf,
 	      "Tilesize %"PRId64" is too small, using alternate calculation %"PRIu64,
               tilesize, tl * tile_rowsize);
 #endif
-    tile_buffsize = tl * tile_rowsize;
-    if (tl != (tile_buffsize / tile_rowsize))
+    if (tile_rowsize != 0 && (tmsize_t)tl > (TIFF_TMSIZE_T_MAX / tile_rowsize))
     {
     	TIFFError("readContigTilesIntoBuffer", "Integer overflow when calculating buffer size.");
         exit(EXIT_FAILURE);
     }
+    tile_buffsize = tl * tile_rowsize;
     }
 
   /* Add 3 padding bytes for extractContigSamplesShifted32bits */
-  if( (size_t) tile_buffsize > 0xFFFFFFFFU - 3U )
+  if( (size_t) tile_buffsize > TIFF_TMSIZE_T_MAX - NUM_BUFF_OVERSIZE_BYTES)
   {
       TIFFError("readContigTilesIntoBuffer", "Integer overflow when calculating buffer size.");
       exit(EXIT_FAILURE);
@@ -1311,12 +1311,12 @@ static int writeBufferToContigTiles (TIFF* out, uint8_t* buf, uint32_t imageleng
 	      "Tilesize %"PRId64" is too small, using alternate calculation %"PRIu32,
               tilesize, tl * tile_rowsize);
 #endif
-    tile_buffsize = tl * tile_rowsize;
-    if (tl != tile_buffsize / tile_rowsize)
+    if (tile_rowsize != 0 && (tmsize_t)tl > (TIFF_TMSIZE_T_MAX / tile_rowsize))
     {
 	TIFFError("writeBufferToContigTiles", "Integer overflow when calculating buffer size");
 	exit(EXIT_FAILURE);
     }
+    tile_buffsize = tl * tile_rowsize;
     }
 
   if( imagewidth == 0 ||
@@ -5032,7 +5032,7 @@ static int readSeparateStripsIntoBuffer (TIFF *in, uint8_t *obuf, uint32_t lengt
   strips_per_sample = nstrips /spp;
 
   /* Add 3 padding bytes for combineSeparateSamples32bits */
-  if( (size_t) stripsize > 0xFFFFFFFFU - 3U )
+  if( (size_t) stripsize > TIFF_TMSIZE_T_MAX - NUM_BUFF_OVERSIZE_BYTES)
   {
       TIFFError("readSeparateStripsIntoBuffer", "Integer overflow when calculating buffer size.");
       exit(EXIT_FAILURE);
@@ -6335,28 +6335,28 @@ loadImage(TIFF* in, struct image_data *image, struct dump_opts *dump, unsigned c
 	TIFFError("loadImage", "File appears to be tiled, but the number of tiles, tile size, or tile rowsize is zero.");
 	exit(EXIT_FAILURE);
     }
-    buffsize = tlsize * ntiles;
-    if (tlsize != (buffsize / ntiles))
+    if (ntiles != 0 && tlsize > (tmsize_t)(TIFF_TMSIZE_T_MAX / ntiles))
     {
 	TIFFError("loadImage", "Integer overflow when calculating buffer size");
 	exit(EXIT_FAILURE);
     }
+    buffsize = tlsize * ntiles;
 
+    if (tl != 0 && ntiles != 0 && tile_rowsize > (tmsize_t)(TIFF_TMSIZE_T_MAX / tl / ntiles))
+    {
+        TIFFError("loadImage", "Integer overflow when calculating buffer size");
+        exit(EXIT_FAILURE);
+    }
     if (buffsize < (tmsize_t)(ntiles * tl * tile_rowsize))
-      {
+    {
       buffsize = ntiles * tl * tile_rowsize;
-      if (ntiles != (buffsize / tl / tile_rowsize))
-      {
-	TIFFError("loadImage", "Integer overflow when calculating buffer size");
-	exit(EXIT_FAILURE);
-      }
       
 #ifdef DEBUG2
       TIFFError("loadImage",
 	        "Tilesize %"PRIu32" is too small, using ntiles * tilelength * tilerowsize %"PRIu32,
                 tlsize, buffsize);
 #endif
-      }
+    }
     
     if (dump->infile != NULL)
       dump_info (dump->infile, dump->format, "", 
@@ -6376,18 +6376,12 @@ loadImage(TIFF* in, struct image_data *image, struct dump_opts *dump, unsigned c
 	exit(EXIT_FAILURE);
     }
 
-    buffsize = stsize * nstrips;
-    if (stsize != (buffsize / nstrips))
+    if (nstrips != 0 && stsize > (tmsize_t)(TIFF_TMSIZE_T_MAX / nstrips))
     {
 	TIFFError("loadImage", "Integer overflow when calculating buffer size");
 	exit(EXIT_FAILURE);
     }
-    buffsize_check = ((length * width * spp * bps) + 7);
-    if (length != ((buffsize_check - 7) / width / spp / bps))
-    {
-	TIFFError("loadImage", "Integer overflow detected.");
-	exit(EXIT_FAILURE);
-    }
+    buffsize = stsize * nstrips;
     /* The buffsize_check and the possible adaptation of buffsize 
      * has to account also for padding of each line to a byte boundary. 
      * This is assumed by mirrorImage() and rotateImage().
@@ -6395,7 +6389,13 @@ loadImage(TIFF* in, struct image_data *image, struct dump_opts *dump, unsigned c
      * need a buffer, which is at least 3 bytes larger than the actual image.
      * Otherwise buffer-overflow might occur there.
      */
-    buffsize_check = length * (uint32_t)(((width * spp * bps) + 7) / 8);
+    if ( (spp != 0 && bps != 0 && width > (uint32_t)((UINT32_MAX - 7) / spp / bps)) ||
+         (width != 0 && spp != 0 && bps != 0 && length > (tmsize_t)(TIFF_TMSIZE_T_MAX / (uint32_t)(((width * spp * bps) + 7) / 8))) )
+    {
+	TIFFError("loadImage", "Integer overflow detected.");
+	exit(EXIT_FAILURE);
+    }
+    buffsize_check = (tmsize_t)length * (uint32_t)(((width * spp * bps) + 7) / 8);
     if (buffsize < buffsize_check)
       {
       buffsize = buffsize_check;