libtiff: fix the FPE in tiffcrop (#415, #427, and #428) (84bd5)

From 84bd5fd331dbba3ff69f7185691cf3aabdfefab3 Mon Sep 17 00:00:00 2001
From: 4ugustus <[EMAIL REDACTED]>
Date: Sat, 11 Jun 2022 09:31:43 +0000
Subject: [PATCH] fix the FPE in tiffcrop (#415, #427, and #428)

---
 libtiff/tif_aux.c |  9 +++++++
 libtiff/tiffiop.h |  1 +
 tools/tiffcrop.c  | 62 ++++++++++++++++++++++++++---------------------
 3 files changed, 44 insertions(+), 28 deletions(-)

diff --git a/libtiff/tif_aux.c b/libtiff/tif_aux.c
index c9f19054..1813452d 100644
--- a/libtiff/tif_aux.c
+++ b/libtiff/tif_aux.c
@@ -409,6 +409,15 @@ int _TIFFSeekOK(TIFF* tif, toff_t off)
     return off <= (~(uint64)0)/2 && TIFFSeekFile(tif,off,SEEK_SET)==off;
 }
 
+uint32 _TIFFClampDoubleToUInt32(double val)
+{
+    if( val < 0 )
+        return 0;
+    if( val > 0xFFFFFFFFU || val != val )
+        return 0xFFFFFFFFU;
+    return (uint32)val;
+}
+
 /* vim: set ts=8 sts=8 sw=8 noet: */
 /*
  * Local Variables:
diff --git a/libtiff/tiffiop.h b/libtiff/tiffiop.h
index 39b54c89..cf50d56c 100644
--- a/libtiff/tiffiop.h
+++ b/libtiff/tiffiop.h
@@ -394,6 +394,7 @@ extern void* _TIFFCheckRealloc(TIFF*, void*, tmsize_t, tmsize_t, const char*);
 
 extern double _TIFFUInt64ToDouble(uint64);
 extern float _TIFFUInt64ToFloat(uint64);
+extern uint32 _TIFFClampDoubleToUInt32(double);
 
 extern float _TIFFClampDoubleToFloat(double);
 
diff --git a/tools/tiffcrop.c b/tools/tiffcrop.c
index 8d3757c7..bc38a07f 100644
--- a/tools/tiffcrop.c
+++ b/tools/tiffcrop.c
@@ -5190,17 +5190,17 @@ computeInputPixelOffsets(struct crop_mask *crop, struct image_data *image,
       {
       if ((crop->res_unit == RESUNIT_INCH) || (crop->res_unit == RESUNIT_CENTIMETER))
         {
-	x1 = (uint32) (crop->corners[i].X1 * scale * xres);
-	x2 = (uint32) (crop->corners[i].X2 * scale * xres);
-	y1 = (uint32) (crop->corners[i].Y1 * scale * yres);
-	y2 = (uint32) (crop->corners[i].Y2 * scale * yres);
+	x1 = _TIFFClampDoubleToUInt32(crop->corners[i].X1 * scale * xres);
+	x2 = _TIFFClampDoubleToUInt32(crop->corners[i].X2 * scale * xres);
+	y1 = _TIFFClampDoubleToUInt32(crop->corners[i].Y1 * scale * yres);
+	y2 = _TIFFClampDoubleToUInt32(crop->corners[i].Y2 * scale * yres);	
         }
       else
         {
-	x1 = (uint32) (crop->corners[i].X1);
-	x2 = (uint32) (crop->corners[i].X2);
-	y1 = (uint32) (crop->corners[i].Y1);
-	y2 = (uint32) (crop->corners[i].Y2);       
+	x1 = _TIFFClampDoubleToUInt32(crop->corners[i].X1);
+	x2 = _TIFFClampDoubleToUInt32(crop->corners[i].X2);
+	y1 = _TIFFClampDoubleToUInt32(crop->corners[i].Y1);
+	y2 = _TIFFClampDoubleToUInt32(crop->corners[i].Y2);
 	}
       if (x1 < 1)
         crop->regionlist[i].x1 = 0;
@@ -5263,17 +5263,17 @@ computeInputPixelOffsets(struct crop_mask *crop, struct image_data *image,
     {
     if (crop->res_unit != RESUNIT_INCH && crop->res_unit != RESUNIT_CENTIMETER)
       { /* User has specified pixels as reference unit */
-      tmargin = (uint32)(crop->margins[0]);
-      lmargin = (uint32)(crop->margins[1]);
-      bmargin = (uint32)(crop->margins[2]);
-      rmargin = (uint32)(crop->margins[3]);
+      tmargin = _TIFFClampDoubleToUInt32(crop->margins[0]);
+      lmargin = _TIFFClampDoubleToUInt32(crop->margins[1]);
+      bmargin = _TIFFClampDoubleToUInt32(crop->margins[2]);
+      rmargin = _TIFFClampDoubleToUInt32(crop->margins[3]);
       }
     else
       { /* inches or centimeters specified */
-      tmargin = (uint32)(crop->margins[0] * scale * yres);
-      lmargin = (uint32)(crop->margins[1] * scale * xres);
-      bmargin = (uint32)(crop->margins[2] * scale * yres);
-      rmargin = (uint32)(crop->margins[3] * scale * xres);
+      tmargin = _TIFFClampDoubleToUInt32(crop->margins[0] * scale * yres);
+      lmargin = _TIFFClampDoubleToUInt32(crop->margins[1] * scale * xres);
+      bmargin = _TIFFClampDoubleToUInt32(crop->margins[2] * scale * yres);
+      rmargin = _TIFFClampDoubleToUInt32(crop->margins[3] * scale * xres);
       }
 
     if ((lmargin + rmargin) > image->width)
@@ -5303,24 +5303,24 @@ computeInputPixelOffsets(struct crop_mask *crop, struct image_data *image,
   if (crop->res_unit != RESUNIT_INCH && crop->res_unit != RESUNIT_CENTIMETER)
     {
     if (crop->crop_mode & CROP_WIDTH)
-      width = (uint32)crop->width;
+      width = _TIFFClampDoubleToUInt32(crop->width);
     else
       width = image->width - lmargin - rmargin;
 
     if (crop->crop_mode & CROP_LENGTH)
-      length  = (uint32)crop->length;
+      length  = _TIFFClampDoubleToUInt32(crop->length);
     else
       length = image->length - tmargin - bmargin;
     }
   else
     {
     if (crop->crop_mode & CROP_WIDTH)
-      width = (uint32)(crop->width * scale * image->xres);
+      width = _TIFFClampDoubleToUInt32(crop->width * scale * image->xres);
     else
       width = image->width - lmargin - rmargin;
 
     if (crop->crop_mode & CROP_LENGTH)
-      length  = (uint32)(crop->length * scale * image->yres);
+      length  = _TIFFClampDoubleToUInt32(crop->length * scale * image->yres);
     else
       length = image->length - tmargin - bmargin;
     }
@@ -5719,13 +5719,13 @@ computeOutputPixelOffsets (struct crop_mask *crop, struct image_data *image,
     {
     if (page->res_unit == RESUNIT_INCH || page->res_unit == RESUNIT_CENTIMETER)
       { /* inches or centimeters specified */
-      hmargin = (uint32)(page->hmargin * scale * page->hres * ((image->bps + 7)/ 8));
-      vmargin = (uint32)(page->vmargin * scale * page->vres * ((image->bps + 7)/ 8));
+      hmargin = _TIFFClampDoubleToUInt32(page->hmargin * scale * page->hres * ((image->bps + 7) / 8));
+      vmargin = _TIFFClampDoubleToUInt32(page->vmargin * scale * page->vres * ((image->bps + 7) / 8));
       }
     else
       { /* Otherwise user has specified pixels as reference unit */
-      hmargin = (uint32)(page->hmargin * scale * ((image->bps + 7)/ 8));
-      vmargin = (uint32)(page->vmargin * scale * ((image->bps + 7)/ 8));
+      hmargin = _TIFFClampDoubleToUInt32(page->hmargin * scale * ((image->bps + 7) / 8));
+      vmargin = _TIFFClampDoubleToUInt32(page->vmargin * scale * ((image->bps + 7) / 8));
       }
 
     if ((hmargin * 2.0) > (pwidth * page->hres))
@@ -5763,13 +5763,13 @@ computeOutputPixelOffsets (struct crop_mask *crop, struct image_data *image,
     {
     if (page->mode & PAGE_MODE_PAPERSIZE )
       {
-      owidth  = (uint32)((pwidth * page->hres) - (hmargin * 2));
-      olength = (uint32)((plength * page->vres) - (vmargin * 2));
+      owidth  = _TIFFClampDoubleToUInt32((pwidth * page->hres) - (hmargin * 2));
+      olength = _TIFFClampDoubleToUInt32((plength * page->vres) - (vmargin * 2));
       }
     else
       {
-      owidth = (uint32)(iwidth - (hmargin * 2 * page->hres));
-      olength = (uint32)(ilength - (vmargin * 2 * page->vres));
+      owidth = _TIFFClampDoubleToUInt32(iwidth - (hmargin * 2 * page->hres));
+      olength = _TIFFClampDoubleToUInt32(ilength - (vmargin * 2 * page->vres));
       }
     }
 
@@ -5778,6 +5778,12 @@ computeOutputPixelOffsets (struct crop_mask *crop, struct image_data *image,
   if (olength > ilength)
     olength = ilength;
 
+  if (owidth == 0 || olength == 0)
+  {
+    TIFFError("computeOutputPixelOffsets", "Integer overflow when calculating the number of pages");
+    exit(EXIT_FAILURE);
+  }
+
   /* Compute the number of pages required for Portrait or Landscape */
   switch (page->orient)
     {