libtiff: * libtiff/tif_fax3.c (Fax3SetupState): Yesterday's fix for

https://github.com/libsdl-org/libtiff/commit/4579a589c91867b80829a8d2f81b011a0fd6d43e

From 4579a589c91867b80829a8d2f81b011a0fd6d43e Mon Sep 17 00:00:00 2001
From: Bob Friesenhahn <[EMAIL REDACTED]>
Date: Wed, 9 Jun 2010 17:16:58 +0000
Subject: [PATCH] * libtiff/tif_fax3.c (Fax3SetupState): Yesterday's fix for
 CVE-2010-1411 was not complete.

* libtiff/tiffiop.h (TIFFSafeMultiply): New macro to safely
multiply two integers.  Returns zero if there is an integer
overflow.

* tools/tiffcp.c (main): Fix more TIFF handle leaks.
---
 ChangeLog          |  9 +++++++++
 libtiff/tif_fax3.c | 23 ++++++++++++++++++-----
 libtiff/tiffiop.h  |  5 ++++-
 tools/tiffcp.c     |  6 ++++--
 4 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 6b280e5d..d15478e3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,14 @@
 2010-06-09  Bob Friesenhahn  <bfriesen@simple.dallas.tx.us>
 
+	* libtiff/tif_fax3.c (Fax3SetupState): Yesterday's fix for
+	CVE-2010-1411 was not complete.
+
+	* libtiff/tiffiop.h (TIFFSafeMultiply): New macro to safely
+	multiply two integers.  Returns zero if there is an integer
+	overflow.
+
+	* tools/tiffcp.c (main): Fix more TIFF handle leaks.
+
 	* libtiff/tif_read.c (TIFFReadBufferSetup): Skip allocating
 	tif_rawdata if tif_rawdatasize becomes zero.
 
diff --git a/libtiff/tif_fax3.c b/libtiff/tif_fax3.c
index 6098562d..9eec4ab7 100644
--- a/libtiff/tif_fax3.c
+++ b/libtiff/tif_fax3.c
@@ -1,4 +1,4 @@
-/* $Id: tif_fax3.c,v 1.43.2.9 2010-06-08 23:29:51 bfriesen Exp $ */
+/* $Id: tif_fax3.c,v 1.43.2.10 2010-06-09 17:16:58 bfriesen Exp $ */
 
 /*
  * Copyright (c) 1990-1997 Sam Leffler
@@ -493,13 +493,26 @@ Fax3SetupState(TIFF* tif)
 	    td->td_compression == COMPRESSION_CCITTFAX4
 	);
 
-	/* TIFFroundup returns zero on internal overflow */
+	/*
+	  Assure that allocation computations do not overflow.
+  
+	  TIFFroundup and TIFFSafeMultiply return zero on integer overflow
+	*/
+	dsp->runs=(uint32*) NULL;
 	nruns = TIFFroundup(rowpixels,32);
 	if (needsRefLine) {
-		nruns *= 2;
+		nruns = TIFFSafeMultiply(uint32,nruns,2);
+	}
+	if ((nruns == 0) || (TIFFSafeMultiply(uint32,nruns,2) == 0)) {
+		TIFFErrorExt(tif->tif_clientdata, tif->tif_name,
+			     "Row pixels integer overflow (rowpixels %u)",
+			     rowpixels);
+		return (0);
 	}
-	dsp->runs = (uint32*) _TIFFCheckMalloc(tif, 2*nruns, sizeof (uint32),
-					  "for Group 3/4 run arrays");
+	dsp->runs = (uint32*) _TIFFCheckMalloc(tif,
+					       TIFFSafeMultiply(uint32,nruns,2),
+					       sizeof (uint32),
+					       "for Group 3/4 run arrays");
 	if (dsp->runs == NULL)
 		return (0);
 	dsp->curruns = dsp->runs;
diff --git a/libtiff/tiffiop.h b/libtiff/tiffiop.h
index 4f39e0a8..fcffcca8 100644
--- a/libtiff/tiffiop.h
+++ b/libtiff/tiffiop.h
@@ -1,4 +1,4 @@
-/* $Id: tiffiop.h,v 1.51.2.3 2010-06-08 23:29:51 bfriesen Exp $ */
+/* $Id: tiffiop.h,v 1.51.2.4 2010-06-09 17:16:58 bfriesen Exp $ */
 
 /*
  * Copyright (c) 1988-1997 Sam Leffler
@@ -242,6 +242,9 @@ struct tiff {
 #define TIFFhowmany8(x) (((x)&0x07)?((uint32)(x)>>3)+1:(uint32)(x)>>3)
 #define	TIFFroundup(x, y) (TIFFhowmany(x,y)*(y))
 
+/* Safe multiply which returns zero if there is an integer overflow */
+#define TIFFSafeMultiply(t,v,m) ((((t)v*m)/(t)m == (t)v) ? (t)v*m : (t)0)
+
 #define TIFFmax(A,B) ((A)>(B)?(A):(B))
 #define TIFFmin(A,B) ((A)<(B)?(A):(B))
 
diff --git a/tools/tiffcp.c b/tools/tiffcp.c
index 6bafb7a1..5e58a6d0 100644
--- a/tools/tiffcp.c
+++ b/tools/tiffcp.c
@@ -1,4 +1,4 @@
-/* $Id: tiffcp.c,v 1.37.2.4 2010-06-08 23:29:51 bfriesen Exp $ */
+/* $Id: tiffcp.c,v 1.37.2.5 2010-06-09 17:16:58 bfriesen Exp $ */
 
 /*
  * Copyright (c) 1988-1997 Sam Leffler
@@ -274,8 +274,10 @@ main(int argc, char* argv[])
 	for (; optind < argc-1 ; optind++) {
                 char *imageCursor = argv[optind];
 		in = openSrcImage (&imageCursor);
-		if (in == NULL)
+		if (in == NULL) {
+			(void) TIFFClose(out);
 			return (-3);
+		}
 		if (diroff != 0 && !TIFFSetSubDirectory(in, diroff)) {
 			TIFFError(TIFFFileName(in),
 			    "Error, setting subdirectory at %#x", diroff);