libtiff: Fix size overflow (zdi-can-1221,CVE-2012-1173) care of Tom Lane @ Red Hat.

https://github.com/libsdl-org/libtiff/commit/fada19fb9c2dfc3eaae38870d77e18d024436eee

From fada19fb9c2dfc3eaae38870d77e18d024436eee Mon Sep 17 00:00:00 2001
From: Frank Warmerdam <[EMAIL REDACTED]>
Date: Fri, 6 Apr 2012 16:47:41 +0000
Subject: [PATCH] Fix size overflow (zdi-can-1221,CVE-2012-1173) care of Tom
 Lane @ Red Hat.

---
 ChangeLog              |  6 ++++++
 libtiff/tif_getimage.c | 22 +++++++++++++++++-----
 libtiff/tiffiop.h      |  4 ++--
 3 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 5d0a7b63..22f6aea0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+2012-03-30  Frank Warmerdam  <warmerdam@google.com>
+
+	* tif_getimage.c: Fix size overflow (zdi-can-1221,CVE-2012-1173)
+	care of Tom Lane @ Red Hat.
+
+
 2012-02-18  Bob Friesenhahn  <bfriesen@simple.dallas.tx.us>
 
 	* libtiff 3.9.6 released.
diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
index 11d59e06..b4f06d19 100644
--- a/libtiff/tif_getimage.c
+++ b/libtiff/tif_getimage.c
@@ -1,4 +1,4 @@
-/* $Id: tif_getimage.c,v 1.63.2.6 2010-07-02 13:38:27 dron Exp $ */
+/* $Id: tif_getimage.c,v 1.63.2.7 2012-04-06 16:47:42 fwarmerdam Exp $ */
 
 /*
  * Copyright (c) 1991-1997 Sam Leffler
@@ -673,18 +673,24 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 	unsigned char* p2;
 	unsigned char* pa;
 	tsize_t tilesize;
+	tsize_t bufsize;
 	int32 fromskew, toskew;
 	int alpha = img->alpha;
 	uint32 nrow;
 	int ret = 1, flip;
 
 	tilesize = TIFFTileSize(tif);
-	buf = (unsigned char*) _TIFFmalloc((alpha?4:3)*tilesize);
+	bufsize = TIFFSafeMultiply(tsize_t,alpha?4:3,tilesize);
+	if (bufsize == 0) {
+		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtTileSeparate");
+		return (0);
+	}
+	buf = (unsigned char*) _TIFFmalloc(bufsize);
 	if (buf == 0) {
 		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for tile buffer");
 		return (0);
 	}
-	_TIFFmemset(buf, 0, (alpha?4:3)*tilesize);
+	_TIFFmemset(buf, 0, bufsize);
 	p0 = buf;
 	p1 = p0 + tilesize;
 	p2 = p1 + tilesize;
@@ -880,17 +886,23 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 	uint32 rowsperstrip, offset_row;
 	uint32 imagewidth = img->width;
 	tsize_t stripsize;
+	tsize_t bufsize;
 	int32 fromskew, toskew;
 	int alpha = img->alpha;
 	int ret = 1, flip;
 
 	stripsize = TIFFStripSize(tif);
-	p0 = buf = (unsigned char *)_TIFFmalloc((alpha?4:3)*stripsize);
+	bufsize = TIFFSafeMultiply(tsize_t,alpha?4:3,stripsize);
+	if (bufsize == 0) {
+		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtStripSeparate");
+		return (0);
+	}
+	p0 = buf = (unsigned char *)_TIFFmalloc(bufsize);
 	if (buf == 0) {
 		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for tile buffer");
 		return (0);
 	}
-	_TIFFmemset(buf, 0, (alpha?4:3)*stripsize);
+	_TIFFmemset(buf, 0, bufsize);
 	p1 = p0 + stripsize;
 	p2 = p1 + stripsize;
 	pa = (alpha?(p2+stripsize):NULL);
diff --git a/libtiff/tiffiop.h b/libtiff/tiffiop.h
index f65f8550..2ecc9946 100644
--- a/libtiff/tiffiop.h
+++ b/libtiff/tiffiop.h
@@ -1,4 +1,4 @@
-/* $Id: tiffiop.h,v 1.51.2.7 2011-03-21 21:09:19 fwarmerdam Exp $ */
+/* $Id: tiffiop.h,v 1.51.2.8 2012-04-06 16:47:42 fwarmerdam Exp $ */
 
 /*
  * Copyright (c) 1988-1997 Sam Leffler
@@ -246,7 +246,7 @@ struct tiff {
 #define	TIFFroundup(x, y) (TIFFhowmany(x,y)*(y))
 
 /* Safe multiply which returns zero if there is an integer overflow */
-#define TIFFSafeMultiply(t,v,m) ((((t)m != (t)0) && (((t)((v*m)/m)) == (t)v)) ? (t)(v*m) : (t)0)
+#define TIFFSafeMultiply(t,v,m) ((((t)(m) != (t)0) && (((t)(((v)*(m))/(m))) == (t)(v))) ? (t)((v)*(m)) : (t)0)
 
 #define TIFFmax(A,B) ((A)>(B)?(A):(B))
 #define TIFFmin(A,B) ((A)<(B)?(A):(B))