libtiff: Use working buffer in PredictorEncodeTile to avoid odifying callers buffer.

From 2fe621014f61a01da56fc6c37f0569925cc4e549 Mon Sep 17 00:00:00 2001
From: Frank Warmerdam <[EMAIL REDACTED]>
Date: Thu, 22 Nov 2007 21:24:50 +0000
Subject: [PATCH] Use working buffer in PredictorEncodeTile to avoid odifying
 callers buffer.   http://trac.osgeo.org/gdal/ticket/1965 Improvements so that
 predictor based encoding and decoding works in read-write update mode
 properly.   http://trac.osgeo.org/gdal/ticket/1948

---
 ChangeLog             |  11 ++++
 libtiff/tif_lzw.c     |  10 ++-
 libtiff/tif_predict.c | 140 ++++++++++++++++++++++++++----------------
 libtiff/tif_predict.h |  16 +++--
 libtiff/tif_zip.c     |   6 +-
 5 files changed, 120 insertions(+), 63 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 3198bbec..fccf2e7b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+2007-11-22  Frank Warmerdam  <warmerdam@pobox.com>
+
+	* tif_predict.c: use working buffer in PredictorEncodeTile to avoid
+	modifying callers buffer. 
+	http://trac.osgeo.org/gdal/ticket/1965
+
+	* tif_predict.c/h, tif_lzw.c, tif_zip.c: Improvements so that 
+	predictor based encoding and decoding works in read-write update
+	mode properly. 
+	http://trac.osgeo.org/gdal/ticket/1948
+
 2007-10-05  Frank Warmerdam  <warmerdam@pobox.com>
 
 	* tools/tiff2pdf.c: Fixed setting of alpha value per report on list.
diff --git a/libtiff/tif_lzw.c b/libtiff/tif_lzw.c
index 9fc0179f..fd20571c 100644
--- a/libtiff/tif_lzw.c
+++ b/libtiff/tif_lzw.c
@@ -1,4 +1,4 @@
-/* $Id: tif_lzw.c,v 1.29 2006-09-27 22:39:00 fwarmerdam Exp $ */
+/* $Id: tif_lzw.c,v 1.29.2.1 2007-11-22 21:24:51 fwarmerdam Exp $ */
 
 /*
  * Copyright (c) 1988-1997 Sam Leffler
@@ -252,7 +252,9 @@ LZWPreDecode(TIFF* tif, tsample_t s)
 	(void) s;
 	assert(sp != NULL);
         if( sp->dec_codetab == NULL )
-            LZWSetupDecode( tif );
+        {
+            tif->tif_setupdecode( tif );
+        }
 
 	/*
 	 * Check for old bit-reversed codes.
@@ -740,7 +742,9 @@ LZWPreEncode(TIFF* tif, tsample_t s)
 	assert(sp != NULL);
         
         if( sp->enc_hashtab == NULL )
-            LZWSetupEncode( tif );
+        {
+            tif->tif_setupencode( tif );
+        }
 
 	sp->lzw_nbits = BITS_MIN;
 	sp->lzw_maxcode = MAXCODE(BITS_MIN);
diff --git a/libtiff/tif_predict.c b/libtiff/tif_predict.c
index 969a6111..5315f433 100644
--- a/libtiff/tif_predict.c
+++ b/libtiff/tif_predict.c
@@ -1,4 +1,4 @@
-/* $Id: tif_predict.c,v 1.11.2.1 2007-04-07 14:58:30 dron Exp $ */
+/* $Id: tif_predict.c,v 1.11.2.2 2007-11-22 21:24:51 fwarmerdam Exp $ */
 
 /*
  * Copyright (c) 1988-1997 Sam Leffler
@@ -105,19 +105,22 @@ PredictorSetupDecode(TIFF* tif)
 
 	if (sp->predictor == 2) {
 		switch (td->td_bitspersample) {
-			case 8:  sp->pfunc = horAcc8; break;
-			case 16: sp->pfunc = horAcc16; break;
+			case 8:  sp->decodepfunc = horAcc8; break;
+			case 16: sp->decodepfunc = horAcc16; break;
 		}
 		/*
 		 * Override default decoding method with one that does the
 		 * predictor stuff.
 		 */
-		sp->coderow = tif->tif_decoderow;
-		tif->tif_decoderow = PredictorDecodeRow;
-		sp->codestrip = tif->tif_decodestrip;
-		tif->tif_decodestrip = PredictorDecodeTile;
-		sp->codetile = tif->tif_decodetile;
-		tif->tif_decodetile = PredictorDecodeTile;
+                if( tif->tif_decoderow != PredictorDecodeRow )
+                {
+                    sp->decoderow = tif->tif_decoderow;
+                    tif->tif_decoderow = PredictorDecodeRow;
+                    sp->decodestrip = tif->tif_decodestrip;
+                    tif->tif_decodestrip = PredictorDecodeTile;
+                    sp->decodetile = tif->tif_decodetile;
+                    tif->tif_decodetile = PredictorDecodeTile;
+                }
 		/*
 		 * If the data is horizontally differenced 16-bit data that
 		 * requires byte-swapping, then it must be byte swapped before
@@ -126,25 +129,28 @@ PredictorSetupDecode(TIFF* tif)
 		 * the library setup when the directory was read.
 		 */
 		if (tif->tif_flags & TIFF_SWAB) {
-			if (sp->pfunc == horAcc16) {
-				sp->pfunc = swabHorAcc16;
+			if (sp->decodepfunc == horAcc16) {
+				sp->decodepfunc = swabHorAcc16;
 				tif->tif_postdecode = _TIFFNoPostDecode;
 			} /* else handle 32-bit case... */
 		}
 	}
 
 	else if (sp->predictor == 3) {
-		sp->pfunc = fpAcc;
+		sp->decodepfunc = fpAcc;
 		/*
 		 * Override default decoding method with one that does the
 		 * predictor stuff.
 		 */
-		sp->coderow = tif->tif_decoderow;
-		tif->tif_decoderow = PredictorDecodeRow;
-		sp->codestrip = tif->tif_decodestrip;
-		tif->tif_decodestrip = PredictorDecodeTile;
-		sp->codetile = tif->tif_decodetile;
-		tif->tif_decodetile = PredictorDecodeTile;
+                if( tif->tif_decoderow != PredictorDecodeRow )
+                {
+                    sp->decoderow = tif->tif_decoderow;
+                    tif->tif_decoderow = PredictorDecodeRow;
+                    sp->decodestrip = tif->tif_decodestrip;
+                    tif->tif_decodestrip = PredictorDecodeTile;
+                    sp->decodetile = tif->tif_decodetile;
+                    tif->tif_decodetile = PredictorDecodeTile;
+                }
 		/*
 		 * The data should not be swapped outside of the floating
 		 * point predictor, the accumulation routine should return
@@ -173,33 +179,39 @@ PredictorSetupEncode(TIFF* tif)
 
 	if (sp->predictor == 2) {
 		switch (td->td_bitspersample) {
-			case 8:  sp->pfunc = horDiff8; break;
-			case 16: sp->pfunc = horDiff16; break;
+			case 8:  sp->encodepfunc = horDiff8; break;
+			case 16: sp->encodepfunc = horDiff16; break;
 		}
 		/*
 		 * Override default encoding method with one that does the
 		 * predictor stuff.
 		 */
-		sp->coderow = tif->tif_encoderow;
-		tif->tif_encoderow = PredictorEncodeRow;
-		sp->codestrip = tif->tif_encodestrip;
-		tif->tif_encodestrip = PredictorEncodeTile;
-		sp->codetile = tif->tif_encodetile;
-		tif->tif_encodetile = PredictorEncodeTile;
+                if( tif->tif_encoderow != PredictorEncodeRow )
+                {
+                    sp->encoderow = tif->tif_encoderow;
+                    tif->tif_encoderow = PredictorEncodeRow;
+                    sp->encodestrip = tif->tif_encodestrip;
+                    tif->tif_encodestrip = PredictorEncodeTile;
+                    sp->encodetile = tif->tif_encodetile;
+                    tif->tif_encodetile = PredictorEncodeTile;
+                }
 	}
 	
 	else if (sp->predictor == 3) {
-		sp->pfunc = fpDiff;
+		sp->encodepfunc = fpDiff;
 		/*
 		 * Override default encoding method with one that does the
 		 * predictor stuff.
 		 */
-		sp->coderow = tif->tif_encoderow;
-		tif->tif_encoderow = PredictorEncodeRow;
-		sp->codestrip = tif->tif_encodestrip;
-		tif->tif_encodestrip = PredictorEncodeTile;
-		sp->codetile = tif->tif_encodetile;
-		tif->tif_encodetile = PredictorEncodeTile;
+                if( tif->tif_encoderow != PredictorEncodeRow )
+                {
+                    sp->encoderow = tif->tif_encoderow;
+                    tif->tif_encoderow = PredictorEncodeRow;
+                    sp->encodestrip = tif->tif_encodestrip;
+                    tif->tif_encodestrip = PredictorEncodeTile;
+                    sp->encodetile = tif->tif_encodetile;
+                    tif->tif_encodetile = PredictorEncodeTile;
+                }
 	}
 
 	return 1;
@@ -337,11 +349,11 @@ PredictorDecodeRow(TIFF* tif, tidata_t op0, tsize_t occ0, tsample_t s)
 	TIFFPredictorState *sp = PredictorState(tif);
 
 	assert(sp != NULL);
-	assert(sp->coderow != NULL);
-	assert(sp->pfunc != NULL);
+	assert(sp->decoderow != NULL);
+	assert(sp->decodepfunc != NULL);
 
-	if ((*sp->coderow)(tif, op0, occ0, s)) {
-		(*sp->pfunc)(tif, op0, occ0);
+	if ((*sp->decoderow)(tif, op0, occ0, s)) {
+		(*sp->decodepfunc)(tif, op0, occ0);
 		return 1;
 	} else
 		return 0;
@@ -360,14 +372,14 @@ PredictorDecodeTile(TIFF* tif, tidata_t op0, tsize_t occ0, tsample_t s)
 	TIFFPredictorState *sp = PredictorState(tif);
 
 	assert(sp != NULL);
-	assert(sp->codetile != NULL);
+	assert(sp->decodetile != NULL);
 
-	if ((*sp->codetile)(tif, op0, occ0, s)) {
+	if ((*sp->decodetile)(tif, op0, occ0, s)) {
 		tsize_t rowsize = sp->rowsize;
 		assert(rowsize > 0);
-		assert(sp->pfunc != NULL);
+		assert(sp->decodepfunc != NULL);
 		while ((long)occ0 > 0) {
-			(*sp->pfunc)(tif, op0, (tsize_t) rowsize);
+			(*sp->decodepfunc)(tif, op0, (tsize_t) rowsize);
 			occ0 -= rowsize;
 			op0 += rowsize;
 		}
@@ -481,33 +493,56 @@ PredictorEncodeRow(TIFF* tif, tidata_t bp, tsize_t cc, tsample_t s)
 	TIFFPredictorState *sp = PredictorState(tif);
 
 	assert(sp != NULL);
-	assert(sp->pfunc != NULL);
-	assert(sp->coderow != NULL);
+	assert(sp->encodepfunc != NULL);
+	assert(sp->encoderow != NULL);
 
 	/* XXX horizontal differencing alters user's data XXX */
-	(*sp->pfunc)(tif, bp, cc);
-	return (*sp->coderow)(tif, bp, cc, s);
+	(*sp->encodepfunc)(tif, bp, cc);
+	return (*sp->encoderow)(tif, bp, cc, s);
 }
 
 static int
 PredictorEncodeTile(TIFF* tif, tidata_t bp0, tsize_t cc0, tsample_t s)
 {
+	static const char module[] = "PredictorEncodeTile";
 	TIFFPredictorState *sp = PredictorState(tif);
+        uint8 *working_copy;
 	tsize_t cc = cc0, rowsize;
-	unsigned char* bp = bp0;
+	unsigned char* bp;
+        int result_code;
 
 	assert(sp != NULL);
-	assert(sp->pfunc != NULL);
-	assert(sp->codetile != NULL);
+	assert(sp->encodepfunc != NULL);
+	assert(sp->encodetile != NULL);
+
+        /* 
+         * Do predictor manipulation in a working buffer to avoid altering
+         * the callers buffer. http://trac.osgeo.org/gdal/ticket/1965
+         */
+        working_copy = (uint8*) _TIFFmalloc(cc0);
+        if( working_copy == NULL )
+        {
+            TIFFErrorExt(tif->tif_clientdata, module, 
+                         "Out of memory allocating %d byte temp buffer.",
+                         cc0 );
+            return 0;
+        }
+        memcpy( working_copy, bp0, cc0 );
+        bp = working_copy;
 
 	rowsize = sp->rowsize;
 	assert(rowsize > 0);
-	while ((long)cc > 0) {
-		(*sp->pfunc)(tif, bp, (tsize_t) rowsize);
+	assert((cc0%rowsize)==0);
+	while (cc > 0) {
+		(*sp->encodepfunc)(tif, bp, rowsize);
 		cc -= rowsize;
 		bp += rowsize;
 	}
-	return (*sp->codetile)(tif, bp0, cc0, s);
+	result_code = (*sp->encodetile)(tif, working_copy, cc0, s);
+
+        _TIFFfree( working_copy );
+
+        return result_code;
 }
 
 #define	FIELD_PREDICTOR	(FIELD_CODEC+0)		/* XXX */
@@ -610,7 +645,8 @@ TIFFPredictorInit(TIFF* tif)
 	tif->tif_setupencode = PredictorSetupEncode;
 
 	sp->predictor = 1;			/* default value */
-	sp->pfunc = NULL;			/* no predictor routine */
+	sp->encodepfunc = NULL;			/* no predictor routine */
+	sp->decodepfunc = NULL;			/* no predictor routine */
 	return 1;
 }
 
diff --git a/libtiff/tif_predict.h b/libtiff/tif_predict.h
index 310cf444..18a83bd4 100644
--- a/libtiff/tif_predict.h
+++ b/libtiff/tif_predict.h
@@ -1,4 +1,4 @@
-/* $Id: tif_predict.h,v 1.3 2006-03-03 14:10:09 dron Exp $ */
+/* $Id: tif_predict.h,v 1.3.2.1 2007-11-22 21:24:51 fwarmerdam Exp $ */
 
 /*
  * Copyright (c) 1995-1997 Sam Leffler
@@ -40,10 +40,16 @@ typedef struct {
 	int		stride;		/* sample stride over data */
 	tsize_t		rowsize;	/* tile/strip row size */
 
-	TIFFPostMethod	pfunc;		/* horizontal differencer/accumulator */
-	TIFFCodeMethod	coderow;	/* parent codec encode/decode row */
-	TIFFCodeMethod	codestrip;	/* parent codec encode/decode strip */
-	TIFFCodeMethod	codetile;	/* parent codec encode/decode tile */
+ 	TIFFCodeMethod  encoderow;	/* parent codec encode/decode row */
+ 	TIFFCodeMethod  encodestrip;	/* parent codec encode/decode strip */
+ 	TIFFCodeMethod  encodetile;	/* parent codec encode/decode tile */ 
+ 	TIFFPostMethod  encodepfunc;	/* horizontal differencer */
+ 
+ 	TIFFCodeMethod  decoderow;	/* parent codec encode/decode row */
+ 	TIFFCodeMethod  decodestrip;	/* parent codec encode/decode strip */
+ 	TIFFCodeMethod  decodetile;	/* parent codec encode/decode tile */ 
+ 	TIFFPostMethod  decodepfunc;	/* horizontal accumulator */
+
 	TIFFVGetMethod	vgetparent;	/* super-class method */
 	TIFFVSetMethod	vsetparent;	/* super-class method */
 	TIFFPrintMethod	printdir;	/* super-class method */
diff --git a/libtiff/tif_zip.c b/libtiff/tif_zip.c
index 6eb07025..1881f162 100644
--- a/libtiff/tif_zip.c
+++ b/libtiff/tif_zip.c
@@ -1,4 +1,4 @@
-/* $Id: tif_zip.c,v 1.11.2.2 2007-09-22 14:51:30 dron Exp $ */
+/* $Id: tif_zip.c,v 1.11.2.3 2007-11-22 21:24:51 fwarmerdam Exp $ */
 
 /*
  * Copyright (c) 1995-1997 Sam Leffler
@@ -119,7 +119,7 @@ ZIPPreDecode(TIFF* tif, tsample_t s)
 	assert(sp != NULL);
 
         if( (sp->state & ZSTATE_INIT_DECODE) == 0 )
-            ZIPSetupDecode(tif);
+            tif->tif_setupdecode( tif );
 
 	sp->stream.next_in = tif->tif_rawdata;
 	sp->stream.avail_in = tif->tif_rawcc;
@@ -197,7 +197,7 @@ ZIPPreEncode(TIFF* tif, tsample_t s)
 	(void) s;
 	assert(sp != NULL);
         if( sp->state != ZSTATE_INIT_ENCODE )
-            ZIPSetupEncode(tif);
+            tif->tif_setupencode( tif );
 
 	sp->stream.next_out = tif->tif_rawdata;
 	sp->stream.avail_out = tif->tif_rawdatasize;