libtiff: Merge branch 'limit_getfilesize' into 'master'

From 0b025324711213a75e38b52f7e7ba60235f108aa Mon Sep 17 00:00:00 2001
From: Even Rouault <[EMAIL REDACTED]>
Date: Tue, 31 Oct 2023 19:47:22 +0100
Subject: [PATCH] tif_dirread.c: only issue TIFFGetFileSize() for large enough
 RAM requests

Ammends 5320c9d89c054fa805d037d84c57da874470b01a

This fixes a performance regression caught by the GDAL regression test
suite.
---
 libtiff/tif_dirread.c | 83 +++++++++++++++++++++++++------------------
 1 file changed, 48 insertions(+), 35 deletions(-)

diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
index 819d6f4b..d6679908 100644
--- a/libtiff/tif_dirread.c
+++ b/libtiff/tif_dirread.c
@@ -1308,19 +1308,22 @@ TIFFReadDirEntryArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry,
     datasize = (*count) * typesize;
     assert((tmsize_t)datasize > 0);
 
-    /* Before allocating a huge amount of memory for corrupted files, check if
-     * size of requested memory is not greater than file size.
-     */
-    uint64_t filesize = TIFFGetFileSize(tif);
-    if (datasize > filesize)
+    if (datasize > 100 * 1024 * 1024)
     {
-        TIFFWarningExtR(tif, "ReadDirEntryArray",
-                        "Requested memory size for tag %d (0x%x) %" PRIu32
-                        " is greater than filesize %" PRIu64
-                        ". Memory not allocated, tag not read",
-                        direntry->tdir_tag, direntry->tdir_tag, datasize,
-                        filesize);
-        return (TIFFReadDirEntryErrAlloc);
+        /* Before allocating a huge amount of memory for corrupted files, check
+         * if size of requested memory is not greater than file size.
+         */
+        const uint64_t filesize = TIFFGetFileSize(tif);
+        if (datasize > filesize)
+        {
+            TIFFWarningExtR(tif, "ReadDirEntryArray",
+                            "Requested memory size for tag %d (0x%x) %" PRIu32
+                            " is greater than filesize %" PRIu64
+                            ". Memory not allocated, tag not read",
+                            direntry->tdir_tag, direntry->tdir_tag, datasize,
+                            filesize);
+            return (TIFFReadDirEntryErrAlloc);
+        }
     }
 
     if (isMapped(tif) && datasize > (uint64_t)tif->tif_size)
@@ -5281,18 +5284,22 @@ static int EstimateStripByteCounts(TIFF *tif, TIFFDirEntry *dir,
     if (!_TIFFFillStrilesInternal(tif, 0))
         return -1;
 
-    /* Before allocating a huge amount of memory for corrupted files, check if
-     * size of requested memory is not greater than file size. */
-    uint64_t filesize = TIFFGetFileSize(tif);
-    uint64_t allocsize = (uint64_t)td->td_nstrips * sizeof(uint64_t);
-    if (allocsize > filesize)
+    const uint64_t allocsize = (uint64_t)td->td_nstrips * sizeof(uint64_t);
+    uint64_t filesize = 0;
+    if (allocsize > 100 * 1024 * 1024)
     {
-        TIFFWarningExtR(tif, module,
-                        "Requested memory size for StripByteCounts of %" PRIu64
-                        " is greater than filesize %" PRIu64
-                        ". Memory not allocated",
-                        allocsize, filesize);
-        return -1;
+        /* Before allocating a huge amount of memory for corrupted files, check
+         * if size of requested memory is not greater than file size. */
+        filesize = TIFFGetFileSize(tif);
+        if (allocsize > filesize)
+        {
+            TIFFWarningExtR(
+                tif, module,
+                "Requested memory size for StripByteCounts of %" PRIu64
+                " is greater than filesize %" PRIu64 ". Memory not allocated",
+                allocsize, filesize);
+            return -1;
+        }
     }
 
     if (td->td_stripbytecount_p)
@@ -5341,6 +5348,8 @@ static int EstimateStripByteCounts(TIFF *tif, TIFFDirEntry *dir,
                 return -1;
             space += datasize;
         }
+        if (filesize == 0)
+            filesize = TIFFGetFileSize(tif);
         if (filesize < space)
             /* we should perhaps return in error ? */
             space = filesize;
@@ -5834,19 +5843,23 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff,
             dircount16 = (uint16_t)dircount64;
             dirsize = 20;
         }
-        /* Before allocating a huge amount of memory for corrupted files, check
-         * if size of requested memory is not greater than file size. */
-        uint64_t filesize = TIFFGetFileSize(tif);
-        uint64_t allocsize = (uint64_t)dircount16 * dirsize;
-        if (allocsize > filesize)
+        const uint64_t allocsize = (uint64_t)dircount16 * dirsize;
+        if (allocsize > 100 * 1024 * 1024)
         {
-            TIFFWarningExtR(
-                tif, module,
-                "Requested memory size for TIFF directory of %" PRIu64
-                " is greater than filesize %" PRIu64
-                ". Memory not allocated, TIFF directory not read",
-                allocsize, filesize);
-            return 0;
+            /* Before allocating a huge amount of memory for corrupted files,
+             * check if size of requested memory is not greater than file size.
+             */
+            const uint64_t filesize = TIFFGetFileSize(tif);
+            if (allocsize > filesize)
+            {
+                TIFFWarningExtR(
+                    tif, module,
+                    "Requested memory size for TIFF directory of %" PRIu64
+                    " is greater than filesize %" PRIu64
+                    ". Memory not allocated, TIFF directory not read",
+                    allocsize, filesize);
+                return 0;
+            }
         }
         origdir = _TIFFCheckMalloc(tif, dircount16, dirsize,
                                    "to read TIFF directory");