libtiff: TIFFScanlineSize64(): fix computation in non-JPEG YCbCr case (#564)

From 706e895071dd2edec55dc750c4fb0a4eca2879a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caol=C3=A1n=20McNamara?= <[EMAIL REDACTED]>
Date: Wed, 29 May 2024 10:08:24 +0000
Subject: [PATCH] TIFFScanlineSize64(): fix computation in non-JPEG YCbCr case
 (#564)

Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=65182
---
 libtiff/tif_strip.c | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/libtiff/tif_strip.c b/libtiff/tif_strip.c
index ee31892a..088dd4bd 100644
--- a/libtiff/tif_strip.c
+++ b/libtiff/tif_strip.c
@@ -294,7 +294,20 @@ uint64_t TIFFScanlineSize64(TIFF *tif)
         else
         {
             uint64_t scanline_samples;
-            scanline_samples = _TIFFMultiply64(tif, td->td_imagewidth,
+            uint32_t scanline_width = td->td_imagewidth;
+
+            if (td->td_photometric == PHOTOMETRIC_YCBCR)
+            {
+                uint16_t subsampling_hor;
+                uint16_t ignored;
+                TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING,
+                                      &subsampling_hor, &ignored);
+                if (subsampling_hor > 1) // roundup width for YCbCr
+                    scanline_width =
+                        TIFFroundup_32(scanline_width, subsampling_hor);
+            }
+
+            scanline_samples = _TIFFMultiply64(tif, scanline_width,
                                                td->td_samplesperpixel, module);
             scanline_size =
                 TIFFhowmany_64(_TIFFMultiply64(tif, scanline_samples,