Fix CVE-2019-2201 (#1770988)

2019-11-12 12:13:58 +01:00 · 2019-11-12 12:13:58 +01:00 · e0497f67e2
parent 5aaca4343d
commit e0497f67e2
2 changed files with 423 additions and 1 deletions
--- a/libjpeg-turbo-CVE-2019-2201.patch
+++ b/libjpeg-turbo-CVE-2019-2201.patch
@ -0,0 +1,417 @@
+From acaf2eaf328e493619193c262b15af6fd8db6762 Mon Sep 17 00:00:00 2001
+From: DRC <information@libjpeg-turbo.org>
+Date: Thu, 11 Jul 2019 15:30:04 -0500
+Subject: [PATCH] TurboJPEG: Properly handle gigapixel images
+
+Prevent several integer overflow issues and subsequent segfaults that
+occurred when attempting to compress or decompress gigapixel images with
+the TurboJPEG API:
+
+- Modify tjBufSize(), tjBufSizeYUV2(), and tjPlaneSizeYUV() to avoid
+  integer overflow when computing the return values and to return an
+  error if such an overflow is unavoidable.
+- Modify tjunittest to validate the above.
+- Modify tjCompress2(), tjEncodeYUVPlanes(), tjDecompress2(), and
+  tjDecodeYUVPlanes() to avoid integer overflow when computing the row
+  pointers in the 64-bit TurboJPEG C API.
+- Modify TJBench (both C and Java versions) to avoid overflowing the
+  size argument to malloc()/new and to fail gracefully if such an
+  overflow is unavoidable.
+
+In general, this allows gigapixel images to be accommodated by the
+64-bit TurboJPEG C API when using automatic JPEG buffer (re)allocation.
+Such images cannot currently be accommodated without automatic JPEG
+buffer (re)allocation, due to the fact that tjAlloc() accepts a 32-bit
+integer argument (oops.)  Such images cannot be accommodated in the
+TurboJPEG Java API due to the fact that Java always uses a signed 32-bit
+integer as an array index.
+
+Fixes #361
+---
+ java/TJBench.java | 11 ++++++++++-
+ tjbench.c         | 43 ++++++++++++++++++++++++++++-------------
+ tjunittest.c      | 37 +++++++++++++++++++++++++++++++++++
+ turbojpeg.c       | 49 ++++++++++++++++++++++++++++-------------------
+ 4 files changed, 106 insertions(+), 34 deletions(-)
+
+diff --git a/java/TJBench.java b/java/TJBench.java
+index 7829e53..cc3178e 100644
+--- a/java/TJBench.java
+++ b/java/TJBench.java
+@@ -121,6 +121,8 @@ final class TJBench {
+     int rindex = TJ.getRedOffset(pixelFormat);
+     int gindex = TJ.getGreenOffset(pixelFormat);
+     int bindex = TJ.getBlueOffset(pixelFormat);
+    if ((long)w[0] * (long)h[0] * (long)ps > (long)Integer.MAX_VALUE)
+      throw new Exception("Image is too large");
+     byte[] dstBuf = new byte[w[0] * h[0] * ps];
+     int pixels = w[0] * h[0], dstPtr = 0, rgbPtr = 0;
+ 
+@@ -175,8 +177,11 @@ final class TJBench {
+ 
+     tjd = new TJDecompressor();
+ 
+-    if (dstBuf == null)
+    if (dstBuf == null) {
+      if ((long)pitch * (long)scaledh > (long)Integer.MAX_VALUE)
+        throw new Exception("Image is too large");
+       dstBuf = new byte[pitch * scaledh];
+    }
+ 
+     /* Set the destination buffer to gray so we know whether the decompressor
+        attempted to write to it */
+@@ -331,6 +336,8 @@ final class TJBench {
+     String pfStr = PIXFORMATSTR[pf];
+     YUVImage yuvImage = null;
+ 
+    if ((long)pitch * (long)h > (long)Integer.MAX_VALUE)
+      throw new Exception("Image is too large");
+     tmpBuf = new byte[pitch * h];
+ 
+     if (quiet == 0)
+@@ -491,6 +498,8 @@ final class TJBench {
+     int tw, th, ttilew, ttileh, tntilesw, tntilesh, tsubsamp;
+ 
+     FileInputStream fis = new FileInputStream(fileName);
+    if (fis.getChannel().size() > (long)Integer.MAX_VALUE)
+      throw new Exception("Image is too large");
+     int srcSize = (int)fis.getChannel().size();
+     srcBuf = new byte[srcSize];
+     fis.read(srcBuf, 0, srcSize);
+diff --git a/tjbench.c b/tjbench.c
+index 0ddcd7b..547e8f5 100644
+--- a/tjbench.c
+++ b/tjbench.c
+@@ -32,6 +32,7 @@
+ #include <ctype.h>
+ #include <math.h>
+ #include <errno.h>
+#include <limits.h>
+ #include <cdjpeg.h>
+ #include "./tjutil.h"
+ #include "./turbojpeg.h"
+@@ -160,7 +161,10 @@ int decomp(unsigned char *srcBuf, unsigned char **jpegBuf,
+     _throwtj("executing tjInitDecompress()");
+ 
+   if (dstBuf == NULL) {
+-    if ((dstBuf = (unsigned char *)malloc(pitch * scaledh)) == NULL)
+    if ((unsigned long long)pitch * (unsigned long long)scaledh >
+        (unsigned long long)((size_t)-1))
+      _throw("allocating destination buffer", "Image is too large");
+    if ((dstBuf = (unsigned char *)malloc((size_t)pitch * scaledh)) == NULL)
+       _throwunix("allocating destination buffer");
+     dstBufAlloc = 1;
+   }
+@@ -171,8 +175,10 @@ int decomp(unsigned char *srcBuf, unsigned char **jpegBuf,
+   if (doYUV) {
+     int width = doTile ? tilew : scaledw;
+     int height = doTile ? tileh : scaledh;
+-    int yuvSize = tjBufSizeYUV2(width, yuvPad, height, subsamp);
+    unsigned long yuvSize = tjBufSizeYUV2(width, yuvPad, height, subsamp);
+ 
+    if (yuvSize == (unsigned long)-1)
+      _throwtj("allocating YUV buffer");
+     if ((yuvBuf = (unsigned char *)malloc(yuvSize)) == NULL)
+       _throwunix("allocating YUV buffer");
+     memset(yuvBuf, 127, yuvSize);
+@@ -266,13 +272,13 @@ int decomp(unsigned char *srcBuf, unsigned char **jpegBuf,
+   if (srcBuf && sf.num == 1 && sf.denom == 1) {
+     if (!quiet) printf("Compression error written to %s.\n", tempStr);
+     if (subsamp == TJ_GRAYSCALE) {
+-      int index, index2;
+      unsigned long index, index2;
+ 
+       for (row = 0, index = 0; row < h; row++, index += pitch) {
+         for (col = 0, index2 = index; col < w; col++, index2 += ps) {
+-          int rindex = index2 + tjRedOffset[pf];
+-          int gindex = index2 + tjGreenOffset[pf];
+-          int bindex = index2 + tjBlueOffset[pf];
+          unsigned long rindex = index2 + tjRedOffset[pf];
+          unsigned long gindex = index2 + tjGreenOffset[pf];
+          unsigned long bindex = index2 + tjBlueOffset[pf];
+           int y = (int)((double)srcBuf[rindex] * 0.299 +
+                         (double)srcBuf[gindex] * 0.587 +
+                         (double)srcBuf[bindex] * 0.114 + 0.5);
+@@ -313,13 +319,16 @@ int fullTest(unsigned char *srcBuf, int w, int h, int subsamp, int jpegQual,
+     *srcPtr2;
+   double start, elapsed, elapsedEncode;
+   int totalJpegSize = 0, row, col, i, tilew = w, tileh = h, retval = 0;
+-  int iter, yuvSize = 0;
+-  unsigned long *jpegSize = NULL;
+  int iter;
+  unsigned long *jpegSize = NULL, yuvSize = 0;
+   int ps = tjPixelSize[pf];
+   int ntilesw = 1, ntilesh = 1, pitch = w * ps;
+   const char *pfStr = pixFormatStr[pf];
+ 
+-  if ((tmpBuf = (unsigned char *)malloc(pitch * h)) == NULL)
+  if ((unsigned long long)pitch * (unsigned long long)h >
+      (unsigned long long)((size_t)-1))
+    _throw("allocating temporary image buffer", "Image is too large");
+  if ((tmpBuf = (unsigned char *)malloc((size_t)pitch * h)) == NULL)
+     _throwunix("allocating temporary image buffer");
+ 
+   if (!quiet)
+@@ -345,6 +354,8 @@ int fullTest(unsigned char *srcBuf, int w, int h, int subsamp, int jpegQual,
+ 
+     if ((flags & TJFLAG_NOREALLOC) != 0)
+       for (i = 0; i < ntilesw * ntilesh; i++) {
+        if (tjBufSize(tilew, tileh, subsamp) > (unsigned long)INT_MAX)
+          _throw("getting buffer size", "Image is too large");
+         if ((jpegBuf[i] = (unsigned char *)
+                           tjAlloc(tjBufSize(tilew, tileh, subsamp))) == NULL)
+           _throwunix("allocating JPEG tiles");
+@@ -362,6 +373,8 @@ int fullTest(unsigned char *srcBuf, int w, int h, int subsamp, int jpegQual,
+ 
+     if (doYUV) {
+       yuvSize = tjBufSizeYUV2(tilew, yuvPad, tileh, subsamp);
+      if (yuvSize == (unsigned long)-1)
+        _throwtj("allocating YUV buffer");
+       if ((yuvBuf = (unsigned char *)malloc(yuvSize)) == NULL)
+         _throwunix("allocating YUV buffer");
+       memset(yuvBuf, 127, yuvSize);
+@@ -436,7 +449,7 @@ int fullTest(unsigned char *srcBuf, int w, int h, int subsamp, int jpegQual,
+       if (doYUV) {
+         printf("Encode YUV    --> Frame rate:         %f fps\n",
+                (double)iter / elapsedEncode);
+-        printf("                  Output image size:  %d bytes\n", yuvSize);
+        printf("                  Output image size:  %lu bytes\n", yuvSize);
+         printf("                  Compression ratio:  %f:1\n",
+                (double)(w * h * ps) / (double)yuvSize);
+         printf("                  Throughput:         %f Megapixels/sec\n",
+@@ -577,8 +590,11 @@ int decompTest(char *fileName)
+       _throwunix("allocating JPEG size array");
+     memset(jpegSize, 0, sizeof(unsigned long) * ntilesw * ntilesh);
+ 
+-    if ((flags & TJFLAG_NOREALLOC) != 0 || !doTile)
+    if ((flags & TJFLAG_NOREALLOC) != 0 &&
+        (doTile || xformOp != TJXOP_NONE || xformOpt != 0 || customFilter))
+       for (i = 0; i < ntilesw * ntilesh; i++) {
+        if (tjBufSize(tilew, tileh, subsamp) > (unsigned long)INT_MAX)
+          _throw("getting buffer size", "Image is too large");
+         if ((jpegBuf[i] = (unsigned char *)
+                           tjAlloc(tjBufSize(tilew, tileh, subsamp))) == NULL)
+           _throwunix("allocating JPEG tiles");
+@@ -684,7 +700,7 @@ int decompTest(char *fileName)
+       }
+     } else {
+       if (quiet == 1) printf("N/A     N/A     ");
+-      tjFree(jpegBuf[0]);
+      if(jpegBuf[0]) tjFree(jpegBuf[0]);
+       jpegBuf[0] = NULL;
+       decompsrc = 1;
+     }
+@@ -699,7 +715,8 @@ int decompTest(char *fileName)
+     } else if (quiet == 1) printf("N/A\n");
+ 
+     for (i = 0; i < ntilesw * ntilesh; i++) {
+-      tjFree(jpegBuf[i]);  jpegBuf[i] = NULL;
+      if(jpegBuf[i]) tjFree(jpegBuf[i]);
+      jpegBuf[i] = NULL;
+     }
+     free(jpegBuf);  jpegBuf = NULL;
+     if (jpegSize) { free(jpegSize);  jpegSize = NULL; }
+diff --git a/tjunittest.c b/tjunittest.c
+index ae72e83..d399107 100644
+--- a/tjunittest.c
+++ b/tjunittest.c
+@@ -552,6 +552,42 @@ bailout:
+ }
+ 
+ 
+#if SIZEOF_SIZE_T == 8
+#define CHECKSIZE(function) { \
+  if ((unsigned long long)size < (unsigned long long)0xFFFFFFFF) \
+    _throw(#function " overflow"); \
+}
+#else
+#define CHECKSIZE(function) { \
+  if (size != (unsigned long)(-1) || \
+      !strcmp(tjGetErrorStr2(NULL), "No error")) \
+    _throw(#function " overflow"); \
+}
+#endif
+
+void overflowTest(void)
+{
+  /* Ensure that the various buffer size functions don't overflow */
+  unsigned long size;
+
+  size = tjBufSize(26755, 26755, TJSAMP_444);
+  CHECKSIZE(tjBufSize());
+  size = TJBUFSIZE(26755, 26755);
+  CHECKSIZE(TJBUFSIZE());
+  size = tjBufSizeYUV2(37838, 1, 37838, TJSAMP_444);
+  CHECKSIZE(tjBufSizeYUV2());
+  size = TJBUFSIZEYUV(37838, 37838, TJSAMP_444);
+  CHECKSIZE(TJBUFSIZEYUV());
+  size = tjBufSizeYUV(37838, 37838, TJSAMP_444);
+  CHECKSIZE(tjBufSizeYUV());
+  size = tjPlaneSizeYUV(0, 65536, 0, 65536, TJSAMP_444);
+  CHECKSIZE(tjPlaneSizeYUV());
+
+bailout:
+  return;
+}
+
+
+ void bufSizeTest(void)
+ {
+   int w, h, i, subsamp;
+@@ -863,6 +899,7 @@ int main(int argc, char *argv[])
+   }
+   if (alloc) printf("Testing automatic buffer allocation\n");
+   if (doYUV) num4bf = 4;
+  overflowTest();
+   doTest(35, 39, _3byteFormats, 2, TJSAMP_444, "test");
+   doTest(39, 41, _4byteFormats, num4bf, TJSAMP_444, "test");
+   doTest(41, 35, _3byteFormats, 2, TJSAMP_422, "test");
+diff --git a/turbojpeg.c b/turbojpeg.c
+index b3caa0d..0dabc5a 100644
+--- a/turbojpeg.c
+++ b/turbojpeg.c
+@@ -489,7 +489,7 @@ DLLEXPORT tjhandle tjInitCompress(void)
+ 
+ DLLEXPORT unsigned long tjBufSize(int width, int height, int jpegSubsamp)
+ {
+-  unsigned long retval = 0;
+  unsigned long long retval = 0;
+   int mcuw, mcuh, chromasf;
+ 
+   if (width < 1 || height < 1 || jpegSubsamp < 0 || jpegSubsamp >= NUMSUBOPT)
+@@ -501,15 +501,17 @@ DLLEXPORT unsigned long tjBufSize(int width, int height, int jpegSubsamp)
+   mcuw = tjMCUWidth[jpegSubsamp];
+   mcuh = tjMCUHeight[jpegSubsamp];
+   chromasf = jpegSubsamp == TJSAMP_GRAY ? 0 : 4 * 64 / (mcuw * mcuh);
+-  retval = PAD(width, mcuw) * PAD(height, mcuh) * (2 + chromasf) + 2048;
+  retval = PAD(width, mcuw) * PAD(height, mcuh) * (2ULL + chromasf) + 2048ULL;
+  if (retval > (unsigned long long)((unsigned long)-1))
+    _throwg("tjBufSize(): Image is too large");
+ 
+ bailout:
+-  return retval;
+  return (unsigned long)retval;
+ }
+ 
+ DLLEXPORT unsigned long TJBUFSIZE(int width, int height)
+ {
+-  unsigned long retval = 0;
+  unsigned long long retval = 0;
+ 
+   if (width < 1 || height < 1)
+     _throwg("TJBUFSIZE(): Invalid argument");
+@@ -517,17 +519,20 @@ DLLEXPORT unsigned long TJBUFSIZE(int width, int height)
+   /* This allows for rare corner cases in which a JPEG image can actually be
+      larger than the uncompressed input (we wouldn't mention it if it hadn't
+      happened before.) */
+-  retval = PAD(width, 16) * PAD(height, 16) * 6 + 2048;
+  retval = PAD(width, 16) * PAD(height, 16) * 6ULL + 2048ULL;
+  if (retval > (unsigned long long)((unsigned long)-1))
+    _throwg("TJBUFSIZE(): Image is too large");
+ 
+ bailout:
+-  return retval;
+  return (unsigned long)retval;
+ }
+ 
+ 
+ DLLEXPORT unsigned long tjBufSizeYUV2(int width, int pad, int height,
+                                       int subsamp)
+ {
+-  int retval = 0, nc, i;
+  unsigned long long retval = 0;
+  int nc, i;
+ 
+   if (subsamp < 0 || subsamp >= NUMSUBOPT)
+     _throwg("tjBufSizeYUV2(): Invalid argument");
+@@ -539,11 +544,13 @@ DLLEXPORT unsigned long tjBufSizeYUV2(int width, int pad, int height,
+     int ph = tjPlaneHeight(i, height, subsamp);
+ 
+     if (pw < 0 || ph < 0) return -1;
+-    else retval += stride * ph;
+    else retval += (unsigned long long)stride * ph;
+   }
+  if (retval > (unsigned long long)((unsigned long)-1))
+    _throwg("tjBufSizeYUV2(): Image is too large");
+ 
+ bailout:
+-  return retval;
+  return (unsigned long)retval;
+ }
+ 
+ DLLEXPORT unsigned long tjBufSizeYUV(int width, int height, int subsamp)
+@@ -602,7 +609,7 @@ bailout:
+ DLLEXPORT unsigned long tjPlaneSizeYUV(int componentID, int width, int stride,
+                                        int height, int subsamp)
+ {
+-  unsigned long retval = 0;
+  unsigned long long retval = 0;
+   int pw, ph;
+ 
+   if (width < 1 || height < 1 || subsamp < 0 || subsamp >= NUMSUBOPT)
+@@ -615,10 +622,12 @@ DLLEXPORT unsigned long tjPlaneSizeYUV(int componentID, int width, int stride,
+   if (stride == 0) stride = pw;
+   else stride = abs(stride);
+ 
+-  retval = stride * (ph - 1) + pw;
+  retval = (unsigned long long)stride * (ph - 1) + pw;
+  if (retval > (unsigned long long)((unsigned long)-1))
+    _throwg("tjPlaneSizeYUV(): Image is too large");
+ 
+ bailout:
+-  return retval;
+  return (unsigned long)retval;
+ }
+ 
+ 
+@@ -670,9 +679,9 @@ DLLEXPORT int tjCompress2(tjhandle handle, const unsigned char *srcBuf,
+   jpeg_start_compress(cinfo, TRUE);
+   for (i = 0; i < height; i++) {
+     if (flags & TJFLAG_BOTTOMUP)
+-      row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * pitch];
+      row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * (size_t)pitch];
+     else
+-      row_pointer[i] = (JSAMPROW)&srcBuf[i * pitch];
+      row_pointer[i] = (JSAMPROW)&srcBuf[i * (size_t)pitch];
+   }
+   while (cinfo->next_scanline < cinfo->image_height)
+     jpeg_write_scanlines(cinfo, &row_pointer[cinfo->next_scanline],
+@@ -781,9 +790,9 @@ DLLEXPORT int tjEncodeYUVPlanes(tjhandle handle, const unsigned char *srcBuf,
+     _throw("tjEncodeYUVPlanes(): Memory allocation failure");
+   for (i = 0; i < height; i++) {
+     if (flags & TJFLAG_BOTTOMUP)
+-      row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * pitch];
+      row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * (size_t)pitch];
+     else
+-      row_pointer[i] = (JSAMPROW)&srcBuf[i * pitch];
+      row_pointer[i] = (JSAMPROW)&srcBuf[i * (size_t)pitch];
+   }
+   if (height < ph0)
+     for (i = height; i < ph0; i++) row_pointer[i] = row_pointer[height - 1];
+@@ -1291,9 +1300,9 @@ DLLEXPORT int tjDecompress2(tjhandle handle, const unsigned char *jpegBuf,
+   }
+   for (i = 0; i < (int)dinfo->output_height; i++) {
+     if (flags & TJFLAG_BOTTOMUP)
+-      row_pointer[i] = &dstBuf[(dinfo->output_height - i - 1) * pitch];
+      row_pointer[i] = &dstBuf[(dinfo->output_height - i - 1) * (size_t)pitch];
+     else
+-      row_pointer[i] = &dstBuf[i * pitch];
+      row_pointer[i] = &dstBuf[i * (size_t)pitch];
+   }
+   while (dinfo->output_scanline < dinfo->output_height)
+     jpeg_read_scanlines(dinfo, &row_pointer[dinfo->output_scanline],
+@@ -1448,9 +1457,9 @@ DLLEXPORT int tjDecodeYUVPlanes(tjhandle handle,
+     _throw("tjDecodeYUVPlanes(): Memory allocation failure");
+   for (i = 0; i < height; i++) {
+     if (flags & TJFLAG_BOTTOMUP)
+-      row_pointer[i] = &dstBuf[(height - i - 1) * pitch];
+      row_pointer[i] = &dstBuf[(height - i - 1) * (size_t)pitch];
+     else
+-      row_pointer[i] = &dstBuf[i * pitch];
+      row_pointer[i] = &dstBuf[i * (size_t)pitch];
+   }
+   if (height < ph0)
+     for (i = height; i < ph0; i++) row_pointer[i] = row_pointer[height - 1];
+-- 
+2.21.0
+
--- a/libjpeg-turbo.spec
+++ b/libjpeg-turbo.spec
@ -1,6 +1,6 @@
 Name:           libjpeg-turbo
 Version:        2.0.2
-Release:        4%{?dist}
+Release:        5%{?dist}
 Summary:        A MMX/SSE2/SIMD accelerated library for manipulating JPEG image files
 License:        IJG
 URL:            http://sourceforge.net/projects/libjpeg-turbo
@ -8,6 +8,7 @@ URL:            http://sourceforge.net/projects/libjpeg-turbo
 Source0:        http://downloads.sourceforge.net/%{name}/%{name}-%{version}.tar.gz
 Patch0:         libjpeg-turbo-cmake.patch
 Patch1:         libjpeg-turbo-CET.patch
+Patch2:         libjpeg-turbo-CVE-2019-2201.patch

 BuildRequires:  gcc
 BuildRequires:  cmake
@ -71,6 +72,7 @@ manipulate JPEG files using the TurboJPEG library.
 %setup -q
 %patch0 -p1
 %patch1 -p1
+%patch2 -p1

 %build
 # NASM object files are missing GNU Property note for Intel CET,
@ -176,6 +178,9 @@ LD_LIBRARY_PATH=%{buildroot}%{_libdir} make test %{?_smp_mflags}
 %{_libdir}/pkgconfig/libturbojpeg.pc

 %changelog
+* Tue Nov 12 2019 Nikola Forró <nforro@redhat.com> - 2.0.2-5
+- Fix CVE-2019-2201 (#1770988)
+
 * Thu Jul 25 2019 Fedora Release Engineering <releng@fedoraproject.org> - 2.0.2-4
 - Rebuilt for https://fedoraproject.org/wiki/Fedora_31_Mass_Rebuild