nixpkgs/pkgs/development/libraries/openjpeg/CVE-2016-9580-and-CVE-2016-9581.patch

From cadff5fb6e73398de26a92e96d3d7cac893af255 Mon Sep 17 00:00:00 2001
From: szukw000 <szukw000@arcor.de>
Date: Fri, 9 Dec 2016 08:29:55 +0100
Subject: [PATCH] These changes repair bugs of #871 and #872

---
 src/bin/jp2/converttif.c | 107 +++++++++++++++++++++++++++++++----------------
 1 file changed, 70 insertions(+), 37 deletions(-)

diff --git a/src/bin/jp2/converttif.c b/src/bin/jp2/converttif.c
index 143d3be..c690f8b 100644
--- a/src/bin/jp2/converttif.c
+++ b/src/bin/jp2/converttif.c
@@ -553,20 +553,18 @@ static void tif_32sto16u(const OPJ_INT32* pSrc, OPJ_UINT16* pDst, OPJ_SIZE_T len
 
 int imagetotif(opj_image_t * image, const char *outfile)
 {
-	int width, height;
-	int bps,adjust, sgnd;
-	int tiPhoto;
+	uint32 width, height, bps, tiPhoto;
+	int adjust, sgnd;
 	TIFF *tif;
 	tdata_t buf;
-	tsize_t strip_size;
+	tmsize_t strip_size, rowStride;
 	OPJ_UINT32 i, numcomps;
-	OPJ_SIZE_T rowStride;
 	OPJ_INT32* buffer32s = NULL;
 	OPJ_INT32 const* planes[4];
 	convert_32s_PXCX cvtPxToCx = NULL;
 	convert_32sXXx_C1R cvt32sToTif = NULL;
 
-	bps = (int)image->comps[0].prec;
+	bps = (uint32)image->comps[0].prec;
 	planes[0] = image->comps[0].data;
 	
 	numcomps = image->numcomps;
@@ -674,13 +672,13 @@ int imagetotif(opj_image_t * image, const char *outfile)
 			break;
 	}
 	sgnd = (int)image->comps[0].sgnd;
-	adjust = sgnd ? 1 << (image->comps[0].prec - 1) : 0;
-	width   = (int)image->comps[0].w;
-	height  = (int)image->comps[0].h;
+	adjust = sgnd ? (int)(1 << (image->comps[0].prec - 1)) : 0;
+	width   = (uint32)image->comps[0].w;
+	height  = (uint32)image->comps[0].h;
 	
 	TIFFSetField(tif, TIFFTAG_IMAGEWIDTH, width);
 	TIFFSetField(tif, TIFFTAG_IMAGELENGTH, height);
-	TIFFSetField(tif, TIFFTAG_SAMPLESPERPIXEL, numcomps);
+	TIFFSetField(tif, TIFFTAG_SAMPLESPERPIXEL, (uint32)numcomps);
 	TIFFSetField(tif, TIFFTAG_BITSPERSAMPLE, bps);
 	TIFFSetField(tif, TIFFTAG_ORIENTATION, ORIENTATION_TOPLEFT);
 	TIFFSetField(tif, TIFFTAG_PLANARCONFIG, PLANARCONFIG_CONTIG);
@@ -688,8 +686,8 @@ int imagetotif(opj_image_t * image, const char *outfile)
 	TIFFSetField(tif, TIFFTAG_ROWSPERSTRIP, 1);
 	
 	strip_size = TIFFStripSize(tif);
-	rowStride = ((OPJ_SIZE_T)width * numcomps * (OPJ_SIZE_T)bps + 7U) / 8U;
-	if (rowStride != (OPJ_SIZE_T)strip_size) {
+	rowStride = (width * numcomps * bps + 7U) / 8U;
+	if (rowStride != strip_size) {
 		fprintf(stderr, "Invalid TIFF strip size\n");
 		TIFFClose(tif);
 		return 1;
@@ -699,7 +697,7 @@ int imagetotif(opj_image_t * image, const char *outfile)
 		TIFFClose(tif);
 		return 1;
 	}
-	buffer32s = (OPJ_INT32 *)malloc((OPJ_SIZE_T)width * numcomps * sizeof(OPJ_INT32));
+	buffer32s = (OPJ_INT32 *)malloc((OPJ_SIZE_T)(width * numcomps * sizeof(OPJ_INT32)));
 	if (buffer32s == NULL) {
 		_TIFFfree(buf);
 		TIFFClose(tif);
@@ -1211,20 +1209,19 @@ opj_image_t* tiftoimage(const char *filename, opj_cparameters_t *parameters)
 	TIFF *tif;
 	tdata_t buf;
 	tstrip_t strip;
-	tsize_t strip_size;
+	tmsize_t strip_size;
 	int j, currentPlane, numcomps = 0, w, h;
 	OPJ_COLOR_SPACE color_space = OPJ_CLRSPC_UNKNOWN;
 	opj_image_cmptparm_t cmptparm[4]; /* RGBA */
 	opj_image_t *image = NULL;
 	int has_alpha = 0;
-	unsigned short tiBps, tiPhoto, tiSf, tiSpp, tiPC;
-	unsigned int tiWidth, tiHeight;
+	uint32 tiBps, tiPhoto, tiSf, tiSpp, tiPC, tiWidth, tiHeight;
 	OPJ_BOOL is_cinema = OPJ_IS_CINEMA(parameters->rsiz);
 	convert_XXx32s_C1R cvtTifTo32s = NULL;
 	convert_32s_CXPX cvtCxToPx = NULL;
 	OPJ_INT32* buffer32s = NULL;
 	OPJ_INT32* planes[4];
-	OPJ_SIZE_T rowStride;
+	tmsize_t rowStride;
 	
 	tif = TIFFOpen(filename, "r");
 	
@@ -1243,22 +1240,35 @@ opj_image_t* tiftoimage(const char *filename, opj_cparameters_t *parameters)
 	TIFFGetField(tif, TIFFTAG_SAMPLESPERPIXEL, &tiSpp);
 	TIFFGetField(tif, TIFFTAG_PHOTOMETRIC, &tiPhoto);
 	TIFFGetField(tif, TIFFTAG_PLANARCONFIG, &tiPC);
-	w= (int)tiWidth;
-	h= (int)tiHeight;
-	
-	if(tiBps > 16U) {
-		fprintf(stderr,"tiftoimage: Bits=%d, Only 1 to 16 bits implemented\n",tiBps);
-		fprintf(stderr,"\tAborting\n");
+
+	if(tiSpp == 0 || tiSpp > 4) { /* should be 1 ... 4 */
+		fprintf(stderr,"tiftoimage: Bad value for samples per pixel == %hu.\n"
+		 "\tAborting.\n", tiSpp);
+		TIFFClose(tif);
+		return NULL;
+	}
+	if(tiBps > 16U || tiBps == 0) {
+		fprintf(stderr,"tiftoimage: Bad values for Bits == %d.\n"
+		 "\tMax. 16 Bits are allowed here.\n\tAborting.\n",tiBps);
 		TIFFClose(tif);
 		return NULL;
 	}
 	if(tiPhoto != PHOTOMETRIC_MINISBLACK && tiPhoto != PHOTOMETRIC_RGB) {
-		fprintf(stderr,"tiftoimage: Bad color format %d.\n\tOnly RGB(A) and GRAY(A) has been implemented\n",(int) tiPhoto);
+		fprintf(stderr,"tiftoimage: Bad color format %d.\n"
+		 "\tOnly RGB(A) and GRAY(A) has been implemented\n",(int) tiPhoto);
 		fprintf(stderr,"\tAborting\n");
 		TIFFClose(tif);
 		return NULL;
 	}
-	
+	if(tiWidth == 0 || tiHeight == 0) {
+		fprintf(stderr,"tiftoimage: Bad values for width(%u) "
+		 "and/or height(%u)\n\tAborting.\n",tiWidth,tiHeight);
+		TIFFClose(tif);
+		return NULL;
+	}
+	w= (int)tiWidth;
+	h= (int)tiHeight;
+
 	switch (tiBps) {
 		case 1:
 		case 2:
@@ -1312,7 +1322,7 @@ opj_image_t* tiftoimage(const char *filename, opj_cparameters_t *parameters)
 		
 		TIFFGetFieldDefaulted(tif, TIFFTAG_EXTRASAMPLES,
 													&extrasamples, &sampleinfo);
-		
+
 		if(extrasamples >= 1)
 		{
 			switch(sampleinfo[0])
@@ -1333,7 +1343,7 @@ opj_image_t* tiftoimage(const char *filename, opj_cparameters_t *parameters)
 		else /* extrasamples == 0 */
 			if(tiSpp == 4 || tiSpp == 2) has_alpha = 1;
 	}
-	
+
 	/* initialize image components */
 	memset(&cmptparm[0], 0, 4 * sizeof(opj_image_cmptparm_t));
 	
@@ -1346,7 +1356,7 @@ opj_image_t* tiftoimage(const char *filename, opj_cparameters_t *parameters)
 	} else {
 		is_cinema = 0U;
 	}
-	
+
 	if(tiPhoto == PHOTOMETRIC_RGB) /* RGB(A) */
 	{
 		numcomps = 3 + has_alpha;
@@ -1384,10 +1394,24 @@ opj_image_t* tiftoimage(const char *filename, opj_cparameters_t *parameters)
 	image->x0 = (OPJ_UINT32)parameters->image_offset_x0;
 	image->y0 = (OPJ_UINT32)parameters->image_offset_y0;
 	image->x1 =	!image->x0 ? (OPJ_UINT32)(w - 1) * (OPJ_UINT32)subsampling_dx + 1 :
-	image->x0 + (OPJ_UINT32)(w - 1) * (OPJ_UINT32)subsampling_dx + 1;
+	 image->x0 + (OPJ_UINT32)(w - 1) * (OPJ_UINT32)subsampling_dx + 1;
+	if(image->x1 <= image->x0) {
+		fprintf(stderr,"tiftoimage: Bad value for image->x1(%d) vs. "
+		 "image->x0(%d)\n\tAborting.\n",image->x1,image->x0);
+		TIFFClose(tif);
+		opj_image_destroy(image);
+		return NULL;
+	}
 	image->y1 =	!image->y0 ? (OPJ_UINT32)(h - 1) * (OPJ_UINT32)subsampling_dy + 1 :
-	image->y0 + (OPJ_UINT32)(h - 1) * (OPJ_UINT32)subsampling_dy + 1;
-
+	 image->y0 + (OPJ_UINT32)(h - 1) * (OPJ_UINT32)subsampling_dy + 1;
+	if(image->y1 <= image->y0) {
+		fprintf(stderr,"tiftoimage: Bad value for image->y1(%d) vs. "
+		 "image->y0(%d)\n\tAborting.\n",image->y1,image->y0);
+		TIFFClose(tif);
+		opj_image_destroy(image);
+		return NULL;
+	}
+	
 	for(j = 0; j < numcomps; j++)
 	{
 		planes[j] = image->comps[j].data;
@@ -1395,15 +1419,15 @@ opj_image_t* tiftoimage(const char *filename, opj_cparameters_t *parameters)
 	image->comps[numcomps - 1].alpha = (OPJ_UINT16)(1 - (numcomps & 1));
 		
 	strip_size = TIFFStripSize(tif);
-	
+
 	buf = _TIFFmalloc(strip_size);
 	if (buf == NULL) {
 		TIFFClose(tif);
 		opj_image_destroy(image);
 		return NULL;
 	}
-	rowStride = ((OPJ_SIZE_T)w * tiSpp * tiBps + 7U) / 8U;
-	buffer32s = (OPJ_INT32 *)malloc((OPJ_SIZE_T)w * tiSpp * sizeof(OPJ_INT32));
+	rowStride = (w * tiSpp * tiBps + 7U) / 8U;
+	buffer32s = (OPJ_INT32 *)malloc((OPJ_SIZE_T)(w * tiSpp * sizeof(OPJ_INT32)));
 	if (buffer32s == NULL) {
 		_TIFFfree(buf);
 		TIFFClose(tif);
@@ -1421,11 +1445,20 @@ opj_image_t* tiftoimage(const char *filename, opj_cparameters_t *parameters)
 		for(; (h > 0) && (strip < TIFFNumberOfStrips(tif)); strip++)
 		{
 				const OPJ_UINT8 *dat8;
-				OPJ_SIZE_T ssize;
+				tmsize_t ssize;
 				
-				ssize = (OPJ_SIZE_T)TIFFReadEncodedStrip(tif, strip, buf, strip_size);
+				ssize = TIFFReadEncodedStrip(tif, strip, buf, strip_size);
+				if(ssize < 1 || ssize > strip_size) {
+					fprintf(stderr,"tiftoimage: Bad value for ssize(%ld) "
+                     "vs. strip_size(%ld).\n\tAborting.\n",ssize,strip_size);
+					_TIFFfree(buf);
+					_TIFFfree(buffer32s);
+					TIFFClose(tif);
+					opj_image_destroy(image);
+					return NULL;
+				}
 				dat8 = (const OPJ_UINT8*)buf;
-				
+
 				while (ssize >= rowStride) {
 					cvtTifTo32s(dat8, buffer32s, (OPJ_SIZE_T)w * tiSpp);
 					cvtCxToPx(buffer32s, planes, (OPJ_SIZE_T)w);