Import OpenBSD reallocarray API which prevents overflow that may occur

with multiplication in malloc and realloc & replace malloc(x*y) with
reallocarray(NULL, x, y).

tjutil.c

@@ -64,3 +64,43 @@ double gettime(void)
 }
 
 #endif
+
+/* OpenBSD reallocarrray API is licensed under: */
+
+/*
+ * Copyright (c) 2008 Otto Moerbeek <[email protected]>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/types.h>
+#include <errno.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+/*
+ * This is sqrt(SIZE_MAX+1), as s1*s2 <= SIZE_MAX
+ * if both s1 < MUL_NO_OVERFLOW and s2 < MUL_NO_OVERFLOW
+ */
+#define MUL_NO_OVERFLOW	((size_t)1 << (sizeof(size_t) * 4))
+
+void *
+reallocarray(void *optr, size_t nmemb, size_t size)
+{
+	if ((nmemb >= MUL_NO_OVERFLOW || size >= MUL_NO_OVERFLOW) &&
+	    nmemb > 0 && SIZE_MAX / nmemb < size) {
+		errno = ENOMEM;
+		return NULL;
+	}
+	return realloc(optr, size * nmemb);
+}

tjutil.h

@@ -45,3 +45,5 @@
 #endif
 
 extern double gettime(void);
+
+extern void *reallocarray(void *optr, size_t nmemb, size_t size);

turbojpeg-jni.c

@@ -1058,15 +1058,15 @@ JNIEXPORT jintArray JNICALL Java_org_libjpegturbo_turbojpeg_TJTransformer_transf
 	if(n!=(*env)->GetArrayLength(env, tobjs))
 		_throwarg("Mismatch between size of transforms array and destination buffers array");
 
-	if((dstBufs=(unsigned char **)malloc(sizeof(unsigned char *)*n))==NULL)
+	if((dstBufs=(unsigned char **)reallocarray(NULL, n, sizeof(unsigned char *)))==NULL)
 		_throwmem();
-	if((jdstBufs=(jbyteArray *)malloc(sizeof(jbyteArray)*n))==NULL)
+	if((jdstBufs=(jbyteArray *)reallocarray(NULL, n, sizeof(jbyteArray)))==NULL)
 		_throwmem();
-	if((dstSizes=(unsigned long *)malloc(sizeof(unsigned long)*n))==NULL)
+	if((dstSizes=(unsigned long *)reallocarray(NULL, n, sizeof(unsigned long)))==NULL)
 		_throwmem();
-	if((t=(tjtransform *)malloc(sizeof(tjtransform)*n))==NULL)
+	if((t=(tjtransform *)reallocarray(NULL, n, sizeof(tjtransform)))==NULL)
 		_throwmem();
-	if((params=(JNICustomFilterParams *)malloc(sizeof(JNICustomFilterParams)*n))
+	if((params=(JNICustomFilterParams *)reallocarray(NULL, n, sizeof(JNICustomFilterParams)))
 		==NULL)
 		_throwmem();
 	for(i=0; i<n; i++)

turbojpeg.c

@@ -782,7 +782,7 @@ DLLEXPORT int DLLCALL tjCompress2(tjhandle handle, const unsigned char *srcBuf,
 		return -1;
 
 	jpeg_start_compress(cinfo, TRUE);
-	if((row_pointer=(JSAMPROW *)malloc(sizeof(JSAMPROW)*height))==NULL)
+	if((row_pointer=(JSAMPROW *)reallocarray(NULL, height, sizeof(JSAMPROW)))==NULL)
 		_throw("tjCompress2(): Memory allocation failure");
 	for(i=0; i<height; i++)
 	{
@@ -909,7 +909,7 @@ DLLEXPORT int DLLCALL tjEncodeYUVPlanes(tjhandle handle,
 	pw0=PAD(width, cinfo->max_h_samp_factor);
 	ph0=PAD(height, cinfo->max_v_samp_factor);
 
-	if((row_pointer=(JSAMPROW *)malloc(sizeof(JSAMPROW)*ph0))==NULL)
+	if((row_pointer=(JSAMPROW *)reallocarray(NULL, ph0, sizeof(JSAMPROW)))==NULL)
 		_throw("tjEncodeYUVPlanes(): Memory allocation failure");
 	for(i=0; i<height; i++)
 	{
@@ -951,7 +951,7 @@ DLLEXPORT int DLLCALL tjEncodeYUVPlanes(tjhandle handle,
 		}
 		pw[i]=pw0*compptr->h_samp_factor/cinfo->max_h_samp_factor;
 		ph[i]=ph0*compptr->v_samp_factor/cinfo->max_v_samp_factor;
-		outbuf[i]=(JSAMPROW *)malloc(sizeof(JSAMPROW)*ph[i]);
+		outbuf[i]=(JSAMPROW *)reallocarray(NULL, ph[i], sizeof(JSAMPROW));
 		if(!outbuf[i]) _throw("tjEncodeYUVPlanes(): Memory allocation failure");
 		ptr=dstPlanes[i];
 		for(row=0; row<ph[i]; row++)
@@ -1109,7 +1109,7 @@ DLLEXPORT int DLLCALL tjCompressFromYUVPlanes(tjhandle handle,
 		if(iw[i]!=pw[i] || ih!=ph[i]) usetmpbuf=1;
 		th[i]=compptr->v_samp_factor*DCTSIZE;
 		tmpbufsize+=iw[i]*th[i];
-		if((inbuf[i]=(JSAMPROW *)malloc(sizeof(JSAMPROW)*ph[i]))==NULL)
+		if((inbuf[i]=(JSAMPROW *)reallocarray(NULL, ph[i], sizeof(JSAMPROW)))==NULL)
 			_throw("tjCompressFromYUVPlanes(): Memory allocation failure");
 		ptr=(JSAMPLE *)srcPlanes[i];
 		for(row=0; row<ph[i]; row++)
@@ -1120,12 +1120,12 @@ DLLEXPORT int DLLCALL tjCompressFromYUVPlanes(tjhandle handle,
 	}
 	if(usetmpbuf)
 	{
-		if((_tmpbuf=(JSAMPLE *)malloc(sizeof(JSAMPLE)*tmpbufsize))==NULL)
+		if((_tmpbuf=(JSAMPLE *)reallocarray(NULL, tmpbufsize, sizeof(JSAMPLE)))==NULL)
 			_throw("tjCompressFromYUVPlanes(): Memory allocation failure");
 		ptr=_tmpbuf;
 		for(i=0; i<cinfo->num_components; i++)
 		{
-			if((tmpbuf[i]=(JSAMPROW *)malloc(sizeof(JSAMPROW)*th[i]))==NULL)
+			if((tmpbuf[i]=(JSAMPROW *)reallocarray(NULL, th[i], sizeof(JSAMPROW)))==NULL)
 				_throw("tjCompressFromYUVPlanes(): Memory allocation failure");
 			for(row=0; row<th[i]; row++)
 			{
@@ -1597,7 +1597,7 @@ DLLEXPORT int DLLCALL tjDecodeYUVPlanes(tjhandle handle,
 	}
 	#endif
 
-	if((row_pointer=(JSAMPROW *)malloc(sizeof(JSAMPROW)*ph0))==NULL)
+	if((row_pointer=(JSAMPROW *)reallocarray(NULL, ph0, sizeof(JSAMPROW)))==NULL)
 		_throw("tjDecodeYUVPlanes(): Memory allocation failure");
 	for(i=0; i<height; i++)
 	{
@@ -1624,7 +1624,7 @@ DLLEXPORT int DLLCALL tjDecodeYUVPlanes(tjhandle handle,
 		}
 		pw[i]=pw0*compptr->h_samp_factor/dinfo->max_h_samp_factor;
 		ph[i]=ph0*compptr->v_samp_factor/dinfo->max_v_samp_factor;
-		inbuf[i]=(JSAMPROW *)malloc(sizeof(JSAMPROW)*ph[i]);
+		inbuf[i]=(JSAMPROW *)reallocarray(NULL, ph[i], sizeof(JSAMPROW));
 		if(!inbuf[i]) _throw("tjDecodeYUVPlanes(): Memory allocation failure");
 		ptr=(JSAMPLE *)srcPlanes[i];
 		for(row=0; row<ph[i]; row++)
@@ -1788,7 +1788,7 @@ DLLEXPORT int DLLCALL tjDecompressToYUVPlanes(tjhandle handle,
 		if(iw[i]!=pw[i] || ih!=ph[i]) usetmpbuf=1;
 		th[i]=compptr->v_samp_factor*dctsize;
 		tmpbufsize+=iw[i]*th[i];
-		if((outbuf[i]=(JSAMPROW *)malloc(sizeof(JSAMPROW)*ph[i]))==NULL)
+		if((outbuf[i]=(JSAMPROW *)reallocarray(NULL, ph[i], sizeof(JSAMPROW)))==NULL)
 			_throw("tjDecompressToYUVPlanes(): Memory allocation failure");
 		ptr=dstPlanes[i];
 		for(row=0; row<ph[i]; row++)
@@ -1799,12 +1799,12 @@ DLLEXPORT int DLLCALL tjDecompressToYUVPlanes(tjhandle handle,
 	}
 	if(usetmpbuf)
 	{
-		if((_tmpbuf=(JSAMPLE *)malloc(sizeof(JSAMPLE)*tmpbufsize))==NULL)
+		if((_tmpbuf=(JSAMPLE *)reallocarray(NULL, tmpbufsize, sizeof(JSAMPLE)))==NULL)
 			_throw("tjDecompressToYUVPlanes(): Memory allocation failure");
 		ptr=_tmpbuf;
 		for(i=0; i<dinfo->num_components; i++)
 		{
-			if((tmpbuf[i]=(JSAMPROW *)malloc(sizeof(JSAMPROW)*th[i]))==NULL)
+			if((tmpbuf[i]=(JSAMPROW *)reallocarray(NULL, th[i], sizeof(JSAMPROW)))==NULL)
 				_throw("tjDecompressToYUVPlanes(): Memory allocation failure");
 			for(row=0; row<th[i]; row++)
 			{
@@ -1994,7 +1994,7 @@ DLLEXPORT int DLLCALL tjTransform(tjhandle handle,
 
 	jpeg_mem_src_tj(dinfo, jpegBuf, jpegSize);
 
-	if((xinfo=(jpeg_transform_info *)malloc(sizeof(jpeg_transform_info)*n))
+	if((xinfo=(jpeg_transform_info *)reallocarray(NULL, n, sizeof(jpeg_transform_info)))
 		==NULL)
 		_throw("tjTransform(): Memory allocation failure");
 	MEMZERO(xinfo, sizeof(jpeg_transform_info)*n);