Fix integer overflow in RGB addframe byte-size calculation and add re…

src/cgif_rgb.c

@@ -542,12 +542,17 @@ static int quantize_and_dither(colHashTable* colhash, const uint8_t* pImageDataR
     if(root == NULL) {
       return -1;
     }
-    float* pImageDataRGBfloat = malloc(fmtChan * numPixel * sizeof(float)); // TBD fmtChan + only when hasAlpha
+    size_t numValues = (size_t)fmtChan * numPixel;
+    if(numPixel != 0 && numValues / numPixel != fmtChan) {
+      free_decision_tree(root);
+      return -1;
+    }
+    float* pImageDataRGBfloat = malloc(numValues * sizeof(float)); // TBD fmtChan + only when hasAlpha
     if(pImageDataRGBfloat == NULL) {
       free_decision_tree(root);
       return -1;
     }
-    for(uint32_t i = 0; i < fmtChan * numPixel; ++i){
+    for(size_t i = 0; i < numValues; ++i){
       pImageDataRGBfloat[i] = pImageDataRGB[i];
     }
     uint8_t transIndex = colMax;
@@ -616,12 +621,17 @@ cgif_result cgif_rgb_addframe(CGIFrgb* pGIF, const CGIFrgb_FrameConfig* pConfig)
     pGIF->curResult = CGIF_ERROR;
     return CGIF_ERROR;
   }
-  pNewBef = malloc(pConfig->fmtChan * MULU16(imageWidth, imageHeight));
+  size_t totalBytes = (size_t)pConfig->fmtChan * numPixel;
+  if(numPixel != 0 && totalBytes / numPixel != pConfig->fmtChan) {
+    pGIF->curResult = CGIF_EALLOC;
+    return pGIF->curResult;
+  }
+  pNewBef = malloc(totalBytes);
   if(pNewBef == NULL) {
     pGIF->curResult = CGIF_EALLOC;
     return pGIF->curResult;
   }
-  memcpy(pNewBef, pConfig->pImageData, pConfig->fmtChan * MULU16(imageWidth, imageHeight));
+  memcpy(pNewBef, pConfig->pImageData, totalBytes);
   fConfig.pLocalPalette = aPalette;
   fConfig.pImageData    = malloc(pGIF->config.width * (uint32_t)pGIF->config.height);
   if(fConfig.pImageData == NULL) {

tests/meson.build

@@ -98,6 +98,14 @@ test_ealloc_rgb_exe = executable(
 )
 test('ealloc_rgb', test_ealloc_rgb_exe, priority : 0)
 
+test_rgb_overflow_trigger_exe = executable(
+  'test_rgb_overflow_trigger',
+  'rgb_overflow_trigger.c',
+  dependencies : [libcgif_dep],
+  include_directories : ['../inc/'],
+)
+test('rgb_overflow_trigger', test_rgb_overflow_trigger_exe, priority : 0)
+
 sha256sumc = find_program('scripts/sha256sum.py')
 # get the ordering right:
 # md5sum check on output GIFs should be run once all of the above tests are done.

tests/rgb_overflow_trigger.c

@@ -0,0 +1,57 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdio.h>
+
+#include "cgif.h"
+
+static int pWriteFn(void* pContext, const uint8_t* pData, const size_t numBytes) {
+  (void)pContext;
+  (void)pData;
+  (void)numBytes;
+  return 0;
+}
+
+/* regression test: fmtChan * numPixel overflows uint32_t for large dimensions with RGBA.
+   on 32-bit platforms size_t is 32 bits, so the overflow check fires and CGIF_EALLOC is returned.
+   on 64-bit platforms the multiplication fits in size_t but processing a 4 GB image
+   would time out on CI, so we skip. */
+int main(void) {
+  if(sizeof(size_t) > 4) {
+    return 0; // skip on 64-bit: no uint32_t overflow possible in size_t
+  }
+
+  const uint16_t width  = 32768;
+  const uint16_t height = 32769; // width * height * 4 > UINT32_MAX
+
+  CGIFrgb_Config gConfig = {0};
+  gConfig.pWriteFn = pWriteFn;
+  gConfig.width    = width;
+  gConfig.height   = height;
+
+  CGIFrgb* pGIF = cgif_rgb_newgif(&gConfig);
+  if(pGIF == NULL) {
+    return 2;
+  }
+
+  size_t bufSize = (size_t)width * height * 4;
+  uint8_t* buf = malloc(bufSize);
+  if(buf == NULL) {
+    cgif_rgb_close(pGIF);
+    return 0; // skip: not enough memory
+  }
+  memset(buf, 0, bufSize);
+
+  CGIFrgb_FrameConfig fConfig = {0};
+  fConfig.pImageData = buf;
+  fConfig.fmtChan    = CGIF_CHAN_FMT_RGBA;
+  fConfig.delay      = 0;
+
+  cgif_result r = cgif_rgb_addframe(pGIF, &fConfig);
+  cgif_rgb_close(pGIF);
+  free(buf);
+  if(r == CGIF_EALLOC) {
+    return 0; // overflow correctly detected
+  }
+  return 1;
+}