From 40e49c57334a1401bba15d6ac65452933715dcf0 Mon Sep 17 00:00:00 2001 From: Wan-Teh Chang Date: Wed, 31 Jul 2024 19:22:53 -0700 Subject: [PATCH] Fix overflows in two functions in src/reformat.c Fix overflows when multiplying with rowBytes in avifImageRGBToYUV() and avifImageYUVAnyToRGBAnySlow(), by storing the various uint32_t rowBytes fields in local variables of the size_t type. Then multiplications with the size_t rowBytes local variables will be done in size_t. Part of the fix to https://github.com/AOMediaCodec/libavif/issues/2271. --- src/reformat.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/reformat.c b/src/reformat.c index 658d5ac8b4..7c4364db92 100644 --- a/src/reformat.c +++ b/src/reformat.c @@ -286,8 +286,8 @@ avifResult avifImageRGBToYUV(avifImage * image, const avifRGBImage * rgb) // Convert an entire 2x2 block to YUV, and populate any fully sampled channels as we go for (int bJ = 0; bJ < blockH; ++bJ) { for (int bI = 0; bI < blockW; ++bI) { - int i = outerI + bI; - int j = outerJ + bJ; + size_t i = outerI + bI; + size_t j = outerJ + bJ; // Unpack RGB into normalized float if (state.rgb.channelBytes > 1) { @@ -420,8 +420,8 @@ avifResult avifImageRGBToYUV(avifImage * image, const avifRGBImage * rgb) const int chromaShiftX = 1; const int chromaShiftY = 1; - int uvI = outerI >> chromaShiftX; - int uvJ = outerJ >> chromaShiftY; + size_t uvI = outerI >> chromaShiftX; + size_t uvJ = outerJ >> chromaShiftY; if (state.yuv.channelBytes > 1) { uint16_t * pU = (uint16_t *)&yuvPlanes[AVIF_CHAN_U][(uvI * 2) + (uvJ * yuvRowBytes[AVIF_CHAN_U])]; *pU = (uint16_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgU); @@ -448,8 +448,8 @@ avifResult avifImageRGBToYUV(avifImage * image, const avifRGBImage * rgb) float avgV = sumV / totalSamples; const int chromaShiftX = 1; - int uvI = outerI >> chromaShiftX; - int uvJ = outerJ + bJ; + size_t uvI = outerI >> chromaShiftX; + size_t uvJ = outerJ + bJ; if (state.yuv.channelBytes > 1) { uint16_t * pU = (uint16_t *)&yuvPlanes[AVIF_CHAN_U][(uvI * 2) + (uvJ * yuvRowBytes[AVIF_CHAN_U])]; *pU = (uint16_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgU);