avcodec/dstdec: Check for overflow in build_filter()

Fixes: signed integer overflow: 1917019860 + 265558963 cannot be represented in type 'int'
Fixes: 48798/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_DST_fuzzer-4833165046317056

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpegSigned-off-by: 's avatarMichael Niedermayer <michael@niedermayer.cc>
parent aa441ac1
......@@ -215,7 +215,7 @@ static uint8_t prob_dst_x_bit(int c)
return (ff_reverse[c & 127] >> 1) + 1;
}
static void build_filter(int16_t table[DST_MAX_ELEMENTS][16][256], const Table *fsets)
static int build_filter(int16_t table[DST_MAX_ELEMENTS][16][256], const Table *fsets)
{
int i, j, k, l;
......@@ -226,14 +226,17 @@ static void build_filter(int16_t table[DST_MAX_ELEMENTS][16][256], const Table *
int total = av_clip(length - j * 8, 0, 8);
for (k = 0; k < 256; k++) {
int v = 0;
int64_t v = 0;
for (l = 0; l < total; l++)
v += (((k >> l) & 1) * 2 - 1) * fsets->coeff[i][j * 8 + l];
if ((int16_t)v != v)
return AVERROR_INVALIDDATA;
table[i][j][k] = v;
}
}
}
return 0;
}
static int decode_frame(AVCodecContext *avctx, AVFrame *frame,
......@@ -328,7 +331,9 @@ static int decode_frame(AVCodecContext *avctx, AVFrame *frame,
return AVERROR_INVALIDDATA;
ac_init(ac, gb);
build_filter(s->filter, &s->fsets);
ret = build_filter(s->filter, &s->fsets);
if (ret < 0)
return ret;
memset(s->status, 0xAA, sizeof(s->status));
memset(dsd, 0, frame->nb_samples * 4 * channels);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment