avfilter/asubprocess: check subprocess output size

Subprocess should not produce more output samples than the number of
input samples provided.
Signed-off-by: Stefan Westerfeld <stefan@space.twc.de>

libavfilter/af_asubprocess.c

@@ -68,6 +68,8 @@ typedef struct ASubProcessContext {
     size_t        sample_buffer_size;
     int           eof;
     int64_t       last_pts;
+    uint64_t      nb_input_samples;
+    uint64_t      nb_output_samples;
     FFFrameQueue  frame_queue;
 } ASubProcessContext;
 
@@ -204,7 +206,7 @@ sp_new(AVFilterContext *ctx)
     return 0;
 }
 
-static int
+static size_t
 sp_can_read(SP *sp)
 {
     int can_read = 0;
@@ -241,7 +243,7 @@ static int update_state(AVFilterContext *ctx)
     ASubProcessContext *s = ctx->priv;
 
     while (s->state != STATE_IN_DATA_CHUNK) {
-        unsigned int can_read = sp_can_read(s->sp);
+        size_t can_read = sp_can_read(s->sp);
 
         if (s->state == STATE_EXPECT_RIFF && can_read >= 12) {
             char buffer[12];
@@ -311,13 +313,23 @@ static int update_state(AVFilterContext *ctx)
             }
         }
         else {
-            if (s->state != STATE_IN_DATA_CHUNK && s->sp->input_count > MAX_INPUT_BEFORE_DATA) {
-                    av_log(ctx, AV_LOG_ERROR, "Input is not a valid wav file: no 'data' chunk found.\n");
+            if (s->sp->input_count > MAX_INPUT_BEFORE_DATA) {
+                    av_log(ctx, AV_LOG_ERROR, "Subprocess output is not a valid wav file: no 'data' chunk found.\n");
                     return AVERROR_INVALIDDATA;
             }
             break;
         }
     }
+    if (s->state == STATE_IN_DATA_CHUNK) {
+        int bytes_per_sample = s->out_bit_depth / 8 * outlink->ch_layout.nb_channels;
+        size_t expect_bytes = (s->nb_input_samples - s->nb_output_samples) * bytes_per_sample;
+        if ((s->nb_input_samples * bytes_per_sample) % 2) // padding byte for odd length input
+            expect_bytes++;
+        if (sp_can_read(s->sp) > expect_bytes) {
+            av_log(ctx, AV_LOG_ERROR, "Subprocess produced more output data than expected.\n");
+            return AVERROR_INVALIDDATA;
+        }
+    }
     return 0;
 }
 
@@ -563,6 +575,7 @@ static int write_frame(AVFilterContext *ctx, ASubProcessContext *s, AVFilterLink
     AVFrame *out;
     int ret = 0;
 
+    s->nb_input_samples += in->nb_samples;
     ret = write_samples(ctx, s, (int32_t *)in->data[0], in->nb_samples * inlink->ch_layout.nb_channels);
     if (av_frame_is_writable(in)) {
         out = in;
@@ -587,17 +600,17 @@ static int try_read_frame(AVFilterContext *ctx)
     if (s->state == STATE_IN_DATA_CHUNK && ff_framequeue_queued_frames(&s->frame_queue)) {
         AVFrame *out = ff_framequeue_peek(&s->frame_queue, 0);
         int sample_size = s->out_bit_depth / 8;
-        int avail = sp_can_read(s->sp) / sample_size / outlink->ch_layout.nb_channels;
 
+        size_t avail = sp_can_read(s->sp) / sample_size / outlink->ch_layout.nb_channels;
         if (avail >= out->nb_samples) {
-            avail = FFMIN(avail, out->nb_samples);
             out = ff_framequeue_take(&s->frame_queue);
-            av_assert0(avail <= s->sample_buffer_size);
-            read_samples(s, (int32_t *)out->data[0], avail * outlink->ch_layout.nb_channels);
+            av_assert0(out->nb_samples <= s->sample_buffer_size);
+            s->nb_output_samples += out->nb_samples;
+            read_samples(s, (int32_t *)out->data[0], out->nb_samples * outlink->ch_layout.nb_channels);
             ret = ff_filter_frame(outlink, out);
             if (ret < 0)
                 return ret;
-            return avail;
+            return out->nb_samples;
         }
     }
 
@@ -648,7 +661,7 @@ static int activate(AVFilterContext *ctx)
 
     if (s->eof && !sp_done(s->sp)) {
         while (!sp_done(s->sp)) {
-            int ret = sp_write(ctx, 0, 0);
+            ret = sp_write(ctx, 0, 0);
             if (ret != 0)
                 return ret;
         }