Reject illegal record sizes in decryption as soon as rs is decoded.

src/vfp.c

@@ -448,7 +448,25 @@ decrypt_init(struct vfp_ctx *ctx, struct ece *ece)
 			return (VFP_OK);
 		}
 	}
+
 	decode_header(hdr->hdr, &rs, &idlen);
+	if (rs < MIN_RS)
+		return (VERR_DEC(ctx, "invalid record size %u", rs));
+	/*
+	 * XXX make max_rs configurable, 0 for unlimited
+	 * A stat should show the rs high watermark.
+	 */
+	if (rs > DEFAULT_MAX_RS)
+		return (VERR_DEC(ctx, "record size %u exceeds max %u", rs,
+				DEFAULT_MAX_RS));
+	if (rs > INT_MAX)
+		/*
+		 * XXX This is because the input params to the libcrypto
+		 * functions are typed as signed int.
+		 */
+		return (VERR_DEC(ctx, "record size %u may not exceed %d", rs,
+				 INT_MAX));
+
 	if (HDR_LEN(hdr) < HDR_PFX_LEN + idlen) {
 		len = (HDR_PFX_LEN + idlen) - HDR_LEN(hdr);
 		vp = suck_bytes(ctx, hdr->next_in, &len);
@@ -467,23 +485,9 @@ decrypt_init(struct vfp_ctx *ctx, struct ece *ece)
 			return (VFP_NULL);
 	}
 	ece->rs = rs;
-
-	if (ece->rs < MIN_RS)
-		return (VERR_DEC(ctx, "invalid record size %u", ece->rs));
-	/*
-	 * XXX make max_rs configurable, 0 for unlimited
-	 * A stat should show the rs high watermark.
-	 */
-	if (ece->rs > DEFAULT_MAX_RS)
-		return (VERR_DEC(ctx, "record size %u exceeds max %u", ece->rs,
-				DEFAULT_MAX_RS));
-	if (ece->rs > INT_MAX)
-		/*
-		 * XXX This is because the input params to the libcrypto
-		 * functions are typed as signed int.
-		 */
-		return (VERR_DEC(ctx, "record size %u may not exceed %d",
-				 ece->rs, INT_MAX));
+	assert(ece->rs >= MIN_RS);
+	assert(ece->rs <= DEFAULT_MAX_RS);
+	assert(ece->rs <= INT_MAX);
 
 	if (crypto_init(ctx, ece->crypto, hdr->hdr, hdr->hdr + HDR_PFX_LEN,
 			idlen, 0) == VFP_ERROR)