fix(rar): add boundary checks to rgb filter (#2210)

`blocklength` should be bigger than `3` (channel count)
`byteoffset` should not be bigger than `2` (does not make sense as per the
last loop)
`src` should not overlap with `dst`.

There is no allocation in this function so it should be safe to return
early.

Security: GHSA-9qqv-q4qw-mf8m

diff --git a/libarchive/archive_read_support_format_rar.c b/libarchive/archive_read_support_format_rar.c
index c8725bcae4edc28e92082e91d3c1d1a46dd5678f..123813084ccb295181aa762a586400302b213dfb 100644 (file)
--- a/libarchive/archive_read_support_format_rar.c
+++ b/libarchive/archive_read_support_format_rar.c
@@ -3681,7 +3681,7 @@ execute_filter_rgb(struct rar_filter *filter, struct rar_virtual_machine *vm)
   uint8_t *src, *dst;
   uint32_t i, j;
 
-  if (blocklength > PROGRAM_WORK_SIZE / 2 || stride > blocklength)
+  if (blocklength > PROGRAM_WORK_SIZE / 2 || stride > blocklength || blocklength < 3 || byteoffset > 2)
     return 0;
 
   src = &vm->memory[0];
@@ -3691,6 +3691,13 @@ execute_filter_rgb(struct rar_filter *filter, struct rar_virtual_machine *vm)
     uint8_t *prev = dst + i - stride;
     for (j = i; j < blocklength; j += 3)
     {
+      /*
+       * The src block should not overlap with the dst block.
+       * If so it would be better to consider this archive is broken.
+       */
+      if (src >= dst)
+        return 0;
+
       if (prev >= dst)
       {
         uint32_t delta1 = abs(prev[3] - prev[0]);