[Feature] Add ASCII85 decode support for PDF text extraction

PDFs may use ASCII85Decode filter for content streams. This was causing
text extraction to fail for such PDFs, resulting in missed URLs and emails.

- Add rspamd_decode_ascii85_buf() in str_util.c
- Add rspamd_util.decode_ascii85() Lua binding
- Add ASCII85Decode filter support in pdf.lua
- Add --raw flag to rspamadm mime urls command

diff --git a/lualib/lua_content/pdf.lua b/lualib/lua_content/pdf.lua
index 2e4dbf01ca893e47ae9213533763e060af591702..4c37b06385fd72109e1091c118cf75f967c1df82 100644 (file)
--- a/lualib/lua_content/pdf.lua
+++ b/lualib/lua_content/pdf.lua
@@ -637,6 +637,8 @@ local function apply_pdf_filter(input, filt)
       return nil
     end
     return lua_util.unhex(to_decode)
+  elseif filt == 'ASCII85Decode' or filt == 'A85' then
+    return rspamd_util.decode_ascii85(input)
   end
 
   return nil

diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua
index 29b3cd7c24823e395dbdc5c96b940ea0852e403e..ea28ac5d0387c7757289dbab27f9cb2750ece8d2 100644 (file)
--- a/lualib/rspamadm/mime.lua
+++ b/lualib/rspamadm/mime.lua
@@ -135,6 +135,8 @@ urls:flag "--count"
     :description "Print count of each printed element"
 urls:flag "-r --reverse"
     :description "Reverse sort order"
+urls:flag "--raw"
+    :description "Load as raw file (for PDFs and other non-email files)"
 
 local modify = parser:command "modify mod m"
                      :description "Modifies MIME message"

diff --git a/src/libutil/str_util.c b/src/libutil/str_util.c
index 8ddd4f4a3ab6102dc29d94b12ee0bee3a2508be3..b73593653ec7b56fc39b48692c5a90b0c5cd256f 100644 (file)
--- a/src/libutil/str_util.c
+++ b/src/libutil/str_util.c
@@ -2932,6 +2932,106 @@ rspamd_decode_uue_buf(const char *in, gsize inlen,
        return (o - out);
 }
 
+gssize
+rspamd_decode_ascii85_buf(const char *in, gsize inlen,
+                                                 unsigned char *out, gsize outlen)
+{
+       const char *p = in;
+       const char *end = in + inlen;
+       unsigned char *o = out;
+       unsigned char *out_end = out + outlen;
+       uint32_t tuple = 0;
+       int count = 0;
+
+       while (p < end) {
+               unsigned char c = *p++;
+
+               /* Skip whitespace */
+               if (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f') {
+                       continue;
+               }
+
+               /* Check for end marker '~>' */
+               if (c == '~') {
+                       if (p < end && *p == '>') {
+                               /* End of ASCII85 data */
+                               break;
+                       }
+                       if (p >= end) {
+                               /* '~' at end of buffer - treat as truncated end marker */
+                               break;
+                       }
+                       /* Invalid: '~' followed by something other than '>' */
+                       return -1;
+               }
+
+               /* Special case: 'z' represents 4 zero bytes */
+               if (c == 'z') {
+                       if (count != 0) {
+                               /* 'z' can only appear between complete groups */
+                               return -1;
+                       }
+                       if (out_end - o < 4) {
+                               return -1;
+                       }
+                       *o++ = 0;
+                       *o++ = 0;
+                       *o++ = 0;
+                       *o++ = 0;
+                       continue;
+               }
+
+               /* Valid ASCII85 characters are '!' (33) to 'u' (117) */
+               if (c < '!' || c > 'u') {
+                       return -1;
+               }
+
+               /* Accumulate the value */
+               tuple = tuple * 85 + (c - '!');
+               count++;
+
+               if (count == 5) {
+                       /* Output 4 bytes (big-endian) */
+                       if (out_end - o < 4) {
+                               return -1;
+                       }
+                       *o++ = (tuple >> 24) & 0xFF;
+                       *o++ = (tuple >> 16) & 0xFF;
+                       *o++ = (tuple >> 8) & 0xFF;
+                       *o++ = tuple & 0xFF;
+                       tuple = 0;
+                       count = 0;
+               }
+       }
+
+       /* Handle final incomplete group */
+       if (count > 0) {
+               /* Pad with 'u' (84) to complete the group */
+               int padding = 5 - count;
+               for (int i = 0; i < padding; i++) {
+                       tuple = tuple * 85 + 84;
+               }
+
+               /* Output (count - 1) bytes */
+               int out_bytes = count - 1;
+               if (out_end - o < out_bytes) {
+                       return -1;
+               }
+
+               if (out_bytes >= 1) {
+                       *o++ = (tuple >> 24) & 0xFF;
+               }
+               if (out_bytes >= 2) {
+                       *o++ = (tuple >> 16) & 0xFF;
+               }
+               if (out_bytes >= 3) {
+                       *o++ = (tuple >> 8) & 0xFF;
+               }
+       }
+
+       return o - out;
+}
+
 #define BITOP(a, b, op) \
        ((a)[(gsize) (b) / (8 * sizeof *(a))] op(gsize) 1 << ((gsize) (b) % (8 * sizeof *(a))))
 

diff --git a/src/libutil/str_util.h b/src/libutil/str_util.h
index e0324a126455e464dadbf0ab8b3866f5ceaf12b3..8de4b361edaa2312d74ee785fc9093e84f13ab50 100644 (file)
--- a/src/libutil/str_util.h
+++ b/src/libutil/str_util.h
@@ -318,6 +318,19 @@ gssize rspamd_decode_qp_buf(const char *in, gsize inlen,
 gssize rspamd_decode_uue_buf(const char *in, gsize inlen,
                                                         char *out, gsize outlen);
 
+/**
+ * Decode ASCII85 (Base85) encoded buffer, input and output must not overlap
+ * ASCII85 encodes 4 bytes as 5 ASCII characters in range '!' to 'u'
+ * Special: 'z' represents 4 null bytes, '~>' marks end of data
+ * @param in input
+ * @param inlen length of input
+ * @param out output
+ * @param outlen length of output
+ * @return real size of decoded output or (-1) if outlen is not enough or invalid input
+ */
+gssize rspamd_decode_ascii85_buf(const char *in, gsize inlen,
+                                                                unsigned char *out, gsize outlen);
+
 /**
  * Decode quoted-printable encoded buffer using rfc2047 format, input and output must not overlap
  * @param in input

diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c
index 8de4412d903c1d5f0eeed389fa82dedde619675a..6bfdd32162953bd65b824a6204b9ffcd3c9a8cc8 100644 (file)
--- a/src/lua/lua_util.c
+++ b/src/lua/lua_util.c
@@ -20,6 +20,7 @@
 #include "libmime/content_type.h"
 #include "libmime/mime_headers.h"
 #include "libutil/hash.h"
+#include "libutil/str_util.h"
 #include "libserver/html/html.h"
 
 #include "lua_parsers.h"
@@ -114,6 +115,14 @@ LUA_FUNCTION_DEF(util, decode_html_entities);
  */
 LUA_FUNCTION_DEF(util, decode_base64);
 
+/***
+ * @function util.decode_ascii85(input)
+ * Decodes data from ASCII85 (Base85) encoding used in PDF files
+ * @param {text or string} input data to decode
+ * @return {rspamd_text} decoded data chunk or nil on error
+ */
+LUA_FUNCTION_DEF(util, decode_ascii85);
+
 /***
  * @function util.encode_base32(input, [b32type = 'default'])
  * Encodes data in base32 breaking lines if needed
@@ -763,6 +772,7 @@ static const struct luaL_reg utillib_f[] = {
        LUA_INTERFACE_DEF(util, decode_qp),
        LUA_INTERFACE_DEF(util, decode_html_entities),
        LUA_INTERFACE_DEF(util, decode_base64),
+       LUA_INTERFACE_DEF(util, decode_ascii85),
        LUA_INTERFACE_DEF(util, encode_base32),
        LUA_INTERFACE_DEF(util, decode_base32),
        LUA_INTERFACE_DEF(util, decode_url),
@@ -1324,6 +1334,53 @@ lua_util_decode_base64(lua_State *L)
        return 1;
 }
 
+static int
+lua_util_decode_ascii85(lua_State *L)
+{
+       LUA_TRACE_POINT;
+       struct rspamd_lua_text *t;
+       const char *s = NULL;
+       gsize inlen = 0;
+       gssize outlen;
+
+       if (lua_type(L, 1) == LUA_TSTRING) {
+               s = luaL_checklstring(L, 1, &inlen);
+       }
+       else if (lua_type(L, 1) == LUA_TUSERDATA) {
+               t = lua_check_text(L, 1);
+
+               if (t != NULL) {
+                       s = t->start;
+                       inlen = t->len;
+               }
+       }
+
+       if (s != NULL && inlen > 0) {
+               /* ASCII85 expands 5 chars to 4 bytes, so output is at most (inlen * 4 / 5) + 4 */
+               gsize max_outlen = (inlen * 4 / 5) + 4;
+               unsigned char *buf = g_malloc(max_outlen);
+
+               outlen = rspamd_decode_ascii85_buf(s, inlen, buf, max_outlen);
+
+               if (outlen >= 0) {
+                       t = lua_newuserdata(L, sizeof(*t));
+                       rspamd_lua_setclass(L, rspamd_text_classname, -1);
+                       t->start = (const char *) buf;
+                       t->len = outlen;
+                       t->flags = RSPAMD_TEXT_FLAG_OWN;
+               }
+               else {
+                       g_free(buf);
+                       lua_pushnil(L);
+               }
+       }
+       else {
+               lua_pushnil(L);
+       }
+
+       return 1;
+}
+
 static int
 lua_util_encode_base32(lua_State *L)
 {