Implement the compression V2 data format for stub and lz4.

Patch V2: Fix minor issues found by Steffan
Patch V3: split wire codes and compression flags
Patch V4: Fix further issues reported by Gert
Patch V5: really fix the issues that should be fixed in v2
Patch V6: fix more minor things

It has been tested against v3 server and again itself. From James Mail:

 Compression V2

   I have observed that compression in many cases, even when
   enabled, often does not produce packet size reduction
   because much of the packet data typically generated by web
   sessions is already compressed.  Further, the single byte that
   precedes the packet and indicates whether or not compression
   occurred has the unfortunate side effect of misaligning the IP
   packet in cases where compression did not occur.  To remedy this,
   I propose a Compression V2 header that is optimized for the
   case where compression does not occur.

   a. No compression occurred and first byte of IP/Ethernet packet
      is NOT 0x50 (0 bytes of overhead and maintains alignment):

        [ uncompressed IP/Ethernet packet ]

   b. No compression occurred and first byte of IP/Ethernet packet
      is 0x50 (2 bytes of overhead but unlikely since no known
      IP packet can begin with 0x50):

        [ 0x50 ] [ 0x00 ] [ uncompressed IP/Ethernet packet ]

   c. Compression occurred (2 bytes of overhead):

        [ 0x50 ] [ compression Alg ID ] [ compressed IP/Ethernet packet ]

      Compression Alg ID is one-byte algorithm identifier
      for LZ4 (0x1), LZO (0x2), or Snappy (0x3).

   This approach has several beneficial effects:

   1. In the common case where compression does not occur, no
      compression op is required, therefore there is zero overhead.

   2. When compression does not occur, the IP/Ethernet packet
      alignment is retained.

   3. This technique does not require any byte swapping with
      the tail of the packet which can potentially incur an
      expensive cache miss.

Acked-by: Steffan Karger <steffan.karger@fox-it.com>
Acked-by: Gert Doering <gert@greenie.muc.de>
Message-Id: <1451842066-13475-1-git-send-email-arne@rfc2549.org>
URL: http://article.gmane.org/gmane.network.openvpn.devel/10925

Signed-off-by: Gert Doering <gert@greenie.muc.de>

diff --git a/src/openvpn/comp-lz4.c b/src/openvpn/comp-lz4.c
index 4651148579b9b3ecde952669c970c5e3cd18f8b8..395f3d2f180a63eb259f2cc011de4b8d1a9092e3 100644 (file)
--- a/src/openvpn/comp-lz4.c
+++ b/src/openvpn/comp-lz4.c
@@ -44,9 +44,6 @@
 
 #include "memdbg.h"
 
-/* Initial command byte to tell our peer if we compressed */
-#define LZ4_COMPRESS_BYTE 0x69
-
 static void
 lz4_compress_init (struct compress_context *compctx)
 {
@@ -55,20 +52,22 @@ lz4_compress_init (struct compress_context *compctx)
 }
 
 static void
-lz4_compress_uninit (struct compress_context *compctx)
+lz4v2_compress_init (struct compress_context *compctx)
 {
+  msg (D_INIT_MEDIUM, "LZ4v2 compression initializing");
 }
 
 static void
-lz4_compress (struct buffer *buf, struct buffer work,
-              struct compress_context *compctx,
-              const struct frame* frame)
+lz4_compress_uninit (struct compress_context *compctx)
 {
-  bool compressed = false;
-
-  if (buf->len <= 0)
-    return;
+}
 
+static bool
+do_lz4_compress (struct buffer *buf,
+                struct buffer *work,
+                struct compress_context *compctx,
+                const struct frame* frame)
+{
   /*
    * In order to attempt compression, length must be at least COMPRESS_THRESHOLD.
    */
@@ -78,33 +77,52 @@ lz4_compress (struct buffer *buf, struct buffer work,
       int zlen_max = ps + COMP_EXTRA_BUFFER (ps);
       int zlen;
 
-      ASSERT (buf_init (&work, FRAME_HEADROOM (frame)));
-      ASSERT (buf_safe (&work, zlen_max));
+      ASSERT (buf_init (work, FRAME_HEADROOM (frame)));
+      ASSERT (buf_safe (work, zlen_max));
 
       if (buf->len > ps)
        {
          dmsg (D_COMP_ERRORS, "LZ4 compression buffer overflow");
          buf->len = 0;
-         return;
+         return false;
        }
 
-      zlen = LZ4_compress_limitedOutput((const char *)BPTR(buf), (char *)BPTR(&work), BLEN(buf), zlen_max );
+      zlen = LZ4_compress_limitedOutput((const char *)BPTR(buf), (char *)BPTR(work), BLEN(buf), zlen_max );
 
       if (zlen <= 0)
        {
          dmsg (D_COMP_ERRORS, "LZ4 compression error");
          buf->len = 0;
-         return;
+         return false;
        }
 
-      ASSERT (buf_safe (&work, zlen));
-      work.len = zlen;
-      compressed = true;
+      ASSERT (buf_safe (work, zlen));
+      work->len = zlen;
+
 
-      dmsg (D_COMP, "LZ4 compress %d -> %d", buf->len, work.len);
+      dmsg (D_COMP, "LZ4 compress %d -> %d", buf->len, work->len);
       compctx->pre_compress += buf->len;
-      compctx->post_compress += work.len;
+      compctx->post_compress += work->len;
+      return true;
     }
+  return false;
+}
+
+
+static void
+lz4_compress (struct buffer *buf, struct buffer work,
+             struct compress_context *compctx,
+             const struct frame* frame)
+{
+  bool compressed;
+  if (buf->len <= 0)
+    return;
+
+  compressed = do_lz4_compress(buf, &work, compctx, frame);
+
+  /* On error do_lz4_compress sets buf len to zero, just return */
+  if (buf->len == 0)
+    return;
 
   /* did compression save us anything? */
   {
@@ -128,13 +146,70 @@ lz4_compress (struct buffer *buf, struct buffer work,
   }
 }
 
+
+static void
+lz4v2_compress (struct buffer *buf, struct buffer work,
+               struct compress_context *compctx,
+               const struct frame* frame)
+{
+  bool compressed;
+  if (buf->len <= 0)
+    return;
+
+  compressed = do_lz4_compress(buf, &work, compctx, frame);
+
+  /* On Error just return */
+  if (buf->len == 0)
+    return;
+
+  /* did compression save us anything?  Include 2 byte compression header
+     in calculation */
+  if (compressed && work.len + 2 < buf->len)
+    {
+      ASSERT(buf_prepend(&work, 2));
+      uint8_t *head = BPTR (&work);
+      head[0] = COMP_ALGV2_INDICATOR_BYTE;
+      head[1] = COMP_ALGV2_LZ4_BYTE;
+      *buf = work;
+    }
+  else
+    {
+      compv2_escape_data_ifneeded(buf);
+    }
+}
+
+void
+do_lz4_decompress(size_t zlen_max,
+                 struct buffer *work,
+                 struct buffer *buf,
+                 struct compress_context *compctx)
+{
+  int uncomp_len;
+  ASSERT (buf_safe (work, zlen_max));
+  uncomp_len = LZ4_decompress_safe((const char *)BPTR(buf), (char *)BPTR(work), (size_t)BLEN(buf), zlen_max);
+  if (uncomp_len <= 0)
+    {
+      dmsg (D_COMP_ERRORS, "LZ4 decompression error: %d", uncomp_len);
+      buf->len = 0;
+      return;
+    }
+
+  ASSERT (buf_safe (work, uncomp_len));
+  work->len = uncomp_len;
+
+  dmsg (D_COMP, "LZ4 decompress %d -> %d", buf->len, work->len);
+  compctx->pre_decompress += buf->len;
+  compctx->post_decompress += work->len;
+
+  *buf = *work;
+}
+
 static void
 lz4_decompress (struct buffer *buf, struct buffer work,
-                struct compress_context *compctx,
-                const struct frame* frame)
+               struct compress_context *compctx,
+               const struct frame* frame)
 {
   size_t zlen_max = EXPANDED_SIZE (frame);
-  int uncomp_len;
   uint8_t c;           /* flag indicating whether or not our peer compressed */
 
   if (buf->len <= 0)
@@ -152,23 +227,7 @@ lz4_decompress (struct buffer *buf, struct buffer work,
 
   if (c == LZ4_COMPRESS_BYTE)  /* packet was compressed */
     {
-      ASSERT (buf_safe (&work, zlen_max));
-      uncomp_len = LZ4_decompress_safe((const char *)BPTR(buf), (char *)BPTR(&work), (size_t)BLEN(buf), zlen_max);
-      if (uncomp_len <= 0)
-       {
-         dmsg (D_COMP_ERRORS, "LZ4 decompression error: %d", uncomp_len);
-         buf->len = 0;
-         return;
-       }
-
-      ASSERT (buf_safe (&work, uncomp_len));
-      work.len = uncomp_len;
-
-      dmsg (D_COMP, "LZ4 decompress %d -> %d", buf->len, work.len);
-      compctx->pre_decompress += buf->len;
-      compctx->post_decompress += work.len;
-
-      *buf = work;
+      do_lz4_decompress(zlen_max, &work, buf, compctx);
     }
   else if (c == NO_COMPRESS_BYTE_SWAP) /* packet was not compressed */
     {
@@ -181,6 +240,52 @@ lz4_decompress (struct buffer *buf, struct buffer work,
     }
 }
 
+static void
+lz4v2_decompress (struct buffer *buf, struct buffer work,
+                 struct compress_context *compctx,
+                 const struct frame* frame)
+{
+  size_t zlen_max = EXPANDED_SIZE (frame);
+  uint8_t c;           /* flag indicating whether or not our peer compressed */
+
+  if (buf->len <= 0)
+    return;
+
+  ASSERT (buf_init (&work, FRAME_HEADROOM (frame)));
+
+  /* do unframing/swap (assumes buf->len > 0) */
+  uint8_t *head = BPTR (buf);
+  c = *head;
+
+  /* Not compressed */
+  if (c != COMP_ALGV2_INDICATOR_BYTE) {
+    return;
+  }
+
+  /* Packet to short to make sense */
+  if (buf->len <= 1)
+    {
+      buf->len=0;
+      return;
+    }
+
+  c = head[1];
+  if (c == COMP_ALGV2_LZ4_BYTE) /* packet was compressed */
+    {
+      buf_advance(buf,2);
+      do_lz4_decompress(zlen_max, &work, buf, compctx);
+    }
+  else if (c == COMP_ALGV2_UNCOMPRESSED_BYTE)
+    {
+      buf_advance(buf,2);
+    }
+  else
+    {
+      dmsg (D_COMP_ERRORS, "Bad LZ4v2 decompression header byte: %d", c);
+      buf->len = 0;
+    }
+}
+
 const struct compress_alg lz4_alg = {
   "lz4",
   lz4_compress_init,
@@ -189,6 +294,14 @@ const struct compress_alg lz4_alg = {
   lz4_decompress
 };
 
+const struct compress_alg lz4v2_alg = {
+  "lz4v2",
+  lz4v2_compress_init,
+  lz4_compress_uninit,
+  lz4v2_compress,
+  lz4v2_decompress
+};
+
 #else
 static void dummy(void) {}
 #endif /* ENABLE_LZ4 */

diff --git a/src/openvpn/comp-lz4.h b/src/openvpn/comp-lz4.h
index ca1dfa9bc76cd832d8b56fe1569f79b849f10189..9d3c6644f655701780fb4bcab8c75e9637f694bd 100644 (file)
--- a/src/openvpn/comp-lz4.h
+++ b/src/openvpn/comp-lz4.h
@@ -31,6 +31,7 @@
 #include "buffer.h"
 
 extern const struct compress_alg lz4_alg;
+extern const struct compress_alg lz4v2_alg;
 
 struct lz4_workspace
 {

diff --git a/src/openvpn/comp.c b/src/openvpn/comp.c
index 706ad7ea8e704679c57398c48b5d7c1e550855e5..b420ea530dac0a7cec6e92732baebafc8d7c66c2 100644 (file)
--- a/src/openvpn/comp.c
+++ b/src/openvpn/comp.c
@@ -50,6 +50,11 @@ comp_init(const struct compress_options *opt)
       compctx->alg = comp_stub_alg;
       (*compctx->alg.compress_init)(compctx);
       break;
+    case COMP_ALGV2_UNCOMPRESSED:
+      ALLOC_OBJ_CLEAR (compctx, struct compress_context);
+      compctx->flags = opt->flags;
+      compctx->alg = compv2_stub_alg;
+      break;
 #ifdef ENABLE_LZO
     case COMP_ALG_LZO:
       ALLOC_OBJ_CLEAR (compctx, struct compress_context);
@@ -65,11 +70,35 @@ comp_init(const struct compress_options *opt)
       compctx->alg = lz4_alg;
       (*compctx->alg.compress_init)(compctx);
       break;
+    case COMP_ALGV2_LZ4:
+      ALLOC_OBJ_CLEAR (compctx, struct compress_context);
+      compctx->flags = opt->flags;
+      compctx->alg = lz4v2_alg;
+      break;
 #endif
     }
   return compctx;
 }
 
+/* In the v2 compression schemes, an uncompressed packet has
+ * has no opcode in front, unless the first byte is 0x50. In this
+ * case the packet needs to be escaped */
+void
+compv2_escape_data_ifneeded (struct buffer *buf)
+{
+    uint8_t *head = BPTR (buf);
+    if (head[0] != COMP_ALGV2_INDICATOR_BYTE)
+       return;
+
+    /* Header is 0x50 */
+    ASSERT(buf_prepend(buf, 2));
+
+    head = BPTR (buf);
+    head[0] = COMP_ALGV2_INDICATOR_BYTE;
+    head[1] = COMP_ALGV2_UNCOMPRESSED;
+}
+
+
 void
 comp_uninit(struct compress_context *compctx)
 {
@@ -120,6 +149,7 @@ comp_generate_peer_info_string(const struct compress_options *opt, struct buffer
        {
 #if defined(ENABLE_LZ4)
          buf_printf (out, "IV_LZ4=1\n");
+         buf_printf (out, "IV_LZ4v2=1\n");
 #endif
 #if defined(ENABLE_LZO)
          buf_printf (out, "IV_LZO=1\n");
@@ -129,6 +159,7 @@ comp_generate_peer_info_string(const struct compress_options *opt, struct buffer
       if (!lzo_avail)
        buf_printf (out, "IV_LZO_STUB=1\n");
       buf_printf (out, "IV_COMP_STUB=1\n");
+      buf_printf (out, "IV_COMP_STUBv2=1\n");
     }
 }
 

diff --git a/src/openvpn/comp.h b/src/openvpn/comp.h
index 716b1c0a8df7bee4cc3bbea5135dde2311844b5c..9ed9532939c58618037f678e9018ef0293995a3b 100644 (file)
--- a/src/openvpn/comp.h
+++ b/src/openvpn/comp.h
@@ -43,12 +43,22 @@
 #define COMP_ALG_SNAPPY 3 /* Snappy algorithm (no longer supported) */
 #define COMP_ALG_LZ4    4 /* LZ4 algorithm */
 
+
+/* algorithm v2 */
+#define COMP_ALGV2_UNCOMPRESSED 10
+#define COMP_ALGV2_LZ4     11
+/*
+#define COMP_ALGV2_LZO     12
+#define COMP_ALGV2_SNAPPY   13
+*/
+
 /* Compression flags */
 #define COMP_F_ADAPTIVE   (1<<0) /* COMP_ALG_LZO only */
 #define COMP_F_ASYM       (1<<1) /* only downlink is compressed, not uplink */
 #define COMP_F_SWAP       (1<<2) /* initial command byte is swapped with last byte in buffer to preserve payload alignment */
 #define COMP_F_ADVERTISE_STUBS_ONLY (1<<3) /* tell server that we only support compression stubs */
 
+
 /*
  * Length of prepended prefix on compressed packets
  */
@@ -57,9 +67,21 @@
 /*
  * Prefix bytes
  */
+
+/* V1 on wire codes */
+/* Initial command byte to tell our peer if we compressed */
+#define LZO_COMPRESS_BYTE 0x66
+#define LZ4_COMPRESS_BYTE 0x69
 #define NO_COMPRESS_BYTE      0xFA
 #define NO_COMPRESS_BYTE_SWAP 0xFB /* to maintain payload alignment, replace this byte with last byte of packet */
 
+/* V2 on wire code */
+#define COMP_ALGV2_INDICATOR_BYTE      0x50
+#define COMP_ALGV2_UNCOMPRESSED_BYTE   0
+#define COMP_ALGV2_LZ4_BYTE            1
+#define COMP_ALGV2_LZO_BYTE            2
+#define COMP_ALGV2_SNAPPY_BYTE         3
+
 /*
  * Compress worst case size expansion (for any algorithm)
  *
@@ -145,6 +167,7 @@ struct compress_context
 };
 
 extern const struct compress_alg comp_stub_alg;
+extern const struct compress_alg compv2_stub_alg;
 
 struct compress_context *comp_init(const struct compress_options *opt);
 
@@ -157,6 +180,8 @@ void comp_print_stats (const struct compress_context *compctx, struct status_out
 
 void comp_generate_peer_info_string(const struct compress_options *opt, struct buffer *out);
 
+void compv2_escape_data_ifneeded (struct buffer *buf);
+
 static inline bool
 comp_enabled(const struct compress_options *info)
 {

diff --git a/src/openvpn/compstub.c b/src/openvpn/compstub.c
index 2ab7163edd605d6c100f4b714367a3f6d7dcf11b..9c6aad2de702a18b350a8d70f09825bc2a4ffaf5 100644 (file)
--- a/src/openvpn/compstub.c
+++ b/src/openvpn/compstub.c
@@ -105,6 +105,57 @@ stub_decompress (struct buffer *buf, struct buffer work,
     }
 }
 
+
+static void
+stubv2_compress (struct buffer *buf, struct buffer work,
+                struct compress_context *compctx,
+                const struct frame* frame)
+{
+    if (buf->len <= 0)
+       return;
+
+    compv2_escape_data_ifneeded (buf);
+}
+
+static void
+stubv2_decompress (struct buffer *buf, struct buffer work,
+                  struct compress_context *compctx,
+                  const struct frame* frame)
+{
+  if (buf->len <= 0)
+    return;
+
+  uint8_t *head = BPTR (buf);
+
+  /* no compression or packet to short*/
+  if (head[0] != COMP_ALGV2_INDICATOR_BYTE)
+    return;
+
+  /* compression header (0x50) is present */
+  buf_advance(buf, 1);
+
+  /* Packet buffer too short (only 1 byte) */
+  if (buf->len <= 0)
+    return;
+
+  head = BPTR (buf);
+  buf_advance(buf, 1);
+
+  if (head[0] != COMP_ALGV2_UNCOMPRESSED_BYTE) {
+    dmsg (D_COMP_ERRORS, "Bad compression stubv2 decompression header byte: %d", *head);
+    buf->len = 0;
+    return;
+  }
+}
+
+const struct compress_alg compv2_stub_alg = {
+  "stubv2",
+  stub_compress_init,
+  stub_compress_uninit,
+  stubv2_compress,
+  stubv2_decompress
+};
+
 const struct compress_alg comp_stub_alg = {
   "stub",
   stub_compress_init,

diff --git a/src/openvpn/lzo.c b/src/openvpn/lzo.c
index daa02ed0e7404458b1c5ac4cb58bc07c25b0e396..25a839bad0ff4cad83a434b82a508f600c27cb72 100644 (file)
--- a/src/openvpn/lzo.c
+++ b/src/openvpn/lzo.c
@@ -42,9 +42,6 @@
 
 #include "memdbg.h"
 
-/* Initial command byte to tell our peer if we compressed */
-#define LZO_COMPRESS_BYTE 0x66
-
 /**
  * Perform adaptive compression housekeeping.
  *

diff --git a/src/openvpn/options.c b/src/openvpn/options.c
index 6861149b3cbbf3da2126b776389d95812bb3f2b1..f154c62ad095d300c31300f350dd4cb73103b4fd 100644 (file)
--- a/src/openvpn/options.c
+++ b/src/openvpn/options.c
@@ -6344,16 +6344,21 @@ add_option (struct options *options,
       VERIFY_PERMISSION (OPT_P_COMP);
       if (p[1])
        {
+         options->comp.flags = 0;
          if (streq (p[1], "stub"))
            {
              options->comp.alg = COMP_ALG_STUB;
              options->comp.flags = (COMP_F_SWAP|COMP_F_ADVERTISE_STUBS_ONLY);
            }
+         else if (streq(p[1], "stub-v2"))
+           {
+             options->comp.alg = COMP_ALGV2_UNCOMPRESSED;
+             options->comp.flags = COMP_F_ADVERTISE_STUBS_ONLY;
+           }
 #if defined(ENABLE_LZO)
          else if (streq (p[1], "lzo"))
            {
              options->comp.alg = COMP_ALG_LZO;
-             options->comp.flags = 0;
            }
 #endif
 #if defined(ENABLE_LZ4)
@@ -6362,6 +6367,10 @@ add_option (struct options *options,
              options->comp.alg = COMP_ALG_LZ4;
              options->comp.flags = COMP_F_SWAP;
            }
+         else if (streq (p[1], "lz4-v2"))
+           {
+             options->comp.alg = COMP_ALGV2_LZ4;
+           }
 #endif
          else
            {