Count zlib buffer memory towards OOM totals.

Part of 11792.

(Uses the zlib-endorsed formula for memory needs for inflate/deflate
from "zconf.h".)

diff --git a/changes/bug11792 b/changes/bug11792
index 1d381891dbfda91ca0056f87d8a43e1cff29c406..7395b4aba59b05b7d91c74460928dbdff9760d26 100644 (file)
--- a/changes/bug11792
+++ b/changes/bug11792
@@ -2,4 +2,10 @@
     - When closing an edge connection because we've run out of memory,
       also count the amount of memory that any tunnelled directory
       connection attached to that connection had consumed. Part of
-      ticket 11792.
\ No newline at end of file
+      ticket 11792.
+
+    - When considering whether we're running low on memory, consider
+      memory that was allocated as part of zlib buffers as well.
+      Count that memory as reclaimed by our OOM handler. Part of
+      ticket 11792.
+

diff --git a/src/common/torgzip.c b/src/common/torgzip.c
index 15451ee30d0ca34e6a6441d4987a9f1af8b14623..67fd742818f9b4cb8aaaea94d0edcae5bbe23622 100644 (file)
--- a/src/common/torgzip.c
+++ b/src/common/torgzip.c
@@ -46,6 +46,12 @@
 
 #include <zlib.h>
 
+static size_t tor_zlib_state_size_precalc(int inflate,
+                                          int windowbits, int memlevel);
+
+/** Total number of bytes allocated for zlib state */
+static size_t total_zlib_allocation = 0;
+
 /** Set to 1 if zlib is a version that supports gzip; set to 0 if it doesn't;
  * set to -1 if we haven't checked yet. */
 static int gzip_is_supported = -1;
@@ -411,6 +417,9 @@ struct tor_zlib_state_t {
   size_t input_so_far;
   /** Number of bytes written so far.  Used to detect zlib bombs. */
   size_t output_so_far;
+
+  /** Approximate number of bytes allocated for this object. */
+  size_t allocation;
 };
 
 /** Construct and return a tor_zlib_state_t object using <b>method</b>.  If
@@ -420,6 +429,7 @@ tor_zlib_state_t *
 tor_zlib_new(int compress, compress_method_t method)
 {
   tor_zlib_state_t *out;
+  int bits;
 
   if (method == GZIP_METHOD && !is_gzip_supported()) {
     /* Old zlib version don't support gzip in inflateInit2 */
@@ -432,14 +442,19 @@ tor_zlib_new(int compress, compress_method_t method)
  out->stream.zfree = Z_NULL;
  out->stream.opaque = NULL;
  out->compress = compress;
+ bits = method_bits(method);
  if (compress) {
    if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
-                    method_bits(method), 8, Z_DEFAULT_STRATEGY) != Z_OK)
+                    bits, 8, Z_DEFAULT_STRATEGY) != Z_OK)
      goto err;
  } else {
-   if (inflateInit2(&out->stream, method_bits(method)) != Z_OK)
+   if (inflateInit2(&out->stream, bits) != Z_OK)
      goto err;
  }
+ out->allocation = tor_zlib_state_size_precalc(!compress, bits, 8);
+
+ total_zlib_allocation += out->allocation;
+
  return out;
 
  err:
@@ -517,6 +532,8 @@ tor_zlib_free(tor_zlib_state_t *state)
   if (!state)
     return;
 
+ total_zlib_allocation -= state->allocation;
+
   if (state->compress)
     deflateEnd(&state->stream);
   else
@@ -525,3 +542,47 @@ tor_zlib_free(tor_zlib_state_t *state)
   tor_free(state);
 }
 
+/** Return an approximate number of bytes used in RAM to hold a state with
+ * window bits <b>windowBits</b> and compression level 'memlevel' */
+static size_t
+tor_zlib_state_size_precalc(int inflate, int windowbits, int memlevel)
+{
+  windowbits &= 15;
+
+#define A_FEW_KILOBYTES 2048
+
+  if (inflate) {
+    /* From zconf.h:
+
+       "The memory requirements for inflate are (in bytes) 1 << windowBits
+       that is, 32K for windowBits=15 (default value) plus a few kilobytes
+       for small objects."
+    */
+    return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) +
+      (1 << 15) + A_FEW_KILOBYTES;
+  } else {
+    /* Also from zconf.h:
+
+       "The memory requirements for deflate are (in bytes):
+            (1 << (windowBits+2)) +  (1 << (memLevel+9))
+        ... plus a few kilobytes for small objects."
+    */
+    return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) +
+      (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES;
+  }
+#undef A_FEW_KILOBYTES
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_zlib_state_size(const tor_zlib_state_t *state)
+{
+  return state->allocation;
+}
+
+/** Return the approximate number of bytes allocated for all zlib states. */
+size_t
+tor_zlib_get_total_allocation(void)
+{
+  return total_zlib_allocation;
+}

diff --git a/src/common/torgzip.h b/src/common/torgzip.h
index 5db03fe6e0ba1897b437d549f682c668deb2a944..d407bf48c60ce9d221b9a1784aaee15e14f263fb 100644 (file)
--- a/src/common/torgzip.h
+++ b/src/common/torgzip.h
@@ -55,5 +55,8 @@ tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state,
                                    int finish);
 void tor_zlib_free(tor_zlib_state_t *state);
 
+size_t tor_zlib_state_size(const tor_zlib_state_t *state);
+size_t tor_zlib_get_total_allocation(void);
+
 #endif
 

diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c
index 9aeb3eb19ac888e0be7f5588650d3c1cd11d22e7..c738f9357fc8877b176cd6d1a5314d74fcae64d3 100644 (file)
--- a/src/or/circuitlist.c
+++ b/src/or/circuitlist.c
@@ -1811,6 +1811,14 @@ marked_circuit_single_conn_free_bytes(connection_t *conn)
     result += buf_allocation(conn->outbuf);
     buf_clear(conn->outbuf);
   }
+  if (conn->type == CONN_TYPE_DIR) {
+    dir_connection_t *dir_conn = TO_DIR_CONN(conn);
+    if (dir_conn->zlib_state) {
+      result += tor_zlib_state_size(dir_conn->zlib_state);
+      tor_zlib_free(dir_conn->zlib_state);
+      dir_conn->zlib_state = NULL;
+    }
+  }
   return result;
 }
 

diff --git a/src/or/relay.c b/src/or/relay.c
index f42602d412cb74751e4342fe1885a5e277a4993c..7d7ef7a8350a18e022200c60fe4fc7fba0ef6c03 100644 (file)
--- a/src/or/relay.c
+++ b/src/or/relay.c
@@ -2433,6 +2433,7 @@ cell_queues_check_size(void)
 {
   size_t alloc = cell_queues_get_total_allocation();
   alloc += buf_get_total_allocation();
+  alloc += tor_zlib_get_total_allocation();
   if (alloc >= get_options()->MaxMemInQueues) {
     circuits_handle_oom(alloc);
     return 1;