From: Stefan Eissing <icing@apache.org>
Date: Tue, 22 Dec 2015 15:52:10 +0000 (+0000)
Subject: rewrote http2 connection state handling, fixed keepalive timeout handling for async... 
X-Git-Tag: 2.5.0-alpha~2504
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7d209de21f4ea8c68a04db6be1d78ca9face4a75;p=thirdparty%2Fapache%2Fhttpd.git

rewrote http2 connection state handling, fixed keepalive timeout handling for async and sync MPMs, cleaned logging, improved scoreboard updates of http2 master connections

git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1721417 13f79535-47bb-0310-9956-ffa450edef68
---

diff --git a/CHANGES b/CHANGES
index cbe47436e6f..c0323386f1e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -2,7 +2,9 @@
 Changes with Apache 2.5.0
 
   *) mod_http2: On async MPMs (event), idle HTTP/2 connections will enter KEEPALIVE
-     state and become eligible for MPM cleanup under load.
+     state and become eligible for MPM cleanup under load. Rewrote connection
+     state handling. Defaults for H2KeepAliveTimeout shortened. Cleaned up logging.
+     Improved scoreboard updates for http2 master threads.
      [Stefan Eissing]
 
   *) mod_http2: new r->subprocess_env variables HTTP2 and H2PUSH, set to "on"
diff --git a/modules/http2/h2_config.c b/modules/http2/h2_config.c
index 329609923fd..a7b63fd66f2 100644
--- a/modules/http2/h2_config.c
+++ b/modules/http2/h2_config.c
@@ -60,12 +60,11 @@ static h2_config defconf = {
     1,                      /* HTTP/2 server push enabled */
     NULL,                   /* map of content-type to priorities */
     5,                      /* normal connection timeout */
-    30,                     /* idle connection timeout */
+    5,                      /* keepalive timeout */
     0,                      /* stream timeout */
 };
 
 static int files_per_session;
-static int async_mpm;
 
 void h2_config_init(apr_pool_t *pool)
 {
@@ -90,14 +89,6 @@ void h2_config_init(apr_pool_t *pool)
             /* don't know anything about it, stay safe */
             break;
     }
-    if (ap_mpm_query(AP_MPMQ_IS_ASYNC, &async_mpm) != APR_SUCCESS) {
-        async_mpm = 0;
-    }
-}
-
-int h2_config_async_mpm(void)
-{
-    return async_mpm;
 }
 
 static void *h2_config_create(apr_pool_t *pool,
diff --git a/modules/http2/h2_config.h b/modules/http2/h2_config.h
index fab9b9fead2..57d926d5f70 100644
--- a/modules/http2/h2_config.h
+++ b/modules/http2/h2_config.h
@@ -95,7 +95,5 @@ void h2_config_init(apr_pool_t *pool);
 const struct h2_priority *h2_config_get_priority(const h2_config *conf, 
                                                  const char *content_type);
        
-int h2_config_async_mpm(void);
-
 #endif /* __mod_h2__h2_config_h__ */
 
diff --git a/modules/http2/h2_conn.c b/modules/http2/h2_conn.c
index b2f2127c1d3..2c8e1e80dd8 100644
--- a/modules/http2/h2_conn.c
+++ b/modules/http2/h2_conn.c
@@ -43,6 +43,7 @@ static struct h2_workers *workers;
 
 static h2_mpm_type_t mpm_type = H2_MPM_UNKNOWN;
 static module *mpm_module;
+static int async_mpm;
 
 static void check_modules(int force) 
 {
@@ -77,17 +78,24 @@ apr_status_t h2_conn_child_init(apr_pool_t *pool, server_rec *s)
 {
     const h2_config *config = h2_config_sget(s);
     apr_status_t status = APR_SUCCESS;
-    int minw = h2_config_geti(config, H2_CONF_MIN_WORKERS);
-    int maxw = h2_config_geti(config, H2_CONF_MAX_WORKERS);    
+    int minw, maxw;
     int max_threads_per_child = 0;
     int idle_secs = 0;
 
-    h2_config_init(pool);
+    check_modules(1);
     
     ap_mpm_query(AP_MPMQ_MAX_THREADS, &max_threads_per_child);
     
-    check_modules(1);
+    status = ap_mpm_query(AP_MPMQ_IS_ASYNC, &async_mpm);
+    if (status != APR_SUCCESS) {
+        ap_log_error(APLOG_MARK, APLOG_ERR, status, s, "querying MPM for async");
+        async_mpm = 0;
+    }
+
+    h2_config_init(pool);
     
+    minw = h2_config_geti(config, H2_CONF_MIN_WORKERS);
+    maxw = h2_config_geti(config, H2_CONF_MAX_WORKERS);    
     if (minw <= 0) {
         minw = max_threads_per_child;
     }
@@ -145,9 +153,8 @@ apr_status_t h2_conn_setup(h2_ctx *ctx, conn_rec *c, request_rec *r)
     return APR_SUCCESS;
 }
 
-apr_status_t h2_conn_process(h2_ctx *ctx, int async)
+static apr_status_t h2_conn_process(h2_ctx *ctx)
 {
-    apr_status_t status;
     h2_session *session;
     
     session = h2_ctx_session_get(ctx);
@@ -155,15 +162,15 @@ apr_status_t h2_conn_process(h2_ctx *ctx, int async)
         session->c->cs->sense = CONN_SENSE_DEFAULT;
     }
 
-    status = h2_session_process(session, async);
+    h2_session_process(session, async_mpm);
 
     session->c->keepalive = AP_CONN_KEEPALIVE;
     if (session->c->cs) {
         session->c->cs->state = CONN_STATE_WRITE_COMPLETION;
     }
     
-    if (status == APR_EOF) {
-        ap_log_cerror( APLOG_MARK, APLOG_DEBUG, status, session->c,
+    if (session->state == H2_SESSION_ST_CLOSING) {
+        ap_log_cerror( APLOG_MARK, APLOG_DEBUG, 0, session->c,
                       "h2_session(%ld): done", session->id);
         /* Make sure this connection gets closed properly. */
         ap_update_child_status_from_conn(session->c->sbh, SERVER_CLOSING, session->c);
@@ -178,9 +185,16 @@ apr_status_t h2_conn_process(h2_ctx *ctx, int async)
 
 apr_status_t h2_conn_run(struct h2_ctx *ctx, conn_rec *c)
 {
+    int mpm_state = 0;
     do {
-        h2_conn_process(ctx, 0);
-    } while (c->keepalive == AP_CONN_KEEPALIVE && !c->aborted);
+        h2_conn_process(ctx);
+        
+        if (ap_mpm_query(AP_MPMQ_MPM_STATE, &mpm_state)) {
+            break;
+        }
+    } while (!async_mpm 
+             && c->keepalive == AP_CONN_KEEPALIVE 
+             && mpm_state != AP_MPMQ_STOPPING);
     
     return DONE;
 }
diff --git a/modules/http2/h2_conn.h b/modules/http2/h2_conn.h
index 87bc98c386a..d59eb46dd95 100644
--- a/modules/http2/h2_conn.h
+++ b/modules/http2/h2_conn.h
@@ -28,16 +28,6 @@ struct h2_task;
  */
 apr_status_t h2_conn_setup(struct h2_ctx *ctx, conn_rec *c, request_rec *r);
 
-/**
- * Process the HTTP/2 connection. Return whenever blocking reads or
- * long writes have to be performed.
- *
- * @param ctx the http2 context to process
- * @return APR_SUCCESS as long as processing needs to continue, APR_EOF
- *         when HTTP/2 session is done.
- */
-apr_status_t h2_conn_process(struct h2_ctx *ctx, int async);
-
 /**
  * Run the HTTP/2 connection in synchronous fashion. 
  * Return when the HTTP/2 session is done
diff --git a/modules/http2/h2_conn_io.c b/modules/http2/h2_conn_io.c
index d0bb7408fe9..2c68db59a82 100644
--- a/modules/http2/h2_conn_io.c
+++ b/modules/http2/h2_conn_io.c
@@ -79,7 +79,7 @@ apr_status_t h2_conn_io_init(h2_conn_io *io, conn_rec *c,
     }
 
     if (APLOGctrace1(c)) {
-        ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, io->connection,
+        ap_log_cerror(APLOG_MARK, APLOG_TRACE4, 0, io->connection,
                       "h2_conn_io(%ld): init, buffering=%d, warmup_size=%ld, cd_secs=%f",
                       io->connection->id, io->buffer_output, (long)io->warmup_size,
                       ((float)io->cooldown_usecs/APR_USEC_PER_SEC));
@@ -106,7 +106,7 @@ static apr_status_t pass_out(apr_bucket_brigade *bb, void *ctx)
     ap_update_child_status(io->connection->sbh, SERVER_BUSY_WRITE, NULL);
     status = apr_brigade_length(bb, 0, &bblen);
     if (status == APR_SUCCESS) {
-        ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, io->connection,
+        ap_log_cerror(APLOG_MARK, APLOG_TRACE4, 0, io->connection,
                       "h2_conn_io(%ld): pass_out brigade %ld bytes",
                       io->connection->id, (long)bblen);
         status = ap_pass_brigade(io->connection->output_filters, bb);
@@ -135,7 +135,7 @@ static apr_status_t bucketeer_buffer(h2_conn_io *io)
         /* long time not written, reset write size */
         io->write_size = WRITE_SIZE_INITIAL;
         io->bytes_written = 0;
-        ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, io->connection,
+        ap_log_cerror(APLOG_MARK, APLOG_TRACE4, 0, io->connection,
                       "h2_conn_io(%ld): timeout write size reset to %ld", 
                       (long)io->connection->id, (long)io->write_size);
     }
@@ -143,7 +143,7 @@ static apr_status_t bucketeer_buffer(h2_conn_io *io)
              && io->bytes_written >= io->warmup_size) {
         /* connection is hot, use max size */
         io->write_size = WRITE_SIZE_MAX;
-        ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, io->connection,
+        ap_log_cerror(APLOG_MARK, APLOG_TRACE4, 0, io->connection,
                       "h2_conn_io(%ld): threshold reached, write size now %ld", 
                       (long)io->connection->id, (long)io->write_size);
     }
@@ -172,7 +172,7 @@ apr_status_t h2_conn_io_write(h2_conn_io *io,
     
     io->unflushed = 1;
     if (io->bufsize > 0) {
-        ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, io->connection,
+        ap_log_cerror(APLOG_MARK, APLOG_TRACE4, 0, io->connection,
                       "h2_conn_io: buffering %ld bytes", (long)length);
                       
         if (!APR_BRIGADE_EMPTY(io->output)) {
@@ -203,7 +203,7 @@ apr_status_t h2_conn_io_write(h2_conn_io *io,
         
     }
     else {
-        ap_log_cerror(APLOG_MARK, APLOG_TRACE2, status, io->connection,
+        ap_log_cerror(APLOG_MARK, APLOG_TRACE4, status, io->connection,
                       "h2_conn_io: writing %ld bytes to brigade", (long)length);
         status = apr_brigade_write(io->output, pass_out, io, buf, length);
     }
@@ -247,7 +247,7 @@ static apr_status_t h2_conn_io_flush_int(h2_conn_io *io, int force)
     if (io->unflushed || force) {
         if (io->buflen > 0) {
             /* something in the buffer, put it in the output brigade */
-            ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, io->connection,
+            ap_log_cerror(APLOG_MARK, APLOG_TRACE4, 0, io->connection,
                           "h2_conn_io: flush, flushing %ld bytes", (long)io->buflen);
             bucketeer_buffer(io);
             io->buflen = 0;
@@ -258,7 +258,7 @@ static apr_status_t h2_conn_io_flush_int(h2_conn_io *io, int force)
                                     apr_bucket_flush_create(io->output->bucket_alloc));
         }
         
-        ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, io->connection,
+        ap_log_cerror(APLOG_MARK, APLOG_TRACE4, 0, io->connection,
                       "h2_conn_io: flush");
         /* Send it out */
         io->unflushed = 0;
@@ -279,17 +279,3 @@ apr_status_t h2_conn_io_pass(h2_conn_io *io)
     return h2_conn_io_flush_int(io, 0);
 }
 
-apr_status_t h2_conn_io_close(h2_conn_io *io, void *session)
-{
-    apr_bucket *b;
-
-    /* Send out anything in our buffers */
-    h2_conn_io_flush_int(io, 0);
-    
-    b = h2_bucket_eoc_create(io->connection->bucket_alloc, session);
-    APR_BRIGADE_INSERT_TAIL(io->output, b);
-    b = apr_bucket_flush_create(io->connection->bucket_alloc);
-    APR_BRIGADE_INSERT_TAIL(io->output, b);
-    return ap_pass_brigade(io->connection->output_filters, io->output);
-    /* and all is gone */
-}
\ No newline at end of file
diff --git a/modules/http2/h2_conn_io.h b/modules/http2/h2_conn_io.h
index 4430b06f4df..e55f08aa5d8 100644
--- a/modules/http2/h2_conn_io.h
+++ b/modules/http2/h2_conn_io.h
@@ -60,6 +60,5 @@ apr_status_t h2_conn_io_consider_flush(h2_conn_io *io);
 
 apr_status_t h2_conn_io_pass(h2_conn_io *io);
 apr_status_t h2_conn_io_flush(h2_conn_io *io);
-apr_status_t h2_conn_io_close(h2_conn_io *io, void *session);
 
 #endif /* defined(__mod_h2__h2_conn_io__) */
diff --git a/modules/http2/h2_filter.c b/modules/http2/h2_filter.c
index 85a8d87ed23..1155ac4dad4 100644
--- a/modules/http2/h2_filter.c
+++ b/modules/http2/h2_filter.c
@@ -58,7 +58,7 @@ static apr_status_t consume_brigade(h2_filter_cin *cin,
                     status = apr_bucket_split(bucket, consumed);
                 }
                 readlen += consumed;
-                cin->last_read = apr_time_now();
+                cin->start_read = apr_time_now();
             }
         }
         apr_bucket_delete(bucket);
@@ -70,22 +70,6 @@ static apr_status_t consume_brigade(h2_filter_cin *cin,
     return status;
 }
 
-static apr_status_t check_time_left(h2_filter_cin *cin,
-                                    apr_time_t *ptime_left)
-{
-    if (cin->timeout_secs > 0) {
-        *ptime_left = (cin->last_read + apr_time_from_sec(cin->timeout_secs)
-                       - apr_time_now());
-        if (*ptime_left <= 0)
-            return APR_TIMEUP;
-        
-        if (*ptime_left < apr_time_from_sec(1)) {
-            *ptime_left = apr_time_from_sec(1);
-        }
-    }
-    return APR_SUCCESS;
-}
-
 h2_filter_cin *h2_filter_cin_create(apr_pool_t *p, h2_filter_cin_cb *cb, void *ctx)
 {
     h2_filter_cin *cin;
@@ -94,7 +78,7 @@ h2_filter_cin *h2_filter_cin_create(apr_pool_t *p, h2_filter_cin_cb *cb, void *c
     cin->pool      = p;
     cin->cb        = cb;
     cin->cb_ctx    = ctx;
-    cin->last_read = UNSET;
+    cin->start_read = UNSET;
     return cin;
 }
 
@@ -112,11 +96,11 @@ apr_status_t h2_filter_core_input(ap_filter_t* f,
     h2_filter_cin *cin = f->ctx;
     apr_status_t status = APR_SUCCESS;
     apr_time_t saved_timeout = UNSET;
-    apr_time_t time_left = UNSET;
     
     ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, f->c,
-                  "core_input: read, block=%d, mode=%d, readbytes=%ld", 
-                  block, mode, (long)readbytes);
+                  "core_input(%ld): read, %s, mode=%d, readbytes=%ld, timeout=%d", 
+                  (long)f->c->id, (block == APR_BLOCK_READ)? "BLOCK_READ" : "NONBLOCK_READ", 
+                  mode, (long)readbytes, cin->timeout_secs);
     
     if (mode == AP_MODE_INIT || mode == AP_MODE_SPECULATIVE) {
         return ap_get_brigade(f->next, brigade, mode, block, readbytes);
@@ -132,9 +116,9 @@ apr_status_t h2_filter_core_input(ap_filter_t* f,
 
     if (!cin->socket) {
         cin->socket = ap_get_conn_socket(f->c);
-        cin->last_read = apr_time_now(); /* first call */
     }
     
+    cin->start_read = apr_time_now();
     if (APR_BRIGADE_EMPTY(cin->bb)) {
         /* We only do a blocking read when we have no streams to process. So,
          * in httpd scoreboard lingo, we are in a KEEPALIVE connection state.
@@ -142,40 +126,30 @@ apr_status_t h2_filter_core_input(ap_filter_t* f,
          * child with NULL request. That way, any current request information
          * in the scoreboard is preserved.
          */
-        status = check_time_left(cin, &time_left);
-        if (status != APR_SUCCESS)
-            goto out;
-        
         if (block == APR_BLOCK_READ) {
-            ap_update_child_status_from_conn(f->c->sbh, 
-                                             SERVER_BUSY_KEEPALIVE, f->c);
-            if (time_left > 0) {
-                status = apr_socket_timeout_get(cin->socket, &saved_timeout);
-                AP_DEBUG_ASSERT(status == APR_SUCCESS);
-                status = apr_socket_timeout_set(cin->socket, H2MIN(time_left, saved_timeout));
-                AP_DEBUG_ASSERT(status == APR_SUCCESS);
+            if (cin->timeout_secs > 0) {
+                apr_time_t t = apr_time_from_sec(cin->timeout_secs);
+                apr_socket_timeout_get(cin->socket, &saved_timeout);
+                apr_socket_timeout_set(cin->socket, H2MIN(t, saved_timeout));
             }
         }
-        else {
-            ap_update_child_status(f->c->sbh, SERVER_BUSY_READ, NULL);
-        }
-        
+        ap_update_child_status_from_conn(f->c->sbh, SERVER_BUSY_READ, f->c);
         status = ap_get_brigade(f->next, cin->bb, AP_MODE_READBYTES,
                                 block, readbytes);
+        if (saved_timeout != UNSET) {
+            apr_socket_timeout_set(cin->socket, saved_timeout);
+        }
+        ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, f->c,
+                      "core_input(%ld): got_brigade", (long)f->c->id);
     }
     
-out:
     switch (status) {
         case APR_SUCCESS:
-            if (saved_timeout != UNSET) {
-                apr_socket_timeout_set(cin->socket, saved_timeout);
-            }
             status = consume_brigade(cin, cin->bb, block);
-            if (status == APR_SUCCESS) {
-                status = check_time_left(cin, &time_left);
-            }
+            break;
         case APR_EOF:
         case APR_EAGAIN:
+        case APR_TIMEUP:
             break;
         default:
             ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, f->c,
diff --git a/modules/http2/h2_filter.h b/modules/http2/h2_filter.h
index e27a82d7ccc..f27c9ce0c8f 100644
--- a/modules/http2/h2_filter.h
+++ b/modules/http2/h2_filter.h
@@ -29,7 +29,7 @@ typedef struct h2_filter_cin {
     void *cb_ctx;
     apr_socket_t *socket;
     int timeout_secs;
-    apr_time_t last_read;
+    apr_time_t start_read;
 } h2_filter_cin;
 
 h2_filter_cin *h2_filter_cin_create(apr_pool_t *p, h2_filter_cin_cb *cb, void *ctx);
diff --git a/modules/http2/h2_h2.c b/modules/http2/h2_h2.c
index 49fab8a7b97..b6a8f9f4d93 100644
--- a/modules/http2/h2_h2.c
+++ b/modules/http2/h2_h2.c
@@ -651,12 +651,7 @@ int h2_h2_process_conn(conn_rec* c)
                 return status;
             }
         }
-        if (h2_config_async_mpm()) {
-            return h2_conn_process(ctx, 1);
-        }
-        else {
-            return h2_conn_run(ctx, c);
-        }
+        return h2_conn_run(ctx, c);
     }
     
     ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, "h2_h2, declined");
diff --git a/modules/http2/h2_io.c b/modules/http2/h2_io.c
index 4b6c013e382..c1d65fb9a1e 100644
--- a/modules/http2/h2_io.c
+++ b/modules/http2/h2_io.c
@@ -221,11 +221,6 @@ apr_status_t h2_io_in_read(h2_io *io, apr_bucket_brigade *bb,
     
     if (io->request->chunked) {
         /* the reader expects HTTP/1.1 chunked encoding */
-        if (!io->tmp) {
-            io->tmp = apr_brigade_create(io->pool, io->bucket_alloc);
-        }
-        apr_brigade_cleanup(io->tmp);
-        
         status = h2_util_move(io->tmp, io->bbin, maxlen, NULL, "h2_io_in_read_chunk");
         if (status == APR_SUCCESS) {
             apr_off_t tmp_len = 0;
@@ -245,6 +240,7 @@ apr_status_t h2_io_in_read(h2_io *io, apr_bucket_brigade *bb,
             else {
                 status = h2_util_move(bb, io->tmp, -1, NULL, "h2_io_in_read_tmp2");
             }
+            apr_brigade_cleanup(io->tmp);
         }
     }
     else {
@@ -274,6 +270,7 @@ apr_status_t h2_io_in_write(h2_io *io, apr_bucket_brigade *bb)
     if (!APR_BRIGADE_EMPTY(bb)) {
         if (!io->bbin) {
             io->bbin = apr_brigade_create(io->pool, io->bucket_alloc);
+            io->tmp = apr_brigade_create(io->pool, io->bucket_alloc);
         }
         return h2_util_move(io->bbin, bb, -1, NULL, "h2_io_in_write");
     }
diff --git a/modules/http2/h2_session.c b/modules/http2/h2_session.c
index fbe4cf7fbb6..6a3d26155cd 100644
--- a/modules/http2/h2_session.c
+++ b/modules/http2/h2_session.c
@@ -22,8 +22,10 @@
 #include <http_core.h>
 #include <http_config.h>
 #include <http_log.h>
+#include <scoreboard.h>
 
 #include "h2_private.h"
+#include "h2_bucket_eoc.h"
 #include "h2_bucket_eos.h"
 #include "h2_config.h"
 #include "h2_ctx.h"
@@ -42,6 +44,9 @@
 #include "h2_version.h"
 #include "h2_workers.h"
 
+#define H2MAX(x,y) ((x) > (y) ? (x) : (y))
+#define H2MIN(x,y) ((x) < (y) ? (x) : (y))
+
 static int frame_print(const nghttp2_frame *frame, char *buffer, size_t maxlen);
 
 static int h2_session_status_from_apr_status(apr_status_t rv)
@@ -454,6 +459,12 @@ static int on_frame_recv_cb(nghttp2_session *ng2s,
                           (int)frame->rst_stream.error_code);
             ++session->streams_reset;
             break;
+        case NGHTTP2_GOAWAY:
+            ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
+                          "h2_session(%ld): GOAWAY errror=%d",
+                          session->id, (int)frame->goaway.error_code);
+            session->client_goaway = 1;
+            break;
         default:
             if (APLOGctrace2(session->c)) {
                 char buffer[256];
@@ -639,6 +650,52 @@ static apr_status_t init_callbacks(conn_rec *c, nghttp2_session_callbacks **pcb)
     return APR_SUCCESS;
 }
 
+static void h2_session_cleanup(h2_session *session)
+{
+    AP_DEBUG_ASSERT(session);
+    /* This is an early cleanup of the session that may
+     * discard what is no longer necessary for *new* streams
+     * and general HTTP/2 processing.
+     * At this point, all frames are in transit or somehwere in
+     * our buffers or passed down output filters.
+     * h2 streams might still being written out.
+     */
+    if (session->c) {
+        h2_ctx_clear(session->c);
+    }
+    if (session->ngh2) {
+        nghttp2_session_del(session->ngh2);
+        session->ngh2 = NULL;
+    }
+    if (session->spare) {
+        apr_pool_destroy(session->spare);
+        session->spare = NULL;
+    }
+}
+
+static void h2_session_destroy(h2_session *session)
+{
+    AP_DEBUG_ASSERT(session);
+    h2_session_cleanup(session);
+
+    if (APLOGctrace1(session->c)) {
+        ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c,
+                      "h2_session(%ld): destroy, %d streams open",
+                      session->id, (int)h2_stream_set_size(session->streams));
+    }
+    if (session->mplx) {
+        h2_mplx_release_and_join(session->mplx, session->iowait);
+        session->mplx = NULL;
+    }
+    if (session->streams) {
+        h2_stream_set_destroy(session->streams);
+        session->streams = NULL;
+    }
+    if (session->pool) {
+        apr_pool_destroy(session->pool);
+    }
+}
+
 static apr_status_t session_pool_cleanup(void *data)
 {
     h2_session *session = data;
@@ -721,6 +778,8 @@ static h2_session *h2_session_create_int(conn_rec *c,
         session->s = h2_ctx_server_get(ctx);
         session->config = h2_config_sget(session->s);
         
+        session->state = H2_SESSION_ST_INIT;
+        
         session->pool = pool;
         apr_pool_pre_cleanup_register(pool, session, session_pool_cleanup);
         
@@ -743,7 +802,6 @@ static h2_session *h2_session_create_int(conn_rec *c,
         
         /* Install the connection input filter that feeds the session */
         session->cin = h2_filter_cin_create(session->pool, h2_session_receive, session);
-        h2_filter_cin_timeout_set(session->cin, session->timeout_secs);
         ap_add_input_filter("H2_IN", session->cin, r, c);
 
         h2_conn_io_init(&session->io, c, session->config, session->pool);
@@ -811,53 +869,6 @@ h2_session *h2_session_rcreate(request_rec *r, h2_ctx *ctx, h2_workers *workers)
     return h2_session_create_int(r->connection, r, ctx, workers);
 }
 
-static void h2_session_cleanup(h2_session *session)
-{
-    AP_DEBUG_ASSERT(session);
-    /* This is an early cleanup of the session that may
-     * discard what is no longer necessary for *new* streams
-     * and general HTTP/2 processing.
-     * At this point, all frames are in transit or somehwere in
-     * our buffers or passed down output filters.
-     * h2 streams might still being written out.
-     */
-    if (session->c) {
-        h2_ctx_clear(session->c);
-    }
-    if (session->ngh2) {
-        nghttp2_session_del(session->ngh2);
-        session->ngh2 = NULL;
-    }
-    if (session->spare) {
-        apr_pool_destroy(session->spare);
-        session->spare = NULL;
-    }
-}
-
-void h2_session_destroy(h2_session *session)
-{
-    AP_DEBUG_ASSERT(session);
-    h2_session_cleanup(session);
-
-    if (APLOGctrace1(session->c)) {
-        ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c,
-                      "h2_session(%ld): destroy, %d streams open",
-                      session->id, (int)h2_stream_set_size(session->streams));
-    }
-    if (session->mplx) {
-        h2_mplx_release_and_join(session->mplx, session->iowait);
-        session->mplx = NULL;
-    }
-    if (session->streams) {
-        h2_stream_set_destroy(session->streams);
-        session->streams = NULL;
-    }
-    if (session->pool) {
-        apr_pool_destroy(session->pool);
-    }
-}
-
-
 void h2_session_eoc_callback(h2_session *session)
 {
     ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c,
@@ -869,36 +880,33 @@ void h2_session_eoc_callback(h2_session *session)
 static apr_status_t h2_session_abort_int(h2_session *session, int reason)
 {
     AP_DEBUG_ASSERT(session);
-    if (!session->aborted) {
-        session->aborted = 1;
-        
-        if (session->ngh2) {            
-            if (NGHTTP2_ERR_EOF == reason) {
-                /* This is our way of indication that the connection is
-                 * gone. No use to send any GOAWAY frames. */
-                nghttp2_session_terminate_session(session->ngh2, reason);
-            }
-            else if (!reason) {
-                nghttp2_submit_goaway(session->ngh2, NGHTTP2_FLAG_NONE, 
-                                      session->max_stream_received, 
-                                      reason, NULL, 0);
-                nghttp2_session_send(session->ngh2);
-            }
-            else {
-                const char *err = nghttp2_strerror(reason);
-                
-                ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
-                              "session(%ld): abort, reason=%d %s",
-                              session->id, reason, err);
-                
-                /* The connection might still be there and we shut down
-                 * with GOAWAY and reason information. */
-                nghttp2_submit_goaway(session->ngh2, NGHTTP2_FLAG_NONE, 
-                                      session->max_stream_received, 
-                                      reason, (const uint8_t *)err, 
-                                      strlen(err));
-                nghttp2_session_send(session->ngh2);
-            }
+    session->aborted = 1;
+    if (session->state != H2_SESSION_ST_CLOSING) {
+        session->state = H2_SESSION_ST_CLOSING;
+        if (session->client_goaway) {
+            /* client sent us a GOAWAY, just terminate */
+            nghttp2_session_terminate_session(session->ngh2, NGHTTP2_ERR_EOF);
+            ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
+                          "session(%ld): closed, GOAWAY from client", session->id);
+        }
+        else if (!reason) {
+            nghttp2_submit_goaway(session->ngh2, NGHTTP2_FLAG_NONE, 
+                                  session->max_stream_received, 
+                                  reason, NULL, 0);
+            nghttp2_session_send(session->ngh2);
+            ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
+                          "session(%ld): closed, no err", session->id);
+        }
+        else {
+            const char *err = nghttp2_strerror(reason);
+            nghttp2_submit_goaway(session->ngh2, NGHTTP2_FLAG_NONE, 
+                                  session->max_stream_received, 
+                                  reason, (const uint8_t *)err, 
+                                  strlen(err));
+            nghttp2_session_send(session->ngh2);
+            ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
+                          "session(%ld): closed, err=%d '%s'",
+                          session->id, reason, err);
         }
         h2_mplx_abort(session->mplx);
     }
@@ -1092,14 +1100,20 @@ h2_stream *h2_session_get_stream(h2_session *session, int stream_id)
     return session->last_stream;
 }
 
-apr_status_t h2_session_close(h2_session *session)
+void h2_session_close(h2_session *session)
 {
+    apr_bucket *b;
+    
     AP_DEBUG_ASSERT(session);
     if (!session->aborted) {
         h2_session_abort_int(session, 0);
     }
     h2_session_cleanup(session);
-    return h2_conn_io_close(&session->io, session);           
+
+    b = h2_bucket_eoc_create(session->c->bucket_alloc, session);
+    h2_conn_io_writeb(&session->io, b);
+    h2_conn_io_flush(&session->io);
+    /* and all is or will be destroyed */
 }
 
 static ssize_t stream_data_cb(nghttp2_session *ng2s,
@@ -1574,7 +1588,6 @@ static apr_status_t h2_session_send(h2_session *session)
         }
     }
     
-    session->wait_micros = 0;
     session->unsent_promises = 0;
     session->unsent_submits = 0;
     
@@ -1594,7 +1607,7 @@ static apr_status_t h2_session_receive(void *ctx, const char *data,
                           "h2_session: nghttp2_session_mem_recv error=%d",
                           (int)n);
             if (nghttp2_is_fatal((int)n)) {
-                h2_session_abort(session, 0, (int)n);
+                h2_session_abort_int(session, (int)n);
                 return APR_EGENERAL;
             }
         }
@@ -1608,12 +1621,13 @@ static apr_status_t h2_session_receive(void *ctx, const char *data,
 static apr_status_t h2_session_read(h2_session *session, int block, int loops)
 {
     apr_status_t status, rstatus = APR_EAGAIN;
+    conn_rec *c = session->c;
     int i;
     
     for (i = 0; i < loops; ++i) {
         /* H2_IN filter handles all incoming data against the session.
          * We just pull at the filter chain to make it happen */
-        status = ap_get_brigade(session->c->input_filters,
+        status = ap_get_brigade(c->input_filters,
                                 session->bbtmp, AP_MODE_READBYTES,
                                 block? APR_BLOCK_READ : APR_NONBLOCK_READ,
                                 APR_BUCKET_BUFF_SIZE);
@@ -1623,7 +1637,6 @@ static apr_status_t h2_session_read(h2_session *session, int block, int loops)
         switch (status) {
             case APR_SUCCESS:
                 /* successful read, reset our idle timers */
-                session->wait_micros = 0;
                 rstatus = APR_SUCCESS;
                 if (block) {
                     /* successfull blocked read, try unblocked to
@@ -1633,6 +1646,8 @@ static apr_status_t h2_session_read(h2_session *session, int block, int loops)
                 break;
             case APR_EAGAIN:
                 return rstatus;
+            case APR_TIMEUP:
+                return status;
             default:
                 if (!i) {
                     /* first attempt failed */
@@ -1640,12 +1655,10 @@ static apr_status_t h2_session_read(h2_session *session, int block, int loops)
                         || APR_STATUS_IS_ECONNABORTED(status)
                         || APR_STATUS_IS_ECONNRESET(status)
                         || APR_STATUS_IS_EOF(status)
-                        || APR_STATUS_IS_TIMEUP(status)
                         || APR_STATUS_IS_EBADF(status)) {
                         /* common status for a client that has left */
-                        ap_log_cerror( APLOG_MARK, APLOG_TRACE1, status, session->c,
-                                      "h2_session(%ld): terminating",
-                                      session->id);
+                        ap_log_cerror( APLOG_MARK, APLOG_TRACE1, status, c,
+                                      "h2_session(%ld): input gone", session->id);
                         /* Stolen from mod_reqtimeout to speed up lingering when
                          * a read timeout happened.
                          */
@@ -1653,7 +1666,7 @@ static apr_status_t h2_session_read(h2_session *session, int block, int loops)
                     }
                     else {
                         /* uncommon status, log on INFO so that we see this */
-                        ap_log_cerror( APLOG_MARK, APLOG_INFO, status, session->c,
+                        ap_log_cerror( APLOG_MARK, APLOG_INFO, status, c,
                                       APLOGNO(02950) 
                                       "h2_session(%ld): error reading, terminating",
                                       session->id);
@@ -1704,203 +1717,273 @@ static const int MAX_WAIT_MICROS = 200 * 1000;
 apr_status_t h2_session_process(h2_session *session, int async)
 {
     apr_status_t status = APR_SUCCESS;
-    int got_streams = 0;
-    int have_written, have_read;
-    int timeout_secs = session->timeout_secs;
-    int keep_alive = 0;
-
-    ap_log_cerror( APLOG_MARK, APLOG_TRACE1, status, session->c,
-                  "h2_session(%ld): process", session->id);
+    conn_rec *c = session->c;
+    int rv, have_written, have_read, remain_secs;
+    const char *reason = "";
 
+    ap_log_cerror( APLOG_MARK, APLOG_TRACE1, status, c,
+                  "h2_session(%ld): process start, async=%d", session->id, async);
+                  
     while (1) {
-        have_read = 0;
-        have_written = 0;
- 
-        if (session->aborted || (!nghttp2_session_want_read(session->ngh2)
-                                 && !nghttp2_session_want_write(session->ngh2))) {
-            ap_log_cerror( APLOG_MARK, APLOG_TRACE1, status, session->c,
-                          "h2_session(%ld): process -> aborted", session->id);
-            h2_conn_io_flush(&session->io);
-            return APR_EOF;
-        }
+        have_read = have_written = 0;
 
-        if (!session->started) {
-            int rv;
-            
-            if (!h2_is_acceptable_connection(session->c, 1)) {
-                nghttp2_submit_goaway(session->ngh2, NGHTTP2_FLAG_NONE, 0,
-                                      NGHTTP2_INADEQUATE_SECURITY, NULL, 0);
-            } 
-            
-            status = h2_session_start(session, &rv);
-            ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, session->c,
-                          "h2_session(%ld): started on %s:%d", session->id,
-                          session->s->server_hostname,
-                          session->c->local_addr->port);
-            if (status != APR_SUCCESS) {
-                h2_session_abort(session, status, rv);
-                h2_session_eoc_callback(session);
-            }
-            session->started = 1;
+        if (session->aborted) {
+            reason = "aborted";
+            status = APR_EOF;
+            goto out;
         }
         
-        got_streams = !h2_stream_set_is_empty(session->streams);
-
-        /* If we want client data, see if some is there. */
-        if (nghttp2_session_want_read(session->ngh2)) {
-            int idle = (session->responses_sent && !got_streams);
-            int may_block = ((session->frames_received <= 1) 
-                             || idle
-                             || (!h2_stream_set_has_unsubmitted(session->streams)
-                                 && !h2_stream_set_has_suspended(session->streams)));
-                                 
-            h2_filter_cin_timeout_set(session->cin, timeout_secs);
-            status = h2_session_read(session, may_block, 10);
-            
-            got_streams = !h2_stream_set_is_empty(session->streams);
-            ap_log_cerror( APLOG_MARK, APLOG_TRACE1, status, session->c,
-                          "h2_session(%ld): process -> read", session->id);
-            if (status == APR_SUCCESS) {
-                have_read = 1;
-                keep_alive = 0;
-                timeout_secs = session->timeout_secs;
-                if (session->reprioritize) {
-                    ap_log_cerror( APLOG_MARK, APLOG_TRACE2, status, session->c,
-                                  "h2_session(%ld): process -> reprioritize", session->id);
-                    h2_mplx_reprioritize(session->mplx, stream_pri_cmp, session);
-                    session->reprioritize = 0;
+        switch (session->state) {
+            case H2_SESSION_ST_INIT:
+                if (!h2_is_acceptable_connection(c, 1)) {
+                    nghttp2_submit_goaway(session->ngh2, NGHTTP2_FLAG_NONE, 0,
+                                          NGHTTP2_INADEQUATE_SECURITY, NULL, 0);
+                } 
+                
+                status = h2_session_start(session, &rv);
+                ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, c,
+                              "h2_session(%ld): started on %s:%d", session->id,
+                              session->s->server_hostname,
+                              c->local_addr->port);
+                if (status != APR_SUCCESS) {
+                    h2_session_abort(session, status, rv);
+                    h2_session_eoc_callback(session);
+                    reason = "start failed";
+                    goto out;
                 }
-            }
-            else if (status == APR_EAGAIN) {
-                /* nothing to read */
-            }
-            else if (APR_STATUS_IS_TIMEUP(status)) {
-                if (may_block) {
-                    /* I think can only happen in mayblock... */
-                    if (async) {
-                        /* timeout reading, return to async MPM with our wish to 
-                         * read and let it handle that. It may just close the 
-                         * connection as it considers it in KEEPALIVE mode. */
-                        ap_log_cerror( APLOG_MARK, APLOG_TRACE1, status, session->c,
-                                      "h2_session(%ld): 1st timeout, return to async "
-                                      "MPM for KEEPALIVE, frames_received=%d, "
-                                      "got_streams=%d, have_written=%d", 
-                                      session->id, (int)session->frames_received,
-                                      (int)got_streams, (int)have_written);
-                        if (session->c->cs) {
-                            session->c->cs->sense = CONN_SENSE_WANT_READ;
-                        }
-                        return APR_SUCCESS;
-                    } 
-                    else if (!keep_alive 
-                             && session->keepalive_secs > timeout_secs) {
-                        keep_alive = 1;
-                        timeout_secs = session->keepalive_secs - timeout_secs;
-                        ap_log_cerror( APLOG_MARK, APLOG_TRACE1, status, session->c,
-                                      "h2_session(%ld): 1st timeout, set "
-                                      "keepalive timeout of + %d seconds", 
-                                      session->id, (int)timeout_secs);
-                        status = APR_EAGAIN;
+                    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, c,
+                                  "h2_session(%ld): INIT -> BUSY", session->id);
+                session->state = H2_SESSION_ST_BUSY;
+                break;
+                
+            case H2_SESSION_ST_IDLE_READ:
+                h2_filter_cin_timeout_set(session->cin, session->timeout_secs);
+                ap_update_child_status(c->sbh, SERVER_BUSY_READ, NULL);
+                status = h2_session_read(session, 1, 10);
+                if (APR_STATUS_IS_TIMEUP(status)) {
+                    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, c,
+                                  "h2_session(%ld): IDLE -> KEEPALIVE", session->id);
+                    session->state = H2_SESSION_ST_KEEPALIVE;
+                }
+                else if (status == APR_SUCCESS) {
+                    /* got something, go busy again */
+                    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, c,
+                                  "h2_session(%ld): IDLE -> BUSY", session->id);
+                    session->state = H2_SESSION_ST_BUSY;
+                }
+                else {
+                    reason = "keepalive error";
+                    goto out;
+                }
+                break;
+                
+            case H2_SESSION_ST_BUSY:
+                if (nghttp2_session_want_read(session->ngh2)) {
+                    status = h2_session_read(session, 0, 10);
+                    if (status == APR_SUCCESS) {
+                        /* got something, continue processing */
+                        have_read = 1;
+                    }
+                    else if (status == APR_EAGAIN) {
+                        /* nothing to read */
                     }
                     else {
-                        /* fall thriough */
-                        ap_log_cerror( APLOG_MARK, APLOG_TRACE1, status, session->c,
-                                      "h2_session(%ld): keepalive expired, close", 
-                                      session->id);
+                        reason = "busy read error";
+                        goto out;
                     }
                 }
-                ap_log_cerror( APLOG_MARK, APLOG_TRACE1, status, session->c,
-                              "h2_session(%ld): timeout", session->id);
-                return status;
-            }
-            else {
-                ap_log_cerror( APLOG_MARK, APLOG_DEBUG, status, session->c,
-                              "h2_session(%ld): failed read", session->id);
-                return status;
-            }
-        }
-            
-        if (got_streams) {
-            ap_log_cerror( APLOG_MARK, APLOG_TRACE2, status, session->c,
-                          "h2_session(%ld): process -> check resume", session->id);
-            /* resume any streams for which data is available again */
-            h2_session_resume_streams_with_data(session);
-            
-            /* Submit any responses/push_promises that are ready */
-            ap_log_cerror( APLOG_MARK, APLOG_TRACE2, status, session->c,
-                          "h2_session(%ld): process -> check submit", session->id);
-            status = h2_session_submit(session);
-            if (status == APR_SUCCESS) {
-                have_written = 1;
-            }
-            else if (status != APR_EAGAIN) {
-                return status;
-            }
-            
-            /* Check that any pending window updates are sent. */
-            ap_log_cerror( APLOG_MARK, APLOG_TRACE2, status, session->c,
-                          "h2_session(%ld): process -> check window_update", session->id);
-            status = h2_mplx_in_update_windows(session->mplx);
-            if (status == APR_SUCCESS) {
-                /* need to flush window updates onto the connection asap */
+                
+                if (!h2_stream_set_is_empty(session->streams)) {
+                    /* resume any streams for which data is available again */
+                    h2_session_resume_streams_with_data(session);
+                    /* Submit any responses/push_promises that are ready */
+                    status = h2_session_submit(session);
+                    if (status == APR_SUCCESS) {
+                        have_written = 1;
+                    }
+                    else if (status != APR_EAGAIN) {
+                        reason = "submit error";
+                        goto out;
+                    }
+                    /* send out window updates for our inputs */
+                    status = h2_mplx_in_update_windows(session->mplx);
+                    if (status != APR_SUCCESS && status != APR_EAGAIN) {
+                        reason = "window update error";
+                        goto out;
+                    }
+                }
+                
+                if (nghttp2_session_want_write(session->ngh2)) {
+                    status = h2_session_send(session);
+                    if (status != APR_SUCCESS) {
+                        reason = "send error";
+                        goto out;
+                    }
+                    have_written = 1;
+                }
+                
+                if (have_read || have_written) {
+                    session->wait_us = 0;
+                }
+                else {
+                    /* nothing for input and output to do. If we remain
+                     * in this state, we go into a tight loop and suck up
+                     * CPU cycles. 
+                     * Ideally, we'd like to do a blocking read, but that
+                     * is not possible if we have scheduled tasks and wait
+                     * for them to produce something. */
+                    if (h2_stream_set_is_empty(session->streams)) {
+                        /* When we have no streams, no task event are possible,
+                         * switch to blocking reads */
+                    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, c,
+                                  "h2_session(%ld): BUSY -> IDLE", session->id);
+                        session->state = H2_SESSION_ST_IDLE_READ;
+                    }
+                    else if (!h2_stream_set_has_unsubmitted(session->streams)
+                             && !h2_stream_set_has_suspended(session->streams)) {
+                        /* none of our streams is waiting for a response or
+                         * new output data from task processing, 
+                         * switch to blocking reads. */
+                    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, c,
+                                  "h2_session(%ld): BUSY -> IDLE", session->id);
+                        session->state = H2_SESSION_ST_IDLE_READ;
+                    }
+                    else {
+                        /* Unable to do blocking reads, as we wait on events from
+                         * task processing in other threads. Do a busy wait with
+                         * backoff timer. */
+                        ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, c,
+                                      "h2_session(%ld): BUSY -> WAIT", session->id);
+                        session->state = H2_SESSION_ST_BUSY_WAIT;
+                    }
+                }
+                break;
+                
+            case H2_SESSION_ST_BUSY_WAIT:
+                session->wait_us = H2MAX(session->wait_us, 10);
+                if (APLOGctrace1(c)) {
+                    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c,
+                                  "h2_session: wait for data, %ld micros", 
+                                  (long)session->wait_us);
+                }
+                
                 h2_conn_io_flush(&session->io);
-            }
-            else if (status != APR_EAGAIN) {
-                return status;
-            }
-        }
-        
-        /* Send data out first, as long as we have some. 
-         * We are a server after all. */
-        if (nghttp2_session_want_write(session->ngh2)) {
-            status = h2_session_send(session);
-            ap_log_cerror( APLOG_MARK, APLOG_TRACE1, status, session->c,
-                          "h2_session(%ld): process -> send", session->id);
-            have_written = 1;
-            if (status != APR_SUCCESS) {
-                ap_log_cerror( APLOG_MARK, APLOG_DEBUG, status, session->c,
-                              "h2_session(%ld): failed send", session->id);
-                return status;
-            }
-        }
-        
-        if (!have_read && !have_written) {
-            if (session->wait_micros == 0) {
-                session->wait_micros = 10;
-            }
-        }
-        
-        if (session->wait_micros > 0 && !session->aborted) {
-            /* Only happens when reading returned EAGAIN and we also
-             * had nothing to write. 
-             * This is a normal state of affairs when streams have been
-             * opened, the client waiting on responses, but our workers
-             * have not produced anything to send yet.
-             */
-            if (APLOGcdebug(session->c)) {
-                ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c,
-                              "h2_session: wait for data, %ld micros", 
-                              (long)session->wait_micros);
-            }
-            
-            h2_conn_io_flush(&session->io);
-            ap_log_cerror( APLOG_MARK, APLOG_TRACE2, status, session->c,
-                          "h2_session(%ld): process -> trywait", session->id);
-            status = h2_mplx_out_trywait(session->mplx, session->wait_micros, 
-                                         session->iowait);
-            if (status == APR_TIMEUP) {
-                if (session->wait_micros < MAX_WAIT_MICROS) {
-                    session->wait_micros *= 2;
+                ap_log_cerror( APLOG_MARK, APLOG_TRACE2, status, c,
+                              "h2_session(%ld): process -> trywait", session->id);
+                status = h2_mplx_out_trywait(session->mplx, session->wait_us, 
+                                             session->iowait);
+                if (status == APR_SUCCESS) {
+                    /* got something, go busy again */
+                    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, c,
+                                  "h2_session(%ld): WAIT -> BUSY", session->id);
+                    session->state = H2_SESSION_ST_BUSY;
                 }
-            }
+                else if (status == APR_TIMEUP) {
+                    if (nghttp2_session_want_read(session->ngh2)) {
+                        status = h2_session_read(session, 0, 1);
+                        if (status == APR_SUCCESS) {
+                            /* got something, go busy again */
+                            session->wait_us = 0;
+                            ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, c,
+                                          "h2_session(%ld): WAIT -> BUSY", session->id);
+                            session->state = H2_SESSION_ST_BUSY;
+                        }
+                        else if (status != APR_EAGAIN) {
+                            reason = "busy read error";
+                            goto out;
+                        }
+                    }
+                    /* nothing, increase timer for graceful backup */
+                    session->wait_us = H2MIN(session->wait_us*2, MAX_WAIT_MICROS);
+                    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, c,
+                                  "h2_session(%ld): WAIT -> BUSY", session->id);
+                    session->state = H2_SESSION_ST_BUSY;
+                }
+                else {
+                    reason = "busy wait error";
+                    goto out;
+                }
+                break;
+                
+            case H2_SESSION_ST_KEEPALIVE:
+                /* Our normal H2Timeout has passed and we are considering to
+                 * extend that with the H2KeepAliveTimeout. This works different
+                 * for async MPMs. */
+                remain_secs = session->keepalive_secs - session->timeout_secs;
+                if (remain_secs <= 0) {
+                    /* keepalive is smaller than normal timeout, close the session */
+                    reason = "keepalive expired";
+                    h2_session_abort_int(session, 0);
+                    goto out;
+                }
+                ap_update_child_status_from_conn(c->sbh, SERVER_BUSY_KEEPALIVE, c);
+                if (async && session->c->cs) {
+                    /* Async MPMs are able to handle keep-alive connections without
+                     * blocking a thread. For this to happen, we need to return from
+                     * processing, indicating the IO event we are waiting for, and
+                     * may be called again if the event happens.
+                     * For now, we let the MPM handle any timing on this, so we
+                     * cannot really enforce the remain_secs here.
+                     */
+                    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, c,
+                                  "h2_session(%ld): async KEEPALIVE -> BUSY", session->id);
+                    session->state = H2_SESSION_ST_BUSY;
+                    session->c->cs->sense = CONN_SENSE_WANT_READ;
+                    reason = "async keepalive";
+                    status = APR_SUCCESS;
+                    goto out;
+                }
+                ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, c,
+                              "h2_session(%ld): KEEPALIVE read", session->id);
+                h2_filter_cin_timeout_set(session->cin, remain_secs);
+                status = h2_session_read(session, 1, 1);
+                if (APR_STATUS_IS_TIMEUP(status)) {
+                    reason = "keepalive expired";
+                    h2_session_abort_int(session, 0);
+                    goto out;
+                }
+                else if (status != APR_SUCCESS) {
+                    reason = "keepalive error";
+                    goto out;
+                }
+                ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, c,
+                              "h2_session(%ld): KEEPALIVE -> BUSY", session->id);
+                session->state = H2_SESSION_ST_BUSY;
+                break;
+                
+            case H2_SESSION_ST_CLOSING:
+                if (nghttp2_session_want_write(session->ngh2)) {
+                    status = h2_session_send(session);
+                    if (status != APR_SUCCESS) {
+                        reason = "send error";
+                        goto out;
+                    }
+                    have_written = 1;
+                }
+                reason = "closing";
+                goto out;
+                
+            default:
+                ap_log_cerror(APLOG_MARK, APLOG_ERR, APR_EGENERAL, c,
+                              "h2_session(%ld): state %d", session->id, session->state);
+                return APR_EGENERAL;
         }
-        
+
+        if (!nghttp2_session_want_read(session->ngh2)
+            && !nghttp2_session_want_write(session->ngh2)) {
+            session->state = H2_SESSION_ST_CLOSING;
+        }        
+
         if (have_written) {
             h2_conn_io_flush(&session->io);
         }
     }
-        
-    ap_log_cerror( APLOG_MARK, APLOG_TRACE1, status, session->c,
-                  "h2_session(%ld): process -> return", session->id);
+out:
+    if (have_written) {
+        h2_conn_io_flush(&session->io);
+    }
+    ap_log_cerror( APLOG_MARK, APLOG_TRACE1, status, c,
+                  "h2_session(%ld): process return, state %d, reason '%s'", 
+                  session->id, session->state, reason);
     return status;
 }
diff --git a/modules/http2/h2_session.h b/modules/http2/h2_session.h
index b1542eddfd2..39f27afbe6c 100644
--- a/modules/http2/h2_session.h
+++ b/modules/http2/h2_session.h
@@ -53,9 +53,16 @@ struct h2_workers;
 
 struct nghttp2_session;
 
-typedef struct h2_session h2_session;
-
-struct h2_session {
+typedef enum {
+    H2_SESSION_ST_INIT,             /* send initial SETTINGS, etc. */
+    H2_SESSION_ST_IDLE_READ,        /* nothing to write, expecting data inc */
+    H2_SESSION_ST_BUSY,             /* read/write without stop */
+    H2_SESSION_ST_BUSY_WAIT,        /* waiting for tasks reporting back */
+    H2_SESSION_ST_KEEPALIVE,        /* nothing to write, normal timeout passed */
+    H2_SESSION_ST_CLOSING,          /* shuting down */
+} h2_session_state;
+
+typedef struct h2_session {
     long id;                        /* identifier of this session, unique
                                      * inside a httpd process */
     conn_rec *c;                    /* the connection this session serves */
@@ -64,11 +71,12 @@ struct h2_session {
     server_rec *s;                  /* server/vhost we're starting on */
     const struct h2_config *config; /* Relevant config for this session */
     
-    unsigned int started      : 1;  /* session startup done */
-    unsigned int aborted      : 1;  /* this session is being aborted */
-    unsigned int reprioritize : 1;  /* scheduled streams priority changed */
-                                     
-    apr_interval_time_t  wait_micros;
+    h2_session_state state;         /* state session is in */
+    unsigned int aborted       : 1; /* aborted processing, emergency exit */
+    unsigned int reprioritize  : 1; /* scheduled streams priority changed */
+    unsigned int client_goaway : 1; /* client sent us a GOAWAY */
+    apr_interval_time_t  wait_us;   /* timout during BUSY_WAIT state, micro secs */
+    
     int unsent_submits;             /* number of submitted, but not yet sent
                                        responses. */
     int unsent_promises;            /* number of submitted, but not yet sent
@@ -104,7 +112,7 @@ struct h2_session {
     
     struct nghttp2_session *ngh2;   /* the nghttp2 session (internal use) */
     struct h2_workers *workers;     /* for executing stream tasks */
-};
+} h2_session;
 
 
 /**
@@ -137,13 +145,6 @@ h2_session *h2_session_rcreate(request_rec *r, struct h2_ctx *ctx,
  */
 apr_status_t h2_session_process(h2_session *session, int async);
 
-/**
- * Destroy the session and all objects it still contains. This will not
- * destroy h2_task instances that have not finished yet. 
- * @param session the session to destroy
- */
-void h2_session_destroy(h2_session *session);
-
 /**
  * Cleanup the session and all objects it still contains. This will not
  * destroy h2_task instances that have not finished yet. 
@@ -161,9 +162,9 @@ void h2_session_eoc_callback(h2_session *session);
 apr_status_t h2_session_abort(h2_session *session, apr_status_t reason, int rv);
 
 /**
- * Called before a session gets destroyed, might flush output etc. 
+ * Close and deallocate the given session.
  */
-apr_status_t h2_session_close(h2_session *session);
+void h2_session_close(h2_session *session);
 
 /* Start submitting the response to a stream request. This is possible
  * once we have all the response headers. */
diff --git a/modules/http2/h2_util.c b/modules/http2/h2_util.c
index 0fd1d7e333a..1ab71fb3c7c 100644
--- a/modules/http2/h2_util.c
+++ b/modules/http2/h2_util.c
@@ -277,7 +277,8 @@ apr_status_t h2_util_move(apr_bucket_brigade *to, apr_bucket_brigade *from,
     
     AP_DEBUG_ASSERT(to);
     AP_DEBUG_ASSERT(from);
-    same_alloc = (to->bucket_alloc == from->bucket_alloc);
+    same_alloc = (to->bucket_alloc == from->bucket_alloc 
+                  || to->p == from->p);
 
     if (!FILE_MOVE) {
         pfile_handles_allowed = NULL;
diff --git a/modules/http2/h2_workers.c b/modules/http2/h2_workers.c
index 33b96de499e..7aec7947491 100644
--- a/modules/http2/h2_workers.c
+++ b/modules/http2/h2_workers.c
@@ -51,7 +51,7 @@ static void cleanup_zombies(h2_workers *workers, int lock)
     while (!H2_WORKER_LIST_EMPTY(&workers->zombies)) {
         h2_worker *zombie = H2_WORKER_LIST_FIRST(&workers->zombies);
         H2_WORKER_REMOVE(zombie);
-        ap_log_error(APLOG_MARK, APLOG_TRACE1, 0, workers->s,
+        ap_log_error(APLOG_MARK, APLOG_TRACE3, 0, workers->s,
                       "h2_workers: cleanup zombie %d", zombie->id);
         h2_worker_destroy(zombie);
     }
@@ -106,7 +106,7 @@ static apr_status_t get_mplx_next(h2_worker *worker, h2_mplx **pm,
     status = apr_thread_mutex_lock(workers->lock);
     if (status == APR_SUCCESS) {
         ++workers->idle_worker_count;
-        ap_log_error(APLOG_MARK, APLOG_TRACE1, 0, workers->s,
+        ap_log_error(APLOG_MARK, APLOG_TRACE3, 0, workers->s,
                      "h2_worker(%d): looking for work", h2_worker_get_id(worker));
         
         while (!task && !h2_worker_is_aborted(worker) && !workers->aborted) {
@@ -147,14 +147,14 @@ static apr_status_t get_mplx_next(h2_worker *worker, h2_mplx **pm,
                     if (now >= (start_wait + max_wait)) {
                         /* waited long enough without getting a task. */
                         if (workers->worker_count > workers->min_size) {
-                            ap_log_error(APLOG_MARK, APLOG_TRACE1, 0, 
+                            ap_log_error(APLOG_MARK, APLOG_TRACE3, 0, 
                                          workers->s,
                                          "h2_workers: aborting idle worker");
                             h2_worker_abort(worker);
                             break;
                         }
                     }
-                    ap_log_error(APLOG_MARK, APLOG_TRACE1, 0, workers->s,
+                    ap_log_error(APLOG_MARK, APLOG_TRACE3, 0, workers->s,
                                  "h2_worker(%d): waiting signal, "
                                  "worker_count=%d", worker->id, 
                                  (int)workers->worker_count);
@@ -162,7 +162,7 @@ static apr_status_t get_mplx_next(h2_worker *worker, h2_mplx **pm,
                                               workers->lock, max_wait);
                 }
                 else {
-                    ap_log_error(APLOG_MARK, APLOG_TRACE1, 0, workers->s,
+                    ap_log_error(APLOG_MARK, APLOG_TRACE3, 0, workers->s,
                                  "h2_worker(%d): waiting signal (eternal), "
                                  "worker_count=%d", worker->id, 
                                  (int)workers->worker_count);
@@ -175,7 +175,7 @@ static apr_status_t get_mplx_next(h2_worker *worker, h2_mplx **pm,
          * needed to give up with more than enough workers.
          */
         if (task) {
-            ap_log_error(APLOG_MARK, APLOG_TRACE1, 0, workers->s,
+            ap_log_error(APLOG_MARK, APLOG_TRACE3, 0, workers->s,
                          "h2_worker(%d): start task(%s)",
                          h2_worker_get_id(worker), task->id);
             /* Since we hand out a reference to the worker, we increase
@@ -206,7 +206,7 @@ static void worker_done(h2_worker *worker, void *ctx)
     h2_workers *workers = (h2_workers *)ctx;
     apr_status_t status = apr_thread_mutex_lock(workers->lock);
     if (status == APR_SUCCESS) {
-        ap_log_error(APLOG_MARK, APLOG_TRACE1, 0, workers->s,
+        ap_log_error(APLOG_MARK, APLOG_TRACE3, 0, workers->s,
                      "h2_worker(%d): done", h2_worker_get_id(worker));
         H2_WORKER_REMOVE(worker);
         --workers->worker_count;
@@ -224,7 +224,7 @@ static apr_status_t add_worker(h2_workers *workers)
     if (!w) {
         return APR_ENOMEM;
     }
-    ap_log_error(APLOG_MARK, APLOG_TRACE1, 0, workers->s,
+    ap_log_error(APLOG_MARK, APLOG_TRACE3, 0, workers->s,
                  "h2_workers: adding worker(%d)", h2_worker_get_id(w));
     ++workers->worker_count;
     H2_WORKER_LIST_INSERT_TAIL(&workers->workers, w);
@@ -335,7 +335,7 @@ apr_status_t h2_workers_register(h2_workers *workers, struct h2_mplx *m)
 {
     apr_status_t status = apr_thread_mutex_lock(workers->lock);
     if (status == APR_SUCCESS) {
-        ap_log_error(APLOG_MARK, APLOG_TRACE2, status, workers->s,
+        ap_log_error(APLOG_MARK, APLOG_TRACE3, status, workers->s,
                      "h2_workers: register mplx(%ld)", m->id);
         if (in_list(workers, m)) {
             status = APR_EAGAIN;
@@ -349,7 +349,7 @@ apr_status_t h2_workers_register(h2_workers *workers, struct h2_mplx *m)
             apr_thread_cond_signal(workers->mplx_added);
         }
         else if (workers->worker_count < workers->max_size) {
-            ap_log_error(APLOG_MARK, APLOG_TRACE1, 0, workers->s,
+            ap_log_error(APLOG_MARK, APLOG_TRACE3, 0, workers->s,
                          "h2_workers: got %d worker, adding 1", 
                          workers->worker_count);
             add_worker(workers);