REGTESTS: disable inter-thread idle connection sharing on sensitive tests

Some regtests involve multiple requests from multiple clients, which can
be dispatched as multiple requests to a server. It turns out that the
idle connection sharing works so well that very quickly few connections
are used, and regularly some of the remaining idle server connections
time out at the moment they were going to be reused, causing those random
"HTTP header incomplete" traces in the logs that make them fail often. In
the end this is only an artefact of the test environment.

And indeed, some tests like normalize-uri which perform a lot of reuse
fail very often, about 20-30% of the times in the CI, and 100% of the
time in local when running 1000 tests in a row. Others like ubase64,
sample_fetches or vary_* fail less often but still a lot in tests.

This patch addresses this by adding "tune.idle-pool.shared off" to all
tests which have at least twice as many requests as clients. It proves
very effective as no single error happens on normalize-uri anymore after
10000 tests. Also 100 full runs of all tests yield no error anymore.

One test is tricky, http_abortonclose, it used to fail ~10 times per
1000 runs and with this workaround still fails once every 1000 runs.
But the test is complex and there's a warning in it mentioning a
possible issue when run in parallel due to a port reuse.

diff --git a/reg-tests/cache/caching_rules.vtc b/reg-tests/cache/caching_rules.vtc
index c20a720e3cc2fac4a7d17c4e0ba7a2f993bfd5e0..114b2fd91ab3fc8bb47d7feedd2d3c2beed2f5ad 100644 (file)
--- a/reg-tests/cache/caching_rules.vtc
+++ b/reg-tests/cache/caching_rules.vtc
@@ -78,6 +78,12 @@ server s2 {
 } -start
 
 haproxy h1 -conf {
+    global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
     defaults
         mode http
         ${no-htx} option http-use-htx

diff --git a/reg-tests/cache/expires.vtc b/reg-tests/cache/expires.vtc
index 51c7487481de52a15312f14be8a5499a1a5b1213..037f09fd0fbf540253c9206bba670c4746580ad8 100644 (file)
--- a/reg-tests/cache/expires.vtc
+++ b/reg-tests/cache/expires.vtc
@@ -40,6 +40,12 @@ server s3 {
 } -start
 
 haproxy h1 -conf {
+       global
+               # WT: limit false-positives causing "HTTP header incomplete" due to
+               # idle server connections being randomly used and randomly expiring
+               # under us.
+               tune.idle-pool.shared off
+
        defaults
                mode http
                ${no-htx} option http-use-htx

diff --git a/reg-tests/cache/if-modified-since.vtc b/reg-tests/cache/if-modified-since.vtc
index e491e4660c22133796800c3553e6ff3cff480a68..8ae1cce5d1d8b31389217b00cc74685a5d9f21f0 100644 (file)
--- a/reg-tests/cache/if-modified-since.vtc
+++ b/reg-tests/cache/if-modified-since.vtc
@@ -40,6 +40,12 @@ server s1 {
 } -start
 
 haproxy h1 -conf {
+       global
+               # WT: limit false-positives causing "HTTP header incomplete" due to
+               # idle server connections being randomly used and randomly expiring
+               # under us.
+               tune.idle-pool.shared off
+
        defaults
                mode http
                ${no-htx} option http-use-htx

diff --git a/reg-tests/cache/if-none-match.vtc b/reg-tests/cache/if-none-match.vtc
index c5391c90f9065f4e9a56d60f3709b98bb35203a7..ba3336a08c958b975f1993c768c61787e46bb2fe 100644 (file)
--- a/reg-tests/cache/if-none-match.vtc
+++ b/reg-tests/cache/if-none-match.vtc
@@ -25,6 +25,12 @@ server s1 {
 } -start
 
 haproxy h1 -conf {
+       global
+               # WT: limit false-positives causing "HTTP header incomplete" due to
+               # idle server connections being randomly used and randomly expiring
+               # under us.
+               tune.idle-pool.shared off
+
        defaults
                mode http
                ${no-htx} option http-use-htx

diff --git a/reg-tests/cache/sample_fetches.vtc b/reg-tests/cache/sample_fetches.vtc
index 73e6e1bf687d2e4da98aa68c432a78f6e166b858..973b3ad61e2dd8185eb534d83d31ab0c48354612 100644 (file)
--- a/reg-tests/cache/sample_fetches.vtc
+++ b/reg-tests/cache/sample_fetches.vtc
@@ -44,6 +44,12 @@ server s3 {
 } -start
 
 haproxy h1 -conf {
+    global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
     defaults
         mode http
         ${no-htx} option http-use-htx

diff --git a/reg-tests/cache/vary.vtc b/reg-tests/cache/vary.vtc
index f3425d00212521cee3c53029648cdc1381809b9c..0b5d139410fa61efc5637c7d86538e5e86c94932 100644 (file)
--- a/reg-tests/cache/vary.vtc
+++ b/reg-tests/cache/vary.vtc
@@ -127,6 +127,12 @@ server s2 {
 } -start
 
 haproxy h1 -conf {
+       global
+               # WT: limit false-positives causing "HTTP header incomplete" due to
+               # idle server connections being randomly used and randomly expiring
+               # under us.
+               tune.idle-pool.shared off
+
        defaults
                mode http
                ${no-htx} option http-use-htx

diff --git a/reg-tests/cache/vary_accept_encoding.vtc b/reg-tests/cache/vary_accept_encoding.vtc
index 185ad0502bf003dd2d006f5f975de01425f35216..852ac293e646192bdd43b738e5f86b1f580ece64 100644 (file)
--- a/reg-tests/cache/vary_accept_encoding.vtc
+++ b/reg-tests/cache/vary_accept_encoding.vtc
@@ -92,6 +92,12 @@ server s1 {
 
 
 haproxy h1 -conf {
+       global
+               # WT: limit false-positives causing "HTTP header incomplete" due to
+               # idle server connections being randomly used and randomly expiring
+               # under us.
+               tune.idle-pool.shared off
+
        defaults
                mode http
                ${no-htx} option http-use-htx

diff --git a/reg-tests/compression/etags_conversion.vtc b/reg-tests/compression/etags_conversion.vtc
index 0cae679592756318ca455ce7b2f1f1e744b86282..48e7955fc4696545a86b74d5696632f3f8912866 100644 (file)
--- a/reg-tests/compression/etags_conversion.vtc
+++ b/reg-tests/compression/etags_conversion.vtc
@@ -106,6 +106,12 @@ server s1 {
 
 
 haproxy h1 -conf {
+    global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
     defaults
         mode http
         ${no-htx} option http-use-htx

diff --git a/reg-tests/compression/vary.vtc b/reg-tests/compression/vary.vtc
index 0a060e4bcc91f29bc2affcb161dd346a2d56cd33..a905f9c9461588c3abba64658e6ae0da5f425079 100644 (file)
--- a/reg-tests/compression/vary.vtc
+++ b/reg-tests/compression/vary.vtc
@@ -61,6 +61,12 @@ server s1 {
 
 
 haproxy h1 -conf {
+    global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
     defaults
         mode http
         ${no-htx} option http-use-htx

diff --git a/reg-tests/converter/json_query.vtc b/reg-tests/converter/json_query.vtc
index ade7b4ccb31d04ce1f4702097c035dc5d4b64ed5..b4209420024fcf9207beac7d8cea2b8c8a92fa1b 100644 (file)
--- a/reg-tests/converter/json_query.vtc
+++ b/reg-tests/converter/json_query.vtc
@@ -9,6 +9,12 @@ server s1 {
 } -repeat 8 -start
 
 haproxy h1 -conf {
+    global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
     defaults
        mode http
        timeout connect 1s

diff --git a/reg-tests/converter/secure_memcmp.vtc b/reg-tests/converter/secure_memcmp.vtc
index f9341f94268b6565a369ff01aaeb461a203a2931..b16e02195c8caaf9536bbdf58fe594d54b95361f 100644 (file)
--- a/reg-tests/converter/secure_memcmp.vtc
+++ b/reg-tests/converter/secure_memcmp.vtc
@@ -16,6 +16,12 @@ server s2 {
 } -repeat 7 -start
 
 haproxy h1 -conf {
+    global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
     defaults
        mode http
        timeout connect 1s

diff --git a/reg-tests/http-messaging/h1_to_h1.vtc b/reg-tests/http-messaging/h1_to_h1.vtc
index 5b02f172433baa84e2571ddaae77a7c56402cf44..c7d00858ea45add32945fd5418420a20548af9cd 100644 (file)
--- a/reg-tests/http-messaging/h1_to_h1.vtc
+++ b/reg-tests/http-messaging/h1_to_h1.vtc
@@ -115,6 +115,12 @@ server s1 {
 } -repeat 3 -start
 
 haproxy h1 -conf {
+    global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
     defaults
        mode http
        ${no-htx} option http-use-htx

diff --git a/reg-tests/http-messaging/h2_to_h1.vtc b/reg-tests/http-messaging/h2_to_h1.vtc
index 481aded12aeea51ca4dba5de71d70f0c284e1995..0d2b1e5f225159bb8b4c503f2c63b085273f4759 100644 (file)
--- a/reg-tests/http-messaging/h2_to_h1.vtc
+++ b/reg-tests/http-messaging/h2_to_h1.vtc
@@ -34,6 +34,12 @@ server s1 {
 } -repeat 2 -start
 
 haproxy h1 -conf {
+    global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
     defaults
        #log stdout format raw daemon
        mode http

diff --git a/reg-tests/http-messaging/http_abortonclose.vtc b/reg-tests/http-messaging/http_abortonclose.vtc
index c0749825717c72990e6756d776ae0c0ad5843f5a..ae0d237ae8fa31a1b7e8c07eade93ca118221826 100644 (file)
--- a/reg-tests/http-messaging/http_abortonclose.vtc
+++ b/reg-tests/http-messaging/http_abortonclose.vtc
@@ -46,6 +46,12 @@ syslog S -level info {
 } -start
 
 haproxy h1 -conf {
+    global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
     defaults
         mode http
         ${no-htx} option http-use-htx

diff --git a/reg-tests/http-messaging/http_bodyless_response.vtc b/reg-tests/http-messaging/http_bodyless_response.vtc
index f45a7d61b8e03f11eb578becba0cfe121862c650..6f36545f265ebaf0c419c1a1a5362095a960455e 100644 (file)
--- a/reg-tests/http-messaging/http_bodyless_response.vtc
+++ b/reg-tests/http-messaging/http_bodyless_response.vtc
@@ -30,6 +30,12 @@ server s1 {
 } -repeat 2 -start
 
 haproxy h1 -conf {
+    global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
     defaults
         mode http
         ${no-htx} option http-use-htx

diff --git a/reg-tests/http-rules/except-forwardfor-originalto.vtc b/reg-tests/http-rules/except-forwardfor-originalto.vtc
index 942c4e2cb817727af6bff8c0d289541644fa5a8d..bb346dda7399bdf89dd6b54a21997aea987d993a 100644 (file)
--- a/reg-tests/http-rules/except-forwardfor-originalto.vtc
+++ b/reg-tests/http-rules/except-forwardfor-originalto.vtc
@@ -7,6 +7,12 @@ varnishtest "Test IPv4/IPv6 except param for the forwardfor and originalto optio
 feature ignore_unknown_macro
 
 haproxy h1 -conf {
+    global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
     defaults
         mode http
         timeout connect 1s

diff --git a/reg-tests/http-rules/http_return.vtc b/reg-tests/http-rules/http_return.vtc
index a50f1fec885d86643cc8ae96ded5088beff63665..8189028047aca21acb8201c3cc77fbe56b91e92b 100644 (file)
--- a/reg-tests/http-rules/http_return.vtc
+++ b/reg-tests/http-rules/http_return.vtc
@@ -6,6 +6,12 @@ varnishtest "Test the HTTP return action"
 feature ignore_unknown_macro
 
 haproxy h1 -conf {
+    global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
     defaults
         mode http
         timeout connect 1s

diff --git a/reg-tests/http-rules/normalize_uri.vtc b/reg-tests/http-rules/normalize_uri.vtc
index 47032d96c0ffc79c8d0b1b41cce7c914c2ff14a4..42c4c428b095492147ec00f1112f539f92a3034d 100644 (file)
--- a/reg-tests/http-rules/normalize_uri.vtc
+++ b/reg-tests/http-rules/normalize_uri.vtc
@@ -12,6 +12,10 @@ server s1 {
 
 haproxy h1 -conf {
     global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
         expose-experimental-directives
 
     defaults

diff --git a/reg-tests/lua/set_var.vtc b/reg-tests/lua/set_var.vtc
index fc21b4133a085e54262fbb4f249be9e876cff508..af4cbf1eb6eade08a4210e2ed229b990767c370e 100644 (file)
--- a/reg-tests/lua/set_var.vtc
+++ b/reg-tests/lua/set_var.vtc
@@ -5,6 +5,12 @@ varnishtest "Lua: set_var"
 feature ignore_unknown_macro
 
 haproxy h1 -conf {
+    global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
     global
         lua-load ${testdir}/set_var.lua
 

diff --git a/reg-tests/lua/txn_get_priv-thread.vtc b/reg-tests/lua/txn_get_priv-thread.vtc
index 77e333dcf11ae634192892a3db015e9342964c3f..6f5d632691fecbc638ca64bab31b1df3899aa7e9 100644 (file)
--- a/reg-tests/lua/txn_get_priv-thread.vtc
+++ b/reg-tests/lua/txn_get_priv-thread.vtc
@@ -7,6 +7,11 @@ feature ignore_unknown_macro
 
 haproxy h1 -conf {
     global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
         lua-load-per-thread ${testdir}/txn_get_priv.lua
         lua-load-per-thread ${testdir}/txn_get_priv-print_r.lua
 

diff --git a/reg-tests/sample_fetches/ubase64.vtc b/reg-tests/sample_fetches/ubase64.vtc
index d6973e3b7a737243f7834878b0a88ea6a3cef9b8..0122bade842422d169c2f5c035c980077d9c9357 100644 (file)
--- a/reg-tests/sample_fetches/ubase64.vtc
+++ b/reg-tests/sample_fetches/ubase64.vtc
@@ -5,6 +5,12 @@ varnishtest "ub64dec sample fetche Test"
 feature ignore_unknown_macro
 
 haproxy h1 -conf {
+    global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
     defaults
         mode http
         timeout connect 1s

diff --git a/reg-tests/webstats/webstats-scope-and-post-change.vtc b/reg-tests/webstats/webstats-scope-and-post-change.vtc
index a77483b57156262c5971bbd144e9397f3239b714..fe8b33333d5751da957832aa811d98eff993f678 100644 (file)
--- a/reg-tests/webstats/webstats-scope-and-post-change.vtc
+++ b/reg-tests/webstats/webstats-scope-and-post-change.vtc
@@ -8,6 +8,11 @@ server s1 {
 
 haproxy h1 -conf {
   global
+    # WT: limit false-positives causing "HTTP header incomplete" due to
+    # idle server connections being randomly used and randomly expiring
+    # under us.
+    tune.idle-pool.shared off
+
     stats socket /tmp/haproxy.socket level admin
 
   defaults