From 749d1b9ebc6fcb79824afd0471a1cfc12ca861b1 Mon Sep 17 00:00:00 2001 From: George Thessalonikefs Date: Fri, 3 Aug 2018 14:00:46 +0000 Subject: [PATCH] - Expose if a query (or a subquery) was ratelimited (not src IP ratelimiting) to libunbound under 'ub_result.was_ratelimited'. This also introduces a change to 'ub_event_callback_type' in libunbound/unbound-event.h. - Tidy pylib tests. git-svn-id: file:///svn/unbound/trunk@4828 be551aaa-1e26-0410-a405-d3ace91eadb9 --- daemon/worker.c | 20 +-- doc/Changelog | 7 + iterator/iterator.c | 2 + libunbound/context.c | 29 ++-- libunbound/libworker.c | 52 +++--- libunbound/unbound-event.h | 2 +- libunbound/unbound.h | 6 + libunbound/worker.h | 6 +- services/authzone.c | 6 +- services/authzone.h | 6 +- services/mesh.c | 19 ++- services/mesh.h | 10 +- smallapp/worker_cb.c | 6 +- .../{pylib.conf => pylib.lookup.conf} | 11 +- testdata/pylib.tdir/pylib.lookup.py | 149 ++++++++++++++-- testdata/pylib.tdir/pylib.post | 4 +- testdata/pylib.tdir/pylib.pre | 13 +- testdata/pylib.tdir/pylib.py | 159 ------------------ testdata/pylib.tdir/pylib.test | 48 ++++-- util/data/msgreply.c | 3 +- util/module.c | 10 ++ util/module.h | 12 ++ validator/autotrust.c | 2 +- validator/autotrust.h | 2 +- 24 files changed, 312 insertions(+), 272 deletions(-) rename testdata/pylib.tdir/{pylib.conf => pylib.lookup.conf} (56%) delete mode 100644 testdata/pylib.tdir/pylib.py diff --git a/daemon/worker.c b/daemon/worker.c index 4c88f048b..ce7b0f25b 100644 --- a/daemon/worker.c +++ b/daemon/worker.c @@ -2007,22 +2007,22 @@ void libworker_handle_control_cmd(struct tube* ATTR_UNUSED(tube), } void libworker_fg_done_cb(void* ATTR_UNUSED(arg), int ATTR_UNUSED(rcode), - sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(s), - char* ATTR_UNUSED(why_bogus)) + sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(s), + char* ATTR_UNUSED(why_bogus), int ATTR_UNUSED(was_ratelimited)) { log_assert(0); } void libworker_bg_done_cb(void* ATTR_UNUSED(arg), int ATTR_UNUSED(rcode), - sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(s), - char* ATTR_UNUSED(why_bogus)) + sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(s), + char* ATTR_UNUSED(why_bogus), int ATTR_UNUSED(was_ratelimited)) { log_assert(0); } void libworker_event_done_cb(void* ATTR_UNUSED(arg), int ATTR_UNUSED(rcode), - sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(s), - char* ATTR_UNUSED(why_bogus)) + sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(s), + char* ATTR_UNUSED(why_bogus), int ATTR_UNUSED(was_ratelimited)) { log_assert(0); } @@ -2035,13 +2035,13 @@ int context_query_cmp(const void* ATTR_UNUSED(a), const void* ATTR_UNUSED(b)) int order_lock_cmp(const void* ATTR_UNUSED(e1), const void* ATTR_UNUSED(e2)) { - log_assert(0); - return 0; + log_assert(0); + return 0; } int codeline_cmp(const void* ATTR_UNUSED(a), const void* ATTR_UNUSED(b)) { - log_assert(0); - return 0; + log_assert(0); + return 0; } diff --git a/doc/Changelog b/doc/Changelog index 73f034fc5..a4882a4e9 100644 --- a/doc/Changelog +++ b/doc/Changelog @@ -1,3 +1,10 @@ +3 August 2018: George + - Expose if a query (or a subquery) was ratelimited (not src IP + ratelimiting) to libunbound under 'ub_result.was_ratelimited'. + This also introduces a change to 'ub_event_callback_type' in + libunbound/unbound-event.h. + - Tidy pylib tests. + 3 August 2018: Wouter - Revert previous change for #4136: because it introduces build problems. diff --git a/iterator/iterator.c b/iterator/iterator.c index 2604856b5..f6bb30bf2 100644 --- a/iterator/iterator.c +++ b/iterator/iterator.c @@ -1399,6 +1399,7 @@ processInitRequest(struct module_qstate* qstate, struct iter_qstate* iq, log_nametypeclass(VERB_ALGO, "ratelimit exceeded with " "delegation point", iq->dp->name, LDNS_RR_TYPE_NS, LDNS_RR_CLASS_IN); + qstate->was_ratelimited = 1; return error_response(qstate, id, LDNS_RCODE_SERVFAIL); } } @@ -2379,6 +2380,7 @@ processQueryTargets(struct module_qstate* qstate, struct iter_qstate* iq, ie->num_queries_ratelimited++; lock_basic_unlock(&ie->queries_ratelimit_lock); verbose(VERB_ALGO, "query exceeded ratelimits"); + qstate->was_ratelimited = 1; return error_response(qstate, id, LDNS_RCODE_SERVFAIL); } } diff --git a/libunbound/context.c b/libunbound/context.c index 20b463f7c..40b5be1e6 100644 --- a/libunbound/context.c +++ b/libunbound/context.c @@ -293,26 +293,29 @@ context_serialize_answer(struct ctx_query* q, int err, sldns_buffer* pkt, * o uint32 id * o uint32 error_code * o uint32 msg_security + * o uint32 was_ratelimited * o uint32 length of why_bogus string (+1 for eos); 0 absent. * o why_bogus_string * o the remainder is the answer msg from resolver lookup. * remainder can be length 0. */ + size_t size_of_uint32s = 6 * sizeof(uint32_t); size_t pkt_len = pkt?sldns_buffer_remaining(pkt):0; size_t wlen = (pkt&&q->res->why_bogus)?strlen(q->res->why_bogus)+1:0; uint8_t* p; - *len = sizeof(uint32_t)*5 + pkt_len + wlen; + *len = size_of_uint32s + pkt_len + wlen; p = (uint8_t*)malloc(*len); if(!p) return NULL; sldns_write_uint32(p, UB_LIBCMD_ANSWER); sldns_write_uint32(p+sizeof(uint32_t), (uint32_t)q->querynum); sldns_write_uint32(p+2*sizeof(uint32_t), (uint32_t)err); sldns_write_uint32(p+3*sizeof(uint32_t), (uint32_t)q->msg_security); - sldns_write_uint32(p+4*sizeof(uint32_t), (uint32_t)wlen); + sldns_write_uint32(p+4*sizeof(uint32_t), (uint32_t)q->res->was_ratelimited); + sldns_write_uint32(p+5*sizeof(uint32_t), (uint32_t)wlen); if(wlen > 0) - memmove(p+5*sizeof(uint32_t), q->res->why_bogus, wlen); + memmove(p+size_of_uint32s, q->res->why_bogus, wlen); if(pkt_len > 0) - memmove(p+5*sizeof(uint32_t)+wlen, + memmove(p+size_of_uint32s+wlen, sldns_buffer_begin(pkt), pkt_len); return p; } @@ -321,21 +324,23 @@ struct ctx_query* context_deserialize_answer(struct ub_ctx* ctx, uint8_t* p, uint32_t len, int* err) { + size_t size_of_uint32s = 6 * sizeof(uint32_t); struct ctx_query* q = NULL ; int id; size_t wlen; - if(len < 5*sizeof(uint32_t)) return NULL; + if(len < size_of_uint32s) return NULL; log_assert( sldns_read_uint32(p) == UB_LIBCMD_ANSWER); id = (int)sldns_read_uint32(p+sizeof(uint32_t)); q = (struct ctx_query*)rbtree_search(&ctx->queries, &id); if(!q) return NULL; *err = (int)sldns_read_uint32(p+2*sizeof(uint32_t)); q->msg_security = sldns_read_uint32(p+3*sizeof(uint32_t)); - wlen = (size_t)sldns_read_uint32(p+4*sizeof(uint32_t)); - if(len > 5*sizeof(uint32_t) && wlen > 0) { - if(len >= 5*sizeof(uint32_t)+wlen) + q->res->was_ratelimited = sldns_read_uint32(p+4*sizeof(uint32_t)); + wlen = (size_t)sldns_read_uint32(p+5*sizeof(uint32_t)); + if(len > size_of_uint32s && wlen > 0) { + if(len >= size_of_uint32s+wlen) q->res->why_bogus = (char*)memdup( - p+5*sizeof(uint32_t), wlen); + p+size_of_uint32s, wlen); if(!q->res->why_bogus) { /* pass malloc failure to the user callback */ q->msg_len = 0; @@ -344,9 +349,9 @@ context_deserialize_answer(struct ub_ctx* ctx, } q->res->why_bogus[wlen-1] = 0; /* zero terminated for sure */ } - if(len > 5*sizeof(uint32_t)+wlen) { - q->msg_len = len - 5*sizeof(uint32_t) - wlen; - q->msg = (uint8_t*)memdup(p+5*sizeof(uint32_t)+wlen, + if(len > size_of_uint32s+wlen) { + q->msg_len = len - size_of_uint32s - wlen; + q->msg = (uint8_t*)memdup(p+size_of_uint32s+wlen, q->msg_len); if(!q->msg) { /* pass malloc failure to the user callback */ diff --git a/libunbound/libworker.c b/libunbound/libworker.c index 3dcaa7818..d984b3f39 100644 --- a/libunbound/libworker.c +++ b/libunbound/libworker.c @@ -521,8 +521,9 @@ libworker_enter_result(struct ub_result* res, sldns_buffer* buf, /** fillup fg results */ static void libworker_fillup_fg(struct ctx_query* q, int rcode, sldns_buffer* buf, - enum sec_status s, char* why_bogus) + enum sec_status s, char* why_bogus, int was_ratelimited) { + q->res->was_ratelimited = was_ratelimited; if(why_bogus) q->res->why_bogus = strdup(why_bogus); if(rcode != 0) { @@ -546,13 +547,13 @@ libworker_fillup_fg(struct ctx_query* q, int rcode, sldns_buffer* buf, void libworker_fg_done_cb(void* arg, int rcode, sldns_buffer* buf, enum sec_status s, - char* why_bogus) + char* why_bogus, int was_ratelimited) { struct ctx_query* q = (struct ctx_query*)arg; /* fg query is done; exit comm base */ comm_base_exit(q->w->base); - libworker_fillup_fg(q, rcode, buf, s, why_bogus); + libworker_fillup_fg(q, rcode, buf, s, why_bogus, was_ratelimited); } /** setup qinfo and edns */ @@ -603,7 +604,7 @@ int libworker_fg(struct ub_ctx* ctx, struct ctx_query* q) NULL, 0, NULL, 0, NULL)) { regional_free_all(w->env->scratch); libworker_fillup_fg(q, LDNS_RCODE_NOERROR, - w->back->udp_buff, sec_status_insecure, NULL); + w->back->udp_buff, sec_status_insecure, NULL, 0); libworker_delete(w); free(qinfo.qname); return UB_NOERROR; @@ -612,7 +613,7 @@ int libworker_fg(struct ub_ctx* ctx, struct ctx_query* q) w->env, &qinfo, &edns, w->back->udp_buff, w->env->scratch)) { regional_free_all(w->env->scratch); libworker_fillup_fg(q, LDNS_RCODE_NOERROR, - w->back->udp_buff, sec_status_insecure, NULL); + w->back->udp_buff, sec_status_insecure, NULL, 0); libworker_delete(w); free(qinfo.qname); return UB_NOERROR; @@ -634,7 +635,7 @@ int libworker_fg(struct ub_ctx* ctx, struct ctx_query* q) void libworker_event_done_cb(void* arg, int rcode, sldns_buffer* buf, - enum sec_status s, char* why_bogus) + enum sec_status s, char* why_bogus, int was_ratelimited) { struct ctx_query* q = (struct ctx_query*)arg; ub_event_callback_type cb = q->cb_event; @@ -657,7 +658,7 @@ libworker_event_done_cb(void* arg, int rcode, sldns_buffer* buf, else if(s == sec_status_secure) sec = 2; (*cb)(cb_arg, rcode, (void*)sldns_buffer_begin(buf), - (int)sldns_buffer_limit(buf), sec, why_bogus); + (int)sldns_buffer_limit(buf), sec, why_bogus, was_ratelimited); } } @@ -684,7 +685,7 @@ int libworker_attach_mesh(struct ub_ctx* ctx, struct ctx_query* q, regional_free_all(w->env->scratch); free(qinfo.qname); libworker_event_done_cb(q, LDNS_RCODE_NOERROR, - w->back->udp_buff, sec_status_insecure, NULL); + w->back->udp_buff, sec_status_insecure, NULL, 0); return UB_NOERROR; } if(ctx->env->auth_zones && auth_zones_answer(ctx->env->auth_zones, @@ -692,7 +693,7 @@ int libworker_attach_mesh(struct ub_ctx* ctx, struct ctx_query* q, regional_free_all(w->env->scratch); free(qinfo.qname); libworker_event_done_cb(q, LDNS_RCODE_NOERROR, - w->back->udp_buff, sec_status_insecure, NULL); + w->back->udp_buff, sec_status_insecure, NULL, 0); return UB_NOERROR; } /* process new query */ @@ -710,7 +711,7 @@ int libworker_attach_mesh(struct ub_ctx* ctx, struct ctx_query* q, /** add result to the bg worker result queue */ static void add_bg_result(struct libworker* w, struct ctx_query* q, sldns_buffer* pkt, - int err, char* reason) + int err, char* reason, int was_ratelimited) { uint8_t* msg = NULL; uint32_t len = 0; @@ -724,19 +725,23 @@ add_bg_result(struct libworker* w, struct ctx_query* q, sldns_buffer* pkt, lock_basic_lock(&w->ctx->cfglock); if(reason) q->res->why_bogus = strdup(reason); + q->res->was_ratelimited = was_ratelimited; if(pkt) { q->msg_len = sldns_buffer_remaining(pkt); q->msg = memdup(sldns_buffer_begin(pkt), q->msg_len); - if(!q->msg) - msg = context_serialize_answer(q, UB_NOMEM, - NULL, &len); - else msg = context_serialize_answer(q, err, - NULL, &len); - } else msg = context_serialize_answer(q, err, NULL, &len); + if(!q->msg) { + msg = context_serialize_answer(q, UB_NOMEM, NULL, &len); + } else { + msg = context_serialize_answer(q, err, NULL, &len); + } + } else { + msg = context_serialize_answer(q, err, NULL, &len); + } lock_basic_unlock(&w->ctx->cfglock); } else { if(reason) q->res->why_bogus = strdup(reason); + q->res->was_ratelimited = was_ratelimited; msg = context_serialize_answer(q, err, pkt, &len); (void)rbtree_delete(&w->ctx->queries, q->node.key); w->ctx->num_async--; @@ -755,7 +760,7 @@ add_bg_result(struct libworker* w, struct ctx_query* q, sldns_buffer* pkt, void libworker_bg_done_cb(void* arg, int rcode, sldns_buffer* buf, enum sec_status s, - char* why_bogus) + char* why_bogus, int was_ratelimited) { struct ctx_query* q = (struct ctx_query*)arg; @@ -773,12 +778,13 @@ libworker_bg_done_cb(void* arg, int rcode, sldns_buffer* buf, enum sec_status s, return; } q->msg_security = s; - if(!buf) + if(!buf) { buf = q->w->env->scratch_buffer; + } if(rcode != 0) { error_encode(buf, rcode, NULL, 0, BIT_RD, NULL); } - add_bg_result(q->w, q, buf, UB_NOERROR, why_bogus); + add_bg_result(q->w, q, buf, UB_NOERROR, why_bogus, was_ratelimited); } @@ -803,7 +809,7 @@ handle_newq(struct libworker* w, uint8_t* buf, uint32_t len) return; } if(!setup_qinfo_edns(w, q, &qinfo, &edns)) { - add_bg_result(w, q, NULL, UB_SYNTAX, NULL); + add_bg_result(w, q, NULL, UB_SYNTAX, NULL, 0); return; } qid = 0; @@ -816,7 +822,7 @@ handle_newq(struct libworker* w, uint8_t* buf, uint32_t len) NULL, 0, NULL, 0, NULL)) { regional_free_all(w->env->scratch); q->msg_security = sec_status_insecure; - add_bg_result(w, q, w->back->udp_buff, UB_NOERROR, NULL); + add_bg_result(w, q, w->back->udp_buff, UB_NOERROR, NULL, 0); free(qinfo.qname); return; } @@ -824,7 +830,7 @@ handle_newq(struct libworker* w, uint8_t* buf, uint32_t len) w->env, &qinfo, &edns, w->back->udp_buff, w->env->scratch)) { regional_free_all(w->env->scratch); q->msg_security = sec_status_insecure; - add_bg_result(w, q, w->back->udp_buff, UB_NOERROR, NULL); + add_bg_result(w, q, w->back->udp_buff, UB_NOERROR, NULL, 0); free(qinfo.qname); return; } @@ -832,7 +838,7 @@ handle_newq(struct libworker* w, uint8_t* buf, uint32_t len) /* process new query */ if(!mesh_new_callback(w->env->mesh, &qinfo, qflags, &edns, w->back->udp_buff, qid, libworker_bg_done_cb, q)) { - add_bg_result(w, q, NULL, UB_NOMEM, NULL); + add_bg_result(w, q, NULL, UB_NOMEM, NULL, 0); } free(qinfo.qname); } diff --git a/libunbound/unbound-event.h b/libunbound/unbound-event.h index d5f0b1a36..4d694b8b4 100644 --- a/libunbound/unbound-event.h +++ b/libunbound/unbound-event.h @@ -170,7 +170,7 @@ struct ub_event { struct ub_event_vmt* vmt; }; -typedef void (*ub_event_callback_type)(void*, int, void*, int, int, char*); +typedef void (*ub_event_callback_type)(void*, int, void*, int, int, char*, int); /** * Create a resolving and validation context. diff --git a/libunbound/unbound.h b/libunbound/unbound.h index 66b40ca73..6bb0ef224 100644 --- a/libunbound/unbound.h +++ b/libunbound/unbound.h @@ -203,6 +203,12 @@ struct ub_result { */ char* why_bogus; + /** + * If the query or one of its subqueries was ratelimited. Useful if + * ratelimiting is enabled and answer is SERVFAIL. + */ + int was_ratelimited; + /** * TTL for the result, in seconds. If the security is bogus, then * you also cannot trust this value. diff --git a/libunbound/worker.h b/libunbound/worker.h index 7d2ede04e..fe1d51878 100644 --- a/libunbound/worker.h +++ b/libunbound/worker.h @@ -89,15 +89,15 @@ void libworker_handle_control_cmd(struct tube* tube, uint8_t* msg, size_t len, /** mesh callback with fg results */ void libworker_fg_done_cb(void* arg, int rcode, sldns_buffer* buf, - enum sec_status s, char* why_bogus); + enum sec_status s, char* why_bogus, int was_ratelimited); /** mesh callback with bg results */ void libworker_bg_done_cb(void* arg, int rcode, sldns_buffer* buf, - enum sec_status s, char* why_bogus); + enum sec_status s, char* why_bogus, int was_ratelimited); /** mesh callback with event results */ void libworker_event_done_cb(void* arg, int rcode, struct sldns_buffer* buf, - enum sec_status s, char* why_bogus); + enum sec_status s, char* why_bogus, int was_ratelimited); /** * Worker signal handler function. User argument is the worker itself. diff --git a/services/authzone.c b/services/authzone.c index 9de43b754..b9576eaae 100644 --- a/services/authzone.c +++ b/services/authzone.c @@ -5150,7 +5150,8 @@ xfr_master_add_addrs(struct auth_master* m, struct ub_packed_rrset_key* rrset, /** callback for task_transfer lookup of host name, of A or AAAA */ void auth_xfer_transfer_lookup_callback(void* arg, int rcode, sldns_buffer* buf, - enum sec_status ATTR_UNUSED(sec), char* ATTR_UNUSED(why_bogus)) + enum sec_status ATTR_UNUSED(sec), char* ATTR_UNUSED(why_bogus), + int ATTR_UNUSED(was_ratelimited)) { struct auth_xfer* xfr = (struct auth_xfer*)arg; struct module_env* env; @@ -6061,7 +6062,8 @@ xfr_probe_send_or_end(struct auth_xfer* xfr, struct module_env* env) /** callback for task_probe lookup of host name, of A or AAAA */ void auth_xfer_probe_lookup_callback(void* arg, int rcode, sldns_buffer* buf, - enum sec_status ATTR_UNUSED(sec), char* ATTR_UNUSED(why_bogus)) + enum sec_status ATTR_UNUSED(sec), char* ATTR_UNUSED(why_bogus), + int ATTR_UNUSED(was_ratelimited)) { struct auth_xfer* xfr = (struct auth_xfer*)arg; struct module_env* env; diff --git a/services/authzone.h b/services/authzone.h index 6b25452dd..39a16966c 100644 --- a/services/authzone.h +++ b/services/authzone.h @@ -646,10 +646,12 @@ int auth_xfer_transfer_http_callback(struct comm_point* c, void* arg, int err, void auth_xfer_probe_timer_callback(void* arg); /** mesh callback for task_probe on lookup of host names */ void auth_xfer_probe_lookup_callback(void* arg, int rcode, - struct sldns_buffer* buf, enum sec_status sec, char* why_bogus); + struct sldns_buffer* buf, enum sec_status sec, char* why_bogus, + int was_ratelimited); /** mesh callback for task_transfer on lookup of host names */ void auth_xfer_transfer_lookup_callback(void* arg, int rcode, - struct sldns_buffer* buf, enum sec_status sec, char* why_bogus); + struct sldns_buffer* buf, enum sec_status sec, char* why_bogus, + int was_ratelimited); /* * Compares two 32-bit serial numbers as defined in RFC1982. Returns diff --git a/services/mesh.c b/services/mesh.c index c18e6867d..55b909ede 100644 --- a/services/mesh.c +++ b/services/mesh.c @@ -632,8 +632,8 @@ void mesh_report_reply(struct mesh_area* mesh, struct outbound_entry* e, mesh_run(mesh, e->qstate->mesh_info, event, e); } -struct mesh_state* -mesh_state_create(struct module_env* env, struct query_info* qinfo, +struct mesh_state* +mesh_state_create(struct module_env* env, struct query_info* qinfo, struct respip_client_info* cinfo, uint16_t qflags, int prime, int valrec) { @@ -694,6 +694,7 @@ mesh_state_create(struct module_env* env, struct query_info* qinfo, mstate->s.no_cache_lookup = 0; mstate->s.no_cache_store = 0; mstate->s.need_refetch = 0; + mstate->s.was_ratelimited = 0; /* init modules */ for(i=0; imesh->mods.num; i++) { @@ -741,7 +742,7 @@ mesh_state_cleanup(struct mesh_state* mstate) mstate->cb_list = cb->next; fptr_ok(fptr_whitelist_mesh_cb(cb->cb)); (*cb->cb)(cb->cb_arg, LDNS_RCODE_SERVFAIL, NULL, - sec_status_unchecked, NULL); + sec_status_unchecked, NULL, 0); mesh->num_reply_addrs--; } } @@ -969,7 +970,8 @@ mesh_do_callback(struct mesh_state* m, int rcode, struct reply_info* rep, { int secure; char* reason = NULL; - /* bogus messages are not made into servfail, sec_status passed + int was_ratelimited = m->s.was_ratelimited; + /* bogus messages are not made into servfail, sec_status passed * to the callback function */ if(rep && rep->security == sec_status_secure) secure = 1; @@ -993,7 +995,8 @@ mesh_do_callback(struct mesh_state* m, int rcode, struct reply_info* rep, r->edns.opt_list = NULL; } fptr_ok(fptr_whitelist_mesh_cb(r->cb)); - (*r->cb)(r->cb_arg, rcode, r->buf, sec_status_unchecked, NULL); + (*r->cb)(r->cb_arg, rcode, r->buf, sec_status_unchecked, NULL, + was_ratelimited); } else { size_t udp_size = r->edns.udp_size; sldns_buffer_clear(r->buf); @@ -1011,11 +1014,11 @@ mesh_do_callback(struct mesh_state* m, int rcode, struct reply_info* rep, { fptr_ok(fptr_whitelist_mesh_cb(r->cb)); (*r->cb)(r->cb_arg, LDNS_RCODE_SERVFAIL, r->buf, - sec_status_unchecked, NULL); + sec_status_unchecked, NULL, 0); } else { fptr_ok(fptr_whitelist_mesh_cb(r->cb)); (*r->cb)(r->cb_arg, LDNS_RCODE_NOERROR, r->buf, - rep->security, reason); + rep->security, reason, was_ratelimited); } } free(reason); @@ -1201,6 +1204,8 @@ void mesh_walk_supers(struct mesh_area* mesh, struct mesh_state* mstate) mesh->mods.mod[ref->s->s.curmod]->inform_super)); (*mesh->mods.mod[ref->s->s.curmod]->inform_super)(&mstate->s, ref->s->s.curmod, &ref->s->s); + /* copy state that is always relevant to super */ + copy_state_to_super(&mstate->s, ref->s->s.curmod, &ref->s->s); } } diff --git a/services/mesh.h b/services/mesh.h index 67749accb..b4ce03e7e 100644 --- a/services/mesh.h +++ b/services/mesh.h @@ -223,10 +223,11 @@ struct mesh_reply { /** * Mesh result callback func. - * called as func(cb_arg, rcode, buffer_with_reply, security, why_bogus); + * called as func(cb_arg, rcode, buffer_with_reply, security, why_bogus, + * was_ratelimited); */ -typedef void (*mesh_cb_func_type)(void*, int, struct sldns_buffer*, enum sec_status, - char*); +typedef void (*mesh_cb_func_type)(void* cb_arg, int rcode, struct sldns_buffer*, + enum sec_status, char* why_bogus, int was_ratelimited); /** * Callback to result routine @@ -242,9 +243,8 @@ struct mesh_cb { uint16_t qflags; /** buffer for reply */ struct sldns_buffer* buf; - /** callback routine for results. if rcode != 0 buf has message. - * called as cb(cb_arg, rcode, buf, sec_state); + * called as cb(cb_arg, rcode, buf, sec_state, why_bogus, was_ratelimited); */ mesh_cb_func_type cb; /** user arg for callback */ diff --git a/smallapp/worker_cb.c b/smallapp/worker_cb.c index dda94cc67..6c3bd0049 100644 --- a/smallapp/worker_cb.c +++ b/smallapp/worker_cb.c @@ -168,21 +168,21 @@ void libworker_handle_control_cmd(struct tube* ATTR_UNUSED(tube), void libworker_fg_done_cb(void* ATTR_UNUSED(arg), int ATTR_UNUSED(rcode), struct sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(s), - char* ATTR_UNUSED(why_bogus)) + char* ATTR_UNUSED(why_bogus), int ATTR_UNUSED(was_ratelimited)) { log_assert(0); } void libworker_bg_done_cb(void* ATTR_UNUSED(arg), int ATTR_UNUSED(rcode), struct sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(s), - char* ATTR_UNUSED(why_bogus)) + char* ATTR_UNUSED(why_bogus), int ATTR_UNUSED(was_ratelimited)) { log_assert(0); } void libworker_event_done_cb(void* ATTR_UNUSED(arg), int ATTR_UNUSED(rcode), struct sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(s), - char* ATTR_UNUSED(why_bogus)) + char* ATTR_UNUSED(why_bogus), int ATTR_UNUSED(was_ratelimited)) { log_assert(0); } diff --git a/testdata/pylib.tdir/pylib.conf b/testdata/pylib.tdir/pylib.lookup.conf similarity index 56% rename from testdata/pylib.tdir/pylib.conf rename to testdata/pylib.tdir/pylib.lookup.conf index 82fa27a68..3f4a8d26d 100644 --- a/testdata/pylib.tdir/pylib.conf +++ b/testdata/pylib.tdir/pylib.lookup.conf @@ -1,7 +1,5 @@ server: verbosity: 2 - # num-threads: 1 - #port: @PORT@ use-syslog: no directory: "" pidfile: "unbound.pid" @@ -10,10 +8,7 @@ server: do-not-query-localhost: no module-config: "validator iterator" -#python: - #python-script: "pylib.py" - -forward-zone: - name: "." - forward-addr: "127.0.0.1@@TOPORT@" +stub-zone: + name: "example.com." + stub-addr: "127.0.0.1@@TOPORT@" diff --git a/testdata/pylib.tdir/pylib.lookup.py b/testdata/pylib.tdir/pylib.lookup.py index 1c829bc70..d45333d76 100755 --- a/testdata/pylib.tdir/pylib.lookup.py +++ b/testdata/pylib.tdir/pylib.lookup.py @@ -1,24 +1,139 @@ #!/usr/bin/env python -''' -Test for unbound lookup. -BSD licensed. -''' +# +# Test for pyunbound lookup. +# BSD licensed. +# +import sys +import time + import unbound -ctx = unbound.ub_ctx() -status = ctx.config("ub.conf") -if status != 0: - print "read config failed ", status - exit(1) +qname = "www.example.com" +qtype = unbound.RR_TYPE_A +qclass = unbound.RR_CLASS_IN + +def create_context(config_file="ub.lookup.conf", async=False): + """ + Create an unbound context to use for testing. + + """ + ctx = unbound.ub_ctx() + status = ctx.config(config_file) + if status != 0: + print("read config failed with status: {}".format(status)) + sys.exit(1) + ctx.set_async(async) + return ctx + + +def callback(data, status, result): + """ + Callback for background workers. + + """ + if status == 0: + data['rcode'] = result.rcode + data['secure'] = result.secure + if result.havedata: + data['data'] = result.data + data['was_ratelimited'] = result.was_ratelimited + data['done'] = True + + +def test_resolve(ctx): + """ + Test resolving a domain with a foreground worker. + + """ + status, result = ctx.resolve(qname, qtype, qclass) + if status == 0 and result.havedata: + print("Resolve: {}".format(result.data.address_list)) + else: + print("Failed resolve with: {}".format(status)) + + +def test_async_resolve(ctx): + """ + Test resolving a domain with a background worker. + + """ + cb_data = dict(done=False) + retval, async_id = ctx.resolve_async(qname, cb_data, callback, qtype, qclass) + while retval == 0 and not cb_data['done']: + time.sleep(0.1) + retval = ctx.process() + + if cb_data.get('data'): + print("Async resolve: {}".format(cb_data['data'].address_list)) + else: + print("Failed async resolve with: {}".format(retval)) + + +def test_ratelimit_fg_on(ctx): + """ + Test resolving a ratelimited domain with a foreground worker. + + """ + ctx.set_option("ratelimit:", "1") + status, result = ctx.resolve(qname, qtype, qclass) + if status == 0 and result.was_ratelimited: + print("Ratelimit-fg-on: pass") + else: + print("Failed ratelimit-fg-on with: {}".format(status)) + + +def test_ratelimit_fg_off(ctx): + """ + Test resolving a non-ratelimited domain with a foreground worker. + + """ + status, result = ctx.resolve(qname, qtype, qclass) + if status == 0 and result.havedata: + print("Ratelimit-fg-off: {}".format(result.data.address_list)) + else: + print("Failed ratelimit-fg-off with: {}".format(status)) + + +def test_ratelimit_bg_on(ctx): + """ + Test resolving a ratelimited domain with a background worker. + + """ + ctx.set_option("ratelimit:", "1") + cb_data = dict(done=False) + retval, async_id = ctx.resolve_async(qname, cb_data, callback, qtype, qclass) + while retval == 0 and not cb_data['done']: + time.sleep(0.1) + retval = ctx.process() + + if cb_data.get('was_ratelimited'): + print("Ratelimit-bg-on: pass") + else: + print("Failed ratelimit-bg-on with: {}".format(status)) + + +def test_ratelimit_bg_off(ctx): + """ + Test resolving a non-ratelimited domain with a background worker. + + """ + cb_data = dict(done=False) + retval, async_id = ctx.resolve_async(qname, cb_data, callback, qtype, qclass) + while retval == 0 and not cb_data['done']: + time.sleep(0.1) + retval = ctx.process() -print "config created" + if cb_data.get('data'): + print("Ratelimit-bg-off: {}".format(cb_data['data'].address_list)) + else: + print("Failed ratelimit-bg-off with: {}".format(status)) -status, result = ctx.resolve("www.example.com", unbound.RR_TYPE_A, unbound.RR_CLASS_IN); -if status == 0 and result.havedata: - print "Result: ", result.data.address_list -else: - print "Failed ", status, " and data ", result -ctx = None +test_resolve(create_context()) +test_async_resolve(create_context(async=True)) +test_ratelimit_fg_on(create_context()) +test_ratelimit_fg_off(create_context()) +test_ratelimit_bg_on(create_context(async=True)) +test_ratelimit_bg_off(create_context(async=True)) -exit(0) +sys.exit(0) diff --git a/testdata/pylib.tdir/pylib.post b/testdata/pylib.tdir/pylib.post index 26db7e80e..875e06d0a 100644 --- a/testdata/pylib.tdir/pylib.post +++ b/testdata/pylib.tdir/pylib.post @@ -10,9 +10,9 @@ PRE="../.." . ../common.sh # if no python; exit if grep "define WITH_PYUNBOUND 1" $PRE/config.h; then - echo "have python module" + echo "have pyunbound" else - echo "no python module" + echo "no pyunbound" exit 0 fi diff --git a/testdata/pylib.tdir/pylib.pre b/testdata/pylib.tdir/pylib.pre index 01ca2b896..30e005906 100644 --- a/testdata/pylib.tdir/pylib.pre +++ b/testdata/pylib.tdir/pylib.pre @@ -8,13 +8,16 @@ PRE="../.." . ../common.sh # if no python; exit if grep "define WITH_PYUNBOUND 1" $PRE/config.h; then - echo "have python module" + echo "have pyunbound" else - echo "no python module" + echo "no pyunbound" exit 0 fi -# get module python local -cp $PRE/pythonmod/unboundmodule.py . + +# Copy the required libraries +cp $PRE/libunbound/python/unbound.py . +cp $PRE/.libs/_unbound* . +cp $PRE/.libs/libunbound* . get_random_port 2 UNBOUND_PORT=$RND_PORT @@ -29,7 +32,7 @@ FWD_PID=$! echo "FWD_PID=$FWD_PID" >> .tpkg.var.test # modify config file -sed -e 's/@PORT\@/'$UNBOUND_PORT'/' -e 's/@TOPORT\@/'$FWD_PORT'/' < pylib.conf > ub.conf +sed -e 's/@PORT\@/'$UNBOUND_PORT'/' -e 's/@TOPORT\@/'$FWD_PORT'/' < pylib.lookup.conf > ub.lookup.conf cat .tpkg.var.test wait_ldns_testns_up fwd.log diff --git a/testdata/pylib.tdir/pylib.py b/testdata/pylib.tdir/pylib.py deleted file mode 100644 index 3f6fed1c6..000000000 --- a/testdata/pylib.tdir/pylib.py +++ /dev/null @@ -1,159 +0,0 @@ -# -*- coding: utf-8 -*- -''' - ubmodule-msg.py: simple response packet logger - - Authors: Zdenek Vasicek (vasicek AT fit.vutbr.cz) - Marek Vavrusa (xvavru00 AT stud.fit.vutbr.cz) - - Copyright (c) 2008. All rights reserved. - - This software is open source. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - Modified for unit test by Wouter Wijngaards, NLnet Labs, 2009. -''' -import os - -def init(id, cfg): - log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, cfg.port, cfg.python_script)) - return True - -def deinit(id): - log_info("pythonmod: deinit called, module id is %d" % id) - return True - -def inform_super(id, qstate, superqstate, qdata): - return True - -def setTTL(qstate, ttl): - """Sets return_msg TTL and all the RRs TTL""" - if qstate.return_msg: - qstate.return_msg.rep.ttl = ttl - if (qstate.return_msg.rep): - for i in range(0,qstate.return_msg.rep.rrset_count): - d = qstate.return_msg.rep.rrsets[i].entry.data - for j in range(0,d.count+d.rrsig_count): - d.rr_ttl[j] = ttl - -def dataHex(data, prefix=""): - res = "" - for i in range(0, (len(data)+15)/16): - res += "%s0x%02X | " % (prefix, i*16) - d = map(lambda x:ord(x), data[i*16:i*16+17]) - for ch in d: - res += "%02X " % ch - for i in range(0,17-len(d)): - res += " " - res += "| " - for ch in d: - if (ch < 32) or (ch > 127): - res += ". " - else: - res += "%c " % ch - res += "\n" - return res - -def printReturnMsg(qstate): - print "Return MSG rep :: flags: %04X, QDcount: %d, Security:%d, TTL=%d" % (qstate.return_msg.rep.flags, qstate.return_msg.rep.qdcount,qstate.return_msg.rep.security, qstate.return_msg.rep.ttl) - print " qinfo :: qname:",qstate.return_msg.qinfo.qname_list, qstate.return_msg.qinfo.qname_str, "type:",qstate.return_msg.qinfo.qtype_str, "class:",qstate.return_msg.qinfo.qclass_str - if (qstate.return_msg.rep): - print "RRSets:",qstate.return_msg.rep.rrset_count - prevkey = None - for i in range(0,qstate.return_msg.rep.rrset_count): - r = qstate.return_msg.rep.rrsets[i] - rk = r.rk - print i,":",rk.dname_list, rk.dname_str, "flags: %04X" % rk.flags, - print "type:",rk.type_str,"(%d)" % ntohs(rk.type), "class:",rk.rrset_class_str,"(%d)" % ntohs(rk.rrset_class) - - d = r.entry.data - print " RRDatas:",d.count+d.rrsig_count - for j in range(0,d.count+d.rrsig_count): - print " ",j,":","TTL=",d.rr_ttl[j],"RR data:" - print dataHex(d.rr_data[j]," ") - - -def operate(id, event, qstate, qdata): - log_info("pythonmod: operate called, id: %d, event:%s" % (id, strmodulevent(event))) - #print "pythonmod: per query data", qdata - - print "Query:", ''.join(map(lambda x:chr(max(32,ord(x))),qstate.qinfo.qname)), qstate.qinfo.qname_list, qstate.qinfo.qname_str, - print "Type:",qstate.qinfo.qtype_str,"(%d)" % qstate.qinfo.qtype, - print "Class:",qstate.qinfo.qclass_str,"(%d)" % qstate.qinfo.qclass - print - - if (event == MODULE_EVENT_NEW or event == MODULE_EVENT_PASS) and (qstate.qinfo.qname_str.endswith("www2.example.com.")): - print qstate.qinfo.qname_str - - qstate.ext_state[id] = MODULE_FINISHED - - msg = DNSMessage(qstate.qinfo.qname_str, RR_TYPE_A, RR_CLASS_IN, PKT_QR | PKT_RA | PKT_AA) #, 300) - #msg.authority.append("xxx.seznam.cz. 10 IN A 192.168.1.1") - #msg.additional.append("yyy.seznam.cz. 10 IN A 1.1.1.2.") - - # answer can be returned to the client without further checking. - - if qstate.qinfo.qtype == RR_TYPE_A: - msg.answer.append("%s 10 IN A 192.168.1.1" % qstate.qinfo.qname_str) - if (qstate.qinfo.qtype == RR_TYPE_SRV) or (qstate.qinfo.qtype == RR_TYPE_ANY): - msg.answer.append("%s 10 IN SRV 0 0 80 neinfo.example.com." % qstate.qinfo.qname_str) - if (qstate.qinfo.qtype == RR_TYPE_TXT) or (qstate.qinfo.qtype == RR_TYPE_ANY): - msg.answer.append("%s 10 IN TXT path=/" % qstate.qinfo.qname_str) - - if not msg.set_return_msg(qstate): - qstate.ext_state[id] = MODULE_ERROR - return True - - #qstate.return_msg.rep.security = 2 #pokud nebude nasledovat validator, je zapotrebi nastavit security, aby nebyl paket zahozen v mesh_send_reply - printReturnMsg(qstate) - - #Authoritative result can't be stored in cache - #if (not storeQueryInCache(qstate, qstate.return_msg.qinfo, qstate.return_msg.rep, 0)): - # print "Can't store in cache" - # qstate.ext_state[id] = MODULE_ERROR - # return False - #print "Store OK" - - qstate.return_rcode = RCODE_NOERROR - return True - - if event == MODULE_EVENT_NEW: - qstate.ext_state[id] = MODULE_WAIT_MODULE - return True - - if event == MODULE_EVENT_MODDONE: - log_info("pythonmod: previous module done") - qstate.ext_state[id] = MODULE_FINISHED - return True - - if event == MODULE_EVENT_PASS: - log_info("pythonmod: event_pass") - qstate.ext_state[id] = MODULE_WAIT_MODULE - return True - - log_err("pythonmod: BAD event") - qstate.ext_state[id] = MODULE_ERROR - return True - -log_info("pythonmod: script loaded.") diff --git a/testdata/pylib.tdir/pylib.test b/testdata/pylib.tdir/pylib.test index 96a846403..9456691aa 100644 --- a/testdata/pylib.tdir/pylib.test +++ b/testdata/pylib.tdir/pylib.test @@ -6,9 +6,9 @@ PRE="../.." if grep "define WITH_PYUNBOUND 1" $PRE/config.h; then - echo "have python module" + echo "have pyunbound" else - echo "no python module" + echo "no pyunbound" exit 0 fi @@ -19,22 +19,50 @@ fi #echo export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:../../.libs:." #export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:../../.libs:." -cp $PRE/libunbound/python/unbound.py . -cp $PRE/.libs/_unbound* . -cp $PRE/.libs/libunbound* . - # do the test echo "> pylib.lookup.py www.example.com." ./pylib.lookup.py www.example.com. | tee outfile echo "> cat logfiles" -cat fwd.log +cat fwd.log echo "> check answer" -if grep "10.20.30.40" outfile; then - echo "OK" +if grep "Resolve: \[.\?10.20.30.40.\?\]" outfile; then + : else - echo "Not OK" + echo "Not OK (resolve)" exit 1 fi +if grep "Async resolve: \[.\?10.20.30.40.\?\]" outfile; then + : +else + echo "Not OK (async resolve)" + exit 1 +fi +if grep "Ratelimit-fg-on: pass" outfile; then + : +else + echo "Not OK (ratelimit-fg-on)" + exit 1 +fi +if grep "Ratelimit-fg-off: \[.\?10.20.30.40.\?\]" outfile; then + : +else + echo "Not OK (ratelimit-fg-off)" + exit 1 +fi +if grep "Ratelimit-bg-on: pass" outfile; then + : +else + echo "Not OK (ratelimit-bg-on)" + exit 1 +fi +if grep "Ratelimit-bg-off: \[.\?10.20.30.40.\?\]" outfile; then + : +else + echo "Not OK (ratelimit-bg-off)" + exit 1 +fi + +echo "OK" exit 0 diff --git a/util/data/msgreply.c b/util/data/msgreply.c index 772f5d1f1..1e16d912c 100644 --- a/util/data/msgreply.c +++ b/util/data/msgreply.c @@ -913,8 +913,9 @@ parse_reply_in_temp_region(sldns_buffer* pkt, struct regional* region, } memset(msg, 0, sizeof(*msg)); sldns_buffer_set_position(pkt, 0); - if(parse_packet(pkt, msg, region) != 0) + if(parse_packet(pkt, msg, region) != 0){ return 0; + } if(!parse_create_msg(pkt, msg, NULL, qi, &rep, region)) { return 0; } diff --git a/util/module.c b/util/module.c index f4b715d14..f16583183 100644 --- a/util/module.c +++ b/util/module.c @@ -236,3 +236,13 @@ log_edns_known_options(enum verbosity_value level, struct module_env* env) } } } + +void +copy_state_to_super(struct module_qstate* qstate, int ATTR_UNUSED(id), + struct module_qstate* super) +{ + /* Overwrite super's was_ratelimited only when it was not set */ + if(!super->was_ratelimited) { + super->was_ratelimited = qstate->was_ratelimited; + } +} diff --git a/util/module.h b/util/module.h index c6e5164de..31c531384 100644 --- a/util/module.h +++ b/util/module.h @@ -621,6 +621,8 @@ struct module_qstate { int no_cache_store; /** whether to refetch a fresh answer on finishing this state*/ int need_refetch; + /** whether the query (or a subquery) was ratelimited */ + int was_ratelimited; /** * Attributes of clients that share the qstate that may affect IP-based @@ -819,4 +821,14 @@ int unique_mesh_state(struct edns_option* list, struct module_env* env); void log_edns_known_options(enum verbosity_value level, struct module_env* env); +/** + * Copy state that may have happened in the subquery and is always relevant to + * the super. + * @param qstate: query state that finished. + * @param id: module id. + * @param super: the qstate to inform. + */ +void copy_state_to_super(struct module_qstate* qstate, int id, + struct module_qstate* super); + #endif /* UTIL_MODULE_H */ diff --git a/validator/autotrust.c b/validator/autotrust.c index 7bc5577f4..a34a7c96c 100644 --- a/validator/autotrust.c +++ b/validator/autotrust.c @@ -2306,7 +2306,7 @@ autr_debug_print(struct val_anchors* anchors) void probe_answer_cb(void* arg, int ATTR_UNUSED(rcode), sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(sec), - char* ATTR_UNUSED(why_bogus)) + char* ATTR_UNUSED(why_bogus), int ATTR_UNUSED(was_ratelimited)) { /* retry was set before the query was done, * re-querytime is set when query succeeded, but that may not diff --git a/validator/autotrust.h b/validator/autotrust.h index c549798f0..057f2b68a 100644 --- a/validator/autotrust.h +++ b/validator/autotrust.h @@ -206,6 +206,6 @@ void autr_debug_print(struct val_anchors* anchors); /** callback for query answer to 5011 probe */ void probe_answer_cb(void* arg, int rcode, struct sldns_buffer* buf, - enum sec_status sec, char* errinf); + enum sec_status sec, char* errinf, int was_ratelimited); #endif /* VALIDATOR_AUTOTRUST_H */ -- 2.47.3