Merge branch 'maint-0.3.1' into maint-0.3.2

diff --cc doc/tor.1.txt
Simple merge

diff --cc src/common/address.c
index dbe129be59423ec2aff71ea79282d8d5330308e8,e6b437e9dd61770dd95497e7477c0fbda34b4a52..c683e90aac0570c08d5af94bb0b902aff07a7bed
--- 1/src/common/address.c
--- 2/src/common/address.c
+++ b/src/common/address.c
@@@ -1193,11 -1194,11 +1196,11 @@@ tor_addr_hash(const tor_addr_t *addr
    case AF_INET:
      return siphash24g(&addr->addr.in_addr.s_addr, 4);
    case AF_UNSPEC:
-     return 0x4e4d5342;
+     return siphash24g(unspec_hash_input, sizeof(unspec_hash_input));
    case AF_INET6:
      return siphash24g(&addr->addr.in6_addr.s6_addr, 16);
 -  default:
      /* LCOV_EXCL_START */
 +  default:
      tor_fragile_assert();
      return 0;
      /* LCOV_EXCL_STOP */

diff --cc src/common/address.h
Simple merge

diff --cc src/common/include.am
index cd5eea34043f5ab65c85c3f50113c3c6f0132228,d12895b107c39b7d049cbffbb384c34aeef53058..715ec0264c00ee538555c5a8a318eb11eef369f5
--- 1/src/common/include.am
--- 2/src/common/include.am
+++ b/src/common/include.am
@@@ -80,8 -80,8 +80,9 @@@ src_common_libor_ctime_testing_a_CFLAG
  
  LIBOR_A_SRC = \
    src/common/address.c                                        \
+   src/common/address_set.c                            \
    src/common/backtrace.c                              \
 +  src/common/buffers.c                                        \
    src/common/compat.c                                 \
    src/common/compat_threads.c                         \
    src/common/compat_time.c                            \
@@@ -148,9 -147,8 +149,10 @@@ src_common_libor_event_testing_a_CFLAG
  
  COMMONHEADERS = \
    src/common/address.h                                \
+   src/common/address_set.h                    \
    src/common/backtrace.h                      \
 +  src/common/buffers.h                                \
 +  src/common/buffers_tls.h                    \
    src/common/aes.h                            \
    src/common/ciphers.inc                      \
    src/common/compat.h                         \

diff --cc src/common/log.c
Simple merge

diff --cc src/common/torlog.h
Simple merge

diff --cc src/or/channel.c
Simple merge

diff --cc src/or/channel.h
Simple merge

diff --cc src/or/channeltls.c
Simple merge

diff --cc src/or/command.c
Simple merge

diff --cc src/or/config.c
Simple merge

diff --cc src/or/connection.c
index d2cf4fb416f6f1d0c8a9ddd0b8db110f35386fb2,fc0646b885b64b3b9d55c434033ff54e183d6035..ed8de05d78418e283e00a163bea43010d9227f75
--- 1/src/or/connection.c
--- 2/src/or/connection.c
+++ b/src/or/connection.c
@@@ -704,7 -696,14 +705,14 @@@ connection_free,(connection_t *conn)
      connection_ap_warn_and_unmark_if_pending_circ(TO_ENTRY_CONN(conn),
                                                    "connection_free");
    }
 -#endif
 +#endif /* 1 */
+ 
+   /* Notify the circuit creation DoS mitigation subsystem that an OR client
+    * connection has been closed. And only do that if we track it. */
+   if (conn->type == CONN_TYPE_OR) {
+     dos_close_client_conn(TO_OR_CONN(conn));
+   }
+ 
    connection_unregister_events(conn);
    connection_free_(conn);
  }

diff --cc src/or/geoip.c
Simple merge

diff --cc src/or/geoip.h
index acf61b97aeca06777493d1acd1a54884b95d0a80,773525ccfec1b48aea15fb85dbd461c81769b3df..753bdbf82a829af32fb77e363a00b0f164864882
--- 1/src/or/geoip.h
--- 2/src/or/geoip.h
+++ b/src/or/geoip.h
@@@ -19,7 -20,30 +20,30 @@@ STATIC int geoip_parse_entry(const cha
  STATIC int geoip_get_country_by_ipv4(uint32_t ipaddr);
  STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr);
  STATIC void clear_geoip_db(void);
 -#endif
 +#endif /* defined(GEOIP_PRIVATE) */
+ 
+ /** Entry in a map from IP address to the last time we've seen an incoming
+  * connection from that IP address. Used by bridges only to track which
+  * countries have them blocked, or the DoS mitigation subsystem if enabled. */
+ typedef struct clientmap_entry_t {
+   HT_ENTRY(clientmap_entry_t) node;
+   tor_addr_t addr;
+   /* Name of pluggable transport used by this client. NULL if no
+      pluggable transport was used. */
+   char *transport_name;
+ 
+   /** Time when we last saw this IP address, in MINUTES since the epoch.
+    *
+    * (This will run out of space around 4011 CE.  If Tor is still in use around
+    * 4000 CE, please remember to add more bits to last_seen_in_minutes.) */
+   unsigned int last_seen_in_minutes:30;
+   unsigned int action:2;
+ 
+   /* This object is used to keep some statistics per client address for the
+    * DoS mitigation subsystem. */
+   dos_client_stats_t dos_stats;
+ } clientmap_entry_t;
+ 
  int should_record_bridge_info(const or_options_t *options);
  int geoip_load_file(sa_family_t family, const char *filename);
  MOCK_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr));

diff --cc src/or/include.am
index 7216aba9af57b32b6c59726da1a06d1fc8cd6439,5d5ff5c41d139ca42ab440d7940b9e72b3c018cc..abf62c2e8feb95424979e6cfd19c6f5d4d4a744a
--- 1/src/or/include.am
--- 2/src/or/include.am
+++ b/src/or/include.am
@@@ -47,8 -48,13 +47,9 @@@ LIBTOR_A_SOURCES = 
        src/or/dirvote.c                                \
        src/or/dns.c                                    \
        src/or/dnsserv.c                                \
+       src/or/dos.c                                    \
        src/or/fp_pair.c                                \
        src/or/geoip.c                                  \
 -      src/or/hs_intropoint.c          \
 -      src/or/hs_circuitmap.c          \
 -      src/or/hs_ntor.c                \
 -      src/or/hs_service.c             \
        src/or/entrynodes.c                             \
        src/or/ext_orport.c                             \
        src/or/hibernate.c                              \

diff --cc src/or/main.c
index c340e4128ba61bdd2e46bdffb03d213c0364868f,197dfd4308c918e608868fdb6a05501a9816db59..036ef1efd4f67a81d6c988284ec9acf3b5777237
--- 1/src/or/main.c
--- 2/src/or/main.c
+++ b/src/or/main.c
@@@ -3268,7 -3242,7 +3269,8 @@@ tor_free_all(int postfork
    protover_free_all();
    bridges_free_all();
    consdiffmgr_free_all();
 +  hs_free_all();
+   dos_free_all();
    if (!postfork) {
      config_free_all();
      or_state_free_all();

diff --cc src/or/networkstatus.c
index 9d9a2b8ad1ccfd96481ab059435035a6362046be,e59069e84efb3fb0afa7cbfd5c09393d8093d411..dd82f668168bf90d8d2d11e9b1daf337b0f2adce
--- 1/src/or/networkstatus.c
--- 2/src/or/networkstatus.c
+++ b/src/or/networkstatus.c
@@@ -1564,22 -1604,15 +1565,23 @@@ notify_control_networkstatus_changed(co
    smartlist_free(changed);
  }
  
 -/* Called when the consensus has changed from old_c to new_c. */
 +/* Called before the consensus changes from old_c to new_c. */
  static void
 -notify_networkstatus_changed(const networkstatus_t *old_c,
 -                             const networkstatus_t *new_c)
 +notify_before_networkstatus_changes(const networkstatus_t *old_c,
 +                                    const networkstatus_t *new_c)
  {
    notify_control_networkstatus_changed(old_c, new_c);
+   dos_consensus_has_changed(new_c);
  }
  
 +/* Called after a new consensus has been put in the global state. It is safe
 + * to use the consensus getters in this function. */
 +static void
 +notify_after_networkstatus_changes(void)
 +{
 +  scheduler_notify_networkstatus_changed();
 +}
 +
  /** Copy all the ancillary information (like router download status and so on)
   * from <b>old_c</b> to <b>new_c</b>. */
  static void

diff --cc src/or/nodelist.c
index f2e979be8bbd0dbebae2e9fb5e86142af5f8539a,c754ebdb6edc7d058ab171f07e72cf8e9ac5ae95..9a477ecf43983d4ff13f6ef519cc8033296171b4
--- 1/src/or/nodelist.c
--- 2/src/or/nodelist.c
+++ b/src/or/nodelist.c
@@@ -38,10 -38,9 +38,11 @@@
   * used for authorities and fallback directories.)
   */
  
 +#define NODELIST_PRIVATE
 +
  #include "or.h"
  #include "address.h"
+ #include "address_set.h"
  #include "config.h"
  #include "control.h"
  #include "dirserv.h"
@@@ -95,14 -92,9 +97,16 @@@ typedef struct nodelist_t 
    smartlist_t *nodes;
    /* Hash table to map from node ID digest to node. */
    HT_HEAD(nodelist_map, node_t) nodes_by_id;
 -
 +  /* Hash table to map from node Ed25519 ID to node.
 +   *
 +   * Whenever a node's routerinfo or microdescriptor is about to change,
 +   * you should remove it from this map with node_remove_from_ed25519_map().
 +   * Whenever a node's routerinfo or microdescriptor has just chaned,
 +   * you should add it to this map with node_add_to_ed25519_map().
 +   */
 +  HT_HEAD(nodelist_ed_map, node_t) nodes_by_ed_id;
+   /* Set of addresses that belong to nodes we believe in. */
+   address_set_t *node_addrs;
  } nodelist_t;
  
  static inline unsigned int
@@@ -442,14 -264,8 +490,16 @@@ nodelist_set_routerinfo(routerinfo_t *r
      dirserv_set_node_flags_from_authoritative_status(node, status);
    }
  
 +  /* Setting the HSDir index requires the ed25519 identity key which can
 +   * only be found either in the ri or md. This is why this is called here.
 +   * Only nodes supporting HSDir=2 protocol version needs this index. */
 +  if (node->rs && node->rs->supports_v3_hsdir) {
 +    node_set_hsdir_index(node,
 +                         networkstatus_get_latest_consensus());
 +  }
 +
+   node_add_to_address_set(node);
+ 
    return node;
  }
  
@@@ -476,21 -292,14 +526,23 @@@ nodelist_add_microdesc(microdesc_t *md
      return NULL;
    node = node_get_mutable_by_id(rs->identity_digest);
    if (node) {
 +    node_remove_from_ed25519_map(node);
      if (node->md)
        node->md->held_by_nodes--;
 +
      node->md = md;
      md->held_by_nodes++;
 +    /* Setting the HSDir index requires the ed25519 identity key which can
 +     * only be found either in the ri or md. This is why this is called here.
 +     * Only nodes supporting HSDir=2 protocol version needs this index. */
 +    if (rs->supports_v3_hsdir) {
 +      node_set_hsdir_index(node, ns);
 +    }
 +    node_add_to_ed25519_map(node);
    }
  
+   node_add_to_address_set(node);
+ 
    return node;
  }
  

diff --cc src/or/nodelist.h
index 754990ac8d635726a7fcc6fec89c28e721a26849,9cd66f60a2d5965a8e54eee2e5ac31e43fbb3b06..8a0c79f86d5e5612042d725373a4093bf3989b82
--- 1/src/or/nodelist.h
--- 2/src/or/nodelist.h
+++ b/src/or/nodelist.h
@@@ -142,16 -132,7 +143,18 @@@ void router_dir_info_changed(void)
  const char *get_dir_info_status_string(void);
  int count_loading_descriptors_progress(void);
  
 -#endif
 +#ifdef NODELIST_PRIVATE
 +
 +#ifdef TOR_UNIT_TESTS
 +
 +STATIC void
 +node_set_hsdir_index(node_t *node, const networkstatus_t *ns);
 +
 +#endif /* defined(TOR_UNIT_TESTS) */
 +
 +#endif /* defined(NODELIST_PRIVATE) */
 +
+ MOCK_DECL(int, get_estimated_address_per_node, (void));
+ 
 +#endif /* !defined(TOR_NODELIST_H) */
  

diff --cc src/or/or.h
index 03e20b80de88e9141d43c2381a4496831eaca433,9e7833386c61437aa60bfbb5168f5f2d032b8229..5128fd2197c43605796753e5c823687b285bd554
--- 1/src/or/or.h
--- 2/src/or/or.h
+++ b/src/or/or.h
@@@ -4624,28 -4583,36 +4628,57 @@@ typedef struct 
     * use the default. */
    int MaxConsensusAgeForDiffs;
  
 +  /** Bool (default: 0). Tells Tor to never try to exec another program.
 +   */
 +  int NoExec;
 +
 +  /** Have the KIST scheduler run every X milliseconds. If less than zero, do
 +   * not use the KIST scheduler but use the old vanilla scheduler instead. If
 +   * zero, do what the consensus says and fall back to using KIST as if this is
 +   * set to "10 msec" if the consensus doesn't say anything. */
 +  int KISTSchedRunInterval;
 +
 +  /** A multiplier for the KIST per-socket limit calculation. */
 +  double KISTSockBufSizeFactor;
 +
 +  /** The list of scheduler type string ordered by priority that is first one
 +   * has to be tried first. Default: KIST,KISTLite,Vanilla */
 +  smartlist_t *Schedulers;
 +  /* An ordered list of scheduler_types mapped from Schedulers. */
 +  smartlist_t *SchedulerTypes_;
++
+   /** Autobool: Is the circuit creation DoS mitigation subsystem enabled? */
+   int DoSCircuitCreationEnabled;
+   /** Minimum concurrent connection needed from one single address before any
+    * defense is used. */
+   int DoSCircuitCreationMinConnections;
+   /** Circuit rate used to refill the token bucket. */
+   int DoSCircuitCreationRate;
+   /** Maximum allowed burst of circuits. Reaching that value, the address is
+    * detected as malicious and a defense might be used. */
+   int DoSCircuitCreationBurst;
+   /** When an address is marked as malicous, what defense should be used
+    * against it. See the dos_cc_defense_type_t enum. */
+   int DoSCircuitCreationDefenseType;
+   /** For how much time (in seconds) the defense is applicable for a malicious
+    * address. A random time delta is added to the defense time of an address
+    * which will be between 1 second and half of this value. */
+   int DoSCircuitCreationDefenseTimePeriod;
+ 
+   /** Autobool: Is the DoS connection mitigation subsystem enabled? */
+   int DoSConnectionEnabled;
+   /** Maximum concurrent connection allowed per address. */
+   int DoSConnectionMaxConcurrentCount;
+   /** When an address is reaches the maximum count, what defense should be
+    * used against it. See the dos_conn_defense_type_t enum. */
+   int DoSConnectionDefenseType;
+ 
+   /** Autobool: Do we refuse single hop client rendezvous? */
+   int DoSRefuseSingleHopClientRendezvous;
  } or_options_t;
  
 +#define LOG_PROTOCOL_WARN (get_protocol_warning_severity_level())
 +
  /** Persistent state for an onion router, as saved to disk. */
  typedef struct {
    uint32_t magic_;

diff --cc src/or/relay.c
Simple merge

diff --cc src/or/rendmid.c
Simple merge

diff --cc src/test/include.am
Simple merge

diff --cc src/test/test.c
Simple merge

diff --cc src/test/test.h
Simple merge