N . Improve memory usage on tight-memory machines.
- Directory-related fixes.
o Remember offset and location of each descriptor in the cache/journal
- - When sending a big pile of descs to a client, don't shove them all
+ o When sending a big pile of descs to a client, don't shove them all
on the buffer at once. Keep a list of the descriptor digests for
the descriptors we still want to send. We might end up truncating
some replies by returning fewer descriptors than were requested (if
somebody requests a desc that we throw away before we deliver it),
but this happens only when somebody wants an obsolete desc, and
clients can already handle truncated replies.
- - But what do we do about compression? That's the part that makes
+ . But what do we do about compression? That's the part that makes
stuff hard.
- - Implement compress/decompress-on-the-fly support.
- - Use it for returning lists of descriptors and lists of
- network status docs.
+ o Implement compress/decompress-on-the-fly support.
+ o Use it for returning lists of descriptors.
+ - Use it for returning lists of network status docs. (This will
+ take a hybrid approach; let's get the other bits working first.)
o Make clients handle missing Content-Length tags. (Oh, they do.)
o Verify that this has happened for a long time.
- - Try a similar trick for spooling out v1 directories.
+ o Try a similar trick for spooling out v1 directories. These we
+ _uncompress_ on the fly.
- Look into pulling serverdescs off buffers as they arrive.
- Mmap cache files where possible.
- Mmap cached-routers file; when building it, go oldest-to-newest.
- Save and mmap v1 directories; store them zipped?
- "bandwidth classes", for incoming vs initiated-here conns.
-N - Asynchronous DNS
+ o Asynchronous DNS
- Security improvements
- Directory guards
static int connection_handle_listener_read(connection_t *conn, int new_type);
static int connection_receiver_bucket_should_increase(connection_t *conn);
static int connection_finished_flushing(connection_t *conn);
+static int connection_flushed_some(connection_t *conn);
static int connection_finished_connecting(connection_t *conn);
static int connection_reached_eof(connection_t *conn);
static int connection_read_to_buf(connection_t *conn, int *max_to_read);
log_warn(LD_BUG, "called on OR conn with non-zeroed identity_digest");
connection_or_remove_from_identity_map(conn);
}
+ if (conn->zlib_state)
+ tor_zlib_free(conn->zlib_state);
+ if (conn->fingerprint_stack) {
+ SMARTLIST_FOREACH(conn->fingerprint_stack, char *, cp, tor_free(cp));
+ smartlist_free(conn->fingerprint_stack);
+ }
+ if (conn->cached_dir)
+ cached_dir_decref(conn->cached_dir);
memset(conn, 0xAA, sizeof(connection_t)); /* poison memory */
tor_free(conn);
}
}
- if (result > 0 && !is_local_IP(conn->addr)) { /* remember it */
- rep_hist_note_bytes_written(result, now);
- global_write_bucket -= result;
+ if (result > 0) {
+ if (!is_local_IP(conn->addr)) { /* remember it */
+ rep_hist_note_bytes_written(result, time(NULL));
+ global_write_bucket -= result;
+ }
+ if (connection_flushed_some(conn) < 0)
+ connection_mark_for_close(conn);
}
if (!connection_wants_to_flush(conn)) { /* it's done flushing */
return;
}
- if (result > 0 && !is_local_IP(conn->addr)) { /* remember it */
- rep_hist_note_bytes_written(result, time(NULL));
- global_write_bucket -= result;
+ if (result > 0) {
+ if (!is_local_IP(conn->addr)) { /* remember it */
+ rep_hist_note_bytes_written(result, time(NULL));
+ global_write_bucket -= result;
+ }
+ if (connection_flushed_some(conn) < 0)
+ connection_mark_for_close(conn);
}
if (!connection_wants_to_flush(conn)) { /* it's done flushing */
}
}
+/** Called whenever we've written data on a connection. */
+static int
+connection_flushed_some(connection_t *conn)
+{
+ if (conn->type == CONN_TYPE_DIR &&
+ conn->state == DIR_CONN_STATE_SERVER_WRITING)
+ return connection_dirserv_flushed_some(conn);
+ else
+ return 0;
+}
+
/** We just finished flushing bytes from conn-\>outbuf, and there
* are no more bytes remaining.
*
if (!strcmp(url,"/tor/") || !strcmp(url,"/tor/dir.z")) { /* dir fetch */
int deflated = !strcmp(url,"/tor/dir.z");
- dlen = dirserv_get_directory(&cp, deflated);
+ cached_dir_t *d = dirserv_get_directory();
- if (dlen == 0) {
+ if (!d) {
log_notice(LD_DIRSERV,"Client asked for the mirrored directory, but we "
"don't have a good one yet. Sending 503 Dir not available.");
write_http_status_line(conn, 503, "Directory unavailable");
tor_free(url);
return 0;
}
+ dlen = deflated ? d->dir_z_len : d->dir_len;
if (global_write_bucket_empty()) {
log_info(LD_DIRSERV,
deflated?"application/octet-stream":"text/plain",
deflated?"deflate":"identity");
connection_write_to_buf(tmp, strlen(tmp), conn);
- connection_write_to_buf(cp, dlen, conn);
+ conn->cached_dir = d;
+ conn->cached_dir_offset = 0;
+ if (deflated)
+ conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD);
+ ++d->refcnt;
+
+ /* Prime the connection with some data. */
+ connection_dirserv_flushed_some(conn);
return 0;
}
int deflated = !strcmp(url+url_len-2, ".z");
int res;
const char *msg;
- smartlist_t *descs = smartlist_create();
const char *request_type = NULL;
if (deflated)
url[url_len-2] = '\0';
- res = dirserv_get_routerdescs(descs, url, &msg);
+ conn->fingerprint_stack = smartlist_create();
+ res = dirserv_get_routerdescs(conn->fingerprint_stack, url, &msg);
if (!strcmpstart(url, "/tor/server/fp/"))
request_type = deflated?"/tor/server/fp.z":"/tor/server/fp";
if (res < 0)
write_http_status_line(conn, 404, msg);
else {
- size_t len = 0;
format_rfc1123_time(date, time(NULL));
- SMARTLIST_FOREACH(descs, signed_descriptor_t *, ri,
- len += ri->signed_descriptor_len);
if (deflated) {
- size_t compressed_len;
- char *compressed;
- char *inp = tor_malloc(len+smartlist_len(descs)+1);
- char *cp = inp;
- SMARTLIST_FOREACH(descs, signed_descriptor_t *, ri,
- {
- const char *body = signed_descriptor_get_body(ri);
- memcpy(cp, body, ri->signed_descriptor_len);
- cp += ri->signed_descriptor_len;
- *cp++ = '\n';
- });
- *cp = '\0';
- /* XXXX This could be way more efficiently handled; let's see if it
- * shows up under oprofile. */
- if (tor_gzip_compress(&compressed, &compressed_len,
- inp, cp-inp, ZLIB_METHOD)<0) {
- tor_free(inp);
- smartlist_free(descs);
- return -1;
- }
- tor_free(inp);
+ conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD);
+ /* // note_request(request_type, compressed_len); XXXX */
tor_snprintf(tmp, sizeof(tmp),
- "HTTP/1.0 200 OK\r\nDate: %s\r\nContent-Length: %d\r\n"
+ "HTTP/1.0 200 OK\r\nDate: %s\r\n"
"Content-Type: application/octet-stream\r\n"
"Content-Encoding: deflate\r\n\r\n",
- date,
- (int)compressed_len);
- note_request(request_type, compressed_len);
+ date);
connection_write_to_buf(tmp, strlen(tmp), conn);
- connection_write_to_buf(compressed, compressed_len, conn);
- tor_free(compressed);
} else {
tor_snprintf(tmp, sizeof(tmp),
- "HTTP/1.0 200 OK\r\nDate: %s\r\nContent-Length: %d\r\n"
+ "HTTP/1.0 200 OK\r\nDate: %s\r\n"
"Content-Type: text/plain\r\n\r\n",
- date,
- (int)len);
- note_request(request_type, len);
+ date);
+ /* note_request(request_type, len); XXXX */
connection_write_to_buf(tmp, strlen(tmp), conn);
- SMARTLIST_FOREACH(descs, signed_descriptor_t *, ri,
- {
- const char *body = signed_descriptor_get_body(ri);
- connection_write_to_buf(body, ri->signed_descriptor_len, conn);
- });
}
+ /* Prime the connection with some data. */
+ connection_dirserv_flushed_some(conn);
}
- smartlist_free(descs);
return 0;
}
directory_handle_command_post(connection_t *conn, char *headers,
char *body, size_t body_len)
{
- const char *cp;
char *origin = NULL;
char *url = NULL;
int r = dirserv_add_descriptor(body, &msg);
tor_assert(msg);
if (r > 0)
- dirserv_get_directory(&cp, 0); /* rebuild and write to disk */
+ dirserv_get_directory(); /* rebuild and write to disk */
switch (r) {
case -2:
case -1:
const char **msg, int should_log);
static int dirserv_thinks_router_is_reachable(routerinfo_t *router,
time_t now);
+static void clear_cached_dir(cached_dir_t *d);
/************** Fingerprint handling code ************/
}
/** Most recently generated encoded signed directory. (auth dirservers only.)*/
-static cached_dir_t the_directory = { NULL, NULL, 0, 0, 0 };
+static cached_dir_t *the_directory = NULL;
/* Used only by non-auth dirservers: The directory and runningrouters we'll
* serve when requested. */
-static cached_dir_t cached_directory = { NULL, NULL, 0, 0, 0 };
-static cached_dir_t cached_runningrouters = { NULL, NULL, 0, 0, 0 };
+static cached_dir_t *cached_directory = NULL;
+static cached_dir_t cached_runningrouters = { NULL, NULL, 0, 0, 0, -1 };
/* Used for other dirservers' v2 network statuses. Map from hexdigest to
* cached_dir_t. */
}
}
+/** DOCDOC */
+void
+cached_dir_decref(cached_dir_t *d)
+{
+ if (!d || --d->refcnt > 0)
+ return;
+ clear_cached_dir(d);
+ tor_free(d);
+}
+
+/** DOCDOC */
+static cached_dir_t *
+new_cached_dir(char *s, time_t published)
+{
+ cached_dir_t *d = tor_malloc_zero(sizeof(cached_dir_t));
+ d->refcnt = 1;
+ d->dir = s;
+ d->dir_len = strlen(s);
+ d->published = published;
+ if (tor_gzip_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len,
+ ZLIB_METHOD)) {
+ log_warn(LD_BUG, "Error compressing directory");
+ }
+ return d;
+}
+
/** Remove all storage held in <b>d</b>, but do not free <b>d</b> itself. */
static void
clear_cached_dir(cached_dir_t *d)
dirserv_set_cached_directory(const char *directory, time_t published,
int is_running_routers)
{
- cached_dir_t *d;
- d = is_running_routers ? &cached_runningrouters : &cached_directory;
- set_cached_dir(d, tor_strdup(directory), published);
+ if (is_running_routers) {
+ set_cached_dir(&cached_runningrouters, tor_strdup(directory), published);
+ } else {
+ cached_dir_decref(cached_directory);
+ cached_directory = new_cached_dir(tor_strdup(directory), published);
+ }
}
/** We've just received a v2 network-status for an authoritative directory
* this kind of object.
**/
static size_t
-dirserv_get_obj(const char **out, int compress,
+dirserv_get_obj(const char **out,
+ int compress,
cached_dir_t *cache_src,
cached_dir_t *auth_src,
time_t dirty, int (*regenerate)(void),
}
}
-/** Set *<b>directory</b> to the most recently generated encoded signed
- * directory, generating a new one as necessary. If not an authoritative
- * directory may return 0 if no directory is yet cached.*/
-size_t
-dirserv_get_directory(const char **directory, int compress)
+/** Return the most recently generated encoded signed directory, generating a
+ * new one as necessary. If not an authoritative directory may return NULL if
+ * no directory is yet cached.*/
+cached_dir_t *
+dirserv_get_directory(void)
{
- return dirserv_get_obj(directory, compress,
- &cached_directory, &the_directory,
- the_directory_is_dirty,
- dirserv_regenerate_directory,
- "server directory", 1);
+ return dirserv_pick_cached_dir_obj(cached_directory, the_directory,
+ the_directory_is_dirty,
+ dirserv_regenerate_directory,
+ "server directory", 1);
}
/**
tor_free(new_directory);
return -1;
}
- set_cached_dir(&the_directory, new_directory, time(NULL));
+ cached_dir_decref(the_directory);
+ the_directory = new_cached_dir(new_directory, time(NULL));
log_info(LD_DIRSERV,"New directory (size %d) has been built.",
- (int)the_directory.dir_len);
+ (int)the_directory->dir_len);
log_debug(LD_DIRSERV,"New directory (size %d):\n%s",
- (int)the_directory.dir_len, the_directory.dir);
+ (int)the_directory->dir_len, the_directory->dir);
the_directory_is_dirty = 0;
/* Save the directory to disk so we re-load it quickly on startup.
*/
- dirserv_set_cached_directory(the_directory.dir, time(NULL), 0);
+ dirserv_set_cached_directory(the_directory->dir, time(NULL), 0);
return 0;
}
/** For authoritative directories: the current (v1) network status */
-static cached_dir_t the_runningrouters = { NULL, NULL, 0, 0, 0 };
+static cached_dir_t the_runningrouters = { NULL, NULL, 0, 0, 0, -1 };
/** Replace the current running-routers list with a newly generated one. */
static int
}
/** For authoritative directories: the current (v2) network status */
-static cached_dir_t the_v2_networkstatus = { NULL, NULL, 0, 0, 0 };
+static cached_dir_t the_v2_networkstatus = { NULL, NULL, 0, 0, 0, -1 };
static int
should_generate_v2_networkstatus(void)
}
}
+/** As dirserv_get_routerdescs(), but instead of getting signed_descriptor_t
+ * pointers, adds copies of digests to fps_out. For a /tor/server/d/ request,
+ * adds descriptor digests; for other requests, adds identity digests.
+ */
+int
+dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key,
+ const char **msg)
+{
+ *msg = NULL;
+
+ if (!strcmp(key, "/tor/server/all")) {
+ routerlist_t *rl = router_get_routerlist();
+ SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
+ smartlist_add(fps_out,
+ tor_memdup(r->cache_info.identity_digest, DIGEST_LEN)));
+ } else if (!strcmp(key, "/tor/server/authority")) {
+ routerinfo_t *ri = router_get_my_routerinfo();
+ if (ri)
+ smartlist_add(fps_out,
+ tor_memdup(ri->cache_info.identity_digest, DIGEST_LEN));
+ } else if (!strcmpstart(key, "/tor/server/d/")) {
+ key += strlen("/tor/server/d/");
+ dir_split_resource_into_fingerprints(key, fps_out, NULL, 1);
+ } else if (!strcmpstart(key, "/tor/server/fp/")) {
+ key += strlen("/tor/server/fp/");
+ dir_split_resource_into_fingerprints(key, fps_out, NULL, 1);
+ } else {
+ *msg = "Key not recognized";
+ return -1;
+ }
+
+ if (!smartlist_len(fps_out)) {
+ *msg = "Servers unavailable";
+ return -1;
+ }
+ return 0;
+}
+
/** Add a signed_descriptor_t to <b>descs_out</b> for each router matching
* <b>key</b>. The key should be either
* - "/tor/server/authority" for our own routerinfo;
}
}
+/** When we're spooling data onto our outbuf, add more whenever we dip
+ * below this threshold. */
+#define DIRSERV_BUFFER_MIN 16384
+
+/** DOCDOC */
+static int
+connection_dirserv_add_servers_to_outbuf(connection_t *conn)
+{
+ int fp;
+
+ if (!strcmpstart(conn->requested_resource, "/tor/server/d/"))
+ fp = 0;
+ else
+ fp = 1;
+
+ while (smartlist_len(conn->fingerprint_stack) &&
+ buf_datalen(conn->outbuf) < DIRSERV_BUFFER_MIN) {
+ char *fp = smartlist_pop_last(conn->fingerprint_stack);
+ signed_descriptor_t *sd = NULL;
+ if (fp) {
+ if (router_digest_is_me(fp)) {
+ sd = &(router_get_my_routerinfo()->cache_info);
+ } else {
+ routerinfo_t *ri = router_get_by_digest(fp);
+ if (ri &&
+ ri->cache_info.published_on > time(NULL)-ROUTER_MAX_AGE_TO_PUBLISH)
+ sd = &ri->cache_info;
+ }
+ } else
+ sd = router_get_by_descriptor_digest(fp);
+ tor_free(fp);
+ if (!sd)
+ continue;
+ if (conn->zlib_state) {
+ write_to_buf_zlib(conn->outbuf, conn->zlib_state,
+ sd->signed_descriptor_body, sd->signed_descriptor_len,
+ 0);
+ } else {
+ write_to_buf(sd->signed_descriptor_body, sd->signed_descriptor_len,
+ conn->outbuf);
+ }
+ }
+
+ if (!smartlist_len(conn->fingerprint_stack)) {
+ /* We just wrote the last one; finish up. */
+ if (conn->zlib_state) {
+ write_to_buf_zlib(conn->outbuf, conn->zlib_state, "", 0, 1);
+ tor_zlib_free(conn->zlib_state);
+ conn->zlib_state = NULL;
+ }
+ smartlist_free(conn->fingerprint_stack);
+ conn->fingerprint_stack = NULL;
+ }
+ return 0;
+}
+
+/** DOCDOC */
+static int
+connection_dirserv_add_dir_bytes_to_outbuf(connection_t *conn)
+{
+ int bytes, remaining;
+
+ bytes = DIRSERV_BUFFER_MIN - buf_datalen(conn->outbuf);
+ tor_assert(bytes > 0);
+ if (bytes < 8192)
+ bytes = 8192;
+ remaining = conn->cached_dir->dir_z_len - conn->cached_dir_offset;
+ if (bytes > remaining)
+ bytes = remaining;
+
+ if (conn->zlib_state) {
+ write_to_buf_zlib(conn->outbuf, conn->zlib_state,
+ conn->cached_dir->dir_z + conn->cached_dir_offset,
+ bytes, bytes == remaining);
+ } else {
+ write_to_buf(conn->cached_dir->dir_z + conn->cached_dir_offset,
+ bytes, conn->outbuf);
+ }
+ conn->cached_dir_offset += bytes;
+ if (bytes == (int)conn->cached_dir->dir_z_len) {
+ /* We just wrote the last one; finish up. */
+ if (conn->zlib_state) {
+ write_to_buf_zlib(conn->outbuf, conn->zlib_state, "", 0, 1);
+ tor_zlib_free(conn->zlib_state);
+ conn->zlib_state = NULL;
+ }
+ cached_dir_decref(conn->cached_dir);
+ conn->cached_dir = NULL;
+ }
+ return 0;
+}
+
+/** Called whenever we have flushed some directory data in state
+ * SERVER_WRITING. */
+int
+connection_dirserv_flushed_some(connection_t *conn)
+{
+ tor_assert(conn->type == CONN_TYPE_DIR);
+ tor_assert(conn->state == DIR_CONN_STATE_SERVER_WRITING);
+
+ if (! (conn->fingerprint_stack || conn->cached_dir)
+ || buf_datalen(conn->outbuf) > DIRSERV_BUFFER_MIN)
+ return 0;
+
+ if (!strcmpstart(conn->requested_resource, "/tor/server/")) {
+ return connection_dirserv_add_servers_to_outbuf(conn);
+ } else if (conn->cached_dir) {
+ return connection_dirserv_add_dir_bytes_to_outbuf(conn);
+ } else {
+ return 0;
+ }
+}
+
/** Release all storage used by the directory server. */
void
dirserv_free_all(void)
smartlist_free(fingerprint_list);
fingerprint_list = NULL;
}
- clear_cached_dir(&the_directory);
+ cached_dir_decref(the_directory);
clear_cached_dir(&the_runningrouters);
clear_cached_dir(&the_v2_networkstatus);
- clear_cached_dir(&cached_directory);
+ cached_dir_decref(cached_directory);
clear_cached_dir(&cached_runningrouters);
if (cached_v2_networkstatus) {
digestmap_free(cached_v2_networkstatus, free_cached_dir);
/* Used only by Dir connections */
char *requested_resource; /**< Which 'resource' did we ask the directory
* for?*/
+/* Used only for server sides of some dir connections. */
+ smartlist_t *fingerprint_stack;
+ struct cached_dir_t *cached_dir;
+ off_t cached_dir_offset;
+ tor_zlib_state_t *zlib_state;
/* Used only by AP connections */
socks_request_t *socks_request; /**< SOCKS structure describing request (AP
size_t dir_len; /**< Length of <b>dir</b> */
size_t dir_z_len; /**< Length of <b>dir_z</b> */
time_t published; /**< When was this object published */
+ int refcnt; /**< Reference count for this cached_dir_t. */
} cached_dir_t;
/** Information need to cache an onion router's descriptor. */
int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen);
int write_to_buf(const char *string, size_t string_len, buf_t *buf);
+int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state,
+ const char *data, size_t data_len, int done);
int fetch_from_buf(char *string, size_t string_len, buf_t *buf);
int fetch_from_buf_http(buf_t *buf,
char **headers_out, size_t max_headerlen,
/********************************* dirserv.c ***************************/
+int connection_dirserv_flushed_some(connection_t *conn);
int dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk);
int dirserv_parse_fingerprint_file(const char *fname);
void dirserv_free_fingerprint_list(void);
crypto_pk_env_t *private_key,
int complete);
void directory_set_dirty(void);
-size_t dirserv_get_directory(const char **cp, int compress);
+cached_dir_t *dirserv_get_directory(void);
size_t dirserv_get_runningrouters(const char **rr, int compress);
void dirserv_set_cached_directory(const char *directory, time_t when,
int is_running_routers);
const char *identity,
time_t published);
void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key);
+int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key,
+ const char **msg);
int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
const char **msg);
void dirserv_orconn_tls_done(const char *address,
int complain);
int dirserv_would_reject_router(routerstatus_t *rs);
void dirserv_free_all(void);
+void cached_dir_decref(cached_dir_t *d);
/********************************* dns.c ***************************/
projects / thirdparty / tor.git / commitdiff
