example/crawler: make it use a few more options

For show, but reasonable

diff --git a/docs/examples/crawler.c b/docs/examples/crawler.c
index e2b58127d2a12232264d298fdd43bdccb7fb2f3a..2ff4ae71459d81f575ae04c43bece5af946e0590 100644 (file)
--- a/docs/examples/crawler.c
+++ b/docs/examples/crawler.c
@@ -99,7 +99,13 @@ CURL *make_handle(char *url)
   curl_easy_setopt(handle, CURLOPT_REDIR_PROTOCOLS_STR, "http,https");
   curl_easy_setopt(handle, CURLOPT_AUTOREFERER, 1L);
   curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 10L);
-  curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, 2L);
+  /* each transfer needs to be done within 20 seconds! */
+  curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, 20000L);
+  /* connect fast or fail */
+  curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, 2000L);
+  /* skip files larger than a gigabyte */
+  curl_easy_setopt(handle, CURLOPT_MAXFILESIZE_LARGE,
+                   (curl_off_t)1024*1024*1024);
   curl_easy_setopt(handle, CURLOPT_COOKIEFILE, "");
   curl_easy_setopt(handle, CURLOPT_FILETIME, 1L);
   curl_easy_setopt(handle, CURLOPT_USERAGENT, "mini crawler");