From aef74ae416a3c8323934745de749be504c215832 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Mon, 19 Jun 2023 09:06:41 +0200 Subject: [PATCH] example/crawler: make it use a few more options For show, but reasonable --- docs/examples/crawler.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/examples/crawler.c b/docs/examples/crawler.c index e2b58127d2..2ff4ae7145 100644 --- a/docs/examples/crawler.c +++ b/docs/examples/crawler.c @@ -99,7 +99,13 @@ CURL *make_handle(char *url) curl_easy_setopt(handle, CURLOPT_REDIR_PROTOCOLS_STR, "http,https"); curl_easy_setopt(handle, CURLOPT_AUTOREFERER, 1L); curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 10L); - curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, 2L); + /* each transfer needs to be done within 20 seconds! */ + curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, 20000L); + /* connect fast or fail */ + curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, 2000L); + /* skip files larger than a gigabyte */ + curl_easy_setopt(handle, CURLOPT_MAXFILESIZE_LARGE, + (curl_off_t)1024*1024*1024); curl_easy_setopt(handle, CURLOPT_COOKIEFILE, ""); curl_easy_setopt(handle, CURLOPT_FILETIME, 1L); curl_easy_setopt(handle, CURLOPT_USERAGENT, "mini crawler"); -- 2.47.3