From: Daniel Stenberg <daniel@haxx.se>
Date: Mon, 19 Jun 2023 07:06:41 +0000 (+0200)
Subject: example/crawler: make it use a few more options
X-Git-Tag: curl-8_2_0~77
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=aef74ae416a3c8323934745de749be504c215832;p=thirdparty%2Fcurl.git

example/crawler: make it use a few more options

For show, but reasonable
---

diff --git a/docs/examples/crawler.c b/docs/examples/crawler.c
index e2b58127d2..2ff4ae7145 100644
--- a/docs/examples/crawler.c
+++ b/docs/examples/crawler.c
@@ -99,7 +99,13 @@ CURL *make_handle(char *url)
   curl_easy_setopt(handle, CURLOPT_REDIR_PROTOCOLS_STR, "http,https");
   curl_easy_setopt(handle, CURLOPT_AUTOREFERER, 1L);
   curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 10L);
-  curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, 2L);
+  /* each transfer needs to be done within 20 seconds! */
+  curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, 20000L);
+  /* connect fast or fail */
+  curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, 2000L);
+  /* skip files larger than a gigabyte */
+  curl_easy_setopt(handle, CURLOPT_MAXFILESIZE_LARGE,
+                   (curl_off_t)1024*1024*1024);
   curl_easy_setopt(handle, CURLOPT_COOKIEFILE, "");
   curl_easy_setopt(handle, CURLOPT_FILETIME, 1L);
   curl_easy_setopt(handle, CURLOPT_USERAGENT, "mini crawler");