1 // Written in the D programming language.
4 Networking client functionality as provided by $(HTTP _curl.haxx.se/libcurl,
5 libcurl). The libcurl library must be installed on the system in order to use
8 $(SCRIPT inhibitQuickIndex = 1;)
12 $(TR $(TH Category) $(TH Functions)
14 $(TR $(TDNW High level) $(TD $(MYREF download) $(MYREF upload) $(MYREF get)
15 $(MYREF post) $(MYREF put) $(MYREF del) $(MYREF options) $(MYREF trace)
16 $(MYREF connect) $(MYREF byLine) $(MYREF byChunk)
17 $(MYREF byLineAsync) $(MYREF byChunkAsync) )
19 $(TR $(TDNW Low level) $(TD $(MYREF HTTP) $(MYREF FTP) $(MYREF
26 You may need to link to the $(B curl) library, e.g. by adding $(D "libs": ["curl"])
27 to your $(B dub.json) file if you are using $(LINK2 http://code.dlang.org, DUB).
30 A DMD compatible libcurl static library can be downloaded from the dlang.org
31 $(LINK2 http://dlang.org/download.html, download page).
33 Compared to using libcurl directly this module allows simpler client code for
34 common uses, requires no unsafe operations, and integrates better with the rest
35 of the language. Futhermore it provides <a href="std_range.html">$(D range)</a>
36 access to protocols supported by libcurl both synchronously and asynchronously.
38 A high level and a low level API are available. The high level API is built
39 entirely on top of the low level one.
41 The high level API is for commonly used functionality such as HTTP/FTP get. The
42 $(LREF byLineAsync) and $(LREF byChunkAsync) provides asynchronous <a
43 href="std_range.html">$(D ranges)</a> that performs the request in another
44 thread while handling a line/chunk in the current thread.
46 The low level API allows for streaming and other advanced features.
48 $(BOOKTABLE Cheat Sheet,
49 $(TR $(TH Function Name) $(TH Description)
51 $(LEADINGROW High level)
52 $(TR $(TDNW $(LREF download)) $(TD $(D
53 download("ftp.digitalmars.com/sieve.ds", "/tmp/downloaded-ftp-file"))
54 downloads file from URL to file system.)
56 $(TR $(TDNW $(LREF upload)) $(TD $(D
57 upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds");)
58 uploads file from file system to URL.)
60 $(TR $(TDNW $(LREF get)) $(TD $(D
61 get("dlang.org")) returns a char[] containing the dlang.org web page.)
63 $(TR $(TDNW $(LREF put)) $(TD $(D
64 put("dlang.org", "Hi")) returns a char[] containing
65 the dlang.org web page. after a HTTP PUT of "hi")
67 $(TR $(TDNW $(LREF post)) $(TD $(D
68 post("dlang.org", "Hi")) returns a char[] containing
69 the dlang.org web page. after a HTTP POST of "hi")
71 $(TR $(TDNW $(LREF byLine)) $(TD $(D
72 byLine("dlang.org")) returns a range of char[] containing the
75 $(TR $(TDNW $(LREF byChunk)) $(TD $(D
76 byChunk("dlang.org", 10)) returns a range of ubyte[10] containing the
79 $(TR $(TDNW $(LREF byLineAsync)) $(TD $(D
80 byLineAsync("dlang.org")) returns a range of char[] containing the dlang.org web
83 $(TR $(TDNW $(LREF byChunkAsync)) $(TD $(D
84 byChunkAsync("dlang.org", 10)) returns a range of ubyte[10] containing the
85 dlang.org web page asynchronously.)
87 $(LEADINGROW Low level
89 $(TR $(TDNW $(LREF HTTP)) $(TD $(D HTTP) struct for advanced usage))
90 $(TR $(TDNW $(LREF FTP)) $(TD $(D FTP) struct for advanced usage))
91 $(TR $(TDNW $(LREF SMTP)) $(TD $(D SMTP) struct for advanced usage))
97 import std.net.curl, std.stdio;
99 // Return a char[] containing the content specified by a URL
100 auto content = get("dlang.org");
102 // Post data and return a char[] containing the content specified by a URL
103 auto content = post("mydomain.com/here.cgi", ["name1" : "value1", "name2" : "value2"]);
105 // Get content of file from ftp server
106 auto content = get("ftp.digitalmars.com/sieve.ds");
108 // Post and print out content line by line. The request is done in another thread.
109 foreach (line; byLineAsync("dlang.org", "Post data"))
112 // Get using a line range and proxy settings
113 auto client = HTTP();
114 client.proxy = "1.2.3.4";
115 foreach (line; byLine("dlang.org", client))
119 For more control than the high level functions provide, use the low level API:
123 import std.net.curl, std.stdio;
125 // GET with custom data receivers
126 auto http = HTTP("dlang.org");
127 http.onReceiveHeader =
128 (in char[] key, in char[] value) { writeln(key, ": ", value); };
129 http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; };
133 First, an instance of the reference-counted HTTP struct is created. Then the
134 custom delegates are set. These will be called whenever the HTTP instance
135 receives a header and a data buffer, respectively. In this simple example, the
136 headers are written to stdout and the data is ignored. If the request should be
137 stopped before it has finished then return something less than data.length from
138 the onReceive callback. See $(LREF onReceiveHeader)/$(LREF onReceive) for more
139 information. Finally the HTTP request is effected by calling perform(), which is
142 Source: $(PHOBOSSRC std/net/_curl.d)
144 Copyright: Copyright Jonas Drewsen 2011-2012
145 License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
146 Authors: Jonas Drewsen. Some of the SMTP code contributed by Jimmy Cao.
148 Credits: The functionally is based on $(HTTP _curl.haxx.se/libcurl, libcurl).
149 LibCurl is licensed under an MIT/X derivative license.
152 Copyright Jonas Drewsen 2011 - 2012.
153 Distributed under the Boost Software License, Version 1.0.
154 (See accompanying file LICENSE_1_0.txt or copy at
155 http://www.boost.org/LICENSE_1_0.txt)
161 import std.concurrency;
163 import std.exception;
165 import std.range.primitives;
166 import std.socket : InternetAddress;
170 import std.internal.cstring;
172 public import etc.c.curl : CurlOption;
176 // Run unit test with the PHOBOS_TEST_ALLOW_NET=1 set in order to
181 import std.socket : Address, INADDR_LOOPBACK, Socket, TcpSocket;
183 private struct TestServer
185 string addr() { return _addr; }
187 void handle(void function(Socket s) dg)
196 static void loop(shared TcpSocket listener)
200 void function(Socket) handler = void;
202 handler = receiveOnly!(typeof(handler));
203 catch (OwnerTerminated)
205 handler((cast() listener).accept);
209 import core.stdc.stdlib : exit, EXIT_FAILURE;
211 exit(EXIT_FAILURE); // Bugzilla 7018
216 private TestServer startServer()
218 auto sock = new TcpSocket;
219 sock.bind(new InternetAddress(INADDR_LOOPBACK, InternetAddress.PORT_ANY));
221 auto addr = sock.localAddress.toString();
222 auto tid = spawn(&TestServer.loop, cast(shared) sock);
223 return TestServer(addr, tid);
226 private ref TestServer testServer()
228 __gshared TestServer server;
229 return initOnce!server(startServer());
232 private struct Request(T)
238 private Request!T recvReq(T=char)(Socket s)
240 import std.algorithm.comparison : min;
241 import std.algorithm.searching : find, canFind;
242 import std.conv : to;
243 import std.regex : ctRegex, matchFirst;
245 ubyte[1024] tmp=void;
250 auto nbytes = s.receive(tmp[]);
253 immutable beg = buf.length > 3 ? buf.length - 3 : 0;
254 buf ~= tmp[0 .. nbytes];
255 auto bdy = buf[beg .. $].find(cast(ubyte[])"\r\n\r\n");
259 auto hdrs = cast(string) buf[0 .. $ - bdy.length];
261 // no support for chunked transfer-encoding
262 if (auto m = hdrs.matchFirst(ctRegex!(`Content-Length: ([0-9]+)`, "i")))
264 import std.uni : asUpperCase;
265 if (hdrs.asUpperCase.canFind("EXPECT: 100-CONTINUE"))
266 s.send(httpContinue);
268 size_t remain = m.captures[1].to!size_t - bdy.length;
271 nbytes = s.receive(tmp[0 .. min(remain, $)]);
273 buf ~= tmp[0 .. nbytes];
281 bdy = buf[hdrs.length + 4 .. $];
282 return typeof(return)(hdrs, cast(immutable(T)[])bdy);
286 private string httpOK(string msg)
288 import std.conv : to;
290 return "HTTP/1.1 200 OK\r\n"~
291 "Content-Type: text/plain\r\n"~
292 "Content-Length: "~msg.length.to!string~"\r\n"~
297 private string httpOK()
299 return "HTTP/1.1 200 OK\r\n"~
300 "Content-Length: 0\r\n"~
304 private string httpNotFound()
306 return "HTTP/1.1 404 Not Found\r\n"~
307 "Content-Length: 0\r\n"~
311 private enum httpContinue = "HTTP/1.1 100 Continue\r\n\r\n";
313 version (StdDdoc) import std.stdio;
315 // Default data timeout for Protocols
316 private enum _defaultDataTimeout = dur!"minutes"(2);
321 CALLBACK_PARAMS = $(TABLE ,
323 $(DDOC_PARAM_ID $(DDOC_PARAM dlTotal))
324 $(DDOC_PARAM_DESC total bytes to download)
327 $(DDOC_PARAM_ID $(DDOC_PARAM dlNow))
328 $(DDOC_PARAM_DESC currently downloaded bytes)
331 $(DDOC_PARAM_ID $(DDOC_PARAM ulTotal))
332 $(DDOC_PARAM_DESC total bytes to upload)
335 $(DDOC_PARAM_ID $(DDOC_PARAM ulNow))
336 $(DDOC_PARAM_DESC currently uploaded bytes)
341 /** Connection type used when the URL should be used to auto detect the protocol.
343 * This struct is used as placeholder for the connection parameter when calling
344 * the high level API and the connection type (HTTP/FTP) should be guessed by
345 * inspecting the URL parameter.
347 * The rules for guessing the protocol are:
348 * 1, if URL starts with ftp://, ftps:// or ftp. then FTP connection is assumed.
349 * 2, HTTP connection otherwise.
353 * import std.net.curl;
354 * // Two requests below will do the same.
357 * // Explicit connection provided
358 * content = get!HTTP("dlang.org");
360 * // Guess connection type by looking at the URL
361 * content = get!AutoProtocol("ftp://foo.com/file");
362 * // and since AutoProtocol is default this is the same as
363 * content = get("ftp://foo.com/file");
364 * // and will end up detecting FTP from the url and be the same as
365 * content = get!FTP("ftp://foo.com/file");
368 struct AutoProtocol { }
370 // Returns true if the url points to an FTP resource
371 private bool isFTPUrl(const(char)[] url)
373 import std.algorithm.searching : startsWith;
374 import std.uni : toLower;
376 return startsWith(url.toLower(), "ftp://", "ftps://", "ftp.") != 0;
379 // Is true if the Conn type is a valid Curl Connection type.
380 private template isCurlConn(Conn)
382 enum auto isCurlConn = is(Conn : HTTP) ||
383 is(Conn : FTP) || is(Conn : AutoProtocol);
386 /** HTTP/FTP download to local file system.
389 * url = resource to download
390 * saveToPath = path to store the downloaded content on local disk
391 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
392 * guess connection type and create a new instance for this call only.
396 * import std.net.curl;
397 * download("d-lang.appspot.com/testUrl2", "/tmp/downloaded-http-file");
400 void download(Conn = AutoProtocol)(const(char)[] url, string saveToPath, Conn conn = Conn())
403 static if (is(Conn : HTTP) || is(Conn : FTP))
405 import std.stdio : File;
407 auto f = File(saveToPath, "wb");
408 conn.onReceive = (ubyte[] data) { f.rawWrite(data); return data.length; };
414 return download!FTP(url, saveToPath, FTP());
416 return download!HTTP(url, saveToPath, HTTP());
422 import std.algorithm.searching : canFind;
423 static import std.file;
425 foreach (host; [testServer.addr, "http://"~testServer.addr])
427 testServer.handle((s) {
428 assert(s.recvReq.hdrs.canFind("GET /"));
429 s.send(httpOK("Hello world"));
431 auto fn = std.file.deleteme;
432 scope (exit) std.file.remove(fn);
434 assert(std.file.readText(fn) == "Hello world");
438 /** Upload file from local files system using the HTTP or FTP protocol.
441 * loadFromPath = path load data from local disk.
442 * url = resource to upload to
443 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
444 * guess connection type and create a new instance for this call only.
448 * import std.net.curl;
449 * upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds");
450 * upload("/tmp/downloaded-http-file", "d-lang.appspot.com/testUrl2");
453 void upload(Conn = AutoProtocol)(string loadFromPath, const(char)[] url, Conn conn = Conn())
456 static if (is(Conn : HTTP))
459 conn.method = HTTP.Method.put;
461 else static if (is(Conn : FTP))
464 conn.handle.set(CurlOption.upload, 1L);
469 return upload!FTP(loadFromPath, url, FTP());
471 return upload!HTTP(loadFromPath, url, HTTP());
474 static if (is(Conn : HTTP) || is(Conn : FTP))
476 import std.stdio : File;
477 auto f = File(loadFromPath, "rb");
478 conn.onSend = buf => f.rawRead(buf).length;
479 immutable sz = f.size;
481 conn.contentLength = sz;
488 import std.algorithm.searching : canFind;
489 static import std.file;
491 foreach (host; [testServer.addr, "http://"~testServer.addr])
493 auto fn = std.file.deleteme;
494 scope (exit) std.file.remove(fn);
495 std.file.write(fn, "upload data\n");
496 testServer.handle((s) {
497 auto req = s.recvReq;
498 assert(req.hdrs.canFind("PUT /path"));
499 assert(req.bdy.canFind("upload data"));
502 upload(fn, host ~ "/path");
506 /** HTTP/FTP get content.
509 * url = resource to get
510 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
511 * guess connection type and create a new instance for this call only.
513 * The template parameter $(D T) specifies the type to return. Possible values
514 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). If asking
515 * for $(D char), content will be converted from the connection character set
516 * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1
517 * by default) to UTF-8.
521 * import std.net.curl;
522 * auto content = get("d-lang.appspot.com/testUrl2");
526 * A T[] range containing the content of the resource pointed to by the URL.
530 * $(D CurlException) on error.
532 * See_Also: $(LREF HTTP.Method)
534 T[] get(Conn = AutoProtocol, T = char)(const(char)[] url, Conn conn = Conn())
535 if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) )
537 static if (is(Conn : HTTP))
539 conn.method = HTTP.Method.get;
540 return _basicHTTP!(T)(url, "", conn);
543 else static if (is(Conn : FTP))
545 return _basicFTP!(T)(url, "", conn);
550 return get!(FTP,T)(url, FTP());
552 return get!(HTTP,T)(url, HTTP());
558 import std.algorithm.searching : canFind;
560 foreach (host; [testServer.addr, "http://"~testServer.addr])
562 testServer.handle((s) {
563 assert(s.recvReq.hdrs.canFind("GET /path"));
564 s.send(httpOK("GETRESPONSE"));
566 auto res = get(host ~ "/path");
567 assert(res == "GETRESPONSE");
572 /** HTTP post content.
575 * url = resource to post to
576 * postDict = data to send as the body of the request. An associative array
577 * of $(D string) is accepted and will be encoded using
578 * www-form-urlencoding
579 * postData = data to send as the body of the request. An array
580 * of an arbitrary type is accepted and will be cast to ubyte[]
582 * conn = HTTP connection to use
583 * T = The template parameter $(D T) specifies the type to return. Possible values
584 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). If asking
585 * for $(D char), content will be converted from the connection character set
586 * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1
587 * by default) to UTF-8.
591 * import std.net.curl;
593 * auto content1 = post("d-lang.appspot.com/testUrl2", ["name1" : "value1", "name2" : "value2"]);
594 * auto content2 = post("d-lang.appspot.com/testUrl2", [1,2,3,4]);
598 * A T[] range containing the content of the resource pointed to by the URL.
600 * See_Also: $(LREF HTTP.Method)
602 T[] post(T = char, PostUnit)(const(char)[] url, const(PostUnit)[] postData, HTTP conn = HTTP())
603 if (is(T == char) || is(T == ubyte))
605 conn.method = HTTP.Method.post;
606 return _basicHTTP!(T)(url, postData, conn);
611 import std.algorithm.searching : canFind;
613 foreach (host; [testServer.addr, "http://"~testServer.addr])
615 testServer.handle((s) {
616 auto req = s.recvReq;
617 assert(req.hdrs.canFind("POST /path"));
618 assert(req.bdy.canFind("POSTBODY"));
619 s.send(httpOK("POSTRESPONSE"));
621 auto res = post(host ~ "/path", "POSTBODY");
622 assert(res == "POSTRESPONSE");
628 import std.algorithm.searching : canFind;
630 auto data = new ubyte[](256);
631 foreach (i, ref ub; data)
634 testServer.handle((s) {
635 auto req = s.recvReq!ubyte;
636 assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4]));
637 assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255]));
638 s.send(httpOK(cast(ubyte[])[17, 27, 35, 41]));
640 auto res = post!ubyte(testServer.addr, data);
641 assert(res == cast(ubyte[])[17, 27, 35, 41]);
645 T[] post(T = char)(const(char)[] url, string[string] postDict, HTTP conn = HTTP())
646 if (is(T == char) || is(T == ubyte))
648 import std.uri : urlEncode;
650 return post(url, urlEncode(postDict), conn);
655 foreach (host; [testServer.addr, "http://" ~ testServer.addr])
657 testServer.handle((s) {
658 auto req = s.recvReq!char;
659 s.send(httpOK(req.bdy));
661 auto res = post(host ~ "/path", ["name1" : "value1", "name2" : "value2"]);
662 assert(res == "name1=value1&name2=value2");
666 /** HTTP/FTP put content.
669 * url = resource to put
670 * putData = data to send as the body of the request. An array
671 * of an arbitrary type is accepted and will be cast to ubyte[]
673 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
674 * guess connection type and create a new instance for this call only.
676 * The template parameter $(D T) specifies the type to return. Possible values
677 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). If asking
678 * for $(D char), content will be converted from the connection character set
679 * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1
680 * by default) to UTF-8.
684 * import std.net.curl;
685 * auto content = put("d-lang.appspot.com/testUrl2",
686 * "Putting this data");
690 * A T[] range containing the content of the resource pointed to by the URL.
692 * See_Also: $(LREF HTTP.Method)
694 T[] put(Conn = AutoProtocol, T = char, PutUnit)(const(char)[] url, const(PutUnit)[] putData,
696 if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) )
698 static if (is(Conn : HTTP))
700 conn.method = HTTP.Method.put;
701 return _basicHTTP!(T)(url, putData, conn);
703 else static if (is(Conn : FTP))
705 return _basicFTP!(T)(url, putData, conn);
710 return put!(FTP,T)(url, putData, FTP());
712 return put!(HTTP,T)(url, putData, HTTP());
718 import std.algorithm.searching : canFind;
720 foreach (host; [testServer.addr, "http://"~testServer.addr])
722 testServer.handle((s) {
723 auto req = s.recvReq;
724 assert(req.hdrs.canFind("PUT /path"));
725 assert(req.bdy.canFind("PUTBODY"));
726 s.send(httpOK("PUTRESPONSE"));
728 auto res = put(host ~ "/path", "PUTBODY");
729 assert(res == "PUTRESPONSE");
734 /** HTTP/FTP delete content.
737 * url = resource to delete
738 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
739 * guess connection type and create a new instance for this call only.
743 * import std.net.curl;
744 * del("d-lang.appspot.com/testUrl2");
747 * See_Also: $(LREF HTTP.Method)
749 void del(Conn = AutoProtocol)(const(char)[] url, Conn conn = Conn())
752 static if (is(Conn : HTTP))
754 conn.method = HTTP.Method.del;
755 _basicHTTP!char(url, cast(void[]) null, conn);
757 else static if (is(Conn : FTP))
759 import std.algorithm.searching : findSplitAfter;
760 import std.conv : text;
762 auto trimmed = url.findSplitAfter("ftp://")[1];
763 auto t = trimmed.findSplitAfter("/");
764 enum minDomainNameLength = 3;
765 enforce!CurlException(t[0].length > minDomainNameLength,
766 text("Invalid FTP URL for delete ", url));
769 enforce!CurlException(!t[1].empty,
770 text("No filename specified to delete for URL ", url));
771 conn.addCommand("DELE " ~ t[1]);
777 return del!FTP(url, FTP());
779 return del!HTTP(url, HTTP());
785 import std.algorithm.searching : canFind;
787 foreach (host; [testServer.addr, "http://"~testServer.addr])
789 testServer.handle((s) {
790 auto req = s.recvReq;
791 assert(req.hdrs.canFind("DELETE /path"));
799 /** HTTP options request.
802 * url = resource make a option call to
803 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
804 * guess connection type and create a new instance for this call only.
806 * The template parameter $(D T) specifies the type to return. Possible values
807 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]).
811 * import std.net.curl;
812 * auto http = HTTP();
813 * options("d-lang.appspot.com/testUrl2", http);
814 * writeln("Allow set to " ~ http.responseHeaders["Allow"]);
818 * A T[] range containing the options of the resource pointed to by the URL.
820 * See_Also: $(LREF HTTP.Method)
822 T[] options(T = char)(const(char)[] url, HTTP conn = HTTP())
823 if (is(T == char) || is(T == ubyte))
825 conn.method = HTTP.Method.options;
826 return _basicHTTP!(T)(url, null, conn);
831 import std.algorithm.searching : canFind;
833 testServer.handle((s) {
834 auto req = s.recvReq;
835 assert(req.hdrs.canFind("OPTIONS /path"));
836 s.send(httpOK("OPTIONSRESPONSE"));
838 auto res = options(testServer.addr ~ "/path");
839 assert(res == "OPTIONSRESPONSE");
843 /** HTTP trace request.
846 * url = resource make a trace call to
847 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
848 * guess connection type and create a new instance for this call only.
850 * The template parameter $(D T) specifies the type to return. Possible values
851 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]).
855 * import std.net.curl;
856 * trace("d-lang.appspot.com/testUrl1");
860 * A T[] range containing the trace info of the resource pointed to by the URL.
862 * See_Also: $(LREF HTTP.Method)
864 T[] trace(T = char)(const(char)[] url, HTTP conn = HTTP())
865 if (is(T == char) || is(T == ubyte))
867 conn.method = HTTP.Method.trace;
868 return _basicHTTP!(T)(url, cast(void[]) null, conn);
873 import std.algorithm.searching : canFind;
875 testServer.handle((s) {
876 auto req = s.recvReq;
877 assert(req.hdrs.canFind("TRACE /path"));
878 s.send(httpOK("TRACERESPONSE"));
880 auto res = trace(testServer.addr ~ "/path");
881 assert(res == "TRACERESPONSE");
885 /** HTTP connect request.
888 * url = resource make a connect to
889 * conn = HTTP connection to use
891 * The template parameter $(D T) specifies the type to return. Possible values
892 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]).
896 * import std.net.curl;
897 * connect("d-lang.appspot.com/testUrl1");
901 * A T[] range containing the connect info of the resource pointed to by the URL.
903 * See_Also: $(LREF HTTP.Method)
905 T[] connect(T = char)(const(char)[] url, HTTP conn = HTTP())
906 if (is(T == char) || is(T == ubyte))
908 conn.method = HTTP.Method.connect;
909 return _basicHTTP!(T)(url, cast(void[]) null, conn);
914 import std.algorithm.searching : canFind;
916 testServer.handle((s) {
917 auto req = s.recvReq;
918 assert(req.hdrs.canFind("CONNECT /path"));
919 s.send(httpOK("CONNECTRESPONSE"));
921 auto res = connect(testServer.addr ~ "/path");
922 assert(res == "CONNECTRESPONSE");
926 /** HTTP patch content.
929 * url = resource to patch
930 * patchData = data to send as the body of the request. An array
931 * of an arbitrary type is accepted and will be cast to ubyte[]
933 * conn = HTTP connection to use
935 * The template parameter $(D T) specifies the type to return. Possible values
936 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]).
940 * auto http = HTTP();
941 * http.addRequestHeader("Content-Type", "application/json");
942 * auto content = patch("d-lang.appspot.com/testUrl2", `{"title": "Patched Title"}`, http);
946 * A T[] range containing the content of the resource pointed to by the URL.
948 * See_Also: $(LREF HTTP.Method)
950 T[] patch(T = char, PatchUnit)(const(char)[] url, const(PatchUnit)[] patchData,
952 if (is(T == char) || is(T == ubyte))
954 conn.method = HTTP.Method.patch;
955 return _basicHTTP!(T)(url, patchData, conn);
960 import std.algorithm.searching : canFind;
962 testServer.handle((s) {
963 auto req = s.recvReq;
964 assert(req.hdrs.canFind("PATCH /path"));
965 assert(req.bdy.canFind("PATCHBODY"));
966 s.send(httpOK("PATCHRESPONSE"));
968 auto res = patch(testServer.addr ~ "/path", "PATCHBODY");
969 assert(res == "PATCHRESPONSE");
974 * Helper function for the high level interface.
976 * It performs an HTTP request using the client which must have
977 * been setup correctly before calling this function.
979 private auto _basicHTTP(T)(const(char)[] url, const(void)[] sendData, HTTP client)
981 import std.algorithm.comparison : min;
982 import std.format : format;
984 immutable doSend = sendData !is null &&
985 (client.method == HTTP.Method.post ||
986 client.method == HTTP.Method.put ||
987 client.method == HTTP.Method.patch);
991 client.onReceiveHeader = null;
992 client.onReceiveStatusLine = null;
993 client.onReceive = null;
997 client.onSend = null;
998 client.handle.onSeek = null;
999 client.contentLength = 0;
1003 HTTP.StatusLine statusLine;
1004 import std.array : appender;
1005 auto content = appender!(ubyte[])();
1006 client.onReceive = (ubyte[] data)
1014 client.contentLength = sendData.length;
1015 auto remainingData = sendData;
1016 client.onSend = delegate size_t(void[] buf)
1018 size_t minLen = min(buf.length, remainingData.length);
1019 if (minLen == 0) return 0;
1020 buf[0 .. minLen] = remainingData[0 .. minLen];
1021 remainingData = remainingData[minLen..$];
1024 client.handle.onSeek = delegate(long offset, CurlSeekPos mode)
1028 case CurlSeekPos.set:
1029 remainingData = sendData[cast(size_t) offset..$];
1032 // As of curl 7.18.0, libcurl will not pass
1033 // anything other than CurlSeekPos.set.
1034 return CurlSeek.cantseek;
1039 client.onReceiveHeader = (in char[] key,
1042 if (key == "content-length")
1044 import std.conv : to;
1045 content.reserve(value.to!size_t);
1048 client.onReceiveStatusLine = (HTTP.StatusLine l) { statusLine = l; };
1050 enforce(statusLine.code / 100 == 2, new HTTPStatusException(statusLine.code,
1051 format("HTTP request returned status code %d (%s)", statusLine.code, statusLine.reason)));
1053 return _decodeContent!T(content.data, client.p.charset);
1058 import std.algorithm.searching : canFind;
1060 testServer.handle((s) {
1061 auto req = s.recvReq;
1062 assert(req.hdrs.canFind("GET /path"));
1063 s.send(httpNotFound());
1065 auto e = collectException!HTTPStatusException(get(testServer.addr ~ "/path"));
1066 assert(e.msg == "HTTP request returned status code 404 (Not Found)");
1067 assert(e.status == 404);
1070 // Bugzilla 14760 - content length must be reset after post
1073 import std.algorithm.searching : canFind;
1075 testServer.handle((s) {
1076 auto req = s.recvReq;
1077 assert(req.hdrs.canFind("POST /"));
1078 assert(req.bdy.canFind("POSTBODY"));
1079 s.send(httpOK("POSTRESPONSE"));
1082 assert(req.hdrs.canFind("TRACE /"));
1083 assert(req.bdy.empty);
1085 ubyte[6] buf = void;
1086 assert(s.receive(buf[]) < 0);
1087 s.send(httpOK("TRACERESPONSE"));
1090 auto res = post(testServer.addr, "POSTBODY", http);
1091 assert(res == "POSTRESPONSE");
1092 res = trace(testServer.addr, http);
1093 assert(res == "TRACERESPONSE");
1096 @system unittest // charset detection and transcoding to T
1098 testServer.handle((s) {
1099 s.send("HTTP/1.1 200 OK\r\n"~
1100 "Content-Length: 4\r\n"~
1101 "Content-Type: text/plain; charset=utf-8\r\n" ~
1105 auto client = HTTP();
1106 auto result = _basicHTTP!char(testServer.addr, "", client);
1107 assert(result == "äbc");
1109 testServer.handle((s) {
1110 s.send("HTTP/1.1 200 OK\r\n"~
1111 "Content-Length: 3\r\n"~
1112 "Content-Type: text/plain; charset=iso-8859-1\r\n" ~
1117 result = _basicHTTP!char(testServer.addr, "", client);
1118 assert(result == "äbc");
1122 * Helper function for the high level interface.
1124 * It performs an FTP request using the client which must have
1125 * been setup correctly before calling this function.
1127 private auto _basicFTP(T)(const(char)[] url, const(void)[] sendData, FTP client)
1129 import std.algorithm.comparison : min;
1133 client.onReceive = null;
1134 if (!sendData.empty)
1135 client.onSend = null;
1140 if (client.encoding.empty)
1141 client.encoding = "ISO-8859-1";
1144 client.onReceive = (ubyte[] data)
1150 if (!sendData.empty)
1152 client.handle.set(CurlOption.upload, 1L);
1153 client.onSend = delegate size_t(void[] buf)
1155 size_t minLen = min(buf.length, sendData.length);
1156 if (minLen == 0) return 0;
1157 buf[0 .. minLen] = sendData[0 .. minLen];
1158 sendData = sendData[minLen..$];
1165 return _decodeContent!T(content, client.encoding);
1168 /* Used by _basicHTTP() and _basicFTP() to decode ubyte[] to
1169 * correct string format
1171 private auto _decodeContent(T)(ubyte[] content, string encoding)
1173 static if (is(T == ubyte))
1179 import std.format : format;
1181 // Optimally just return the utf8 encoded content
1182 if (encoding == "UTF-8")
1183 return cast(char[])(content);
1185 // The content has to be re-encoded to utf8
1186 auto scheme = EncodingScheme.create(encoding);
1187 enforce!CurlException(scheme !is null,
1188 format("Unknown encoding '%s'", encoding));
1190 auto strInfo = decodeString(content, scheme);
1191 enforce!CurlException(strInfo[0] != size_t.max,
1192 format("Invalid encoding sequence for encoding '%s'",
1199 alias KeepTerminator = Flag!"keepTerminator";
1201 struct ByLineBuffer(Char)
1206 ubyte[] decodeRemainder;
1208 bool append(const(ubyte)[] data)
1210 byLineBuffer ~= data;
1213 @property bool linePresent()
1215 return byLinePresent;
1222 // Decode ubyte[] into Char[] until a Terminator is found.
1223 // If not Terminator is found and EOF is false then raise an
1226 return byLineBuffer;
1231 /** HTTP/FTP fetch content as a range of lines.
1233 * A range of lines is returned when the request is complete. If the method or
1234 * other request properties is to be customized then set the $(D conn) parameter
1235 * with a HTTP/FTP instance that has these properties set.
1239 * import std.net.curl, std.stdio;
1240 * foreach (line; byLine("dlang.org"))
1245 * url = The url to receive content from
1246 * keepTerminator = $(D Yes.keepTerminator) signals that the line terminator should be
1247 * returned as part of the lines in the range.
1248 * terminator = The character that terminates a line
1249 * conn = The connection to use e.g. HTTP or FTP.
1252 * A range of Char[] with the content of the resource pointer to by the URL
1254 auto byLine(Conn = AutoProtocol, Terminator = char, Char = char)
1255 (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator,
1256 Terminator terminator = '\n', Conn conn = Conn())
1257 if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator)
1259 static struct SyncLineInputRange
1262 private Char[] lines;
1263 private Char[] current;
1264 private bool currentValid;
1265 private bool keepTerminator;
1266 private Terminator terminator;
1268 this(Char[] lines, bool kt, Terminator terminator)
1271 this.keepTerminator = kt;
1272 this.terminator = terminator;
1273 currentValid = true;
1277 @property @safe bool empty()
1279 return !currentValid;
1282 @property @safe Char[] front()
1284 enforce!CurlException(currentValid, "Cannot call front() on empty range");
1290 import std.algorithm.searching : findSplitAfter, findSplit;
1292 enforce!CurlException(currentValid, "Cannot call popFront() on empty range");
1295 currentValid = false;
1301 auto r = findSplitAfter(lines, [ terminator ]);
1315 auto r = findSplit(lines, [ terminator ]);
1322 auto result = _getForRange!Char(url, conn);
1323 return SyncLineInputRange(result, keepTerminator == Yes.keepTerminator, terminator);
1328 import std.algorithm.comparison : equal;
1330 foreach (host; [testServer.addr, "http://"~testServer.addr])
1332 testServer.handle((s) {
1333 auto req = s.recvReq;
1334 s.send(httpOK("Line1\nLine2\nLine3"));
1336 assert(byLine(host).equal(["Line1", "Line2", "Line3"]));
1340 /** HTTP/FTP fetch content as a range of chunks.
1342 * A range of chunks is returned when the request is complete. If the method or
1343 * other request properties is to be customized then set the $(D conn) parameter
1344 * with a HTTP/FTP instance that has these properties set.
1348 * import std.net.curl, std.stdio;
1349 * foreach (chunk; byChunk("dlang.org", 100))
1350 * writeln(chunk); // chunk is ubyte[100]
1354 * url = The url to receive content from
1355 * chunkSize = The size of each chunk
1356 * conn = The connection to use e.g. HTTP or FTP.
1359 * A range of ubyte[chunkSize] with the content of the resource pointer to by the URL
1361 auto byChunk(Conn = AutoProtocol)
1362 (const(char)[] url, size_t chunkSize = 1024, Conn conn = Conn())
1363 if (isCurlConn!(Conn))
1365 static struct SyncChunkInputRange
1367 private size_t chunkSize;
1368 private ubyte[] _bytes;
1369 private size_t offset;
1371 this(ubyte[] bytes, size_t chunkSize)
1373 this._bytes = bytes;
1374 this.chunkSize = chunkSize;
1377 @property @safe auto empty()
1379 return offset == _bytes.length;
1382 @property ubyte[] front()
1384 size_t nextOffset = offset + chunkSize;
1385 if (nextOffset > _bytes.length) nextOffset = _bytes.length;
1386 return _bytes[offset .. nextOffset];
1389 @safe void popFront()
1391 offset += chunkSize;
1392 if (offset > _bytes.length) offset = _bytes.length;
1396 auto result = _getForRange!ubyte(url, conn);
1397 return SyncChunkInputRange(result, chunkSize);
1402 import std.algorithm.comparison : equal;
1404 foreach (host; [testServer.addr, "http://"~testServer.addr])
1406 testServer.handle((s) {
1407 auto req = s.recvReq;
1408 s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5]));
1410 assert(byChunk(host, 2).equal([[0, 1], [2, 3], [4, 5]]));
1414 private T[] _getForRange(T,Conn)(const(char)[] url, Conn conn)
1416 static if (is(Conn : HTTP))
1418 conn.method = conn.method == HTTP.Method.undefined ? HTTP.Method.get : conn.method;
1419 return _basicHTTP!(T)(url, null, conn);
1421 else static if (is(Conn : FTP))
1423 return _basicFTP!(T)(url, null, conn);
1428 return get!(FTP,T)(url, FTP());
1430 return get!(HTTP,T)(url, HTTP());
1435 Main thread part of the message passing protocol used for all async
1438 private mixin template WorkerThreadProtocol(Unit, alias units)
1440 @property bool empty()
1443 return state == State.done;
1446 @property Unit[] front()
1448 import std.format : format;
1450 assert(state == State.gotUnits,
1451 format("Expected %s but got $s",
1452 State.gotUnits, state));
1458 import std.format : format;
1460 assert(state == State.gotUnits,
1461 format("Expected %s but got $s",
1462 State.gotUnits, state));
1463 state = State.needUnits;
1464 // Send to worker thread for buffer reuse
1465 workerTid.send(cast(immutable(Unit)[]) units);
1469 /** Wait for duration or until data is available and return true if data is
1472 bool wait(Duration d)
1474 import std.datetime.stopwatch : StopWatch;
1476 if (state == State.gotUnits)
1479 enum noDur = dur!"hnsecs"(0);
1482 while (state != State.gotUnits && d > noDur)
1484 final switch (state)
1486 case State.needUnits:
1488 (Tid origin, CurlMessage!(immutable(Unit)[]) _data)
1490 if (origin != workerTid)
1492 units = cast(Unit[]) _data.data;
1493 state = State.gotUnits;
1496 (Tid origin, CurlMessage!bool f)
1498 if (origin != workerTid)
1505 case State.gotUnits: return true;
1512 return state == State.gotUnits;
1523 void tryEnsureUnits()
1527 final switch (state)
1529 case State.needUnits:
1531 (Tid origin, CurlMessage!(immutable(Unit)[]) _data)
1533 if (origin != workerTid)
1535 units = cast(Unit[]) _data.data;
1536 state = State.gotUnits;
1539 (Tid origin, CurlMessage!bool f)
1541 if (origin != workerTid)
1548 case State.gotUnits: return;
1556 // @@@@BUG 15831@@@@
1557 // this should be inside byLineAsync
1558 // Range that reads one line at a time asynchronously.
1559 private static struct AsyncLineInputRange(Char)
1561 private Char[] line;
1562 mixin WorkerThreadProtocol!(Char, line);
1564 private Tid workerTid;
1565 private State running;
1567 private this(Tid tid, size_t transmitBuffers, size_t bufferSize)
1570 state = State.needUnits;
1572 // Send buffers to other thread for it to use. Since no mechanism is in
1573 // place for moving ownership a cast to shared is done here and casted
1574 // back to non-shared in the receiving end.
1575 foreach (i ; 0 .. transmitBuffers)
1577 auto arr = new Char[](bufferSize);
1578 workerTid.send(cast(immutable(Char[]))arr);
1583 /** HTTP/FTP fetch content as a range of lines asynchronously.
1585 * A range of lines is returned immediately and the request that fetches the
1586 * lines is performed in another thread. If the method or other request
1587 * properties is to be customized then set the $(D conn) parameter with a
1588 * HTTP/FTP instance that has these properties set.
1590 * If $(D postData) is non-_null the method will be set to $(D post) for HTTP
1593 * The background thread will buffer up to transmitBuffers number of lines
1594 * before it stops receiving data from network. When the main thread reads the
1595 * lines from the range it frees up buffers and allows for the background thread
1596 * to receive more data from the network.
1598 * If no data is available and the main thread accesses the range it will block
1599 * until data becomes available. An exception to this is the $(D wait(Duration)) method on
1600 * the $(LREF AsyncLineInputRange). This method will wait at maximum for the
1601 * specified duration and return true if data is available.
1605 * import std.net.curl, std.stdio;
1606 * // Get some pages in the background
1607 * auto range1 = byLineAsync("www.google.com");
1608 * auto range2 = byLineAsync("www.wikipedia.org");
1609 * foreach (line; byLineAsync("dlang.org"))
1612 * // Lines already fetched in the background and ready
1613 * foreach (line; range1) writeln(line);
1614 * foreach (line; range2) writeln(line);
1618 * import std.net.curl, std.stdio;
1619 * // Get a line in a background thread and wait in
1620 * // main thread for 2 seconds for it to arrive.
1621 * auto range3 = byLineAsync("dlang.com");
1622 * if (range3.wait(dur!"seconds"(2)))
1623 * writeln(range3.front);
1625 * writeln("No line received after 2 seconds!");
1629 * url = The url to receive content from
1630 * postData = Data to HTTP Post
1631 * keepTerminator = $(D Yes.keepTerminator) signals that the line terminator should be
1632 * returned as part of the lines in the range.
1633 * terminator = The character that terminates a line
1634 * transmitBuffers = The number of lines buffered asynchronously
1635 * conn = The connection to use e.g. HTTP or FTP.
1638 * A range of Char[] with the content of the resource pointer to by the
1641 auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char, PostUnit)
1642 (const(char)[] url, const(PostUnit)[] postData,
1643 KeepTerminator keepTerminator = No.keepTerminator,
1644 Terminator terminator = '\n',
1645 size_t transmitBuffers = 10, Conn conn = Conn())
1646 if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator)
1648 static if (is(Conn : AutoProtocol))
1651 return byLineAsync(url, postData, keepTerminator,
1652 terminator, transmitBuffers, FTP());
1654 return byLineAsync(url, postData, keepTerminator,
1655 terminator, transmitBuffers, HTTP());
1659 // 50 is just an arbitrary number for now
1660 setMaxMailboxSize(thisTid, 50, OnCrowding.block);
1661 auto tid = spawn(&_spawnAsync!(Conn, Char, Terminator));
1663 tid.send(terminator);
1664 tid.send(keepTerminator == Yes.keepTerminator);
1666 _asyncDuplicateConnection(url, conn, postData, tid);
1668 return AsyncLineInputRange!Char(tid, transmitBuffers,
1669 Conn.defaultAsyncStringBufferSize);
1674 auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char)
1675 (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator,
1676 Terminator terminator = '\n',
1677 size_t transmitBuffers = 10, Conn conn = Conn())
1679 static if (is(Conn : AutoProtocol))
1682 return byLineAsync(url, cast(void[]) null, keepTerminator,
1683 terminator, transmitBuffers, FTP());
1685 return byLineAsync(url, cast(void[]) null, keepTerminator,
1686 terminator, transmitBuffers, HTTP());
1690 return byLineAsync(url, cast(void[]) null, keepTerminator,
1691 terminator, transmitBuffers, conn);
1697 import std.algorithm.comparison : equal;
1699 foreach (host; [testServer.addr, "http://"~testServer.addr])
1701 testServer.handle((s) {
1702 auto req = s.recvReq;
1703 s.send(httpOK("Line1\nLine2\nLine3"));
1705 assert(byLineAsync(host).equal(["Line1", "Line2", "Line3"]));
1709 // @@@@BUG 15831@@@@
1710 // this should be inside byLineAsync
1711 // Range that reads one chunk at a time asynchronously.
1712 private static struct AsyncChunkInputRange
1714 private ubyte[] chunk;
1715 mixin WorkerThreadProtocol!(ubyte, chunk);
1717 private Tid workerTid;
1718 private State running;
1720 private this(Tid tid, size_t transmitBuffers, size_t chunkSize)
1723 state = State.needUnits;
1725 // Send buffers to other thread for it to use. Since no mechanism is in
1726 // place for moving ownership a cast to shared is done here and a cast
1727 // back to non-shared in the receiving end.
1728 foreach (i ; 0 .. transmitBuffers)
1730 ubyte[] arr = new ubyte[](chunkSize);
1731 workerTid.send(cast(immutable(ubyte[]))arr);
1736 /** HTTP/FTP fetch content as a range of chunks asynchronously.
1738 * A range of chunks is returned immediately and the request that fetches the
1739 * chunks is performed in another thread. If the method or other request
1740 * properties is to be customized then set the $(D conn) parameter with a
1741 * HTTP/FTP instance that has these properties set.
1743 * If $(D postData) is non-_null the method will be set to $(D post) for HTTP
1746 * The background thread will buffer up to transmitBuffers number of chunks
1747 * before is stops receiving data from network. When the main thread reads the
1748 * chunks from the range it frees up buffers and allows for the background
1749 * thread to receive more data from the network.
1751 * If no data is available and the main thread access the range it will block
1752 * until data becomes available. An exception to this is the $(D wait(Duration))
1753 * method on the $(LREF AsyncChunkInputRange). This method will wait at maximum for the specified
1754 * duration and return true if data is available.
1758 * import std.net.curl, std.stdio;
1759 * // Get some pages in the background
1760 * auto range1 = byChunkAsync("www.google.com", 100);
1761 * auto range2 = byChunkAsync("www.wikipedia.org");
1762 * foreach (chunk; byChunkAsync("dlang.org"))
1763 * writeln(chunk); // chunk is ubyte[100]
1765 * // Chunks already fetched in the background and ready
1766 * foreach (chunk; range1) writeln(chunk);
1767 * foreach (chunk; range2) writeln(chunk);
1771 * import std.net.curl, std.stdio;
1772 * // Get a line in a background thread and wait in
1773 * // main thread for 2 seconds for it to arrive.
1774 * auto range3 = byChunkAsync("dlang.com", 10);
1775 * if (range3.wait(dur!"seconds"(2)))
1776 * writeln(range3.front);
1778 * writeln("No chunk received after 2 seconds!");
1782 * url = The url to receive content from
1783 * postData = Data to HTTP Post
1784 * chunkSize = The size of the chunks
1785 * transmitBuffers = The number of chunks buffered asynchronously
1786 * conn = The connection to use e.g. HTTP or FTP.
1789 * A range of ubyte[chunkSize] with the content of the resource pointer to by
1792 auto byChunkAsync(Conn = AutoProtocol, PostUnit)
1793 (const(char)[] url, const(PostUnit)[] postData,
1794 size_t chunkSize = 1024, size_t transmitBuffers = 10,
1796 if (isCurlConn!(Conn))
1798 static if (is(Conn : AutoProtocol))
1801 return byChunkAsync(url, postData, chunkSize,
1802 transmitBuffers, FTP());
1804 return byChunkAsync(url, postData, chunkSize,
1805 transmitBuffers, HTTP());
1809 // 50 is just an arbitrary number for now
1810 setMaxMailboxSize(thisTid, 50, OnCrowding.block);
1811 auto tid = spawn(&_spawnAsync!(Conn, ubyte));
1814 _asyncDuplicateConnection(url, conn, postData, tid);
1816 return AsyncChunkInputRange(tid, transmitBuffers, chunkSize);
1821 auto byChunkAsync(Conn = AutoProtocol)
1823 size_t chunkSize = 1024, size_t transmitBuffers = 10,
1825 if (isCurlConn!(Conn))
1827 static if (is(Conn : AutoProtocol))
1830 return byChunkAsync(url, cast(void[]) null, chunkSize,
1831 transmitBuffers, FTP());
1833 return byChunkAsync(url, cast(void[]) null, chunkSize,
1834 transmitBuffers, HTTP());
1838 return byChunkAsync(url, cast(void[]) null, chunkSize,
1839 transmitBuffers, conn);
1845 import std.algorithm.comparison : equal;
1847 foreach (host; [testServer.addr, "http://"~testServer.addr])
1849 testServer.handle((s) {
1850 auto req = s.recvReq;
1851 s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5]));
1853 assert(byChunkAsync(host, 2).equal([[0, 1], [2, 3], [4, 5]]));
1858 /* Used by byLineAsync/byChunkAsync to duplicate an existing connection
1859 * that can be used exclusively in a spawned thread.
1861 private void _asyncDuplicateConnection(Conn, PostData)
1862 (const(char)[] url, Conn conn, PostData postData, Tid tid)
1864 // no move semantic available in std.concurrency ie. must use casting.
1865 auto connDup = conn.dup();
1868 static if ( is(Conn : HTTP) )
1870 connDup.p.headersOut = null;
1871 connDup.method = conn.method == HTTP.Method.undefined ?
1872 HTTP.Method.get : conn.method;
1873 if (postData !is null)
1875 if (connDup.method == HTTP.Method.put)
1877 connDup.handle.set(CurlOption.infilesize_large,
1883 connDup.method = HTTP.Method.post;
1884 connDup.handle.set(CurlOption.postfieldsize_large,
1887 connDup.handle.set(CurlOption.copypostfields,
1888 cast(void*) postData.ptr);
1890 tid.send(cast(ulong) connDup.handle.handle);
1891 tid.send(connDup.method);
1895 enforce!CurlException(postData is null,
1896 "Cannot put ftp data using byLineAsync()");
1897 tid.send(cast(ulong) connDup.handle.handle);
1898 tid.send(HTTP.Method.undefined);
1900 connDup.p.curl.handle = null; // make sure handle is not freed
1904 Mixin template for all supported curl protocols. This is the commom
1905 functionallity such as timeouts and network interface settings. This should
1906 really be in the HTTP/FTP/SMTP structs but the documentation tool does not
1907 support a mixin to put its doc strings where a mixin is done. Therefore docs
1908 in this template is copied into each of HTTP/FTP/SMTP below.
1910 private mixin template Protocol()
1913 /// Value to return from $(D onSend)/$(D onReceive) delegates in order to
1915 alias requestPause = CurlReadFunc.pause;
1917 /// Value to return from onSend delegate in order to abort a request
1918 alias requestAbort = CurlReadFunc.abort;
1920 static uint defaultAsyncStringBufferSize = 100;
1923 The curl handle used by this connection.
1925 @property ref Curl handle() return
1931 True if the instance is stopped. A stopped instance is not usable.
1933 @property bool isStopped()
1935 return p.curl.stopped;
1938 /// Stop and invalidate this instance.
1945 This will print request information to stderr.
1947 @property void verbose(bool on)
1949 p.curl.set(CurlOption.verbose, on ? 1L : 0L);
1952 // Connection settings
1954 /// Set timeout for activity on connection.
1955 @property void dataTimeout(Duration d)
1957 p.curl.set(CurlOption.low_speed_limit, 1);
1958 p.curl.set(CurlOption.low_speed_time, d.total!"seconds");
1961 /** Set maximum time an operation is allowed to take.
1962 This includes dns resolution, connecting, data transfer, etc.
1964 @property void operationTimeout(Duration d)
1966 p.curl.set(CurlOption.timeout_ms, d.total!"msecs");
1969 /// Set timeout for connecting.
1970 @property void connectTimeout(Duration d)
1972 p.curl.set(CurlOption.connecttimeout_ms, d.total!"msecs");
1978 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy)
1980 @property void proxy(const(char)[] host)
1982 p.curl.set(CurlOption.proxy, host);
1986 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port)
1988 @property void proxyPort(ushort port)
1990 p.curl.set(CurlOption.proxyport, cast(long) port);
1994 alias CurlProxy = etc.c.curl.CurlProxy;
1997 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type)
1999 @property void proxyType(CurlProxy type)
2001 p.curl.set(CurlOption.proxytype, cast(long) type);
2004 /// DNS lookup timeout.
2005 @property void dnsTimeout(Duration d)
2007 p.curl.set(CurlOption.dns_cache_timeout, d.total!"msecs");
2011 * The network interface to use in form of the the IP of the interface.
2015 * theprotocol.netInterface = "192.168.1.32";
2016 * theprotocol.netInterface = [ 192, 168, 1, 32 ];
2019 * See: $(REF InternetAddress, std,socket)
2021 @property void netInterface(const(char)[] i)
2023 p.curl.set(CurlOption.intrface, i);
2027 @property void netInterface(const(ubyte)[4] i)
2029 import std.format : format;
2030 const str = format("%d.%d.%d.%d", i[0], i[1], i[2], i[3]);
2035 @property void netInterface(InternetAddress i)
2037 netInterface = i.toAddrString();
2041 Set the local outgoing port to use.
2043 port = the first outgoing port number to try and use
2045 @property void localPort(ushort port)
2047 p.curl.set(CurlOption.localport, cast(long) port);
2051 Set the no proxy flag for the specified host names.
2053 test = a list of comma host names that do not require
2054 proxy to get reached
2056 void setNoProxy(string hosts)
2058 p.curl.set(CurlOption.noproxy, hosts);
2062 Set the local outgoing port range to use.
2063 This can be used together with the localPort property.
2065 range = if the first port is occupied then try this many
2066 port number forwards
2068 @property void localPortRange(ushort range)
2070 p.curl.set(CurlOption.localportrange, cast(long) range);
2073 /** Set the tcp no-delay socket option on or off.
2074 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay)
2076 @property void tcpNoDelay(bool on)
2078 p.curl.set(CurlOption.tcp_nodelay, cast(long) (on ? 1 : 0) );
2081 /** Sets whether SSL peer certificates should be verified.
2082 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYPEER, verifypeer)
2084 @property void verifyPeer(bool on)
2086 p.curl.set(CurlOption.ssl_verifypeer, on ? 1 : 0);
2089 /** Sets whether the host within an SSL certificate should be verified.
2090 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYHOST, verifypeer)
2092 @property void verifyHost(bool on)
2094 p.curl.set(CurlOption.ssl_verifyhost, on ? 2 : 0);
2097 // Authentication settings
2100 Set the user name, password and optionally domain for authentication
2103 Some protocols may need authentication in some cases. Use this
2104 function to provide credentials.
2107 username = the username
2108 password = the password
2109 domain = used for NTLM authentication only and is set to the NTLM domain
2112 void setAuthentication(const(char)[] username, const(char)[] password,
2113 const(char)[] domain = "")
2115 import std.format : format;
2117 username = format("%s/%s", domain, username);
2118 p.curl.set(CurlOption.userpwd, format("%s:%s", username, password));
2123 import std.algorithm.searching : canFind;
2125 testServer.handle((s) {
2126 auto req = s.recvReq;
2127 assert(req.hdrs.canFind("GET /"));
2128 assert(req.hdrs.canFind("Basic dXNlcjpwYXNz"));
2132 auto http = HTTP(testServer.addr);
2133 http.onReceive = (ubyte[] data) { return data.length; };
2134 http.setAuthentication("user", "pass");
2138 http.setNoProxy("www.example.com");
2142 Set the user name and password for proxy authentication.
2145 username = the username
2146 password = the password
2148 void setProxyAuthentication(const(char)[] username, const(char)[] password)
2150 import std.array : replace;
2151 import std.format : format;
2153 p.curl.set(CurlOption.proxyuserpwd,
2155 username.replace(":", "%3A"),
2156 password.replace(":", "%3A"))
2161 * The event handler that gets called when data is needed for sending. The
2162 * length of the $(D void[]) specifies the maximum number of bytes that can
2166 * The callback returns the number of elements in the buffer that have been
2167 * filled and are ready to send.
2168 * The special value $(D .abortRequest) can be returned in order to abort the
2170 * The special value $(D .pauseRequest) can be returned in order to pause the
2175 * import std.net.curl;
2176 * string msg = "Hello world";
2177 * auto client = HTTP("dlang.org");
2178 * client.onSend = delegate size_t(void[] data)
2180 * auto m = cast(void[]) msg;
2181 * size_t length = m.length > data.length ? data.length : m.length;
2182 * if (length == 0) return 0;
2183 * data[0 .. length] = m[0 .. length];
2184 * msg = msg[length..$];
2190 @property void onSend(size_t delegate(void[]) callback)
2192 p.curl.clear(CurlOption.postfields); // cannot specify data when using callback
2193 p.curl.onSend = callback;
2197 * The event handler that receives incoming data. Be sure to copy the
2198 * incoming ubyte[] since it is not guaranteed to be valid after the
2202 * The callback returns the number of incoming bytes read. If the entire array is
2203 * not read the request will abort.
2204 * The special value .pauseRequest can be returned in order to pause the
2209 * import std.net.curl, std.stdio;
2210 * auto client = HTTP("dlang.org");
2211 * client.onReceive = (ubyte[] data)
2213 * writeln("Got data", to!(const(char)[])(data));
2214 * return data.length;
2219 @property void onReceive(size_t delegate(ubyte[]) callback)
2221 p.curl.onReceive = callback;
2225 * The event handler that gets called to inform of upload/download progress.
2228 * dlTotal = total bytes to download
2229 * dlNow = currently downloaded bytes
2230 * ulTotal = total bytes to upload
2231 * ulNow = currently uploaded bytes
2234 * Return 0 from the callback to signal success, return non-zero to abort
2239 * import std.net.curl, std.stdio;
2240 * auto client = HTTP("dlang.org");
2241 * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t ult)
2243 * writeln("Progress: downloaded ", dln, " of ", dl);
2244 * writeln("Progress: uploaded ", uln, " of ", ul);
2249 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow,
2250 size_t ulTotal, size_t ulNow) callback)
2252 p.curl.onProgress = callback;
2257 Decode $(D ubyte[]) array using the provided EncodingScheme up to maxChars
2258 Returns: Tuple of ubytes read and the $(D Char[]) characters decoded.
2259 Not all ubytes are guaranteed to be read in case of decoding error.
2261 private Tuple!(size_t,Char[])
2262 decodeString(Char = char)(const(ubyte)[] data,
2263 EncodingScheme scheme,
2264 size_t maxChars = size_t.max)
2267 immutable startLen = data.length;
2268 size_t charsDecoded = 0;
2269 while (data.length && charsDecoded < maxChars)
2271 immutable dchar dc = scheme.safeDecode(data);
2272 if (dc == INVALID_SEQUENCE)
2274 return typeof(return)(size_t.max, cast(Char[]) null);
2279 return typeof(return)(startLen-data.length, res);
2283 Decode $(D ubyte[]) array using the provided $(D EncodingScheme) until a the
2284 line terminator specified is found. The basesrc parameter is effectively
2285 prepended to src as the first thing.
2287 This function is used for decoding as much of the src buffer as
2288 possible until either the terminator is found or decoding fails. If
2289 it fails as the last data in the src it may mean that the src buffer
2290 were missing some bytes in order to represent a correct code
2291 point. Upon the next call to this function more bytes have been
2292 received from net and the failing bytes should be given as the
2293 basesrc parameter. It is done this way to minimize data copying.
2295 Returns: true if a terminator was found
2296 Not all ubytes are guaranteed to be read in case of decoding error.
2297 any decoded chars will be inserted into dst.
2299 private bool decodeLineInto(Terminator, Char = char)(ref const(ubyte)[] basesrc,
2300 ref const(ubyte)[] src,
2302 EncodingScheme scheme,
2303 Terminator terminator)
2305 import std.algorithm.searching : endsWith;
2307 // if there is anything in the basesrc then try to decode that
2309 if (basesrc.length != 0)
2311 // Try to ensure 4 entries in the basesrc by copying from src.
2312 immutable blen = basesrc.length;
2313 immutable len = (basesrc.length + src.length) >= 4 ?
2314 4 : basesrc.length + src.length;
2315 basesrc.length = len;
2317 immutable dchar dc = scheme.safeDecode(basesrc);
2318 if (dc == INVALID_SEQUENCE)
2320 enforce!CurlException(len != 4, "Invalid code sequence");
2324 src = src[len-basesrc.length-blen .. $]; // remove used ubytes from src
2331 dchar dc = scheme.safeDecode(src);
2332 if (dc == INVALID_SEQUENCE)
2336 // The invalid sequence was in the end of the src. Maybe there
2337 // just need to be more bytes available so these last bytes are
2338 // put back to src for later use.
2346 if (dst.endsWith(terminator))
2349 return false; // no terminator found
2353 * HTTP client functionality.
2357 * import std.net.curl, std.stdio;
2359 * // Get with custom data receivers
2360 * auto http = HTTP("dlang.org");
2361 * http.onReceiveHeader =
2362 * (in char[] key, in char[] value) { writeln(key ~ ": " ~ value); };
2363 * http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; };
2366 * // Put with data senders
2367 * auto msg = "Hello world";
2368 * http.contentLength = msg.length;
2369 * http.onSend = (void[] data)
2371 * auto m = cast(void[]) msg;
2372 * size_t len = m.length > data.length ? data.length : m.length;
2373 * if (len == 0) return len;
2374 * data[0 .. len] = m[0 .. len];
2375 * msg = msg[len..$];
2381 * http.method = HTTP.Method.get;
2382 * http.url = "http://upload.wikimedia.org/wikipedia/commons/"
2383 * "5/53/Wikipedia-logo-en-big.png";
2384 * http.onReceive = (ubyte[] data) { return data.length; };
2385 * http.onProgress = (size_t dltotal, size_t dlnow,
2386 * size_t ultotal, size_t ulnow)
2388 * writeln("Progress ", dltotal, ", ", dlnow, ", ", ultotal, ", ", ulnow);
2394 * See_Also: $(_HTTP www.ietf.org/rfc/rfc2616.txt, RFC2616)
2401 import std.datetime.systime : SysTime;
2403 /// Authentication method equal to $(REF CurlAuth, etc,c,curl)
2404 alias AuthMethod = CurlAuth;
2406 static private uint defaultMaxRedirects = 10;
2412 if (headersOut !is null)
2413 Curl.curl.slist_free_all(headersOut);
2414 if (curl.handle !is null) // work around RefCounted/emplace bug
2418 curl_slist* headersOut;
2419 string[string] headersIn;
2422 /// The status line of the final sub-request in a request.
2424 private void delegate(StatusLine) onReceiveStatusLine;
2426 /// The HTTP method to use.
2427 Method method = Method.undefined;
2429 @system @property void onReceiveHeader(void delegate(in char[] key,
2430 in char[] value) callback)
2432 import std.algorithm.searching : startsWith;
2433 import std.conv : to;
2434 import std.regex : regex, match;
2435 import std.uni : toLower;
2437 // Wrap incoming callback in order to separate http status line from
2438 // http headers. On redirected requests there may be several such
2439 // status lines. The last one is the one recorded.
2440 auto dg = (in char[] header)
2442 import std.utf : UTFException;
2450 if (header.startsWith("HTTP/"))
2454 const m = match(header, regex(r"^HTTP/(\d+)\.(\d+) (\d+) (.*)$"));
2457 // Invalid status line
2461 status.majorVersion = to!ushort(m.captures[1]);
2462 status.minorVersion = to!ushort(m.captures[2]);
2463 status.code = to!ushort(m.captures[3]);
2464 status.reason = m.captures[4].idup;
2465 if (onReceiveStatusLine != null)
2466 onReceiveStatusLine(status);
2471 // Normal http header
2472 auto m = match(cast(char[]) header, regex("(.*?): (.*)$"));
2474 auto fieldName = m.captures[1].toLower().idup;
2475 if (fieldName == "content-type")
2477 auto mct = match(cast(char[]) m.captures[2],
2478 regex("charset=([^;]*)", "i"));
2479 if (!mct.empty && mct.captures.length > 1)
2480 charset = mct.captures[1].idup;
2483 if (!m.empty && callback !is null)
2484 callback(fieldName, m.captures[2]);
2485 headersIn[fieldName] = m.captures[2].idup;
2487 catch (UTFException e)
2489 //munch it - a header should be all ASCII, any "wrong UTF" is broken header
2493 curl.onReceiveHeader = dg;
2497 private RefCounted!Impl p;
2499 /** Time condition enumeration as an alias of $(REF CurlTimeCond, etc,c,curl)
2501 $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25)
2503 alias TimeCond = CurlTimeCond;
2506 Constructor taking the url as parameter.
2508 static HTTP opCall(const(char)[] url)
2517 static HTTP opCall()
2529 copy.p.method = p.method;
2530 curl_slist* cur = p.headersOut;
2531 curl_slist* newlist = null;
2534 newlist = Curl.curl.slist_append(newlist, cur.data);
2537 copy.p.headersOut = newlist;
2538 copy.p.curl.set(CurlOption.httpheader, copy.p.headersOut);
2539 copy.p.curl = p.curl.dup();
2540 copy.dataTimeout = _defaultDataTimeout;
2541 copy.onReceiveHeader = null;
2545 private void initialize()
2547 p.curl.initialize();
2548 maxRedirects = HTTP.defaultMaxRedirects;
2549 p.charset = "ISO-8859-1"; // Default charset defined in HTTP RFC
2550 p.method = Method.undefined;
2551 setUserAgent(HTTP.defaultUserAgent);
2552 dataTimeout = _defaultDataTimeout;
2553 onReceiveHeader = null;
2559 Perform a http request.
2561 After the HTTP client has been setup and possibly assigned callbacks the
2562 $(D perform()) method will start performing the request towards the
2566 throwOnError = whether to throw an exception or return a CurlCode on error
2568 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError)
2573 final switch (p.method)
2576 p.curl.set(CurlOption.nobody, 1L);
2577 opt = CurlOption.nobody;
2579 case Method.undefined:
2581 p.curl.set(CurlOption.httpget, 1L);
2582 opt = CurlOption.httpget;
2585 p.curl.set(CurlOption.post, 1L);
2586 opt = CurlOption.post;
2589 p.curl.set(CurlOption.upload, 1L);
2590 opt = CurlOption.upload;
2593 p.curl.set(CurlOption.customrequest, "DELETE");
2594 opt = CurlOption.customrequest;
2596 case Method.options:
2597 p.curl.set(CurlOption.customrequest, "OPTIONS");
2598 opt = CurlOption.customrequest;
2601 p.curl.set(CurlOption.customrequest, "TRACE");
2602 opt = CurlOption.customrequest;
2604 case Method.connect:
2605 p.curl.set(CurlOption.customrequest, "CONNECT");
2606 opt = CurlOption.customrequest;
2609 p.curl.set(CurlOption.customrequest, "PATCH");
2610 opt = CurlOption.customrequest;
2614 scope (exit) p.curl.clear(opt);
2615 return p.curl.perform(throwOnError);
2618 /// The URL to specify the location of the resource.
2619 @property void url(const(char)[] url)
2621 import std.algorithm.searching : startsWith;
2622 import std.uni : toLower;
2623 if (!startsWith(url.toLower(), "http://", "https://"))
2624 url = "http://" ~ url;
2625 p.curl.set(CurlOption.url, url);
2628 /// Set the CA certificate bundle file to use for SSL peer verification
2629 @property void caInfo(const(char)[] caFile)
2631 p.curl.set(CurlOption.cainfo, caFile);
2634 // This is a workaround for mixed in content not having its
2638 /// Value to return from $(D onSend)/$(D onReceive) delegates in order to
2640 alias requestPause = CurlReadFunc.pause;
2642 /// Value to return from onSend delegate in order to abort a request
2643 alias requestAbort = CurlReadFunc.abort;
2646 True if the instance is stopped. A stopped instance is not usable.
2648 @property bool isStopped();
2650 /// Stop and invalidate this instance.
2654 This will print request information to stderr.
2656 @property void verbose(bool on);
2658 // Connection settings
2660 /// Set timeout for activity on connection.
2661 @property void dataTimeout(Duration d);
2663 /** Set maximum time an operation is allowed to take.
2664 This includes dns resolution, connecting, data transfer, etc.
2666 @property void operationTimeout(Duration d);
2668 /// Set timeout for connecting.
2669 @property void connectTimeout(Duration d);
2674 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy)
2676 @property void proxy(const(char)[] host);
2679 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port)
2681 @property void proxyPort(ushort port);
2684 alias CurlProxy = etc.c.curl.CurlProxy;
2687 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type)
2689 @property void proxyType(CurlProxy type);
2691 /// DNS lookup timeout.
2692 @property void dnsTimeout(Duration d);
2695 * The network interface to use in form of the the IP of the interface.
2699 * theprotocol.netInterface = "192.168.1.32";
2700 * theprotocol.netInterface = [ 192, 168, 1, 32 ];
2703 * See: $(REF InternetAddress, std,socket)
2705 @property void netInterface(const(char)[] i);
2708 @property void netInterface(const(ubyte)[4] i);
2711 @property void netInterface(InternetAddress i);
2714 Set the local outgoing port to use.
2716 port = the first outgoing port number to try and use
2718 @property void localPort(ushort port);
2721 Set the local outgoing port range to use.
2722 This can be used together with the localPort property.
2724 range = if the first port is occupied then try this many
2725 port number forwards
2727 @property void localPortRange(ushort range);
2729 /** Set the tcp no-delay socket option on or off.
2730 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay)
2732 @property void tcpNoDelay(bool on);
2734 // Authentication settings
2737 Set the user name, password and optionally domain for authentication
2740 Some protocols may need authentication in some cases. Use this
2741 function to provide credentials.
2744 username = the username
2745 password = the password
2746 domain = used for NTLM authentication only and is set to the NTLM domain
2749 void setAuthentication(const(char)[] username, const(char)[] password,
2750 const(char)[] domain = "");
2753 Set the user name and password for proxy authentication.
2756 username = the username
2757 password = the password
2759 void setProxyAuthentication(const(char)[] username, const(char)[] password);
2762 * The event handler that gets called when data is needed for sending. The
2763 * length of the $(D void[]) specifies the maximum number of bytes that can
2767 * The callback returns the number of elements in the buffer that have been
2768 * filled and are ready to send.
2769 * The special value $(D .abortRequest) can be returned in order to abort the
2771 * The special value $(D .pauseRequest) can be returned in order to pause the
2776 * import std.net.curl;
2777 * string msg = "Hello world";
2778 * auto client = HTTP("dlang.org");
2779 * client.onSend = delegate size_t(void[] data)
2781 * auto m = cast(void[]) msg;
2782 * size_t length = m.length > data.length ? data.length : m.length;
2783 * if (length == 0) return 0;
2784 * data[0 .. length] = m[0 .. length];
2785 * msg = msg[length..$];
2791 @property void onSend(size_t delegate(void[]) callback);
2794 * The event handler that receives incoming data. Be sure to copy the
2795 * incoming ubyte[] since it is not guaranteed to be valid after the
2799 * The callback returns the incoming bytes read. If not the entire array is
2800 * the request will abort.
2801 * The special value .pauseRequest can be returned in order to pause the
2806 * import std.net.curl, std.stdio;
2807 * auto client = HTTP("dlang.org");
2808 * client.onReceive = (ubyte[] data)
2810 * writeln("Got data", to!(const(char)[])(data));
2811 * return data.length;
2816 @property void onReceive(size_t delegate(ubyte[]) callback);
2819 * Register an event handler that gets called to inform of
2820 * upload/download progress.
2822 * Callback_parameters:
2823 * $(CALLBACK_PARAMS)
2825 * Callback_returns: Return 0 to signal success, return non-zero to
2830 * import std.net.curl, std.stdio;
2831 * auto client = HTTP("dlang.org");
2832 * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t ult)
2834 * writeln("Progress: downloaded ", dln, " of ", dl);
2835 * writeln("Progress: uploaded ", uln, " of ", ul);
2840 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow,
2841 size_t ulTotal, size_t ulNow) callback);
2844 /** Clear all outgoing headers.
2846 void clearRequestHeaders()
2848 if (p.headersOut !is null)
2849 Curl.curl.slist_free_all(p.headersOut);
2850 p.headersOut = null;
2851 p.curl.clear(CurlOption.httpheader);
2854 /** Add a header e.g. "X-CustomField: Something is fishy".
2856 * There is no remove header functionality. Do a $(LREF clearRequestHeaders)
2857 * and set the needed headers instead.
2861 * import std.net.curl;
2862 * auto client = HTTP();
2863 * client.addRequestHeader("X-Custom-ABC", "This is the custom value");
2864 * auto content = get("dlang.org", client);
2867 void addRequestHeader(const(char)[] name, const(char)[] value)
2869 import std.format : format;
2870 import std.uni : icmp;
2872 if (icmp(name, "User-Agent") == 0)
2873 return setUserAgent(value);
2874 string nv = format("%s: %s", name, value);
2875 p.headersOut = Curl.curl.slist_append(p.headersOut,
2876 nv.tempCString().buffPtr);
2877 p.curl.set(CurlOption.httpheader, p.headersOut);
2881 * The default "User-Agent" value send with a request.
2882 * It has the form "Phobos-std.net.curl/$(I PHOBOS_VERSION) (libcurl/$(I CURL_VERSION))"
2884 static string defaultUserAgent() @property
2886 import std.compiler : version_major, version_minor;
2887 import std.format : format, sformat;
2889 // http://curl.haxx.se/docs/versions.html
2890 enum fmt = "Phobos-std.net.curl/%d.%03d (libcurl/%d.%d.%d)";
2891 enum maxLen = fmt.length - "%d%03d%d%d%d".length + 10 + 10 + 3 + 3 + 3;
2893 static char[maxLen] buf = void;
2894 static string userAgent;
2896 if (!userAgent.length)
2898 auto curlVer = Curl.curl.version_info(CURLVERSION_NOW).version_num;
2899 userAgent = cast(immutable) sformat(
2900 buf, fmt, version_major, version_minor,
2901 curlVer >> 16 & 0xFF, curlVer >> 8 & 0xFF, curlVer & 0xFF);
2906 /** Set the value of the user agent request header field.
2908 * By default a request has it's "User-Agent" field set to $(LREF
2909 * defaultUserAgent) even if $(D setUserAgent) was never called. Pass
2910 * an empty string to suppress the "User-Agent" field altogether.
2912 void setUserAgent(const(char)[] userAgent)
2914 p.curl.set(CurlOption.useragent, userAgent);
2918 * Get various timings defined in $(REF CurlInfo, etc, c, curl).
2919 * The value is usable only if the return value is equal to $(D etc.c.curl.CurlError.ok).
2922 * timing = one of the timings defined in $(REF CurlInfo, etc, c, curl).
2924 * $(D etc.c.curl.CurlInfo.namelookup_time),
2925 * $(D etc.c.curl.CurlInfo.connect_time),
2926 * $(D etc.c.curl.CurlInfo.pretransfer_time),
2927 * $(D etc.c.curl.CurlInfo.starttransfer_time),
2928 * $(D etc.c.curl.CurlInfo.redirect_time),
2929 * $(D etc.c.curl.CurlInfo.appconnect_time),
2930 * $(D etc.c.curl.CurlInfo.total_time).
2931 * val = the actual value of the inquired timing.
2934 * The return code of the operation. The value stored in val
2935 * should be used only if the return value is $(D etc.c.curl.CurlInfo.ok).
2939 * import std.net.curl;
2940 * import etc.c.curl : CurlError, CurlInfo;
2942 * auto client = HTTP("dlang.org");
2948 * code = http.getTiming(CurlInfo.namelookup_time, val);
2949 * assert(code == CurlError.ok);
2952 CurlCode getTiming(CurlInfo timing, ref double val)
2954 return p.curl.getTiming(timing, val);
2957 /** The headers read from a successful response.
2960 @property string[string] responseHeaders()
2965 /// HTTP method used.
2966 @property void method(Method m)
2972 @property Method method()
2978 HTTP status line of last response. One call to perform may
2979 result in several requests because of redirection.
2981 @property StatusLine statusLine()
2986 /// Set the active cookie string e.g. "name1=value1;name2=value2"
2987 void setCookie(const(char)[] cookie)
2989 p.curl.set(CurlOption.cookie, cookie);
2992 /// Set a file path to where a cookie jar should be read/stored.
2993 void setCookieJar(const(char)[] path)
2995 p.curl.set(CurlOption.cookiefile, path);
2997 p.curl.set(CurlOption.cookiejar, path);
3000 /// Flush cookie jar to disk.
3001 void flushCookieJar()
3003 p.curl.set(CurlOption.cookielist, "FLUSH");
3006 /// Clear session cookies.
3007 void clearSessionCookies()
3009 p.curl.set(CurlOption.cookielist, "SESS");
3012 /// Clear all cookies.
3013 void clearAllCookies()
3015 p.curl.set(CurlOption.cookielist, "ALL");
3019 Set time condition on the request.
3022 cond = $(D CurlTimeCond.{none,ifmodsince,ifunmodsince,lastmod})
3023 timestamp = Timestamp for the condition
3025 $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25)
3027 void setTimeCondition(HTTP.TimeCond cond, SysTime timestamp)
3029 p.curl.set(CurlOption.timecondition, cond);
3030 p.curl.set(CurlOption.timevalue, timestamp.toUnixTime());
3033 /** Specifying data to post when not using the onSend callback.
3035 * The data is NOT copied by the library. Content-Type will default to
3036 * application/octet-stream. Data is not converted or encoded by this
3041 * import std.net.curl, std.stdio;
3042 * auto http = HTTP("http://www.mydomain.com");
3043 * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; };
3044 * http.postData = [1,2,3,4,5];
3048 @property void postData(const(void)[] data)
3050 setPostData(data, "application/octet-stream");
3053 /** Specifying data to post when not using the onSend callback.
3055 * The data is NOT copied by the library. Content-Type will default to
3056 * text/plain. Data is not converted or encoded by this method.
3060 * import std.net.curl, std.stdio;
3061 * auto http = HTTP("http://www.mydomain.com");
3062 * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; };
3063 * http.postData = "The quick....";
3067 @property void postData(const(char)[] data)
3069 setPostData(data, "text/plain");
3073 * Specify data to post when not using the onSend callback, with
3074 * user-specified Content-Type.
3076 * data = Data to post.
3077 * contentType = MIME type of the data, for example, "text/plain" or
3078 * "application/octet-stream". See also:
3079 * $(LINK2 http://en.wikipedia.org/wiki/Internet_media_type,
3080 * Internet media type) on Wikipedia.
3082 * import std.net.curl;
3083 * auto http = HTTP("http://onlineform.example.com");
3084 * auto data = "app=login&username=bob&password=s00perS3kret";
3085 * http.setPostData(data, "application/x-www-form-urlencoded");
3086 * http.onReceive = (ubyte[] data) { return data.length; };
3090 void setPostData(const(void)[] data, string contentType)
3092 // cannot use callback when specifying data directly so it is disabled here.
3093 p.curl.clear(CurlOption.readfunction);
3094 addRequestHeader("Content-Type", contentType);
3095 p.curl.set(CurlOption.postfields, cast(void*) data.ptr);
3096 p.curl.set(CurlOption.postfieldsize, data.length);
3097 if (method == Method.undefined)
3098 method = Method.post;
3103 import std.algorithm.searching : canFind;
3105 testServer.handle((s) {
3106 auto req = s.recvReq!ubyte;
3107 assert(req.hdrs.canFind("POST /path"));
3108 assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4]));
3109 assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255]));
3110 s.send(httpOK(cast(ubyte[])[17, 27, 35, 41]));
3112 auto data = new ubyte[](256);
3113 foreach (i, ref ub; data)
3116 auto http = HTTP(testServer.addr~"/path");
3117 http.postData = data;
3119 http.onReceive = (data) { res ~= data; return data.length; };
3121 assert(res == cast(ubyte[])[17, 27, 35, 41]);
3125 * Set the event handler that receives incoming headers.
3127 * The callback will receive a header field key, value as parameter. The
3128 * $(D const(char)[]) arrays are not valid after the delegate has returned.
3132 * import std.net.curl, std.stdio;
3133 * auto http = HTTP("dlang.org");
3134 * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; };
3135 * http.onReceiveHeader = (in char[] key, in char[] value) { writeln(key, " = ", value); };
3139 @property void onReceiveHeader(void delegate(in char[] key,
3140 in char[] value) callback)
3142 p.onReceiveHeader = callback;
3146 Callback for each received StatusLine.
3148 Notice that several callbacks can be done for each call to
3149 $(D perform()) due to redirections.
3151 See_Also: $(LREF StatusLine)
3153 @property void onReceiveStatusLine(void delegate(StatusLine) callback)
3155 p.onReceiveStatusLine = callback;
3159 The content length in bytes when using request that has content
3160 e.g. POST/PUT and not using chunked transfer. Is set as the
3161 "Content-Length" header. Set to ulong.max to reset to chunked transfer.
3163 @property void contentLength(ulong len)
3165 import std.conv : to;
3169 // Force post if necessary
3170 if (p.method != Method.put && p.method != Method.post &&
3171 p.method != Method.patch)
3172 p.method = Method.post;
3174 if (p.method == Method.post || p.method == Method.patch)
3175 lenOpt = CurlOption.postfieldsize_large;
3177 lenOpt = CurlOption.infilesize_large;
3179 if (size_t.max != ulong.max && len == size_t.max)
3180 len = ulong.max; // check size_t.max for backwards compat, turn into error
3182 if (len == ulong.max)
3184 // HTTP 1.1 supports requests with no length header set.
3185 addRequestHeader("Transfer-Encoding", "chunked");
3186 addRequestHeader("Expect", "100-continue");
3190 p.curl.set(lenOpt, to!curl_off_t(len));
3195 Authentication method as specified in $(LREF AuthMethod).
3197 @property void authenticationMethod(AuthMethod authMethod)
3199 p.curl.set(CurlOption.httpauth, cast(long) authMethod);
3203 Set max allowed redirections using the location header.
3204 uint.max for infinite.
3206 @property void maxRedirects(uint maxRedirs)
3208 if (maxRedirs == uint.max)
3211 p.curl.set(CurlOption.followlocation, 0);
3215 p.curl.set(CurlOption.followlocation, 1);
3216 p.curl.set(CurlOption.maxredirs, maxRedirs);
3220 /** <a name="HTTP.Method"/>The standard HTTP methods :
3221 * $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.1, _RFC2616 Section 5.1.1)
3238 HTTP status line ie. the first line returned in an HTTP response.
3240 If authentication or redirections are done then the status will be for
3241 the last response received.
3245 ushort majorVersion; /// Major HTTP version ie. 1 in HTTP/1.0.
3246 ushort minorVersion; /// Minor HTTP version ie. 0 in HTTP/1.0.
3247 ushort code; /// HTTP status line code e.g. 200.
3248 string reason; /// HTTP status line reason string.
3250 /// Reset this status line
3260 string toString() const
3262 import std.format : format;
3263 return format("%s %s (%s.%s)",
3264 code, reason, majorVersion, minorVersion);
3270 @system unittest // charset/Charset/CHARSET/...
3272 import std.meta : AliasSeq;
3274 foreach (c; AliasSeq!("charset", "Charset", "CHARSET", "CharSet", "charSet",
3275 "ChArSeT", "cHaRsEt"))
3277 testServer.handle((s) {
3278 s.send("HTTP/1.1 200 OK\r\n"~
3279 "Content-Length: 0\r\n"~
3280 "Content-Type: text/plain; " ~ c ~ "=foo\r\n" ~
3284 auto http = HTTP(testServer.addr);
3286 assert(http.p.charset == "foo");
3292 code = http.getTiming(CurlInfo.total_time, val);
3293 assert(code == CurlError.ok);
3294 code = http.getTiming(CurlInfo.namelookup_time, val);
3295 assert(code == CurlError.ok);
3296 code = http.getTiming(CurlInfo.connect_time, val);
3297 assert(code == CurlError.ok);
3298 code = http.getTiming(CurlInfo.pretransfer_time, val);
3299 assert(code == CurlError.ok);
3300 code = http.getTiming(CurlInfo.starttransfer_time, val);
3301 assert(code == CurlError.ok);
3302 code = http.getTiming(CurlInfo.redirect_time, val);
3303 assert(code == CurlError.ok);
3304 code = http.getTiming(CurlInfo.appconnect_time, val);
3305 assert(code == CurlError.ok);
3310 FTP client functionality.
3312 See_Also: $(HTTP tools.ietf.org/html/rfc959, RFC959)
3323 if (commands !is null)
3324 Curl.curl.slist_free_all(commands);
3325 if (curl.handle !is null) // work around RefCounted/emplace bug
3328 curl_slist* commands;
3333 private RefCounted!Impl p;
3336 FTP access to the specified url.
3338 static FTP opCall(const(char)[] url)
3359 copy.p.encoding = p.encoding;
3360 copy.p.curl = p.curl.dup();
3361 curl_slist* cur = p.commands;
3362 curl_slist* newlist = null;
3365 newlist = Curl.curl.slist_append(newlist, cur.data);
3368 copy.p.commands = newlist;
3369 copy.p.curl.set(CurlOption.postquote, copy.p.commands);
3370 copy.dataTimeout = _defaultDataTimeout;
3374 private void initialize()
3376 p.curl.initialize();
3377 p.encoding = "ISO-8859-1";
3378 dataTimeout = _defaultDataTimeout;
3382 Performs the ftp request as it has been configured.
3384 After a FTP client has been setup and possibly assigned callbacks the $(D
3385 perform()) method will start performing the actual communication with the
3389 throwOnError = whether to throw an exception or return a CurlCode on error
3391 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError)
3393 return p.curl.perform(throwOnError);
3396 /// The URL to specify the location of the resource.
3397 @property void url(const(char)[] url)
3399 import std.algorithm.searching : startsWith;
3400 import std.uni : toLower;
3402 if (!startsWith(url.toLower(), "ftp://", "ftps://"))
3403 url = "ftp://" ~ url;
3404 p.curl.set(CurlOption.url, url);
3407 // This is a workaround for mixed in content not having its
3411 /// Value to return from $(D onSend)/$(D onReceive) delegates in order to
3413 alias requestPause = CurlReadFunc.pause;
3415 /// Value to return from onSend delegate in order to abort a request
3416 alias requestAbort = CurlReadFunc.abort;
3419 True if the instance is stopped. A stopped instance is not usable.
3421 @property bool isStopped();
3423 /// Stop and invalidate this instance.
3427 This will print request information to stderr.
3429 @property void verbose(bool on);
3431 // Connection settings
3433 /// Set timeout for activity on connection.
3434 @property void dataTimeout(Duration d);
3436 /** Set maximum time an operation is allowed to take.
3437 This includes dns resolution, connecting, data transfer, etc.
3439 @property void operationTimeout(Duration d);
3441 /// Set timeout for connecting.
3442 @property void connectTimeout(Duration d);
3447 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy)
3449 @property void proxy(const(char)[] host);
3452 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port)
3454 @property void proxyPort(ushort port);
3457 alias CurlProxy = etc.c.curl.CurlProxy;
3460 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type)
3462 @property void proxyType(CurlProxy type);
3464 /// DNS lookup timeout.
3465 @property void dnsTimeout(Duration d);
3468 * The network interface to use in form of the the IP of the interface.
3472 * theprotocol.netInterface = "192.168.1.32";
3473 * theprotocol.netInterface = [ 192, 168, 1, 32 ];
3476 * See: $(REF InternetAddress, std,socket)
3478 @property void netInterface(const(char)[] i);
3481 @property void netInterface(const(ubyte)[4] i);
3484 @property void netInterface(InternetAddress i);
3487 Set the local outgoing port to use.
3489 port = the first outgoing port number to try and use
3491 @property void localPort(ushort port);
3494 Set the local outgoing port range to use.
3495 This can be used together with the localPort property.
3497 range = if the first port is occupied then try this many
3498 port number forwards
3500 @property void localPortRange(ushort range);
3502 /** Set the tcp no-delay socket option on or off.
3503 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay)
3505 @property void tcpNoDelay(bool on);
3507 // Authentication settings
3510 Set the user name, password and optionally domain for authentication
3513 Some protocols may need authentication in some cases. Use this
3514 function to provide credentials.
3517 username = the username
3518 password = the password
3519 domain = used for NTLM authentication only and is set to the NTLM domain
3522 void setAuthentication(const(char)[] username, const(char)[] password,
3523 const(char)[] domain = "");
3526 Set the user name and password for proxy authentication.
3529 username = the username
3530 password = the password
3532 void setProxyAuthentication(const(char)[] username, const(char)[] password);
3535 * The event handler that gets called when data is needed for sending. The
3536 * length of the $(D void[]) specifies the maximum number of bytes that can
3540 * The callback returns the number of elements in the buffer that have been
3541 * filled and are ready to send.
3542 * The special value $(D .abortRequest) can be returned in order to abort the
3544 * The special value $(D .pauseRequest) can be returned in order to pause the
3548 @property void onSend(size_t delegate(void[]) callback);
3551 * The event handler that receives incoming data. Be sure to copy the
3552 * incoming ubyte[] since it is not guaranteed to be valid after the
3556 * The callback returns the incoming bytes read. If not the entire array is
3557 * the request will abort.
3558 * The special value .pauseRequest can be returned in order to pause the
3562 @property void onReceive(size_t delegate(ubyte[]) callback);
3565 * The event handler that gets called to inform of upload/download progress.
3567 * Callback_parameters:
3568 * $(CALLBACK_PARAMS)
3571 * Return 0 from the callback to signal success, return non-zero to
3574 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow,
3575 size_t ulTotal, size_t ulNow) callback);
3578 /** Clear all commands send to ftp server.
3580 void clearCommands()
3582 if (p.commands !is null)
3583 Curl.curl.slist_free_all(p.commands);
3585 p.curl.clear(CurlOption.postquote);
3588 /** Add a command to send to ftp server.
3590 * There is no remove command functionality. Do a $(LREF clearCommands) and
3591 * set the needed commands instead.
3595 * import std.net.curl;
3596 * auto client = FTP();
3597 * client.addCommand("RNFR my_file.txt");
3598 * client.addCommand("RNTO my_renamed_file.txt");
3599 * upload("my_file.txt", "ftp.digitalmars.com", client);
3602 void addCommand(const(char)[] command)
3604 p.commands = Curl.curl.slist_append(p.commands,
3605 command.tempCString().buffPtr);
3606 p.curl.set(CurlOption.postquote, p.commands);
3609 /// Connection encoding. Defaults to ISO-8859-1.
3610 @property void encoding(string name)
3616 @property string encoding()
3622 The content length in bytes of the ftp data.
3624 @property void contentLength(ulong len)
3626 import std.conv : to;
3627 p.curl.set(CurlOption.infilesize_large, to!curl_off_t(len));
3631 * Get various timings defined in $(REF CurlInfo, etc, c, curl).
3632 * The value is usable only if the return value is equal to $(D etc.c.curl.CurlError.ok).
3635 * timing = one of the timings defined in $(REF CurlInfo, etc, c, curl).
3637 * $(D etc.c.curl.CurlInfo.namelookup_time),
3638 * $(D etc.c.curl.CurlInfo.connect_time),
3639 * $(D etc.c.curl.CurlInfo.pretransfer_time),
3640 * $(D etc.c.curl.CurlInfo.starttransfer_time),
3641 * $(D etc.c.curl.CurlInfo.redirect_time),
3642 * $(D etc.c.curl.CurlInfo.appconnect_time),
3643 * $(D etc.c.curl.CurlInfo.total_time).
3644 * val = the actual value of the inquired timing.
3647 * The return code of the operation. The value stored in val
3648 * should be used only if the return value is $(D etc.c.curl.CurlInfo.ok).
3652 * import std.net.curl;
3653 * import etc.c.curl : CurlError, CurlInfo;
3655 * auto client = FTP();
3656 * client.addCommand("RNFR my_file.txt");
3657 * client.addCommand("RNTO my_renamed_file.txt");
3658 * upload("my_file.txt", "ftp.digitalmars.com", client);
3663 * code = http.getTiming(CurlInfo.namelookup_time, val);
3664 * assert(code == CurlError.ok);
3667 CurlCode getTiming(CurlInfo timing, ref double val)
3669 return p.curl.getTiming(timing, val);
3674 auto client = FTP();
3679 code = client.getTiming(CurlInfo.total_time, val);
3680 assert(code == CurlError.ok);
3681 code = client.getTiming(CurlInfo.namelookup_time, val);
3682 assert(code == CurlError.ok);
3683 code = client.getTiming(CurlInfo.connect_time, val);
3684 assert(code == CurlError.ok);
3685 code = client.getTiming(CurlInfo.pretransfer_time, val);
3686 assert(code == CurlError.ok);
3687 code = client.getTiming(CurlInfo.starttransfer_time, val);
3688 assert(code == CurlError.ok);
3689 code = client.getTiming(CurlInfo.redirect_time, val);
3690 assert(code == CurlError.ok);
3691 code = client.getTiming(CurlInfo.appconnect_time, val);
3692 assert(code == CurlError.ok);
3697 * Basic SMTP protocol support.
3701 * import std.net.curl;
3703 * // Send an email with SMTPS
3704 * auto smtp = SMTP("smtps://smtp.gmail.com");
3705 * smtp.setAuthentication("from.addr@gmail.com", "password");
3706 * smtp.mailTo = ["<to.addr@gmail.com>"];
3707 * smtp.mailFrom = "<from.addr@gmail.com>";
3708 * smtp.message = "Example Message";
3712 * See_Also: $(HTTP www.ietf.org/rfc/rfc2821.txt, RFC2821)
3722 if (curl.handle !is null) // work around RefCounted/emplace bug
3727 @property void message(string msg)
3729 import std.algorithm.comparison : min;
3731 auto _message = msg;
3733 This delegate reads the message text and copies it.
3735 curl.onSend = delegate size_t(void[] data)
3737 if (!msg.length) return 0;
3738 size_t to_copy = min(data.length, _message.length);
3739 data[0 .. to_copy] = (cast(void[])_message)[0 .. to_copy];
3740 _message = _message[to_copy..$];
3746 private RefCounted!Impl p;
3749 Sets to the URL of the SMTP server.
3751 static SMTP opCall(const(char)[] url)
3760 static SMTP opCall()
3767 /+ TODO: The other structs have this function.
3772 copy.p.encoding = p.encoding;
3773 copy.p.curl = p.curl.dup();
3774 curl_slist* cur = p.commands;
3775 curl_slist* newlist = null;
3778 newlist = Curl.curl.slist_append(newlist, cur.data);
3781 copy.p.commands = newlist;
3782 copy.p.curl.set(CurlOption.postquote, copy.p.commands);
3783 copy.dataTimeout = _defaultDataTimeout;
3789 Performs the request as configured.
3791 throwOnError = whether to throw an exception or return a CurlCode on error
3793 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError)
3795 return p.curl.perform(throwOnError);
3798 /// The URL to specify the location of the resource.
3799 @property void url(const(char)[] url)
3801 import std.algorithm.searching : startsWith;
3802 import std.uni : toLower;
3804 auto lowered = url.toLower();
3806 if (lowered.startsWith("smtps://"))
3808 p.curl.set(CurlOption.use_ssl, CurlUseSSL.all);
3812 enforce!CurlException(lowered.startsWith("smtp://"),
3813 "The url must be for the smtp protocol.");
3815 p.curl.set(CurlOption.url, url);
3818 private void initialize()
3820 p.curl.initialize();
3821 p.curl.set(CurlOption.upload, 1L);
3822 dataTimeout = _defaultDataTimeout;
3827 // This is a workaround for mixed in content not having its
3831 /// Value to return from $(D onSend)/$(D onReceive) delegates in order to
3833 alias requestPause = CurlReadFunc.pause;
3835 /// Value to return from onSend delegate in order to abort a request
3836 alias requestAbort = CurlReadFunc.abort;
3839 True if the instance is stopped. A stopped instance is not usable.
3841 @property bool isStopped();
3843 /// Stop and invalidate this instance.
3847 This will print request information to stderr.
3849 @property void verbose(bool on);
3851 // Connection settings
3853 /// Set timeout for activity on connection.
3854 @property void dataTimeout(Duration d);
3856 /** Set maximum time an operation is allowed to take.
3857 This includes dns resolution, connecting, data transfer, etc.
3859 @property void operationTimeout(Duration d);
3861 /// Set timeout for connecting.
3862 @property void connectTimeout(Duration d);
3867 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy)
3869 @property void proxy(const(char)[] host);
3872 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port)
3874 @property void proxyPort(ushort port);
3877 alias CurlProxy = etc.c.curl.CurlProxy;
3880 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type)
3882 @property void proxyType(CurlProxy type);
3884 /// DNS lookup timeout.
3885 @property void dnsTimeout(Duration d);
3888 * The network interface to use in form of the the IP of the interface.
3892 * theprotocol.netInterface = "192.168.1.32";
3893 * theprotocol.netInterface = [ 192, 168, 1, 32 ];
3896 * See: $(REF InternetAddress, std,socket)
3898 @property void netInterface(const(char)[] i);
3901 @property void netInterface(const(ubyte)[4] i);
3904 @property void netInterface(InternetAddress i);
3907 Set the local outgoing port to use.
3909 port = the first outgoing port number to try and use
3911 @property void localPort(ushort port);
3914 Set the local outgoing port range to use.
3915 This can be used together with the localPort property.
3917 range = if the first port is occupied then try this many
3918 port number forwards
3920 @property void localPortRange(ushort range);
3922 /** Set the tcp no-delay socket option on or off.
3923 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay)
3925 @property void tcpNoDelay(bool on);
3927 // Authentication settings
3930 Set the user name, password and optionally domain for authentication
3933 Some protocols may need authentication in some cases. Use this
3934 function to provide credentials.
3937 username = the username
3938 password = the password
3939 domain = used for NTLM authentication only and is set to the NTLM domain
3942 void setAuthentication(const(char)[] username, const(char)[] password,
3943 const(char)[] domain = "");
3946 Set the user name and password for proxy authentication.
3949 username = the username
3950 password = the password
3952 void setProxyAuthentication(const(char)[] username, const(char)[] password);
3955 * The event handler that gets called when data is needed for sending. The
3956 * length of the $(D void[]) specifies the maximum number of bytes that can
3960 * The callback returns the number of elements in the buffer that have been
3961 * filled and are ready to send.
3962 * The special value $(D .abortRequest) can be returned in order to abort the
3964 * The special value $(D .pauseRequest) can be returned in order to pause the
3967 @property void onSend(size_t delegate(void[]) callback);
3970 * The event handler that receives incoming data. Be sure to copy the
3971 * incoming ubyte[] since it is not guaranteed to be valid after the
3975 * The callback returns the incoming bytes read. If not the entire array is
3976 * the request will abort.
3977 * The special value .pauseRequest can be returned in order to pause the
3980 @property void onReceive(size_t delegate(ubyte[]) callback);
3983 * The event handler that gets called to inform of upload/download progress.
3985 * Callback_parameters:
3986 * $(CALLBACK_PARAMS)
3989 * Return 0 from the callback to signal success, return non-zero to
3992 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow,
3993 size_t ulTotal, size_t ulNow) callback);
3997 Setter for the sender's email address.
3999 @property void mailFrom()(const(char)[] sender)
4001 assert(!sender.empty, "Sender must not be empty");
4002 p.curl.set(CurlOption.mail_from, sender);
4006 Setter for the recipient email addresses.
4008 void mailTo()(const(char)[][] recipients...)
4010 assert(!recipients.empty, "Recipient must not be empty");
4011 curl_slist* recipients_list = null;
4012 foreach (recipient; recipients)
4015 Curl.curl.slist_append(recipients_list,
4016 recipient.tempCString().buffPtr);
4018 p.curl.set(CurlOption.mail_rcpt, recipients_list);
4022 Sets the message body text.
4025 @property void message(string msg)
4032 Exception thrown on errors in std.net.curl functions.
4034 class CurlException : Exception
4038 msg = The message for the exception.
4039 file = The file where the exception occurred.
4040 line = The line number where the exception occurred.
4041 next = The previous exception in the chain of exceptions, if any.
4045 string file = __FILE__,
4046 size_t line = __LINE__,
4047 Throwable next = null)
4049 super(msg, file, line, next);
4054 Exception thrown on timeout errors in std.net.curl functions.
4056 class CurlTimeoutException : CurlException
4060 msg = The message for the exception.
4061 file = The file where the exception occurred.
4062 line = The line number where the exception occurred.
4063 next = The previous exception in the chain of exceptions, if any.
4067 string file = __FILE__,
4068 size_t line = __LINE__,
4069 Throwable next = null)
4071 super(msg, file, line, next);
4076 Exception thrown on HTTP request failures, e.g. 404 Not Found.
4078 class HTTPStatusException : CurlException
4082 status = The HTTP status code.
4083 msg = The message for the exception.
4084 file = The file where the exception occurred.
4085 line = The line number where the exception occurred.
4086 next = The previous exception in the chain of exceptions, if any.
4091 string file = __FILE__,
4092 size_t line = __LINE__,
4093 Throwable next = null)
4095 super(msg, file, line, next);
4096 this.status = status;
4099 immutable int status; /// The HTTP status code
4102 /// Equal to $(REF CURLcode, etc,c,curl)
4103 alias CurlCode = CURLcode;
4105 import std.typecons : Flag, Yes, No;
4106 /// Flag to specify whether or not an exception is thrown on error.
4107 alias ThrowOnError = Flag!"throwOnError";
4109 private struct CurlAPI
4114 import core.stdc.config : c_long;
4115 CURLcode function(c_long flags) global_init;
4116 void function() global_cleanup;
4117 curl_version_info_data * function(CURLversion) version_info;
4118 CURL* function() easy_init;
4119 CURLcode function(CURL *curl, CURLoption option,...) easy_setopt;
4120 CURLcode function(CURL *curl) easy_perform;
4121 CURLcode function(CURL *curl, CURLINFO info,...) easy_getinfo;
4122 CURL* function(CURL *curl) easy_duphandle;
4123 char* function(CURLcode) easy_strerror;
4124 CURLcode function(CURL *handle, int bitmask) easy_pause;
4125 void function(CURL *curl) easy_cleanup;
4126 curl_slist* function(curl_slist *, char *) slist_append;
4127 void function(curl_slist *) slist_free_all;
4130 __gshared void* _handle;
4132 static ref API instance() @property
4134 import std.concurrency : initOnce;
4135 initOnce!_handle(loadAPI());
4139 static void* loadAPI()
4143 import core.sys.posix.dlfcn : dlsym, dlopen, dlclose, RTLD_LAZY;
4144 alias loadSym = dlsym;
4146 else version (Windows)
4148 import core.sys.windows.windows : GetProcAddress, GetModuleHandleA,
4150 alias loadSym = GetProcAddress;
4153 static assert(0, "unimplemented");
4157 handle = dlopen(null, RTLD_LAZY);
4158 else version (Windows)
4159 handle = GetModuleHandleA(null);
4160 assert(handle !is null);
4162 // try to load curl from the executable to allow static linking
4163 if (loadSym(handle, "curl_global_init") is null)
4165 import std.format : format;
4170 static immutable names = ["libcurl.4.dylib"];
4171 else version (Posix)
4173 static immutable names = ["libcurl.so", "libcurl.so.4",
4174 "libcurl-gnutls.so.4", "libcurl-nss.so.4", "libcurl.so.3"];
4176 else version (Windows)
4177 static immutable names = ["libcurl.dll", "curl.dll"];
4179 foreach (name; names)
4182 handle = dlopen(name.ptr, RTLD_LAZY);
4183 else version (Windows)
4184 handle = LoadLibraryA(name.ptr);
4185 if (handle !is null) break;
4188 enforce!CurlException(handle !is null, "Failed to load curl, tried %(%s, %).".format(names));
4191 foreach (i, FP; typeof(API.tupleof))
4193 enum name = __traits(identifier, _api.tupleof[i]);
4194 auto p = enforce!CurlException(loadSym(handle, "curl_"~name),
4195 "Couldn't load curl_"~name~" from libcurl.");
4196 _api.tupleof[i] = cast(FP) p;
4199 enforce!CurlException(!_api.global_init(CurlGlobal.all),
4200 "Failed to initialize libcurl");
4202 static extern(C) void cleanup()
4204 if (_handle is null) return;
4205 _api.global_cleanup();
4208 import core.sys.posix.dlfcn : dlclose;
4211 else version (Windows)
4213 import core.sys.windows.windows : FreeLibrary;
4214 FreeLibrary(_handle);
4217 static assert(0, "unimplemented");
4222 import core.stdc.stdlib : atexit;
4230 Wrapper to provide a better interface to libcurl than using the plain C API.
4231 It is recommended to use the $(D HTTP)/$(D FTP) etc. structs instead unless
4232 raw access to libcurl is needed.
4234 Warning: This struct uses interior pointers for callbacks. Only allocate it
4235 on the stack if you never move or copy it. This also means passing by reference
4236 when passing Curl to other functions. Otherwise always allocate on
4241 alias OutData = void[];
4242 alias InData = ubyte[];
4243 private bool _stopped;
4245 private static auto ref curl() @property { return CurlAPI.instance; }
4247 // A handle should not be used by two threads simultaneously
4248 private CURL* handle;
4250 // May also return $(D CURL_READFUNC_ABORT) or $(D CURL_READFUNC_PAUSE)
4251 private size_t delegate(OutData) _onSend;
4252 private size_t delegate(InData) _onReceive;
4253 private void delegate(in char[]) _onReceiveHeader;
4254 private CurlSeek delegate(long,CurlSeekPos) _onSeek;
4255 private int delegate(curl_socket_t,CurlSockType) _onSocketOption;
4256 private int delegate(size_t dltotal, size_t dlnow,
4257 size_t ultotal, size_t ulnow) _onProgress;
4259 alias requestPause = CurlReadFunc.pause;
4260 alias requestAbort = CurlReadFunc.abort;
4263 Initialize the instance by creating a working curl handle.
4267 enforce!CurlException(!handle, "Curl instance already initialized");
4268 handle = curl.easy_init();
4269 enforce!CurlException(handle, "Curl instance couldn't be initialized");
4271 set(CurlOption.nosignal, 1);
4275 @property bool stopped() const
4281 Duplicate this handle.
4283 The new handle will have all options set as the one it was duplicated
4284 from. An exception to this is that all options that cannot be shared
4285 across threads are reset thereby making it safe to use the duplicate
4291 copy.handle = curl.easy_duphandle(handle);
4292 copy._stopped = false;
4295 auto tt = AliasSeq!(file, writefunction, writeheader,
4296 headerfunction, infile, readfunction, ioctldata, ioctlfunction,
4297 seekdata, seekfunction, sockoptdata, sockoptfunction,
4298 opensocketdata, opensocketfunction, progressdata,
4299 progressfunction, debugdata, debugfunction, interleavedata,
4300 interleavefunction, chunk_data, chunk_bgn_function,
4301 chunk_end_function, fnmatch_data, fnmatch_function, cookiejar, postfields);
4303 foreach (option; tt)
4307 // The options are only supported by libcurl when it has been built
4308 // against certain versions of OpenSSL - if your libcurl uses an old
4309 // OpenSSL, or uses an entirely different SSL engine, attempting to
4310 // clear these normally will raise an exception
4311 copy.clearIfSupported(CurlOption.ssl_ctx_function);
4312 copy.clearIfSupported(CurlOption.ssh_keydata);
4314 // Enable for curl version > 7.21.7
4315 static if (LIBCURL_VERSION_MAJOR >= 7 &&
4316 LIBCURL_VERSION_MINOR >= 21 &&
4317 LIBCURL_VERSION_PATCH >= 7)
4319 copy.clear(CurlOption.closesocketdata);
4320 copy.clear(CurlOption.closesocketfunction);
4323 copy.set(CurlOption.nosignal, 1);
4325 // copy.clear(CurlOption.ssl_ctx_data); Let ssl function be shared
4326 // copy.clear(CurlOption.ssh_keyfunction); Let key function be shared
4329 Allow sharing of conv functions
4330 copy.clear(CurlOption.conv_to_network_function);
4331 copy.clear(CurlOption.conv_from_network_function);
4332 copy.clear(CurlOption.conv_from_utf8_function);
4338 private void _check(CurlCode code)
4340 enforce!CurlTimeoutException(code != CurlError.operation_timedout,
4343 enforce!CurlException(code == CurlError.ok,
4347 private string errorString(CurlCode code)
4349 import core.stdc.string : strlen;
4350 import std.format : format;
4352 auto msgZ = curl.easy_strerror(code);
4353 // doing the following (instead of just using std.conv.to!string) avoids 1 allocation
4354 return format("%s on handle %s", msgZ[0 .. strlen(msgZ)], handle);
4357 private void throwOnStopped(string message = null)
4359 auto def = "Curl instance called after being cleaned up";
4360 enforce!CurlException(!stopped,
4361 message == null ? def : message);
4365 Stop and invalidate this curl instance.
4366 Warning: Do not call this from inside a callback handler e.g. $(D onReceive).
4372 curl.easy_cleanup(this.handle);
4377 Pausing and continuing transfers.
4379 void pause(bool sendingPaused, bool receivingPaused)
4382 _check(curl.easy_pause(this.handle,
4383 (sendingPaused ? CurlPause.send_cont : CurlPause.send) |
4384 (receivingPaused ? CurlPause.recv_cont : CurlPause.recv)));
4388 Set a string curl option.
4390 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation
4393 void set(CurlOption option, const(char)[] value)
4396 _check(curl.easy_setopt(this.handle, option, value.tempCString().buffPtr));
4400 Set a long curl option.
4402 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation
4405 void set(CurlOption option, long value)
4408 _check(curl.easy_setopt(this.handle, option, value));
4412 Set a void* curl option.
4414 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation
4417 void set(CurlOption option, void* value)
4420 _check(curl.easy_setopt(this.handle, option, value));
4424 Clear a pointer option.
4426 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation
4428 void clear(CurlOption option)
4431 _check(curl.easy_setopt(this.handle, option, null));
4435 Clear a pointer option. Does not raise an exception if the underlying
4436 libcurl does not support the option. Use sparingly.
4438 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation
4440 void clearIfSupported(CurlOption option)
4443 auto rval = curl.easy_setopt(this.handle, option, null);
4444 if (rval != CurlError.unknown_option && rval != CurlError.not_built_in)
4449 perform the curl request by doing the HTTP,FTP etc. as it has
4450 been setup beforehand.
4453 throwOnError = whether to throw an exception or return a CurlCode on error
4455 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError)
4458 CurlCode code = curl.easy_perform(this.handle);
4465 Get the various timings like name lookup time, total time, connect time etc.
4466 The timed category is passed through the timing parameter while the timing
4467 value is stored at val. The value is usable only if res is equal to
4468 $(D etc.c.curl.CurlError.ok).
4470 CurlCode getTiming(CurlInfo timing, ref double val)
4473 code = curl.easy_getinfo(handle, timing, &val);
4478 * The event handler that receives incoming data.
4481 * callback = the callback that receives the $(D ubyte[]) data.
4482 * Be sure to copy the incoming data and not store
4486 * The callback returns the incoming bytes read. If not the entire array is
4487 * the request will abort.
4488 * The special value HTTP.pauseRequest can be returned in order to pause the
4493 * import std.net.curl, std.stdio;
4495 * curl.initialize();
4496 * curl.set(CurlOption.url, "http://dlang.org");
4497 * curl.onReceive = (ubyte[] data) { writeln("Got data", to!(const(char)[])(data)); return data.length;};
4501 @property void onReceive(size_t delegate(InData) callback)
4503 _onReceive = (InData id)
4505 throwOnStopped("Receive callback called on cleaned up Curl instance");
4506 return callback(id);
4508 set(CurlOption.file, cast(void*) &this);
4509 set(CurlOption.writefunction, cast(void*) &Curl._receiveCallback);
4513 * The event handler that receives incoming headers for protocols
4514 * that uses headers.
4517 * callback = the callback that receives the header string.
4518 * Make sure the callback copies the incoming params if
4519 * it needs to store it because they are references into
4520 * the backend and may very likely change.
4524 * import std.net.curl, std.stdio;
4526 * curl.initialize();
4527 * curl.set(CurlOption.url, "http://dlang.org");
4528 * curl.onReceiveHeader = (in char[] header) { writeln(header); };
4532 @property void onReceiveHeader(void delegate(in char[]) callback)
4534 _onReceiveHeader = (in char[] od)
4536 throwOnStopped("Receive header callback called on "~
4537 "cleaned up Curl instance");
4540 set(CurlOption.writeheader, cast(void*) &this);
4541 set(CurlOption.headerfunction,
4542 cast(void*) &Curl._receiveHeaderCallback);
4546 * The event handler that gets called when data is needed for sending.
4549 * callback = the callback that has a $(D void[]) buffer to be filled
4552 * The callback returns the number of elements in the buffer that have been
4553 * filled and are ready to send.
4554 * The special value $(D Curl.abortRequest) can be returned in
4555 * order to abort the current request.
4556 * The special value $(D Curl.pauseRequest) can be returned in order to
4557 * pause the current request.
4561 * import std.net.curl;
4563 * curl.initialize();
4564 * curl.set(CurlOption.url, "http://dlang.org");
4566 * string msg = "Hello world";
4567 * curl.onSend = (void[] data)
4569 * auto m = cast(void[]) msg;
4570 * size_t length = m.length > data.length ? data.length : m.length;
4571 * if (length == 0) return 0;
4572 * data[0 .. length] = m[0 .. length];
4573 * msg = msg[length..$];
4579 @property void onSend(size_t delegate(OutData) callback)
4581 _onSend = (OutData od)
4583 throwOnStopped("Send callback called on cleaned up Curl instance");
4584 return callback(od);
4586 set(CurlOption.infile, cast(void*) &this);
4587 set(CurlOption.readfunction, cast(void*) &Curl._sendCallback);
4591 * The event handler that gets called when the curl backend needs to seek
4592 * the data to be sent.
4595 * callback = the callback that receives a seek offset and a seek position
4596 * $(REF CurlSeekPos, etc,c,curl)
4599 * The callback returns the success state of the seeking
4600 * $(REF CurlSeek, etc,c,curl)
4604 * import std.net.curl;
4606 * curl.initialize();
4607 * curl.set(CurlOption.url, "http://dlang.org");
4608 * curl.onSeek = (long p, CurlSeekPos sp)
4610 * return CurlSeek.cantseek;
4615 @property void onSeek(CurlSeek delegate(long, CurlSeekPos) callback)
4617 _onSeek = (long ofs, CurlSeekPos sp)
4619 throwOnStopped("Seek callback called on cleaned up Curl instance");
4620 return callback(ofs, sp);
4622 set(CurlOption.seekdata, cast(void*) &this);
4623 set(CurlOption.seekfunction, cast(void*) &Curl._seekCallback);
4627 * The event handler that gets called when the net socket has been created
4628 * but a $(D connect()) call has not yet been done. This makes it possible to set
4629 * misc. socket options.
4632 * callback = the callback that receives the socket and socket type
4633 * $(REF CurlSockType, etc,c,curl)
4636 * Return 0 from the callback to signal success, return 1 to signal error
4637 * and make curl close the socket
4641 * import std.net.curl;
4643 * curl.initialize();
4644 * curl.set(CurlOption.url, "http://dlang.org");
4645 * curl.onSocketOption = delegate int(curl_socket_t s, CurlSockType t) { /+ do stuff +/ };
4649 @property void onSocketOption(int delegate(curl_socket_t,
4650 CurlSockType) callback)
4652 _onSocketOption = (curl_socket_t sock, CurlSockType st)
4654 throwOnStopped("Socket option callback called on "~
4655 "cleaned up Curl instance");
4656 return callback(sock, st);
4658 set(CurlOption.sockoptdata, cast(void*) &this);
4659 set(CurlOption.sockoptfunction,
4660 cast(void*) &Curl._socketOptionCallback);
4664 * The event handler that gets called to inform of upload/download progress.
4667 * callback = the callback that receives the (total bytes to download,
4668 * currently downloaded bytes, total bytes to upload, currently uploaded
4672 * Return 0 from the callback to signal success, return non-zero to abort
4677 * import std.net.curl;
4679 * curl.initialize();
4680 * curl.set(CurlOption.url, "http://dlang.org");
4681 * curl.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t uln)
4683 * writeln("Progress: downloaded bytes ", dlnow, " of ", dltotal);
4684 * writeln("Progress: uploaded bytes ", ulnow, " of ", ultotal);
4689 @property void onProgress(int delegate(size_t dlTotal,
4692 size_t ulNow) callback)
4694 _onProgress = (size_t dlt, size_t dln, size_t ult, size_t uln)
4696 throwOnStopped("Progress callback called on cleaned "~
4697 "up Curl instance");
4698 return callback(dlt, dln, ult, uln);
4700 set(CurlOption.noprogress, 0);
4701 set(CurlOption.progressdata, cast(void*) &this);
4702 set(CurlOption.progressfunction, cast(void*) &Curl._progressCallback);
4705 // Internal C callbacks to register with libcurl
4706 extern (C) private static
4707 size_t _receiveCallback(const char* str,
4708 size_t size, size_t nmemb, void* ptr)
4710 auto b = cast(Curl*) ptr;
4711 if (b._onReceive != null)
4712 return b._onReceive(cast(InData)(str[0 .. size*nmemb]));
4716 extern (C) private static
4717 size_t _receiveHeaderCallback(const char* str,
4718 size_t size, size_t nmemb, void* ptr)
4720 import std.string : chomp;
4722 auto b = cast(Curl*) ptr;
4723 auto s = str[0 .. size*nmemb].chomp();
4724 if (b._onReceiveHeader != null)
4725 b._onReceiveHeader(s);
4730 extern (C) private static
4731 size_t _sendCallback(char *str, size_t size, size_t nmemb, void *ptr)
4733 Curl* b = cast(Curl*) ptr;
4734 auto a = cast(void[]) str[0 .. size*nmemb];
4735 if (b._onSend == null)
4737 return b._onSend(a);
4740 extern (C) private static
4741 int _seekCallback(void *ptr, curl_off_t offset, int origin)
4743 auto b = cast(Curl*) ptr;
4744 if (b._onSeek == null)
4745 return CurlSeek.cantseek;
4747 // origin: CurlSeekPos.set/current/end
4748 // return: CurlSeek.ok/fail/cantseek
4749 return b._onSeek(cast(long) offset, cast(CurlSeekPos) origin);
4752 extern (C) private static
4753 int _socketOptionCallback(void *ptr,
4754 curl_socket_t curlfd, curlsocktype purpose)
4756 auto b = cast(Curl*) ptr;
4757 if (b._onSocketOption == null)
4760 // return: 0 ok, 1 fail
4761 return b._onSocketOption(curlfd, cast(CurlSockType) purpose);
4764 extern (C) private static
4765 int _progressCallback(void *ptr,
4766 double dltotal, double dlnow,
4767 double ultotal, double ulnow)
4769 auto b = cast(Curl*) ptr;
4770 if (b._onProgress == null)
4773 // return: 0 ok, 1 fail
4774 return b._onProgress(cast(size_t) dltotal, cast(size_t) dlnow,
4775 cast(size_t) ultotal, cast(size_t) ulnow);
4780 // Internal messages send between threads.
4781 // The data is wrapped in this struct in order to ensure that
4782 // other std.concurrency.receive calls does not pick up our messages
4784 private struct CurlMessage(T)
4789 private static CurlMessage!T curlMessage(T)(T data)
4791 return CurlMessage!T(data);
4794 // Pool of to be used for reusing buffers
4795 private struct Pool(Data)
4797 private struct Entry
4802 private Entry* root;
4803 private Entry* freeList;
4805 @safe @property bool empty()
4807 return root == null;
4810 @safe nothrow void push(Data d)
4812 if (freeList == null)
4814 // Allocate new Entry since there is no one
4815 // available in the freeList
4816 freeList = new Entry;
4819 Entry* oldroot = root;
4821 freeList = freeList.next;
4822 root.next = oldroot;
4827 enforce!Exception(root != null, "pop() called on empty pool");
4830 root.next = freeList;
4837 // Shared function for reading incoming chunks of data and
4838 // sending the to a parent thread
4839 private static size_t _receiveAsyncChunks(ubyte[] data, ref ubyte[] outdata,
4840 Pool!(ubyte[]) freeBuffers,
4841 ref ubyte[] buffer, Tid fromTid,
4844 immutable datalen = data.length;
4846 // Copy data to fill active buffer
4850 // Make sure a buffer is present
4851 while ( outdata.empty && freeBuffers.empty)
4853 // Active buffer is invalid and there are no
4854 // available buffers in the pool. Wait for buffers
4855 // to return from main thread in order to reuse
4857 receive((immutable(ubyte)[] buf)
4859 buffer = cast(ubyte[]) buf;
4862 (bool flag) { aborted = true; }
4864 if (aborted) return cast(size_t) 0;
4868 buffer = freeBuffers.pop();
4873 auto copyBytes = outdata.length < data.length ?
4874 outdata.length : data.length;
4876 outdata[0 .. copyBytes] = data[0 .. copyBytes];
4877 outdata = outdata[copyBytes..$];
4878 data = data[copyBytes..$];
4881 fromTid.send(thisTid, curlMessage(cast(immutable(ubyte)[])buffer));
4888 private static void _finalizeAsyncChunks(ubyte[] outdata, ref ubyte[] buffer,
4893 // Resize the last buffer
4894 buffer.length = buffer.length - outdata.length;
4895 fromTid.send(thisTid, curlMessage(cast(immutable(ubyte)[])buffer));
4900 // Shared function for reading incoming lines of data and sending the to a
4902 private static size_t _receiveAsyncLines(Terminator, Unit)
4903 (const(ubyte)[] data, ref EncodingScheme encodingScheme,
4904 bool keepTerminator, Terminator terminator,
4905 ref const(ubyte)[] leftOverBytes, ref bool bufferValid,
4906 ref Pool!(Unit[]) freeBuffers, ref Unit[] buffer,
4907 Tid fromTid, ref bool aborted)
4909 import std.format : format;
4911 immutable datalen = data.length;
4913 // Terminator is specified and buffers should be resized as determined by
4916 // Copy data to active buffer until terminator is found.
4918 // Decode as many lines as possible
4922 // Make sure a buffer is present
4923 while (!bufferValid && freeBuffers.empty)
4925 // Active buffer is invalid and there are no available buffers in
4926 // the pool. Wait for buffers to return from main thread in order to
4928 receive((immutable(Unit)[] buf)
4930 buffer = cast(Unit[]) buf;
4932 buffer.assumeSafeAppend();
4935 (bool flag) { aborted = true; }
4937 if (aborted) return cast(size_t) 0;
4941 buffer = freeBuffers.pop();
4945 // Try to read a line from left over bytes from last onReceive plus the
4946 // newly received bytes.
4949 if (decodeLineInto(leftOverBytes, data, buffer,
4950 encodingScheme, terminator))
4954 fromTid.send(thisTid,
4955 curlMessage(cast(immutable(Unit)[])buffer));
4959 static if (isArray!Terminator)
4960 fromTid.send(thisTid,
4961 curlMessage(cast(immutable(Unit)[])
4962 buffer[0..$-terminator.length]));
4964 fromTid.send(thisTid,
4965 curlMessage(cast(immutable(Unit)[])
4968 bufferValid = false;
4972 // Could not decode an entire line. Save
4973 // bytes left in data for next call to
4974 // onReceive. Can be up to a max of 4 bytes.
4975 enforce!CurlException(data.length <= 4,
4977 "Too many bytes left not decoded %s"~
4978 " > 4. Maybe the charset specified in"~
4979 " headers does not match "~
4980 "the actual content downloaded?",
4982 leftOverBytes ~= data;
4986 catch (CurlException ex)
4988 prioritySend(fromTid, cast(immutable(CurlException))ex);
4989 return cast(size_t) 0;
4997 void _finalizeAsyncLines(Unit)(bool bufferValid, Unit[] buffer, Tid fromTid)
4999 if (bufferValid && buffer.length != 0)
5000 fromTid.send(thisTid, curlMessage(cast(immutable(Unit)[])buffer[0..$]));
5004 // Spawn a thread for handling the reading of incoming data in the
5005 // background while the delegate is executing. This will optimize
5006 // throughput by allowing simultaneous input (this struct) and
5007 // output (e.g. AsyncHTTPLineOutputRange).
5008 private static void _spawnAsync(Conn, Unit, Terminator = void)()
5010 Tid fromTid = receiveOnly!Tid();
5012 // Get buffer to read into
5013 Pool!(Unit[]) freeBuffers; // Free list of buffer objects
5015 // Number of bytes filled into active buffer
5017 bool aborted = false;
5019 EncodingScheme encodingScheme;
5020 static if ( !is(Terminator == void))
5022 // Only lines reading will receive a terminator
5023 const terminator = receiveOnly!Terminator();
5024 const keepTerminator = receiveOnly!bool();
5026 // max number of bytes to carry over from an onReceive
5027 // callback. This is 4 because it is the max code units to
5028 // decode a code point in the supported encodings.
5029 auto leftOverBytes = new const(ubyte)[4];
5030 leftOverBytes.length = 0;
5031 auto bufferValid = false;
5038 // no move semantic available in std.concurrency ie. must use casting.
5039 auto connDup = cast(CURL*) receiveOnly!ulong();
5040 auto client = Conn();
5041 client.p.curl.handle = connDup;
5043 // receive a method for both ftp and http but just use it for http
5044 auto method = receiveOnly!(HTTP.Method)();
5046 client.onReceive = (ubyte[] data)
5048 // If no terminator is specified the chunk size is fixed.
5049 static if ( is(Terminator == void) )
5050 return _receiveAsyncChunks(data, outdata, freeBuffers, buffer,
5053 return _receiveAsyncLines(data, encodingScheme,
5054 keepTerminator, terminator, leftOverBytes,
5055 bufferValid, freeBuffers, buffer,
5059 static if ( is(Conn == HTTP) )
5061 client.method = method;
5062 // register dummy header handler
5063 client.onReceiveHeader = (in char[] key, in char[] value)
5065 if (key == "content-type")
5066 encodingScheme = EncodingScheme.create(client.p.charset);
5071 encodingScheme = EncodingScheme.create(client.encoding);
5074 // Start the request
5078 code = client.perform(No.throwOnError);
5080 catch (Exception ex)
5082 prioritySend(fromTid, cast(immutable(Exception)) ex);
5083 fromTid.send(thisTid, curlMessage(true)); // signal done
5087 if (code != CurlError.ok)
5089 if (aborted && (code == CurlError.aborted_by_callback ||
5090 code == CurlError.write_error))
5092 fromTid.send(thisTid, curlMessage(true)); // signal done
5095 prioritySend(fromTid, cast(immutable(CurlException))
5096 new CurlException(client.p.curl.errorString(code)));
5098 fromTid.send(thisTid, curlMessage(true)); // signal done
5102 // Send remaining data that is not a full chunk size
5103 static if ( is(Terminator == void) )
5104 _finalizeAsyncChunks(outdata, buffer, fromTid);
5106 _finalizeAsyncLines(bufferValid, buffer, fromTid);
5108 fromTid.send(thisTid, curlMessage(true)); // signal done