]>
Commit | Line | Data |
---|---|---|
03385ed3 | 1 | // Written in the D programming language. |
2 | ||
3 | /** | |
4 | Networking client functionality as provided by $(HTTP _curl.haxx.se/libcurl, | |
5 | libcurl). The libcurl library must be installed on the system in order to use | |
6 | this module. | |
7 | ||
8 | $(SCRIPT inhibitQuickIndex = 1;) | |
9 | ||
10 | $(DIVC quickindex, | |
11 | $(BOOKTABLE , | |
12 | $(TR $(TH Category) $(TH Functions) | |
13 | ) | |
14 | $(TR $(TDNW High level) $(TD $(MYREF download) $(MYREF upload) $(MYREF get) | |
15 | $(MYREF post) $(MYREF put) $(MYREF del) $(MYREF options) $(MYREF trace) | |
16 | $(MYREF connect) $(MYREF byLine) $(MYREF byChunk) | |
17 | $(MYREF byLineAsync) $(MYREF byChunkAsync) ) | |
18 | ) | |
19 | $(TR $(TDNW Low level) $(TD $(MYREF HTTP) $(MYREF FTP) $(MYREF | |
20 | SMTP) ) | |
21 | ) | |
22 | ) | |
23 | ) | |
24 | ||
25 | Note: | |
26 | You may need to link to the $(B curl) library, e.g. by adding $(D "libs": ["curl"]) | |
27 | to your $(B dub.json) file if you are using $(LINK2 http://code.dlang.org, DUB). | |
28 | ||
29 | Windows x86 note: | |
30 | A DMD compatible libcurl static library can be downloaded from the dlang.org | |
31 | $(LINK2 http://dlang.org/download.html, download page). | |
32 | ||
33 | Compared to using libcurl directly this module allows simpler client code for | |
34 | common uses, requires no unsafe operations, and integrates better with the rest | |
35 | of the language. Futhermore it provides <a href="std_range.html">$(D range)</a> | |
36 | access to protocols supported by libcurl both synchronously and asynchronously. | |
37 | ||
38 | A high level and a low level API are available. The high level API is built | |
39 | entirely on top of the low level one. | |
40 | ||
41 | The high level API is for commonly used functionality such as HTTP/FTP get. The | |
42 | $(LREF byLineAsync) and $(LREF byChunkAsync) provides asynchronous <a | |
43 | href="std_range.html">$(D ranges)</a> that performs the request in another | |
44 | thread while handling a line/chunk in the current thread. | |
45 | ||
46 | The low level API allows for streaming and other advanced features. | |
47 | ||
48 | $(BOOKTABLE Cheat Sheet, | |
49 | $(TR $(TH Function Name) $(TH Description) | |
50 | ) | |
51 | $(LEADINGROW High level) | |
52 | $(TR $(TDNW $(LREF download)) $(TD $(D | |
53 | download("ftp.digitalmars.com/sieve.ds", "/tmp/downloaded-ftp-file")) | |
54 | downloads file from URL to file system.) | |
55 | ) | |
56 | $(TR $(TDNW $(LREF upload)) $(TD $(D | |
57 | upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds");) | |
58 | uploads file from file system to URL.) | |
59 | ) | |
60 | $(TR $(TDNW $(LREF get)) $(TD $(D | |
61 | get("dlang.org")) returns a char[] containing the dlang.org web page.) | |
62 | ) | |
63 | $(TR $(TDNW $(LREF put)) $(TD $(D | |
64 | put("dlang.org", "Hi")) returns a char[] containing | |
65 | the dlang.org web page. after a HTTP PUT of "hi") | |
66 | ) | |
67 | $(TR $(TDNW $(LREF post)) $(TD $(D | |
68 | post("dlang.org", "Hi")) returns a char[] containing | |
69 | the dlang.org web page. after a HTTP POST of "hi") | |
70 | ) | |
71 | $(TR $(TDNW $(LREF byLine)) $(TD $(D | |
72 | byLine("dlang.org")) returns a range of char[] containing the | |
73 | dlang.org web page.) | |
74 | ) | |
75 | $(TR $(TDNW $(LREF byChunk)) $(TD $(D | |
76 | byChunk("dlang.org", 10)) returns a range of ubyte[10] containing the | |
77 | dlang.org web page.) | |
78 | ) | |
79 | $(TR $(TDNW $(LREF byLineAsync)) $(TD $(D | |
80 | byLineAsync("dlang.org")) returns a range of char[] containing the dlang.org web | |
81 | page asynchronously.) | |
82 | ) | |
83 | $(TR $(TDNW $(LREF byChunkAsync)) $(TD $(D | |
84 | byChunkAsync("dlang.org", 10)) returns a range of ubyte[10] containing the | |
85 | dlang.org web page asynchronously.) | |
86 | ) | |
87 | $(LEADINGROW Low level | |
88 | ) | |
89 | $(TR $(TDNW $(LREF HTTP)) $(TD $(D HTTP) struct for advanced usage)) | |
90 | $(TR $(TDNW $(LREF FTP)) $(TD $(D FTP) struct for advanced usage)) | |
91 | $(TR $(TDNW $(LREF SMTP)) $(TD $(D SMTP) struct for advanced usage)) | |
92 | ) | |
93 | ||
94 | ||
95 | Example: | |
96 | --- | |
97 | import std.net.curl, std.stdio; | |
98 | ||
99 | // Return a char[] containing the content specified by a URL | |
100 | auto content = get("dlang.org"); | |
101 | ||
102 | // Post data and return a char[] containing the content specified by a URL | |
103 | auto content = post("mydomain.com/here.cgi", ["name1" : "value1", "name2" : "value2"]); | |
104 | ||
105 | // Get content of file from ftp server | |
106 | auto content = get("ftp.digitalmars.com/sieve.ds"); | |
107 | ||
108 | // Post and print out content line by line. The request is done in another thread. | |
109 | foreach (line; byLineAsync("dlang.org", "Post data")) | |
110 | writeln(line); | |
111 | ||
112 | // Get using a line range and proxy settings | |
113 | auto client = HTTP(); | |
114 | client.proxy = "1.2.3.4"; | |
115 | foreach (line; byLine("dlang.org", client)) | |
116 | writeln(line); | |
117 | --- | |
118 | ||
119 | For more control than the high level functions provide, use the low level API: | |
120 | ||
121 | Example: | |
122 | --- | |
123 | import std.net.curl, std.stdio; | |
124 | ||
125 | // GET with custom data receivers | |
126 | auto http = HTTP("dlang.org"); | |
127 | http.onReceiveHeader = | |
128 | (in char[] key, in char[] value) { writeln(key, ": ", value); }; | |
129 | http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; }; | |
130 | http.perform(); | |
131 | --- | |
132 | ||
133 | First, an instance of the reference-counted HTTP struct is created. Then the | |
134 | custom delegates are set. These will be called whenever the HTTP instance | |
135 | receives a header and a data buffer, respectively. In this simple example, the | |
136 | headers are written to stdout and the data is ignored. If the request should be | |
137 | stopped before it has finished then return something less than data.length from | |
138 | the onReceive callback. See $(LREF onReceiveHeader)/$(LREF onReceive) for more | |
139 | information. Finally the HTTP request is effected by calling perform(), which is | |
140 | synchronous. | |
141 | ||
142 | Source: $(PHOBOSSRC std/net/_curl.d) | |
143 | ||
144 | Copyright: Copyright Jonas Drewsen 2011-2012 | |
145 | License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0). | |
146 | Authors: Jonas Drewsen. Some of the SMTP code contributed by Jimmy Cao. | |
147 | ||
148 | Credits: The functionally is based on $(HTTP _curl.haxx.se/libcurl, libcurl). | |
149 | LibCurl is licensed under an MIT/X derivative license. | |
150 | */ | |
151 | /* | |
152 | Copyright Jonas Drewsen 2011 - 2012. | |
153 | Distributed under the Boost Software License, Version 1.0. | |
154 | (See accompanying file LICENSE_1_0.txt or copy at | |
155 | http://www.boost.org/LICENSE_1_0.txt) | |
156 | */ | |
157 | module std.net.curl; | |
158 | ||
159 | import core.thread; | |
160 | import etc.c.curl; | |
161 | import std.concurrency; | |
162 | import std.encoding; | |
163 | import std.exception; | |
164 | import std.meta; | |
165 | import std.range.primitives; | |
166 | import std.socket : InternetAddress; | |
167 | import std.traits; | |
168 | import std.typecons; | |
169 | ||
170 | import std.internal.cstring; | |
171 | ||
172 | public import etc.c.curl : CurlOption; | |
173 | ||
174 | version (unittest) | |
175 | { | |
176 | // Run unit test with the PHOBOS_TEST_ALLOW_NET=1 set in order to | |
177 | // allow net traffic | |
178 | import std.range; | |
179 | import std.stdio; | |
180 | ||
181 | import std.socket : Address, INADDR_LOOPBACK, Socket, TcpSocket; | |
182 | ||
183 | private struct TestServer | |
184 | { | |
185 | string addr() { return _addr; } | |
186 | ||
187 | void handle(void function(Socket s) dg) | |
188 | { | |
189 | tid.send(dg); | |
190 | } | |
191 | ||
192 | private: | |
193 | string _addr; | |
194 | Tid tid; | |
195 | ||
196 | static void loop(shared TcpSocket listener) | |
197 | { | |
198 | try while (true) | |
199 | { | |
200 | void function(Socket) handler = void; | |
201 | try | |
202 | handler = receiveOnly!(typeof(handler)); | |
203 | catch (OwnerTerminated) | |
204 | return; | |
205 | handler((cast() listener).accept); | |
206 | } | |
207 | catch (Throwable e) | |
208 | { | |
209 | import core.stdc.stdlib : exit, EXIT_FAILURE; | |
210 | stderr.writeln(e); | |
211 | exit(EXIT_FAILURE); // Bugzilla 7018 | |
212 | } | |
213 | } | |
214 | } | |
215 | ||
216 | private TestServer startServer() | |
217 | { | |
218 | auto sock = new TcpSocket; | |
219 | sock.bind(new InternetAddress(INADDR_LOOPBACK, InternetAddress.PORT_ANY)); | |
220 | sock.listen(1); | |
221 | auto addr = sock.localAddress.toString(); | |
222 | auto tid = spawn(&TestServer.loop, cast(shared) sock); | |
223 | return TestServer(addr, tid); | |
224 | } | |
225 | ||
226 | private ref TestServer testServer() | |
227 | { | |
228 | __gshared TestServer server; | |
229 | return initOnce!server(startServer()); | |
230 | } | |
231 | ||
232 | private struct Request(T) | |
233 | { | |
234 | string hdrs; | |
235 | immutable(T)[] bdy; | |
236 | } | |
237 | ||
238 | private Request!T recvReq(T=char)(Socket s) | |
239 | { | |
240 | import std.algorithm.comparison : min; | |
241 | import std.algorithm.searching : find, canFind; | |
242 | import std.conv : to; | |
243 | import std.regex : ctRegex, matchFirst; | |
244 | ||
245 | ubyte[1024] tmp=void; | |
246 | ubyte[] buf; | |
247 | ||
248 | while (true) | |
249 | { | |
250 | auto nbytes = s.receive(tmp[]); | |
251 | assert(nbytes >= 0); | |
252 | ||
253 | immutable beg = buf.length > 3 ? buf.length - 3 : 0; | |
254 | buf ~= tmp[0 .. nbytes]; | |
255 | auto bdy = buf[beg .. $].find(cast(ubyte[])"\r\n\r\n"); | |
256 | if (bdy.empty) | |
257 | continue; | |
258 | ||
259 | auto hdrs = cast(string) buf[0 .. $ - bdy.length]; | |
260 | bdy.popFrontN(4); | |
261 | // no support for chunked transfer-encoding | |
262 | if (auto m = hdrs.matchFirst(ctRegex!(`Content-Length: ([0-9]+)`, "i"))) | |
263 | { | |
264 | import std.uni : asUpperCase; | |
265 | if (hdrs.asUpperCase.canFind("EXPECT: 100-CONTINUE")) | |
266 | s.send(httpContinue); | |
267 | ||
268 | size_t remain = m.captures[1].to!size_t - bdy.length; | |
269 | while (remain) | |
270 | { | |
271 | nbytes = s.receive(tmp[0 .. min(remain, $)]); | |
272 | assert(nbytes >= 0); | |
273 | buf ~= tmp[0 .. nbytes]; | |
274 | remain -= nbytes; | |
275 | } | |
276 | } | |
277 | else | |
278 | { | |
279 | assert(bdy.empty); | |
280 | } | |
281 | bdy = buf[hdrs.length + 4 .. $]; | |
282 | return typeof(return)(hdrs, cast(immutable(T)[])bdy); | |
283 | } | |
284 | } | |
285 | ||
286 | private string httpOK(string msg) | |
287 | { | |
288 | import std.conv : to; | |
289 | ||
290 | return "HTTP/1.1 200 OK\r\n"~ | |
291 | "Content-Type: text/plain\r\n"~ | |
292 | "Content-Length: "~msg.length.to!string~"\r\n"~ | |
293 | "\r\n"~ | |
294 | msg; | |
295 | } | |
296 | ||
297 | private string httpOK() | |
298 | { | |
299 | return "HTTP/1.1 200 OK\r\n"~ | |
300 | "Content-Length: 0\r\n"~ | |
301 | "\r\n"; | |
302 | } | |
303 | ||
304 | private string httpNotFound() | |
305 | { | |
306 | return "HTTP/1.1 404 Not Found\r\n"~ | |
307 | "Content-Length: 0\r\n"~ | |
308 | "\r\n"; | |
309 | } | |
310 | ||
311 | private enum httpContinue = "HTTP/1.1 100 Continue\r\n\r\n"; | |
312 | } | |
313 | version (StdDdoc) import std.stdio; | |
314 | ||
315 | // Default data timeout for Protocols | |
316 | private enum _defaultDataTimeout = dur!"minutes"(2); | |
317 | ||
318 | /** | |
319 | Macros: | |
320 | ||
321 | CALLBACK_PARAMS = $(TABLE , | |
322 | $(DDOC_PARAM_ROW | |
323 | $(DDOC_PARAM_ID $(DDOC_PARAM dlTotal)) | |
324 | $(DDOC_PARAM_DESC total bytes to download) | |
325 | ) | |
326 | $(DDOC_PARAM_ROW | |
327 | $(DDOC_PARAM_ID $(DDOC_PARAM dlNow)) | |
328 | $(DDOC_PARAM_DESC currently downloaded bytes) | |
329 | ) | |
330 | $(DDOC_PARAM_ROW | |
331 | $(DDOC_PARAM_ID $(DDOC_PARAM ulTotal)) | |
332 | $(DDOC_PARAM_DESC total bytes to upload) | |
333 | ) | |
334 | $(DDOC_PARAM_ROW | |
335 | $(DDOC_PARAM_ID $(DDOC_PARAM ulNow)) | |
336 | $(DDOC_PARAM_DESC currently uploaded bytes) | |
337 | ) | |
338 | ) | |
339 | */ | |
340 | ||
341 | /** Connection type used when the URL should be used to auto detect the protocol. | |
342 | * | |
343 | * This struct is used as placeholder for the connection parameter when calling | |
344 | * the high level API and the connection type (HTTP/FTP) should be guessed by | |
345 | * inspecting the URL parameter. | |
346 | * | |
347 | * The rules for guessing the protocol are: | |
348 | * 1, if URL starts with ftp://, ftps:// or ftp. then FTP connection is assumed. | |
349 | * 2, HTTP connection otherwise. | |
350 | * | |
351 | * Example: | |
352 | * --- | |
353 | * import std.net.curl; | |
354 | * // Two requests below will do the same. | |
355 | * string content; | |
356 | * | |
357 | * // Explicit connection provided | |
358 | * content = get!HTTP("dlang.org"); | |
359 | * | |
360 | * // Guess connection type by looking at the URL | |
361 | * content = get!AutoProtocol("ftp://foo.com/file"); | |
362 | * // and since AutoProtocol is default this is the same as | |
363 | * content = get("ftp://foo.com/file"); | |
364 | * // and will end up detecting FTP from the url and be the same as | |
365 | * content = get!FTP("ftp://foo.com/file"); | |
366 | * --- | |
367 | */ | |
368 | struct AutoProtocol { } | |
369 | ||
370 | // Returns true if the url points to an FTP resource | |
371 | private bool isFTPUrl(const(char)[] url) | |
372 | { | |
373 | import std.algorithm.searching : startsWith; | |
374 | import std.uni : toLower; | |
375 | ||
376 | return startsWith(url.toLower(), "ftp://", "ftps://", "ftp.") != 0; | |
377 | } | |
378 | ||
379 | // Is true if the Conn type is a valid Curl Connection type. | |
380 | private template isCurlConn(Conn) | |
381 | { | |
382 | enum auto isCurlConn = is(Conn : HTTP) || | |
383 | is(Conn : FTP) || is(Conn : AutoProtocol); | |
384 | } | |
385 | ||
386 | /** HTTP/FTP download to local file system. | |
387 | * | |
388 | * Params: | |
389 | * url = resource to download | |
390 | * saveToPath = path to store the downloaded content on local disk | |
391 | * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will | |
392 | * guess connection type and create a new instance for this call only. | |
393 | * | |
394 | * Example: | |
395 | * ---- | |
396 | * import std.net.curl; | |
397 | * download("d-lang.appspot.com/testUrl2", "/tmp/downloaded-http-file"); | |
398 | * ---- | |
399 | */ | |
400 | void download(Conn = AutoProtocol)(const(char)[] url, string saveToPath, Conn conn = Conn()) | |
401 | if (isCurlConn!Conn) | |
402 | { | |
403 | static if (is(Conn : HTTP) || is(Conn : FTP)) | |
404 | { | |
405 | import std.stdio : File; | |
406 | conn.url = url; | |
407 | auto f = File(saveToPath, "wb"); | |
408 | conn.onReceive = (ubyte[] data) { f.rawWrite(data); return data.length; }; | |
409 | conn.perform(); | |
410 | } | |
411 | else | |
412 | { | |
413 | if (isFTPUrl(url)) | |
414 | return download!FTP(url, saveToPath, FTP()); | |
415 | else | |
416 | return download!HTTP(url, saveToPath, HTTP()); | |
417 | } | |
418 | } | |
419 | ||
420 | @system unittest | |
421 | { | |
422 | import std.algorithm.searching : canFind; | |
423 | static import std.file; | |
424 | ||
425 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
426 | { | |
427 | testServer.handle((s) { | |
428 | assert(s.recvReq.hdrs.canFind("GET /")); | |
429 | s.send(httpOK("Hello world")); | |
430 | }); | |
431 | auto fn = std.file.deleteme; | |
432 | scope (exit) std.file.remove(fn); | |
433 | download(host, fn); | |
434 | assert(std.file.readText(fn) == "Hello world"); | |
435 | } | |
436 | } | |
437 | ||
438 | /** Upload file from local files system using the HTTP or FTP protocol. | |
439 | * | |
440 | * Params: | |
441 | * loadFromPath = path load data from local disk. | |
442 | * url = resource to upload to | |
443 | * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will | |
444 | * guess connection type and create a new instance for this call only. | |
445 | * | |
446 | * Example: | |
447 | * ---- | |
448 | * import std.net.curl; | |
449 | * upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds"); | |
450 | * upload("/tmp/downloaded-http-file", "d-lang.appspot.com/testUrl2"); | |
451 | * ---- | |
452 | */ | |
453 | void upload(Conn = AutoProtocol)(string loadFromPath, const(char)[] url, Conn conn = Conn()) | |
454 | if (isCurlConn!Conn) | |
455 | { | |
456 | static if (is(Conn : HTTP)) | |
457 | { | |
458 | conn.url = url; | |
459 | conn.method = HTTP.Method.put; | |
460 | } | |
461 | else static if (is(Conn : FTP)) | |
462 | { | |
463 | conn.url = url; | |
464 | conn.handle.set(CurlOption.upload, 1L); | |
465 | } | |
466 | else | |
467 | { | |
468 | if (isFTPUrl(url)) | |
469 | return upload!FTP(loadFromPath, url, FTP()); | |
470 | else | |
471 | return upload!HTTP(loadFromPath, url, HTTP()); | |
472 | } | |
473 | ||
474 | static if (is(Conn : HTTP) || is(Conn : FTP)) | |
475 | { | |
476 | import std.stdio : File; | |
477 | auto f = File(loadFromPath, "rb"); | |
478 | conn.onSend = buf => f.rawRead(buf).length; | |
479 | immutable sz = f.size; | |
480 | if (sz != ulong.max) | |
481 | conn.contentLength = sz; | |
482 | conn.perform(); | |
483 | } | |
484 | } | |
485 | ||
486 | @system unittest | |
487 | { | |
488 | import std.algorithm.searching : canFind; | |
489 | static import std.file; | |
490 | ||
491 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
492 | { | |
493 | auto fn = std.file.deleteme; | |
494 | scope (exit) std.file.remove(fn); | |
495 | std.file.write(fn, "upload data\n"); | |
496 | testServer.handle((s) { | |
497 | auto req = s.recvReq; | |
498 | assert(req.hdrs.canFind("PUT /path")); | |
499 | assert(req.bdy.canFind("upload data")); | |
500 | s.send(httpOK()); | |
501 | }); | |
502 | upload(fn, host ~ "/path"); | |
503 | } | |
504 | } | |
505 | ||
506 | /** HTTP/FTP get content. | |
507 | * | |
508 | * Params: | |
509 | * url = resource to get | |
510 | * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will | |
511 | * guess connection type and create a new instance for this call only. | |
512 | * | |
513 | * The template parameter $(D T) specifies the type to return. Possible values | |
514 | * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). If asking | |
515 | * for $(D char), content will be converted from the connection character set | |
516 | * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1 | |
517 | * by default) to UTF-8. | |
518 | * | |
519 | * Example: | |
520 | * ---- | |
521 | * import std.net.curl; | |
522 | * auto content = get("d-lang.appspot.com/testUrl2"); | |
523 | * ---- | |
524 | * | |
525 | * Returns: | |
526 | * A T[] range containing the content of the resource pointed to by the URL. | |
527 | * | |
528 | * Throws: | |
529 | * | |
530 | * $(D CurlException) on error. | |
531 | * | |
532 | * See_Also: $(LREF HTTP.Method) | |
533 | */ | |
534 | T[] get(Conn = AutoProtocol, T = char)(const(char)[] url, Conn conn = Conn()) | |
535 | if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) ) | |
536 | { | |
537 | static if (is(Conn : HTTP)) | |
538 | { | |
539 | conn.method = HTTP.Method.get; | |
540 | return _basicHTTP!(T)(url, "", conn); | |
541 | ||
542 | } | |
543 | else static if (is(Conn : FTP)) | |
544 | { | |
545 | return _basicFTP!(T)(url, "", conn); | |
546 | } | |
547 | else | |
548 | { | |
549 | if (isFTPUrl(url)) | |
550 | return get!(FTP,T)(url, FTP()); | |
551 | else | |
552 | return get!(HTTP,T)(url, HTTP()); | |
553 | } | |
554 | } | |
555 | ||
556 | @system unittest | |
557 | { | |
558 | import std.algorithm.searching : canFind; | |
559 | ||
560 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
561 | { | |
562 | testServer.handle((s) { | |
563 | assert(s.recvReq.hdrs.canFind("GET /path")); | |
564 | s.send(httpOK("GETRESPONSE")); | |
565 | }); | |
566 | auto res = get(host ~ "/path"); | |
567 | assert(res == "GETRESPONSE"); | |
568 | } | |
569 | } | |
570 | ||
571 | ||
572 | /** HTTP post content. | |
573 | * | |
574 | * Params: | |
575 | * url = resource to post to | |
576 | * postDict = data to send as the body of the request. An associative array | |
577 | * of $(D string) is accepted and will be encoded using | |
578 | * www-form-urlencoding | |
579 | * postData = data to send as the body of the request. An array | |
580 | * of an arbitrary type is accepted and will be cast to ubyte[] | |
581 | * before sending it. | |
582 | * conn = HTTP connection to use | |
583 | * T = The template parameter $(D T) specifies the type to return. Possible values | |
584 | * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). If asking | |
585 | * for $(D char), content will be converted from the connection character set | |
586 | * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1 | |
587 | * by default) to UTF-8. | |
588 | * | |
589 | * Examples: | |
590 | * ---- | |
591 | * import std.net.curl; | |
592 | * | |
593 | * auto content1 = post("d-lang.appspot.com/testUrl2", ["name1" : "value1", "name2" : "value2"]); | |
594 | * auto content2 = post("d-lang.appspot.com/testUrl2", [1,2,3,4]); | |
595 | * ---- | |
596 | * | |
597 | * Returns: | |
598 | * A T[] range containing the content of the resource pointed to by the URL. | |
599 | * | |
600 | * See_Also: $(LREF HTTP.Method) | |
601 | */ | |
602 | T[] post(T = char, PostUnit)(const(char)[] url, const(PostUnit)[] postData, HTTP conn = HTTP()) | |
603 | if (is(T == char) || is(T == ubyte)) | |
604 | { | |
605 | conn.method = HTTP.Method.post; | |
606 | return _basicHTTP!(T)(url, postData, conn); | |
607 | } | |
608 | ||
609 | @system unittest | |
610 | { | |
611 | import std.algorithm.searching : canFind; | |
612 | ||
613 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
614 | { | |
615 | testServer.handle((s) { | |
616 | auto req = s.recvReq; | |
617 | assert(req.hdrs.canFind("POST /path")); | |
618 | assert(req.bdy.canFind("POSTBODY")); | |
619 | s.send(httpOK("POSTRESPONSE")); | |
620 | }); | |
621 | auto res = post(host ~ "/path", "POSTBODY"); | |
622 | assert(res == "POSTRESPONSE"); | |
623 | } | |
624 | } | |
625 | ||
626 | @system unittest | |
627 | { | |
628 | import std.algorithm.searching : canFind; | |
629 | ||
630 | auto data = new ubyte[](256); | |
631 | foreach (i, ref ub; data) | |
632 | ub = cast(ubyte) i; | |
633 | ||
634 | testServer.handle((s) { | |
635 | auto req = s.recvReq!ubyte; | |
636 | assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4])); | |
637 | assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255])); | |
638 | s.send(httpOK(cast(ubyte[])[17, 27, 35, 41])); | |
639 | }); | |
640 | auto res = post!ubyte(testServer.addr, data); | |
641 | assert(res == cast(ubyte[])[17, 27, 35, 41]); | |
642 | } | |
643 | ||
644 | /// ditto | |
645 | T[] post(T = char)(const(char)[] url, string[string] postDict, HTTP conn = HTTP()) | |
646 | if (is(T == char) || is(T == ubyte)) | |
647 | { | |
648 | import std.uri : urlEncode; | |
649 | ||
650 | return post(url, urlEncode(postDict), conn); | |
651 | } | |
652 | ||
653 | @system unittest | |
654 | { | |
655 | foreach (host; [testServer.addr, "http://" ~ testServer.addr]) | |
656 | { | |
657 | testServer.handle((s) { | |
658 | auto req = s.recvReq!char; | |
659 | s.send(httpOK(req.bdy)); | |
660 | }); | |
661 | auto res = post(host ~ "/path", ["name1" : "value1", "name2" : "value2"]); | |
22163f0d | 662 | assert(res == "name1=value1&name2=value2" || res == "name2=value2&name1=value1"); |
03385ed3 | 663 | } |
664 | } | |
665 | ||
666 | /** HTTP/FTP put content. | |
667 | * | |
668 | * Params: | |
669 | * url = resource to put | |
670 | * putData = data to send as the body of the request. An array | |
671 | * of an arbitrary type is accepted and will be cast to ubyte[] | |
672 | * before sending it. | |
673 | * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will | |
674 | * guess connection type and create a new instance for this call only. | |
675 | * | |
676 | * The template parameter $(D T) specifies the type to return. Possible values | |
677 | * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). If asking | |
678 | * for $(D char), content will be converted from the connection character set | |
679 | * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1 | |
680 | * by default) to UTF-8. | |
681 | * | |
682 | * Example: | |
683 | * ---- | |
684 | * import std.net.curl; | |
685 | * auto content = put("d-lang.appspot.com/testUrl2", | |
686 | * "Putting this data"); | |
687 | * ---- | |
688 | * | |
689 | * Returns: | |
690 | * A T[] range containing the content of the resource pointed to by the URL. | |
691 | * | |
692 | * See_Also: $(LREF HTTP.Method) | |
693 | */ | |
694 | T[] put(Conn = AutoProtocol, T = char, PutUnit)(const(char)[] url, const(PutUnit)[] putData, | |
695 | Conn conn = Conn()) | |
696 | if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) ) | |
697 | { | |
698 | static if (is(Conn : HTTP)) | |
699 | { | |
700 | conn.method = HTTP.Method.put; | |
701 | return _basicHTTP!(T)(url, putData, conn); | |
702 | } | |
703 | else static if (is(Conn : FTP)) | |
704 | { | |
705 | return _basicFTP!(T)(url, putData, conn); | |
706 | } | |
707 | else | |
708 | { | |
709 | if (isFTPUrl(url)) | |
710 | return put!(FTP,T)(url, putData, FTP()); | |
711 | else | |
712 | return put!(HTTP,T)(url, putData, HTTP()); | |
713 | } | |
714 | } | |
715 | ||
716 | @system unittest | |
717 | { | |
718 | import std.algorithm.searching : canFind; | |
719 | ||
720 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
721 | { | |
722 | testServer.handle((s) { | |
723 | auto req = s.recvReq; | |
724 | assert(req.hdrs.canFind("PUT /path")); | |
725 | assert(req.bdy.canFind("PUTBODY")); | |
726 | s.send(httpOK("PUTRESPONSE")); | |
727 | }); | |
728 | auto res = put(host ~ "/path", "PUTBODY"); | |
729 | assert(res == "PUTRESPONSE"); | |
730 | } | |
731 | } | |
732 | ||
733 | ||
734 | /** HTTP/FTP delete content. | |
735 | * | |
736 | * Params: | |
737 | * url = resource to delete | |
738 | * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will | |
739 | * guess connection type and create a new instance for this call only. | |
740 | * | |
741 | * Example: | |
742 | * ---- | |
743 | * import std.net.curl; | |
744 | * del("d-lang.appspot.com/testUrl2"); | |
745 | * ---- | |
746 | * | |
747 | * See_Also: $(LREF HTTP.Method) | |
748 | */ | |
749 | void del(Conn = AutoProtocol)(const(char)[] url, Conn conn = Conn()) | |
750 | if (isCurlConn!Conn) | |
751 | { | |
752 | static if (is(Conn : HTTP)) | |
753 | { | |
754 | conn.method = HTTP.Method.del; | |
755 | _basicHTTP!char(url, cast(void[]) null, conn); | |
756 | } | |
757 | else static if (is(Conn : FTP)) | |
758 | { | |
759 | import std.algorithm.searching : findSplitAfter; | |
760 | import std.conv : text; | |
761 | ||
762 | auto trimmed = url.findSplitAfter("ftp://")[1]; | |
763 | auto t = trimmed.findSplitAfter("/"); | |
764 | enum minDomainNameLength = 3; | |
765 | enforce!CurlException(t[0].length > minDomainNameLength, | |
766 | text("Invalid FTP URL for delete ", url)); | |
767 | conn.url = t[0]; | |
768 | ||
769 | enforce!CurlException(!t[1].empty, | |
770 | text("No filename specified to delete for URL ", url)); | |
771 | conn.addCommand("DELE " ~ t[1]); | |
772 | conn.perform(); | |
773 | } | |
774 | else | |
775 | { | |
776 | if (isFTPUrl(url)) | |
777 | return del!FTP(url, FTP()); | |
778 | else | |
779 | return del!HTTP(url, HTTP()); | |
780 | } | |
781 | } | |
782 | ||
783 | @system unittest | |
784 | { | |
785 | import std.algorithm.searching : canFind; | |
786 | ||
787 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
788 | { | |
789 | testServer.handle((s) { | |
790 | auto req = s.recvReq; | |
791 | assert(req.hdrs.canFind("DELETE /path")); | |
792 | s.send(httpOK()); | |
793 | }); | |
794 | del(host ~ "/path"); | |
795 | } | |
796 | } | |
797 | ||
798 | ||
799 | /** HTTP options request. | |
800 | * | |
801 | * Params: | |
802 | * url = resource make a option call to | |
803 | * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will | |
804 | * guess connection type and create a new instance for this call only. | |
805 | * | |
806 | * The template parameter $(D T) specifies the type to return. Possible values | |
807 | * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). | |
808 | * | |
809 | * Example: | |
810 | * ---- | |
811 | * import std.net.curl; | |
812 | * auto http = HTTP(); | |
813 | * options("d-lang.appspot.com/testUrl2", http); | |
814 | * writeln("Allow set to " ~ http.responseHeaders["Allow"]); | |
815 | * ---- | |
816 | * | |
817 | * Returns: | |
818 | * A T[] range containing the options of the resource pointed to by the URL. | |
819 | * | |
820 | * See_Also: $(LREF HTTP.Method) | |
821 | */ | |
822 | T[] options(T = char)(const(char)[] url, HTTP conn = HTTP()) | |
823 | if (is(T == char) || is(T == ubyte)) | |
824 | { | |
825 | conn.method = HTTP.Method.options; | |
826 | return _basicHTTP!(T)(url, null, conn); | |
827 | } | |
828 | ||
829 | @system unittest | |
830 | { | |
831 | import std.algorithm.searching : canFind; | |
832 | ||
833 | testServer.handle((s) { | |
834 | auto req = s.recvReq; | |
835 | assert(req.hdrs.canFind("OPTIONS /path")); | |
836 | s.send(httpOK("OPTIONSRESPONSE")); | |
837 | }); | |
838 | auto res = options(testServer.addr ~ "/path"); | |
839 | assert(res == "OPTIONSRESPONSE"); | |
840 | } | |
841 | ||
842 | ||
843 | /** HTTP trace request. | |
844 | * | |
845 | * Params: | |
846 | * url = resource make a trace call to | |
847 | * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will | |
848 | * guess connection type and create a new instance for this call only. | |
849 | * | |
850 | * The template parameter $(D T) specifies the type to return. Possible values | |
851 | * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). | |
852 | * | |
853 | * Example: | |
854 | * ---- | |
855 | * import std.net.curl; | |
856 | * trace("d-lang.appspot.com/testUrl1"); | |
857 | * ---- | |
858 | * | |
859 | * Returns: | |
860 | * A T[] range containing the trace info of the resource pointed to by the URL. | |
861 | * | |
862 | * See_Also: $(LREF HTTP.Method) | |
863 | */ | |
864 | T[] trace(T = char)(const(char)[] url, HTTP conn = HTTP()) | |
865 | if (is(T == char) || is(T == ubyte)) | |
866 | { | |
867 | conn.method = HTTP.Method.trace; | |
868 | return _basicHTTP!(T)(url, cast(void[]) null, conn); | |
869 | } | |
870 | ||
871 | @system unittest | |
872 | { | |
873 | import std.algorithm.searching : canFind; | |
874 | ||
875 | testServer.handle((s) { | |
876 | auto req = s.recvReq; | |
877 | assert(req.hdrs.canFind("TRACE /path")); | |
878 | s.send(httpOK("TRACERESPONSE")); | |
879 | }); | |
880 | auto res = trace(testServer.addr ~ "/path"); | |
881 | assert(res == "TRACERESPONSE"); | |
882 | } | |
883 | ||
884 | ||
885 | /** HTTP connect request. | |
886 | * | |
887 | * Params: | |
888 | * url = resource make a connect to | |
889 | * conn = HTTP connection to use | |
890 | * | |
891 | * The template parameter $(D T) specifies the type to return. Possible values | |
892 | * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). | |
893 | * | |
894 | * Example: | |
895 | * ---- | |
896 | * import std.net.curl; | |
897 | * connect("d-lang.appspot.com/testUrl1"); | |
898 | * ---- | |
899 | * | |
900 | * Returns: | |
901 | * A T[] range containing the connect info of the resource pointed to by the URL. | |
902 | * | |
903 | * See_Also: $(LREF HTTP.Method) | |
904 | */ | |
905 | T[] connect(T = char)(const(char)[] url, HTTP conn = HTTP()) | |
906 | if (is(T == char) || is(T == ubyte)) | |
907 | { | |
908 | conn.method = HTTP.Method.connect; | |
909 | return _basicHTTP!(T)(url, cast(void[]) null, conn); | |
910 | } | |
911 | ||
912 | @system unittest | |
913 | { | |
914 | import std.algorithm.searching : canFind; | |
915 | ||
916 | testServer.handle((s) { | |
917 | auto req = s.recvReq; | |
918 | assert(req.hdrs.canFind("CONNECT /path")); | |
919 | s.send(httpOK("CONNECTRESPONSE")); | |
920 | }); | |
921 | auto res = connect(testServer.addr ~ "/path"); | |
922 | assert(res == "CONNECTRESPONSE"); | |
923 | } | |
924 | ||
925 | ||
926 | /** HTTP patch content. | |
927 | * | |
928 | * Params: | |
929 | * url = resource to patch | |
930 | * patchData = data to send as the body of the request. An array | |
931 | * of an arbitrary type is accepted and will be cast to ubyte[] | |
932 | * before sending it. | |
933 | * conn = HTTP connection to use | |
934 | * | |
935 | * The template parameter $(D T) specifies the type to return. Possible values | |
936 | * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). | |
937 | * | |
938 | * Example: | |
939 | * ---- | |
940 | * auto http = HTTP(); | |
941 | * http.addRequestHeader("Content-Type", "application/json"); | |
942 | * auto content = patch("d-lang.appspot.com/testUrl2", `{"title": "Patched Title"}`, http); | |
943 | * ---- | |
944 | * | |
945 | * Returns: | |
946 | * A T[] range containing the content of the resource pointed to by the URL. | |
947 | * | |
948 | * See_Also: $(LREF HTTP.Method) | |
949 | */ | |
950 | T[] patch(T = char, PatchUnit)(const(char)[] url, const(PatchUnit)[] patchData, | |
951 | HTTP conn = HTTP()) | |
952 | if (is(T == char) || is(T == ubyte)) | |
953 | { | |
954 | conn.method = HTTP.Method.patch; | |
955 | return _basicHTTP!(T)(url, patchData, conn); | |
956 | } | |
957 | ||
958 | @system unittest | |
959 | { | |
960 | import std.algorithm.searching : canFind; | |
961 | ||
962 | testServer.handle((s) { | |
963 | auto req = s.recvReq; | |
964 | assert(req.hdrs.canFind("PATCH /path")); | |
965 | assert(req.bdy.canFind("PATCHBODY")); | |
966 | s.send(httpOK("PATCHRESPONSE")); | |
967 | }); | |
968 | auto res = patch(testServer.addr ~ "/path", "PATCHBODY"); | |
969 | assert(res == "PATCHRESPONSE"); | |
970 | } | |
971 | ||
972 | ||
973 | /* | |
974 | * Helper function for the high level interface. | |
975 | * | |
976 | * It performs an HTTP request using the client which must have | |
977 | * been setup correctly before calling this function. | |
978 | */ | |
979 | private auto _basicHTTP(T)(const(char)[] url, const(void)[] sendData, HTTP client) | |
980 | { | |
981 | import std.algorithm.comparison : min; | |
982 | import std.format : format; | |
983 | ||
984 | immutable doSend = sendData !is null && | |
985 | (client.method == HTTP.Method.post || | |
986 | client.method == HTTP.Method.put || | |
987 | client.method == HTTP.Method.patch); | |
988 | ||
989 | scope (exit) | |
990 | { | |
991 | client.onReceiveHeader = null; | |
992 | client.onReceiveStatusLine = null; | |
993 | client.onReceive = null; | |
994 | ||
995 | if (doSend) | |
996 | { | |
997 | client.onSend = null; | |
998 | client.handle.onSeek = null; | |
999 | client.contentLength = 0; | |
1000 | } | |
1001 | } | |
1002 | client.url = url; | |
1003 | HTTP.StatusLine statusLine; | |
1004 | import std.array : appender; | |
1005 | auto content = appender!(ubyte[])(); | |
1006 | client.onReceive = (ubyte[] data) | |
1007 | { | |
1008 | content ~= data; | |
1009 | return data.length; | |
1010 | }; | |
1011 | ||
1012 | if (doSend) | |
1013 | { | |
1014 | client.contentLength = sendData.length; | |
1015 | auto remainingData = sendData; | |
1016 | client.onSend = delegate size_t(void[] buf) | |
1017 | { | |
1018 | size_t minLen = min(buf.length, remainingData.length); | |
1019 | if (minLen == 0) return 0; | |
1020 | buf[0 .. minLen] = remainingData[0 .. minLen]; | |
1021 | remainingData = remainingData[minLen..$]; | |
1022 | return minLen; | |
1023 | }; | |
1024 | client.handle.onSeek = delegate(long offset, CurlSeekPos mode) | |
1025 | { | |
1026 | switch (mode) | |
1027 | { | |
1028 | case CurlSeekPos.set: | |
1029 | remainingData = sendData[cast(size_t) offset..$]; | |
1030 | return CurlSeek.ok; | |
1031 | default: | |
1032 | // As of curl 7.18.0, libcurl will not pass | |
1033 | // anything other than CurlSeekPos.set. | |
1034 | return CurlSeek.cantseek; | |
1035 | } | |
1036 | }; | |
1037 | } | |
1038 | ||
1039 | client.onReceiveHeader = (in char[] key, | |
1040 | in char[] value) | |
1041 | { | |
1042 | if (key == "content-length") | |
1043 | { | |
1044 | import std.conv : to; | |
1045 | content.reserve(value.to!size_t); | |
1046 | } | |
1047 | }; | |
1048 | client.onReceiveStatusLine = (HTTP.StatusLine l) { statusLine = l; }; | |
1049 | client.perform(); | |
1050 | enforce(statusLine.code / 100 == 2, new HTTPStatusException(statusLine.code, | |
1051 | format("HTTP request returned status code %d (%s)", statusLine.code, statusLine.reason))); | |
1052 | ||
1053 | return _decodeContent!T(content.data, client.p.charset); | |
1054 | } | |
1055 | ||
1056 | @system unittest | |
1057 | { | |
1058 | import std.algorithm.searching : canFind; | |
1059 | ||
1060 | testServer.handle((s) { | |
1061 | auto req = s.recvReq; | |
1062 | assert(req.hdrs.canFind("GET /path")); | |
1063 | s.send(httpNotFound()); | |
1064 | }); | |
1065 | auto e = collectException!HTTPStatusException(get(testServer.addr ~ "/path")); | |
1066 | assert(e.msg == "HTTP request returned status code 404 (Not Found)"); | |
1067 | assert(e.status == 404); | |
1068 | } | |
1069 | ||
1070 | // Bugzilla 14760 - content length must be reset after post | |
1071 | @system unittest | |
1072 | { | |
1073 | import std.algorithm.searching : canFind; | |
1074 | ||
1075 | testServer.handle((s) { | |
1076 | auto req = s.recvReq; | |
1077 | assert(req.hdrs.canFind("POST /")); | |
1078 | assert(req.bdy.canFind("POSTBODY")); | |
1079 | s.send(httpOK("POSTRESPONSE")); | |
1080 | ||
1081 | req = s.recvReq; | |
1082 | assert(req.hdrs.canFind("TRACE /")); | |
1083 | assert(req.bdy.empty); | |
1084 | s.blocking = false; | |
1085 | ubyte[6] buf = void; | |
1086 | assert(s.receive(buf[]) < 0); | |
1087 | s.send(httpOK("TRACERESPONSE")); | |
1088 | }); | |
1089 | auto http = HTTP(); | |
1090 | auto res = post(testServer.addr, "POSTBODY", http); | |
1091 | assert(res == "POSTRESPONSE"); | |
1092 | res = trace(testServer.addr, http); | |
1093 | assert(res == "TRACERESPONSE"); | |
1094 | } | |
1095 | ||
1096 | @system unittest // charset detection and transcoding to T | |
1097 | { | |
1098 | testServer.handle((s) { | |
1099 | s.send("HTTP/1.1 200 OK\r\n"~ | |
1100 | "Content-Length: 4\r\n"~ | |
1101 | "Content-Type: text/plain; charset=utf-8\r\n" ~ | |
1102 | "\r\n" ~ | |
1103 | "äbc"); | |
1104 | }); | |
1105 | auto client = HTTP(); | |
1106 | auto result = _basicHTTP!char(testServer.addr, "", client); | |
1107 | assert(result == "äbc"); | |
1108 | ||
1109 | testServer.handle((s) { | |
1110 | s.send("HTTP/1.1 200 OK\r\n"~ | |
1111 | "Content-Length: 3\r\n"~ | |
1112 | "Content-Type: text/plain; charset=iso-8859-1\r\n" ~ | |
1113 | "\r\n" ~ | |
1114 | 0xE4 ~ "bc"); | |
1115 | }); | |
1116 | client = HTTP(); | |
1117 | result = _basicHTTP!char(testServer.addr, "", client); | |
1118 | assert(result == "äbc"); | |
1119 | } | |
1120 | ||
1121 | /* | |
1122 | * Helper function for the high level interface. | |
1123 | * | |
1124 | * It performs an FTP request using the client which must have | |
1125 | * been setup correctly before calling this function. | |
1126 | */ | |
1127 | private auto _basicFTP(T)(const(char)[] url, const(void)[] sendData, FTP client) | |
1128 | { | |
1129 | import std.algorithm.comparison : min; | |
1130 | ||
1131 | scope (exit) | |
1132 | { | |
1133 | client.onReceive = null; | |
1134 | if (!sendData.empty) | |
1135 | client.onSend = null; | |
1136 | } | |
1137 | ||
1138 | ubyte[] content; | |
1139 | ||
1140 | if (client.encoding.empty) | |
1141 | client.encoding = "ISO-8859-1"; | |
1142 | ||
1143 | client.url = url; | |
1144 | client.onReceive = (ubyte[] data) | |
1145 | { | |
1146 | content ~= data; | |
1147 | return data.length; | |
1148 | }; | |
1149 | ||
1150 | if (!sendData.empty) | |
1151 | { | |
1152 | client.handle.set(CurlOption.upload, 1L); | |
1153 | client.onSend = delegate size_t(void[] buf) | |
1154 | { | |
1155 | size_t minLen = min(buf.length, sendData.length); | |
1156 | if (minLen == 0) return 0; | |
1157 | buf[0 .. minLen] = sendData[0 .. minLen]; | |
1158 | sendData = sendData[minLen..$]; | |
1159 | return minLen; | |
1160 | }; | |
1161 | } | |
1162 | ||
1163 | client.perform(); | |
1164 | ||
1165 | return _decodeContent!T(content, client.encoding); | |
1166 | } | |
1167 | ||
1168 | /* Used by _basicHTTP() and _basicFTP() to decode ubyte[] to | |
1169 | * correct string format | |
1170 | */ | |
1171 | private auto _decodeContent(T)(ubyte[] content, string encoding) | |
1172 | { | |
1173 | static if (is(T == ubyte)) | |
1174 | { | |
1175 | return content; | |
1176 | } | |
1177 | else | |
1178 | { | |
1179 | import std.format : format; | |
1180 | ||
1181 | // Optimally just return the utf8 encoded content | |
1182 | if (encoding == "UTF-8") | |
1183 | return cast(char[])(content); | |
1184 | ||
1185 | // The content has to be re-encoded to utf8 | |
1186 | auto scheme = EncodingScheme.create(encoding); | |
1187 | enforce!CurlException(scheme !is null, | |
1188 | format("Unknown encoding '%s'", encoding)); | |
1189 | ||
1190 | auto strInfo = decodeString(content, scheme); | |
1191 | enforce!CurlException(strInfo[0] != size_t.max, | |
1192 | format("Invalid encoding sequence for encoding '%s'", | |
1193 | encoding)); | |
1194 | ||
1195 | return strInfo[1]; | |
1196 | } | |
1197 | } | |
1198 | ||
1199 | alias KeepTerminator = Flag!"keepTerminator"; | |
1200 | /+ | |
1201 | struct ByLineBuffer(Char) | |
1202 | { | |
1203 | bool linePresent; | |
1204 | bool EOF; | |
1205 | Char[] buffer; | |
1206 | ubyte[] decodeRemainder; | |
1207 | ||
1208 | bool append(const(ubyte)[] data) | |
1209 | { | |
1210 | byLineBuffer ~= data; | |
1211 | } | |
1212 | ||
1213 | @property bool linePresent() | |
1214 | { | |
1215 | return byLinePresent; | |
1216 | } | |
1217 | ||
1218 | Char[] get() | |
1219 | { | |
1220 | if (!linePresent) | |
1221 | { | |
1222 | // Decode ubyte[] into Char[] until a Terminator is found. | |
1223 | // If not Terminator is found and EOF is false then raise an | |
1224 | // exception. | |
1225 | } | |
1226 | return byLineBuffer; | |
1227 | } | |
1228 | ||
1229 | } | |
1230 | ++/ | |
1231 | /** HTTP/FTP fetch content as a range of lines. | |
1232 | * | |
1233 | * A range of lines is returned when the request is complete. If the method or | |
1234 | * other request properties is to be customized then set the $(D conn) parameter | |
1235 | * with a HTTP/FTP instance that has these properties set. | |
1236 | * | |
1237 | * Example: | |
1238 | * ---- | |
1239 | * import std.net.curl, std.stdio; | |
1240 | * foreach (line; byLine("dlang.org")) | |
1241 | * writeln(line); | |
1242 | * ---- | |
1243 | * | |
1244 | * Params: | |
1245 | * url = The url to receive content from | |
1246 | * keepTerminator = $(D Yes.keepTerminator) signals that the line terminator should be | |
1247 | * returned as part of the lines in the range. | |
1248 | * terminator = The character that terminates a line | |
1249 | * conn = The connection to use e.g. HTTP or FTP. | |
1250 | * | |
1251 | * Returns: | |
1252 | * A range of Char[] with the content of the resource pointer to by the URL | |
1253 | */ | |
1254 | auto byLine(Conn = AutoProtocol, Terminator = char, Char = char) | |
1255 | (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator, | |
1256 | Terminator terminator = '\n', Conn conn = Conn()) | |
1257 | if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator) | |
1258 | { | |
1259 | static struct SyncLineInputRange | |
1260 | { | |
1261 | ||
1262 | private Char[] lines; | |
1263 | private Char[] current; | |
1264 | private bool currentValid; | |
1265 | private bool keepTerminator; | |
1266 | private Terminator terminator; | |
1267 | ||
1268 | this(Char[] lines, bool kt, Terminator terminator) | |
1269 | { | |
1270 | this.lines = lines; | |
1271 | this.keepTerminator = kt; | |
1272 | this.terminator = terminator; | |
1273 | currentValid = true; | |
1274 | popFront(); | |
1275 | } | |
1276 | ||
1277 | @property @safe bool empty() | |
1278 | { | |
1279 | return !currentValid; | |
1280 | } | |
1281 | ||
1282 | @property @safe Char[] front() | |
1283 | { | |
1284 | enforce!CurlException(currentValid, "Cannot call front() on empty range"); | |
1285 | return current; | |
1286 | } | |
1287 | ||
1288 | void popFront() | |
1289 | { | |
1290 | import std.algorithm.searching : findSplitAfter, findSplit; | |
1291 | ||
1292 | enforce!CurlException(currentValid, "Cannot call popFront() on empty range"); | |
1293 | if (lines.empty) | |
1294 | { | |
1295 | currentValid = false; | |
1296 | return; | |
1297 | } | |
1298 | ||
1299 | if (keepTerminator) | |
1300 | { | |
1301 | auto r = findSplitAfter(lines, [ terminator ]); | |
1302 | if (r[0].empty) | |
1303 | { | |
1304 | current = r[1]; | |
1305 | lines = r[0]; | |
1306 | } | |
1307 | else | |
1308 | { | |
1309 | current = r[0]; | |
1310 | lines = r[1]; | |
1311 | } | |
1312 | } | |
1313 | else | |
1314 | { | |
1315 | auto r = findSplit(lines, [ terminator ]); | |
1316 | current = r[0]; | |
1317 | lines = r[2]; | |
1318 | } | |
1319 | } | |
1320 | } | |
1321 | ||
1322 | auto result = _getForRange!Char(url, conn); | |
1323 | return SyncLineInputRange(result, keepTerminator == Yes.keepTerminator, terminator); | |
1324 | } | |
1325 | ||
1326 | @system unittest | |
1327 | { | |
1328 | import std.algorithm.comparison : equal; | |
1329 | ||
1330 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
1331 | { | |
1332 | testServer.handle((s) { | |
1333 | auto req = s.recvReq; | |
1334 | s.send(httpOK("Line1\nLine2\nLine3")); | |
1335 | }); | |
1336 | assert(byLine(host).equal(["Line1", "Line2", "Line3"])); | |
1337 | } | |
1338 | } | |
1339 | ||
1340 | /** HTTP/FTP fetch content as a range of chunks. | |
1341 | * | |
1342 | * A range of chunks is returned when the request is complete. If the method or | |
1343 | * other request properties is to be customized then set the $(D conn) parameter | |
1344 | * with a HTTP/FTP instance that has these properties set. | |
1345 | * | |
1346 | * Example: | |
1347 | * ---- | |
1348 | * import std.net.curl, std.stdio; | |
1349 | * foreach (chunk; byChunk("dlang.org", 100)) | |
1350 | * writeln(chunk); // chunk is ubyte[100] | |
1351 | * ---- | |
1352 | * | |
1353 | * Params: | |
1354 | * url = The url to receive content from | |
1355 | * chunkSize = The size of each chunk | |
1356 | * conn = The connection to use e.g. HTTP or FTP. | |
1357 | * | |
1358 | * Returns: | |
1359 | * A range of ubyte[chunkSize] with the content of the resource pointer to by the URL | |
1360 | */ | |
1361 | auto byChunk(Conn = AutoProtocol) | |
1362 | (const(char)[] url, size_t chunkSize = 1024, Conn conn = Conn()) | |
1363 | if (isCurlConn!(Conn)) | |
1364 | { | |
1365 | static struct SyncChunkInputRange | |
1366 | { | |
1367 | private size_t chunkSize; | |
1368 | private ubyte[] _bytes; | |
1369 | private size_t offset; | |
1370 | ||
1371 | this(ubyte[] bytes, size_t chunkSize) | |
1372 | { | |
1373 | this._bytes = bytes; | |
1374 | this.chunkSize = chunkSize; | |
1375 | } | |
1376 | ||
1377 | @property @safe auto empty() | |
1378 | { | |
1379 | return offset == _bytes.length; | |
1380 | } | |
1381 | ||
1382 | @property ubyte[] front() | |
1383 | { | |
1384 | size_t nextOffset = offset + chunkSize; | |
1385 | if (nextOffset > _bytes.length) nextOffset = _bytes.length; | |
1386 | return _bytes[offset .. nextOffset]; | |
1387 | } | |
1388 | ||
1389 | @safe void popFront() | |
1390 | { | |
1391 | offset += chunkSize; | |
1392 | if (offset > _bytes.length) offset = _bytes.length; | |
1393 | } | |
1394 | } | |
1395 | ||
1396 | auto result = _getForRange!ubyte(url, conn); | |
1397 | return SyncChunkInputRange(result, chunkSize); | |
1398 | } | |
1399 | ||
1400 | @system unittest | |
1401 | { | |
1402 | import std.algorithm.comparison : equal; | |
1403 | ||
1404 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
1405 | { | |
1406 | testServer.handle((s) { | |
1407 | auto req = s.recvReq; | |
1408 | s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5])); | |
1409 | }); | |
1410 | assert(byChunk(host, 2).equal([[0, 1], [2, 3], [4, 5]])); | |
1411 | } | |
1412 | } | |
1413 | ||
1414 | private T[] _getForRange(T,Conn)(const(char)[] url, Conn conn) | |
1415 | { | |
1416 | static if (is(Conn : HTTP)) | |
1417 | { | |
1418 | conn.method = conn.method == HTTP.Method.undefined ? HTTP.Method.get : conn.method; | |
1419 | return _basicHTTP!(T)(url, null, conn); | |
1420 | } | |
1421 | else static if (is(Conn : FTP)) | |
1422 | { | |
1423 | return _basicFTP!(T)(url, null, conn); | |
1424 | } | |
1425 | else | |
1426 | { | |
1427 | if (isFTPUrl(url)) | |
1428 | return get!(FTP,T)(url, FTP()); | |
1429 | else | |
1430 | return get!(HTTP,T)(url, HTTP()); | |
1431 | } | |
1432 | } | |
1433 | ||
1434 | /* | |
1435 | Main thread part of the message passing protocol used for all async | |
1436 | curl protocols. | |
1437 | */ | |
1438 | private mixin template WorkerThreadProtocol(Unit, alias units) | |
1439 | { | |
1440 | @property bool empty() | |
1441 | { | |
1442 | tryEnsureUnits(); | |
1443 | return state == State.done; | |
1444 | } | |
1445 | ||
1446 | @property Unit[] front() | |
1447 | { | |
1448 | import std.format : format; | |
1449 | tryEnsureUnits(); | |
1450 | assert(state == State.gotUnits, | |
1451 | format("Expected %s but got $s", | |
1452 | State.gotUnits, state)); | |
1453 | return units; | |
1454 | } | |
1455 | ||
1456 | void popFront() | |
1457 | { | |
1458 | import std.format : format; | |
1459 | tryEnsureUnits(); | |
1460 | assert(state == State.gotUnits, | |
1461 | format("Expected %s but got $s", | |
1462 | State.gotUnits, state)); | |
1463 | state = State.needUnits; | |
1464 | // Send to worker thread for buffer reuse | |
1465 | workerTid.send(cast(immutable(Unit)[]) units); | |
1466 | units = null; | |
1467 | } | |
1468 | ||
1469 | /** Wait for duration or until data is available and return true if data is | |
1470 | available | |
1471 | */ | |
1472 | bool wait(Duration d) | |
1473 | { | |
1474 | import std.datetime.stopwatch : StopWatch; | |
1475 | ||
1476 | if (state == State.gotUnits) | |
1477 | return true; | |
1478 | ||
1479 | enum noDur = dur!"hnsecs"(0); | |
1480 | StopWatch sw; | |
1481 | sw.start(); | |
1482 | while (state != State.gotUnits && d > noDur) | |
1483 | { | |
1484 | final switch (state) | |
1485 | { | |
1486 | case State.needUnits: | |
1487 | receiveTimeout(d, | |
1488 | (Tid origin, CurlMessage!(immutable(Unit)[]) _data) | |
1489 | { | |
1490 | if (origin != workerTid) | |
1491 | return false; | |
1492 | units = cast(Unit[]) _data.data; | |
1493 | state = State.gotUnits; | |
1494 | return true; | |
1495 | }, | |
1496 | (Tid origin, CurlMessage!bool f) | |
1497 | { | |
1498 | if (origin != workerTid) | |
1499 | return false; | |
1500 | state = state.done; | |
1501 | return true; | |
1502 | } | |
1503 | ); | |
1504 | break; | |
1505 | case State.gotUnits: return true; | |
1506 | case State.done: | |
1507 | return false; | |
1508 | } | |
1509 | d -= sw.peek(); | |
1510 | sw.reset(); | |
1511 | } | |
1512 | return state == State.gotUnits; | |
1513 | } | |
1514 | ||
1515 | enum State | |
1516 | { | |
1517 | needUnits, | |
1518 | gotUnits, | |
1519 | done | |
1520 | } | |
1521 | State state; | |
1522 | ||
1523 | void tryEnsureUnits() | |
1524 | { | |
1525 | while (true) | |
1526 | { | |
1527 | final switch (state) | |
1528 | { | |
1529 | case State.needUnits: | |
1530 | receive( | |
1531 | (Tid origin, CurlMessage!(immutable(Unit)[]) _data) | |
1532 | { | |
1533 | if (origin != workerTid) | |
1534 | return false; | |
1535 | units = cast(Unit[]) _data.data; | |
1536 | state = State.gotUnits; | |
1537 | return true; | |
1538 | }, | |
1539 | (Tid origin, CurlMessage!bool f) | |
1540 | { | |
1541 | if (origin != workerTid) | |
1542 | return false; | |
1543 | state = state.done; | |
1544 | return true; | |
1545 | } | |
1546 | ); | |
1547 | break; | |
1548 | case State.gotUnits: return; | |
1549 | case State.done: | |
1550 | return; | |
1551 | } | |
1552 | } | |
1553 | } | |
1554 | } | |
1555 | ||
1556 | // @@@@BUG 15831@@@@ | |
1557 | // this should be inside byLineAsync | |
1558 | // Range that reads one line at a time asynchronously. | |
1559 | private static struct AsyncLineInputRange(Char) | |
1560 | { | |
1561 | private Char[] line; | |
1562 | mixin WorkerThreadProtocol!(Char, line); | |
1563 | ||
1564 | private Tid workerTid; | |
1565 | private State running; | |
1566 | ||
1567 | private this(Tid tid, size_t transmitBuffers, size_t bufferSize) | |
1568 | { | |
1569 | workerTid = tid; | |
1570 | state = State.needUnits; | |
1571 | ||
1572 | // Send buffers to other thread for it to use. Since no mechanism is in | |
1573 | // place for moving ownership a cast to shared is done here and casted | |
1574 | // back to non-shared in the receiving end. | |
1575 | foreach (i ; 0 .. transmitBuffers) | |
1576 | { | |
1577 | auto arr = new Char[](bufferSize); | |
1578 | workerTid.send(cast(immutable(Char[]))arr); | |
1579 | } | |
1580 | } | |
1581 | } | |
1582 | ||
1583 | /** HTTP/FTP fetch content as a range of lines asynchronously. | |
1584 | * | |
1585 | * A range of lines is returned immediately and the request that fetches the | |
1586 | * lines is performed in another thread. If the method or other request | |
1587 | * properties is to be customized then set the $(D conn) parameter with a | |
1588 | * HTTP/FTP instance that has these properties set. | |
1589 | * | |
1590 | * If $(D postData) is non-_null the method will be set to $(D post) for HTTP | |
1591 | * requests. | |
1592 | * | |
1593 | * The background thread will buffer up to transmitBuffers number of lines | |
1594 | * before it stops receiving data from network. When the main thread reads the | |
1595 | * lines from the range it frees up buffers and allows for the background thread | |
1596 | * to receive more data from the network. | |
1597 | * | |
1598 | * If no data is available and the main thread accesses the range it will block | |
1599 | * until data becomes available. An exception to this is the $(D wait(Duration)) method on | |
1600 | * the $(LREF AsyncLineInputRange). This method will wait at maximum for the | |
1601 | * specified duration and return true if data is available. | |
1602 | * | |
1603 | * Example: | |
1604 | * ---- | |
1605 | * import std.net.curl, std.stdio; | |
1606 | * // Get some pages in the background | |
1607 | * auto range1 = byLineAsync("www.google.com"); | |
1608 | * auto range2 = byLineAsync("www.wikipedia.org"); | |
1609 | * foreach (line; byLineAsync("dlang.org")) | |
1610 | * writeln(line); | |
1611 | * | |
1612 | * // Lines already fetched in the background and ready | |
1613 | * foreach (line; range1) writeln(line); | |
1614 | * foreach (line; range2) writeln(line); | |
1615 | * ---- | |
1616 | * | |
1617 | * ---- | |
1618 | * import std.net.curl, std.stdio; | |
1619 | * // Get a line in a background thread and wait in | |
1620 | * // main thread for 2 seconds for it to arrive. | |
1621 | * auto range3 = byLineAsync("dlang.com"); | |
1622 | * if (range3.wait(dur!"seconds"(2))) | |
1623 | * writeln(range3.front); | |
1624 | * else | |
1625 | * writeln("No line received after 2 seconds!"); | |
1626 | * ---- | |
1627 | * | |
1628 | * Params: | |
1629 | * url = The url to receive content from | |
1630 | * postData = Data to HTTP Post | |
1631 | * keepTerminator = $(D Yes.keepTerminator) signals that the line terminator should be | |
1632 | * returned as part of the lines in the range. | |
1633 | * terminator = The character that terminates a line | |
1634 | * transmitBuffers = The number of lines buffered asynchronously | |
1635 | * conn = The connection to use e.g. HTTP or FTP. | |
1636 | * | |
1637 | * Returns: | |
1638 | * A range of Char[] with the content of the resource pointer to by the | |
1639 | * URL. | |
1640 | */ | |
1641 | auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char, PostUnit) | |
1642 | (const(char)[] url, const(PostUnit)[] postData, | |
1643 | KeepTerminator keepTerminator = No.keepTerminator, | |
1644 | Terminator terminator = '\n', | |
1645 | size_t transmitBuffers = 10, Conn conn = Conn()) | |
1646 | if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator) | |
1647 | { | |
1648 | static if (is(Conn : AutoProtocol)) | |
1649 | { | |
1650 | if (isFTPUrl(url)) | |
1651 | return byLineAsync(url, postData, keepTerminator, | |
1652 | terminator, transmitBuffers, FTP()); | |
1653 | else | |
1654 | return byLineAsync(url, postData, keepTerminator, | |
1655 | terminator, transmitBuffers, HTTP()); | |
1656 | } | |
1657 | else | |
1658 | { | |
1659 | // 50 is just an arbitrary number for now | |
1660 | setMaxMailboxSize(thisTid, 50, OnCrowding.block); | |
1661 | auto tid = spawn(&_spawnAsync!(Conn, Char, Terminator)); | |
1662 | tid.send(thisTid); | |
1663 | tid.send(terminator); | |
1664 | tid.send(keepTerminator == Yes.keepTerminator); | |
1665 | ||
1666 | _asyncDuplicateConnection(url, conn, postData, tid); | |
1667 | ||
1668 | return AsyncLineInputRange!Char(tid, transmitBuffers, | |
1669 | Conn.defaultAsyncStringBufferSize); | |
1670 | } | |
1671 | } | |
1672 | ||
1673 | /// ditto | |
1674 | auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char) | |
1675 | (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator, | |
1676 | Terminator terminator = '\n', | |
1677 | size_t transmitBuffers = 10, Conn conn = Conn()) | |
1678 | { | |
1679 | static if (is(Conn : AutoProtocol)) | |
1680 | { | |
1681 | if (isFTPUrl(url)) | |
1682 | return byLineAsync(url, cast(void[]) null, keepTerminator, | |
1683 | terminator, transmitBuffers, FTP()); | |
1684 | else | |
1685 | return byLineAsync(url, cast(void[]) null, keepTerminator, | |
1686 | terminator, transmitBuffers, HTTP()); | |
1687 | } | |
1688 | else | |
1689 | { | |
1690 | return byLineAsync(url, cast(void[]) null, keepTerminator, | |
1691 | terminator, transmitBuffers, conn); | |
1692 | } | |
1693 | } | |
1694 | ||
1695 | @system unittest | |
1696 | { | |
1697 | import std.algorithm.comparison : equal; | |
1698 | ||
1699 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
1700 | { | |
1701 | testServer.handle((s) { | |
1702 | auto req = s.recvReq; | |
1703 | s.send(httpOK("Line1\nLine2\nLine3")); | |
1704 | }); | |
1705 | assert(byLineAsync(host).equal(["Line1", "Line2", "Line3"])); | |
1706 | } | |
1707 | } | |
1708 | ||
1709 | // @@@@BUG 15831@@@@ | |
1710 | // this should be inside byLineAsync | |
1711 | // Range that reads one chunk at a time asynchronously. | |
1712 | private static struct AsyncChunkInputRange | |
1713 | { | |
1714 | private ubyte[] chunk; | |
1715 | mixin WorkerThreadProtocol!(ubyte, chunk); | |
1716 | ||
1717 | private Tid workerTid; | |
1718 | private State running; | |
1719 | ||
1720 | private this(Tid tid, size_t transmitBuffers, size_t chunkSize) | |
1721 | { | |
1722 | workerTid = tid; | |
1723 | state = State.needUnits; | |
1724 | ||
1725 | // Send buffers to other thread for it to use. Since no mechanism is in | |
1726 | // place for moving ownership a cast to shared is done here and a cast | |
1727 | // back to non-shared in the receiving end. | |
1728 | foreach (i ; 0 .. transmitBuffers) | |
1729 | { | |
1730 | ubyte[] arr = new ubyte[](chunkSize); | |
1731 | workerTid.send(cast(immutable(ubyte[]))arr); | |
1732 | } | |
1733 | } | |
1734 | } | |
1735 | ||
1736 | /** HTTP/FTP fetch content as a range of chunks asynchronously. | |
1737 | * | |
1738 | * A range of chunks is returned immediately and the request that fetches the | |
1739 | * chunks is performed in another thread. If the method or other request | |
1740 | * properties is to be customized then set the $(D conn) parameter with a | |
1741 | * HTTP/FTP instance that has these properties set. | |
1742 | * | |
1743 | * If $(D postData) is non-_null the method will be set to $(D post) for HTTP | |
1744 | * requests. | |
1745 | * | |
1746 | * The background thread will buffer up to transmitBuffers number of chunks | |
1747 | * before is stops receiving data from network. When the main thread reads the | |
1748 | * chunks from the range it frees up buffers and allows for the background | |
1749 | * thread to receive more data from the network. | |
1750 | * | |
1751 | * If no data is available and the main thread access the range it will block | |
1752 | * until data becomes available. An exception to this is the $(D wait(Duration)) | |
1753 | * method on the $(LREF AsyncChunkInputRange). This method will wait at maximum for the specified | |
1754 | * duration and return true if data is available. | |
1755 | * | |
1756 | * Example: | |
1757 | * ---- | |
1758 | * import std.net.curl, std.stdio; | |
1759 | * // Get some pages in the background | |
1760 | * auto range1 = byChunkAsync("www.google.com", 100); | |
1761 | * auto range2 = byChunkAsync("www.wikipedia.org"); | |
1762 | * foreach (chunk; byChunkAsync("dlang.org")) | |
1763 | * writeln(chunk); // chunk is ubyte[100] | |
1764 | * | |
1765 | * // Chunks already fetched in the background and ready | |
1766 | * foreach (chunk; range1) writeln(chunk); | |
1767 | * foreach (chunk; range2) writeln(chunk); | |
1768 | * ---- | |
1769 | * | |
1770 | * ---- | |
1771 | * import std.net.curl, std.stdio; | |
1772 | * // Get a line in a background thread and wait in | |
1773 | * // main thread for 2 seconds for it to arrive. | |
1774 | * auto range3 = byChunkAsync("dlang.com", 10); | |
1775 | * if (range3.wait(dur!"seconds"(2))) | |
1776 | * writeln(range3.front); | |
1777 | * else | |
1778 | * writeln("No chunk received after 2 seconds!"); | |
1779 | * ---- | |
1780 | * | |
1781 | * Params: | |
1782 | * url = The url to receive content from | |
1783 | * postData = Data to HTTP Post | |
1784 | * chunkSize = The size of the chunks | |
1785 | * transmitBuffers = The number of chunks buffered asynchronously | |
1786 | * conn = The connection to use e.g. HTTP or FTP. | |
1787 | * | |
1788 | * Returns: | |
1789 | * A range of ubyte[chunkSize] with the content of the resource pointer to by | |
1790 | * the URL. | |
1791 | */ | |
1792 | auto byChunkAsync(Conn = AutoProtocol, PostUnit) | |
1793 | (const(char)[] url, const(PostUnit)[] postData, | |
1794 | size_t chunkSize = 1024, size_t transmitBuffers = 10, | |
1795 | Conn conn = Conn()) | |
1796 | if (isCurlConn!(Conn)) | |
1797 | { | |
1798 | static if (is(Conn : AutoProtocol)) | |
1799 | { | |
1800 | if (isFTPUrl(url)) | |
1801 | return byChunkAsync(url, postData, chunkSize, | |
1802 | transmitBuffers, FTP()); | |
1803 | else | |
1804 | return byChunkAsync(url, postData, chunkSize, | |
1805 | transmitBuffers, HTTP()); | |
1806 | } | |
1807 | else | |
1808 | { | |
1809 | // 50 is just an arbitrary number for now | |
1810 | setMaxMailboxSize(thisTid, 50, OnCrowding.block); | |
1811 | auto tid = spawn(&_spawnAsync!(Conn, ubyte)); | |
1812 | tid.send(thisTid); | |
1813 | ||
1814 | _asyncDuplicateConnection(url, conn, postData, tid); | |
1815 | ||
1816 | return AsyncChunkInputRange(tid, transmitBuffers, chunkSize); | |
1817 | } | |
1818 | } | |
1819 | ||
1820 | /// ditto | |
1821 | auto byChunkAsync(Conn = AutoProtocol) | |
1822 | (const(char)[] url, | |
1823 | size_t chunkSize = 1024, size_t transmitBuffers = 10, | |
1824 | Conn conn = Conn()) | |
1825 | if (isCurlConn!(Conn)) | |
1826 | { | |
1827 | static if (is(Conn : AutoProtocol)) | |
1828 | { | |
1829 | if (isFTPUrl(url)) | |
1830 | return byChunkAsync(url, cast(void[]) null, chunkSize, | |
1831 | transmitBuffers, FTP()); | |
1832 | else | |
1833 | return byChunkAsync(url, cast(void[]) null, chunkSize, | |
1834 | transmitBuffers, HTTP()); | |
1835 | } | |
1836 | else | |
1837 | { | |
1838 | return byChunkAsync(url, cast(void[]) null, chunkSize, | |
1839 | transmitBuffers, conn); | |
1840 | } | |
1841 | } | |
1842 | ||
1843 | @system unittest | |
1844 | { | |
1845 | import std.algorithm.comparison : equal; | |
1846 | ||
1847 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
1848 | { | |
1849 | testServer.handle((s) { | |
1850 | auto req = s.recvReq; | |
1851 | s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5])); | |
1852 | }); | |
1853 | assert(byChunkAsync(host, 2).equal([[0, 1], [2, 3], [4, 5]])); | |
1854 | } | |
1855 | } | |
1856 | ||
1857 | ||
1858 | /* Used by byLineAsync/byChunkAsync to duplicate an existing connection | |
1859 | * that can be used exclusively in a spawned thread. | |
1860 | */ | |
1861 | private void _asyncDuplicateConnection(Conn, PostData) | |
1862 | (const(char)[] url, Conn conn, PostData postData, Tid tid) | |
1863 | { | |
1864 | // no move semantic available in std.concurrency ie. must use casting. | |
1865 | auto connDup = conn.dup(); | |
1866 | connDup.url = url; | |
1867 | ||
1868 | static if ( is(Conn : HTTP) ) | |
1869 | { | |
1870 | connDup.p.headersOut = null; | |
1871 | connDup.method = conn.method == HTTP.Method.undefined ? | |
1872 | HTTP.Method.get : conn.method; | |
1873 | if (postData !is null) | |
1874 | { | |
1875 | if (connDup.method == HTTP.Method.put) | |
1876 | { | |
1877 | connDup.handle.set(CurlOption.infilesize_large, | |
1878 | postData.length); | |
1879 | } | |
1880 | else | |
1881 | { | |
1882 | // post | |
1883 | connDup.method = HTTP.Method.post; | |
1884 | connDup.handle.set(CurlOption.postfieldsize_large, | |
1885 | postData.length); | |
1886 | } | |
1887 | connDup.handle.set(CurlOption.copypostfields, | |
1888 | cast(void*) postData.ptr); | |
1889 | } | |
1890 | tid.send(cast(ulong) connDup.handle.handle); | |
1891 | tid.send(connDup.method); | |
1892 | } | |
1893 | else | |
1894 | { | |
1895 | enforce!CurlException(postData is null, | |
1896 | "Cannot put ftp data using byLineAsync()"); | |
1897 | tid.send(cast(ulong) connDup.handle.handle); | |
1898 | tid.send(HTTP.Method.undefined); | |
1899 | } | |
1900 | connDup.p.curl.handle = null; // make sure handle is not freed | |
1901 | } | |
1902 | ||
1903 | /* | |
1904 | Mixin template for all supported curl protocols. This is the commom | |
1905 | functionallity such as timeouts and network interface settings. This should | |
1906 | really be in the HTTP/FTP/SMTP structs but the documentation tool does not | |
1907 | support a mixin to put its doc strings where a mixin is done. Therefore docs | |
1908 | in this template is copied into each of HTTP/FTP/SMTP below. | |
1909 | */ | |
1910 | private mixin template Protocol() | |
1911 | { | |
1912 | ||
1913 | /// Value to return from $(D onSend)/$(D onReceive) delegates in order to | |
1914 | /// pause a request | |
1915 | alias requestPause = CurlReadFunc.pause; | |
1916 | ||
1917 | /// Value to return from onSend delegate in order to abort a request | |
1918 | alias requestAbort = CurlReadFunc.abort; | |
1919 | ||
1920 | static uint defaultAsyncStringBufferSize = 100; | |
1921 | ||
1922 | /** | |
1923 | The curl handle used by this connection. | |
1924 | */ | |
1925 | @property ref Curl handle() return | |
1926 | { | |
1927 | return p.curl; | |
1928 | } | |
1929 | ||
1930 | /** | |
1931 | True if the instance is stopped. A stopped instance is not usable. | |
1932 | */ | |
1933 | @property bool isStopped() | |
1934 | { | |
1935 | return p.curl.stopped; | |
1936 | } | |
1937 | ||
1938 | /// Stop and invalidate this instance. | |
1939 | void shutdown() | |
1940 | { | |
1941 | p.curl.shutdown(); | |
1942 | } | |
1943 | ||
1944 | /** Set verbose. | |
1945 | This will print request information to stderr. | |
1946 | */ | |
1947 | @property void verbose(bool on) | |
1948 | { | |
1949 | p.curl.set(CurlOption.verbose, on ? 1L : 0L); | |
1950 | } | |
1951 | ||
1952 | // Connection settings | |
1953 | ||
1954 | /// Set timeout for activity on connection. | |
1955 | @property void dataTimeout(Duration d) | |
1956 | { | |
1957 | p.curl.set(CurlOption.low_speed_limit, 1); | |
1958 | p.curl.set(CurlOption.low_speed_time, d.total!"seconds"); | |
1959 | } | |
1960 | ||
1961 | /** Set maximum time an operation is allowed to take. | |
1962 | This includes dns resolution, connecting, data transfer, etc. | |
1963 | */ | |
1964 | @property void operationTimeout(Duration d) | |
1965 | { | |
1966 | p.curl.set(CurlOption.timeout_ms, d.total!"msecs"); | |
1967 | } | |
1968 | ||
1969 | /// Set timeout for connecting. | |
1970 | @property void connectTimeout(Duration d) | |
1971 | { | |
1972 | p.curl.set(CurlOption.connecttimeout_ms, d.total!"msecs"); | |
1973 | } | |
1974 | ||
1975 | // Network settings | |
1976 | ||
1977 | /** Proxy | |
1978 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) | |
1979 | */ | |
1980 | @property void proxy(const(char)[] host) | |
1981 | { | |
1982 | p.curl.set(CurlOption.proxy, host); | |
1983 | } | |
1984 | ||
1985 | /** Proxy port | |
1986 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) | |
1987 | */ | |
1988 | @property void proxyPort(ushort port) | |
1989 | { | |
1990 | p.curl.set(CurlOption.proxyport, cast(long) port); | |
1991 | } | |
1992 | ||
1993 | /// Type of proxy | |
1994 | alias CurlProxy = etc.c.curl.CurlProxy; | |
1995 | ||
1996 | /** Proxy type | |
1997 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) | |
1998 | */ | |
1999 | @property void proxyType(CurlProxy type) | |
2000 | { | |
2001 | p.curl.set(CurlOption.proxytype, cast(long) type); | |
2002 | } | |
2003 | ||
2004 | /// DNS lookup timeout. | |
2005 | @property void dnsTimeout(Duration d) | |
2006 | { | |
2007 | p.curl.set(CurlOption.dns_cache_timeout, d.total!"msecs"); | |
2008 | } | |
2009 | ||
2010 | /** | |
2011 | * The network interface to use in form of the the IP of the interface. | |
2012 | * | |
2013 | * Example: | |
2014 | * ---- | |
2015 | * theprotocol.netInterface = "192.168.1.32"; | |
2016 | * theprotocol.netInterface = [ 192, 168, 1, 32 ]; | |
2017 | * ---- | |
2018 | * | |
2019 | * See: $(REF InternetAddress, std,socket) | |
2020 | */ | |
2021 | @property void netInterface(const(char)[] i) | |
2022 | { | |
2023 | p.curl.set(CurlOption.intrface, i); | |
2024 | } | |
2025 | ||
2026 | /// ditto | |
2027 | @property void netInterface(const(ubyte)[4] i) | |
2028 | { | |
2029 | import std.format : format; | |
2030 | const str = format("%d.%d.%d.%d", i[0], i[1], i[2], i[3]); | |
2031 | netInterface = str; | |
2032 | } | |
2033 | ||
2034 | /// ditto | |
2035 | @property void netInterface(InternetAddress i) | |
2036 | { | |
2037 | netInterface = i.toAddrString(); | |
2038 | } | |
2039 | ||
2040 | /** | |
2041 | Set the local outgoing port to use. | |
2042 | Params: | |
2043 | port = the first outgoing port number to try and use | |
2044 | */ | |
2045 | @property void localPort(ushort port) | |
2046 | { | |
2047 | p.curl.set(CurlOption.localport, cast(long) port); | |
2048 | } | |
2049 | ||
2050 | /** | |
2051 | Set the no proxy flag for the specified host names. | |
2052 | Params: | |
2053 | test = a list of comma host names that do not require | |
2054 | proxy to get reached | |
2055 | */ | |
2056 | void setNoProxy(string hosts) | |
2057 | { | |
2058 | p.curl.set(CurlOption.noproxy, hosts); | |
2059 | } | |
2060 | ||
2061 | /** | |
2062 | Set the local outgoing port range to use. | |
2063 | This can be used together with the localPort property. | |
2064 | Params: | |
2065 | range = if the first port is occupied then try this many | |
2066 | port number forwards | |
2067 | */ | |
2068 | @property void localPortRange(ushort range) | |
2069 | { | |
2070 | p.curl.set(CurlOption.localportrange, cast(long) range); | |
2071 | } | |
2072 | ||
2073 | /** Set the tcp no-delay socket option on or off. | |
2074 | See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) | |
2075 | */ | |
2076 | @property void tcpNoDelay(bool on) | |
2077 | { | |
2078 | p.curl.set(CurlOption.tcp_nodelay, cast(long) (on ? 1 : 0) ); | |
2079 | } | |
2080 | ||
2081 | /** Sets whether SSL peer certificates should be verified. | |
2082 | See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYPEER, verifypeer) | |
2083 | */ | |
2084 | @property void verifyPeer(bool on) | |
2085 | { | |
2086 | p.curl.set(CurlOption.ssl_verifypeer, on ? 1 : 0); | |
2087 | } | |
2088 | ||
2089 | /** Sets whether the host within an SSL certificate should be verified. | |
2090 | See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYHOST, verifypeer) | |
2091 | */ | |
2092 | @property void verifyHost(bool on) | |
2093 | { | |
2094 | p.curl.set(CurlOption.ssl_verifyhost, on ? 2 : 0); | |
2095 | } | |
2096 | ||
2097 | // Authentication settings | |
2098 | ||
2099 | /** | |
2100 | Set the user name, password and optionally domain for authentication | |
2101 | purposes. | |
2102 | ||
2103 | Some protocols may need authentication in some cases. Use this | |
2104 | function to provide credentials. | |
2105 | ||
2106 | Params: | |
2107 | username = the username | |
2108 | password = the password | |
2109 | domain = used for NTLM authentication only and is set to the NTLM domain | |
2110 | name | |
2111 | */ | |
2112 | void setAuthentication(const(char)[] username, const(char)[] password, | |
2113 | const(char)[] domain = "") | |
2114 | { | |
2115 | import std.format : format; | |
2116 | if (!domain.empty) | |
2117 | username = format("%s/%s", domain, username); | |
2118 | p.curl.set(CurlOption.userpwd, format("%s:%s", username, password)); | |
2119 | } | |
2120 | ||
2121 | @system unittest | |
2122 | { | |
2123 | import std.algorithm.searching : canFind; | |
2124 | ||
2125 | testServer.handle((s) { | |
2126 | auto req = s.recvReq; | |
2127 | assert(req.hdrs.canFind("GET /")); | |
2128 | assert(req.hdrs.canFind("Basic dXNlcjpwYXNz")); | |
2129 | s.send(httpOK()); | |
2130 | }); | |
2131 | ||
2132 | auto http = HTTP(testServer.addr); | |
2133 | http.onReceive = (ubyte[] data) { return data.length; }; | |
2134 | http.setAuthentication("user", "pass"); | |
2135 | http.perform(); | |
2136 | ||
2137 | // Bugzilla 17540 | |
2138 | http.setNoProxy("www.example.com"); | |
2139 | } | |
2140 | ||
2141 | /** | |
2142 | Set the user name and password for proxy authentication. | |
2143 | ||
2144 | Params: | |
2145 | username = the username | |
2146 | password = the password | |
2147 | */ | |
2148 | void setProxyAuthentication(const(char)[] username, const(char)[] password) | |
2149 | { | |
2150 | import std.array : replace; | |
2151 | import std.format : format; | |
2152 | ||
2153 | p.curl.set(CurlOption.proxyuserpwd, | |
2154 | format("%s:%s", | |
2155 | username.replace(":", "%3A"), | |
2156 | password.replace(":", "%3A")) | |
2157 | ); | |
2158 | } | |
2159 | ||
2160 | /** | |
2161 | * The event handler that gets called when data is needed for sending. The | |
2162 | * length of the $(D void[]) specifies the maximum number of bytes that can | |
2163 | * be sent. | |
2164 | * | |
2165 | * Returns: | |
2166 | * The callback returns the number of elements in the buffer that have been | |
2167 | * filled and are ready to send. | |
2168 | * The special value $(D .abortRequest) can be returned in order to abort the | |
2169 | * current request. | |
2170 | * The special value $(D .pauseRequest) can be returned in order to pause the | |
2171 | * current request. | |
2172 | * | |
2173 | * Example: | |
2174 | * ---- | |
2175 | * import std.net.curl; | |
2176 | * string msg = "Hello world"; | |
2177 | * auto client = HTTP("dlang.org"); | |
2178 | * client.onSend = delegate size_t(void[] data) | |
2179 | * { | |
2180 | * auto m = cast(void[]) msg; | |
2181 | * size_t length = m.length > data.length ? data.length : m.length; | |
2182 | * if (length == 0) return 0; | |
2183 | * data[0 .. length] = m[0 .. length]; | |
2184 | * msg = msg[length..$]; | |
2185 | * return length; | |
2186 | * }; | |
2187 | * client.perform(); | |
2188 | * ---- | |
2189 | */ | |
2190 | @property void onSend(size_t delegate(void[]) callback) | |
2191 | { | |
2192 | p.curl.clear(CurlOption.postfields); // cannot specify data when using callback | |
2193 | p.curl.onSend = callback; | |
2194 | } | |
2195 | ||
2196 | /** | |
2197 | * The event handler that receives incoming data. Be sure to copy the | |
2198 | * incoming ubyte[] since it is not guaranteed to be valid after the | |
2199 | * callback returns. | |
2200 | * | |
2201 | * Returns: | |
2202 | * The callback returns the number of incoming bytes read. If the entire array is | |
2203 | * not read the request will abort. | |
2204 | * The special value .pauseRequest can be returned in order to pause the | |
2205 | * current request. | |
2206 | * | |
2207 | * Example: | |
2208 | * ---- | |
2209 | * import std.net.curl, std.stdio; | |
2210 | * auto client = HTTP("dlang.org"); | |
2211 | * client.onReceive = (ubyte[] data) | |
2212 | * { | |
2213 | * writeln("Got data", to!(const(char)[])(data)); | |
2214 | * return data.length; | |
2215 | * }; | |
2216 | * client.perform(); | |
2217 | * ---- | |
2218 | */ | |
2219 | @property void onReceive(size_t delegate(ubyte[]) callback) | |
2220 | { | |
2221 | p.curl.onReceive = callback; | |
2222 | } | |
2223 | ||
2224 | /** | |
2225 | * The event handler that gets called to inform of upload/download progress. | |
2226 | * | |
2227 | * Params: | |
2228 | * dlTotal = total bytes to download | |
2229 | * dlNow = currently downloaded bytes | |
2230 | * ulTotal = total bytes to upload | |
2231 | * ulNow = currently uploaded bytes | |
2232 | * | |
2233 | * Returns: | |
2234 | * Return 0 from the callback to signal success, return non-zero to abort | |
2235 | * transfer | |
2236 | * | |
2237 | * Example: | |
2238 | * ---- | |
2239 | * import std.net.curl, std.stdio; | |
2240 | * auto client = HTTP("dlang.org"); | |
2241 | * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t ult) | |
2242 | * { | |
2243 | * writeln("Progress: downloaded ", dln, " of ", dl); | |
2244 | * writeln("Progress: uploaded ", uln, " of ", ul); | |
2245 | * }; | |
2246 | * client.perform(); | |
2247 | * ---- | |
2248 | */ | |
2249 | @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, | |
2250 | size_t ulTotal, size_t ulNow) callback) | |
2251 | { | |
2252 | p.curl.onProgress = callback; | |
2253 | } | |
2254 | } | |
2255 | ||
2256 | /* | |
2257 | Decode $(D ubyte[]) array using the provided EncodingScheme up to maxChars | |
2258 | Returns: Tuple of ubytes read and the $(D Char[]) characters decoded. | |
2259 | Not all ubytes are guaranteed to be read in case of decoding error. | |
2260 | */ | |
2261 | private Tuple!(size_t,Char[]) | |
2262 | decodeString(Char = char)(const(ubyte)[] data, | |
2263 | EncodingScheme scheme, | |
2264 | size_t maxChars = size_t.max) | |
2265 | { | |
2266 | Char[] res; | |
2267 | immutable startLen = data.length; | |
2268 | size_t charsDecoded = 0; | |
2269 | while (data.length && charsDecoded < maxChars) | |
2270 | { | |
2271 | immutable dchar dc = scheme.safeDecode(data); | |
2272 | if (dc == INVALID_SEQUENCE) | |
2273 | { | |
2274 | return typeof(return)(size_t.max, cast(Char[]) null); | |
2275 | } | |
2276 | charsDecoded++; | |
2277 | res ~= dc; | |
2278 | } | |
2279 | return typeof(return)(startLen-data.length, res); | |
2280 | } | |
2281 | ||
2282 | /* | |
2283 | Decode $(D ubyte[]) array using the provided $(D EncodingScheme) until a the | |
2284 | line terminator specified is found. The basesrc parameter is effectively | |
2285 | prepended to src as the first thing. | |
2286 | ||
2287 | This function is used for decoding as much of the src buffer as | |
2288 | possible until either the terminator is found or decoding fails. If | |
2289 | it fails as the last data in the src it may mean that the src buffer | |
2290 | were missing some bytes in order to represent a correct code | |
2291 | point. Upon the next call to this function more bytes have been | |
2292 | received from net and the failing bytes should be given as the | |
2293 | basesrc parameter. It is done this way to minimize data copying. | |
2294 | ||
2295 | Returns: true if a terminator was found | |
2296 | Not all ubytes are guaranteed to be read in case of decoding error. | |
2297 | any decoded chars will be inserted into dst. | |
2298 | */ | |
2299 | private bool decodeLineInto(Terminator, Char = char)(ref const(ubyte)[] basesrc, | |
2300 | ref const(ubyte)[] src, | |
2301 | ref Char[] dst, | |
2302 | EncodingScheme scheme, | |
2303 | Terminator terminator) | |
2304 | { | |
2305 | import std.algorithm.searching : endsWith; | |
2306 | ||
2307 | // if there is anything in the basesrc then try to decode that | |
2308 | // first. | |
2309 | if (basesrc.length != 0) | |
2310 | { | |
2311 | // Try to ensure 4 entries in the basesrc by copying from src. | |
2312 | immutable blen = basesrc.length; | |
2313 | immutable len = (basesrc.length + src.length) >= 4 ? | |
2314 | 4 : basesrc.length + src.length; | |
2315 | basesrc.length = len; | |
2316 | ||
2317 | immutable dchar dc = scheme.safeDecode(basesrc); | |
2318 | if (dc == INVALID_SEQUENCE) | |
2319 | { | |
2320 | enforce!CurlException(len != 4, "Invalid code sequence"); | |
2321 | return false; | |
2322 | } | |
2323 | dst ~= dc; | |
2324 | src = src[len-basesrc.length-blen .. $]; // remove used ubytes from src | |
2325 | basesrc.length = 0; | |
2326 | } | |
2327 | ||
2328 | while (src.length) | |
2329 | { | |
2330 | const lsrc = src; | |
2331 | dchar dc = scheme.safeDecode(src); | |
2332 | if (dc == INVALID_SEQUENCE) | |
2333 | { | |
2334 | if (src.empty) | |
2335 | { | |
2336 | // The invalid sequence was in the end of the src. Maybe there | |
2337 | // just need to be more bytes available so these last bytes are | |
2338 | // put back to src for later use. | |
2339 | src = lsrc; | |
2340 | return false; | |
2341 | } | |
2342 | dc = '?'; | |
2343 | } | |
2344 | dst ~= dc; | |
2345 | ||
2346 | if (dst.endsWith(terminator)) | |
2347 | return true; | |
2348 | } | |
2349 | return false; // no terminator found | |
2350 | } | |
2351 | ||
2352 | /** | |
2353 | * HTTP client functionality. | |
2354 | * | |
2355 | * Example: | |
2356 | * --- | |
2357 | * import std.net.curl, std.stdio; | |
2358 | * | |
2359 | * // Get with custom data receivers | |
2360 | * auto http = HTTP("dlang.org"); | |
2361 | * http.onReceiveHeader = | |
2362 | * (in char[] key, in char[] value) { writeln(key ~ ": " ~ value); }; | |
2363 | * http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; }; | |
2364 | * http.perform(); | |
2365 | * | |
2366 | * // Put with data senders | |
2367 | * auto msg = "Hello world"; | |
2368 | * http.contentLength = msg.length; | |
2369 | * http.onSend = (void[] data) | |
2370 | * { | |
2371 | * auto m = cast(void[]) msg; | |
2372 | * size_t len = m.length > data.length ? data.length : m.length; | |
2373 | * if (len == 0) return len; | |
2374 | * data[0 .. len] = m[0 .. len]; | |
2375 | * msg = msg[len..$]; | |
2376 | * return len; | |
2377 | * }; | |
2378 | * http.perform(); | |
2379 | * | |
2380 | * // Track progress | |
2381 | * http.method = HTTP.Method.get; | |
2382 | * http.url = "http://upload.wikimedia.org/wikipedia/commons/" | |
2383 | * "5/53/Wikipedia-logo-en-big.png"; | |
2384 | * http.onReceive = (ubyte[] data) { return data.length; }; | |
2385 | * http.onProgress = (size_t dltotal, size_t dlnow, | |
2386 | * size_t ultotal, size_t ulnow) | |
2387 | * { | |
2388 | * writeln("Progress ", dltotal, ", ", dlnow, ", ", ultotal, ", ", ulnow); | |
2389 | * return 0; | |
2390 | * }; | |
2391 | * http.perform(); | |
2392 | * --- | |
2393 | * | |
2394 | * See_Also: $(_HTTP www.ietf.org/rfc/rfc2616.txt, RFC2616) | |
2395 | * | |
2396 | */ | |
2397 | struct HTTP | |
2398 | { | |
2399 | mixin Protocol; | |
2400 | ||
2401 | import std.datetime.systime : SysTime; | |
2402 | ||
2403 | /// Authentication method equal to $(REF CurlAuth, etc,c,curl) | |
2404 | alias AuthMethod = CurlAuth; | |
2405 | ||
2406 | static private uint defaultMaxRedirects = 10; | |
2407 | ||
2408 | private struct Impl | |
2409 | { | |
2410 | ~this() | |
2411 | { | |
2412 | if (headersOut !is null) | |
2413 | Curl.curl.slist_free_all(headersOut); | |
2414 | if (curl.handle !is null) // work around RefCounted/emplace bug | |
2415 | curl.shutdown(); | |
2416 | } | |
2417 | Curl curl; | |
2418 | curl_slist* headersOut; | |
2419 | string[string] headersIn; | |
2420 | string charset; | |
2421 | ||
2422 | /// The status line of the final sub-request in a request. | |
2423 | StatusLine status; | |
2424 | private void delegate(StatusLine) onReceiveStatusLine; | |
2425 | ||
2426 | /// The HTTP method to use. | |
2427 | Method method = Method.undefined; | |
2428 | ||
2429 | @system @property void onReceiveHeader(void delegate(in char[] key, | |
2430 | in char[] value) callback) | |
2431 | { | |
2432 | import std.algorithm.searching : startsWith; | |
2433 | import std.conv : to; | |
2434 | import std.regex : regex, match; | |
2435 | import std.uni : toLower; | |
2436 | ||
2437 | // Wrap incoming callback in order to separate http status line from | |
2438 | // http headers. On redirected requests there may be several such | |
2439 | // status lines. The last one is the one recorded. | |
2440 | auto dg = (in char[] header) | |
2441 | { | |
2442 | import std.utf : UTFException; | |
2443 | try | |
2444 | { | |
2445 | if (header.empty) | |
2446 | { | |
2447 | // header delimiter | |
2448 | return; | |
2449 | } | |
2450 | if (header.startsWith("HTTP/")) | |
2451 | { | |
2452 | headersIn.clear(); | |
2453 | ||
2454 | const m = match(header, regex(r"^HTTP/(\d+)\.(\d+) (\d+) (.*)$")); | |
2455 | if (m.empty) | |
2456 | { | |
2457 | // Invalid status line | |
2458 | } | |
2459 | else | |
2460 | { | |
2461 | status.majorVersion = to!ushort(m.captures[1]); | |
2462 | status.minorVersion = to!ushort(m.captures[2]); | |
2463 | status.code = to!ushort(m.captures[3]); | |
2464 | status.reason = m.captures[4].idup; | |
2465 | if (onReceiveStatusLine != null) | |
2466 | onReceiveStatusLine(status); | |
2467 | } | |
2468 | return; | |
2469 | } | |
2470 | ||
2471 | // Normal http header | |
2472 | auto m = match(cast(char[]) header, regex("(.*?): (.*)$")); | |
2473 | ||
2474 | auto fieldName = m.captures[1].toLower().idup; | |
2475 | if (fieldName == "content-type") | |
2476 | { | |
2477 | auto mct = match(cast(char[]) m.captures[2], | |
2478 | regex("charset=([^;]*)", "i")); | |
2479 | if (!mct.empty && mct.captures.length > 1) | |
2480 | charset = mct.captures[1].idup; | |
2481 | } | |
2482 | ||
2483 | if (!m.empty && callback !is null) | |
2484 | callback(fieldName, m.captures[2]); | |
2485 | headersIn[fieldName] = m.captures[2].idup; | |
2486 | } | |
2487 | catch (UTFException e) | |
2488 | { | |
2489 | //munch it - a header should be all ASCII, any "wrong UTF" is broken header | |
2490 | } | |
2491 | }; | |
2492 | ||
2493 | curl.onReceiveHeader = dg; | |
2494 | } | |
2495 | } | |
2496 | ||
2497 | private RefCounted!Impl p; | |
2498 | ||
2499 | /** Time condition enumeration as an alias of $(REF CurlTimeCond, etc,c,curl) | |
2500 | ||
2501 | $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25) | |
2502 | */ | |
2503 | alias TimeCond = CurlTimeCond; | |
2504 | ||
2505 | /** | |
2506 | Constructor taking the url as parameter. | |
2507 | */ | |
2508 | static HTTP opCall(const(char)[] url) | |
2509 | { | |
2510 | HTTP http; | |
2511 | http.initialize(); | |
2512 | http.url = url; | |
2513 | return http; | |
2514 | } | |
2515 | ||
2516 | /// | |
2517 | static HTTP opCall() | |
2518 | { | |
2519 | HTTP http; | |
2520 | http.initialize(); | |
2521 | return http; | |
2522 | } | |
2523 | ||
2524 | /// | |
2525 | HTTP dup() | |
2526 | { | |
2527 | HTTP copy; | |
2528 | copy.initialize(); | |
2529 | copy.p.method = p.method; | |
2530 | curl_slist* cur = p.headersOut; | |
2531 | curl_slist* newlist = null; | |
2532 | while (cur) | |
2533 | { | |
2534 | newlist = Curl.curl.slist_append(newlist, cur.data); | |
2535 | cur = cur.next; | |
2536 | } | |
2537 | copy.p.headersOut = newlist; | |
2538 | copy.p.curl.set(CurlOption.httpheader, copy.p.headersOut); | |
2539 | copy.p.curl = p.curl.dup(); | |
2540 | copy.dataTimeout = _defaultDataTimeout; | |
2541 | copy.onReceiveHeader = null; | |
2542 | return copy; | |
2543 | } | |
2544 | ||
2545 | private void initialize() | |
2546 | { | |
2547 | p.curl.initialize(); | |
2548 | maxRedirects = HTTP.defaultMaxRedirects; | |
2549 | p.charset = "ISO-8859-1"; // Default charset defined in HTTP RFC | |
2550 | p.method = Method.undefined; | |
2551 | setUserAgent(HTTP.defaultUserAgent); | |
2552 | dataTimeout = _defaultDataTimeout; | |
2553 | onReceiveHeader = null; | |
2554 | verifyPeer = true; | |
2555 | verifyHost = true; | |
2556 | } | |
2557 | ||
2558 | /** | |
2559 | Perform a http request. | |
2560 | ||
2561 | After the HTTP client has been setup and possibly assigned callbacks the | |
2562 | $(D perform()) method will start performing the request towards the | |
2563 | specified server. | |
2564 | ||
2565 | Params: | |
2566 | throwOnError = whether to throw an exception or return a CurlCode on error | |
2567 | */ | |
2568 | CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) | |
2569 | { | |
2570 | p.status.reset(); | |
2571 | ||
2572 | CurlOption opt; | |
2573 | final switch (p.method) | |
2574 | { | |
2575 | case Method.head: | |
2576 | p.curl.set(CurlOption.nobody, 1L); | |
2577 | opt = CurlOption.nobody; | |
2578 | break; | |
2579 | case Method.undefined: | |
2580 | case Method.get: | |
2581 | p.curl.set(CurlOption.httpget, 1L); | |
2582 | opt = CurlOption.httpget; | |
2583 | break; | |
2584 | case Method.post: | |
2585 | p.curl.set(CurlOption.post, 1L); | |
2586 | opt = CurlOption.post; | |
2587 | break; | |
2588 | case Method.put: | |
2589 | p.curl.set(CurlOption.upload, 1L); | |
2590 | opt = CurlOption.upload; | |
2591 | break; | |
2592 | case Method.del: | |
2593 | p.curl.set(CurlOption.customrequest, "DELETE"); | |
2594 | opt = CurlOption.customrequest; | |
2595 | break; | |
2596 | case Method.options: | |
2597 | p.curl.set(CurlOption.customrequest, "OPTIONS"); | |
2598 | opt = CurlOption.customrequest; | |
2599 | break; | |
2600 | case Method.trace: | |
2601 | p.curl.set(CurlOption.customrequest, "TRACE"); | |
2602 | opt = CurlOption.customrequest; | |
2603 | break; | |
2604 | case Method.connect: | |
2605 | p.curl.set(CurlOption.customrequest, "CONNECT"); | |
2606 | opt = CurlOption.customrequest; | |
2607 | break; | |
2608 | case Method.patch: | |
2609 | p.curl.set(CurlOption.customrequest, "PATCH"); | |
2610 | opt = CurlOption.customrequest; | |
2611 | break; | |
2612 | } | |
2613 | ||
2614 | scope (exit) p.curl.clear(opt); | |
2615 | return p.curl.perform(throwOnError); | |
2616 | } | |
2617 | ||
2618 | /// The URL to specify the location of the resource. | |
2619 | @property void url(const(char)[] url) | |
2620 | { | |
2621 | import std.algorithm.searching : startsWith; | |
2622 | import std.uni : toLower; | |
2623 | if (!startsWith(url.toLower(), "http://", "https://")) | |
2624 | url = "http://" ~ url; | |
2625 | p.curl.set(CurlOption.url, url); | |
2626 | } | |
2627 | ||
2628 | /// Set the CA certificate bundle file to use for SSL peer verification | |
2629 | @property void caInfo(const(char)[] caFile) | |
2630 | { | |
2631 | p.curl.set(CurlOption.cainfo, caFile); | |
2632 | } | |
2633 | ||
2634 | // This is a workaround for mixed in content not having its | |
2635 | // docs mixed in. | |
2636 | version (StdDdoc) | |
2637 | { | |
2638 | /// Value to return from $(D onSend)/$(D onReceive) delegates in order to | |
2639 | /// pause a request | |
2640 | alias requestPause = CurlReadFunc.pause; | |
2641 | ||
2642 | /// Value to return from onSend delegate in order to abort a request | |
2643 | alias requestAbort = CurlReadFunc.abort; | |
2644 | ||
2645 | /** | |
2646 | True if the instance is stopped. A stopped instance is not usable. | |
2647 | */ | |
2648 | @property bool isStopped(); | |
2649 | ||
2650 | /// Stop and invalidate this instance. | |
2651 | void shutdown(); | |
2652 | ||
2653 | /** Set verbose. | |
2654 | This will print request information to stderr. | |
2655 | */ | |
2656 | @property void verbose(bool on); | |
2657 | ||
2658 | // Connection settings | |
2659 | ||
2660 | /// Set timeout for activity on connection. | |
2661 | @property void dataTimeout(Duration d); | |
2662 | ||
2663 | /** Set maximum time an operation is allowed to take. | |
2664 | This includes dns resolution, connecting, data transfer, etc. | |
2665 | */ | |
2666 | @property void operationTimeout(Duration d); | |
2667 | ||
2668 | /// Set timeout for connecting. | |
2669 | @property void connectTimeout(Duration d); | |
2670 | ||
2671 | // Network settings | |
2672 | ||
2673 | /** Proxy | |
2674 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) | |
2675 | */ | |
2676 | @property void proxy(const(char)[] host); | |
2677 | ||
2678 | /** Proxy port | |
2679 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) | |
2680 | */ | |
2681 | @property void proxyPort(ushort port); | |
2682 | ||
2683 | /// Type of proxy | |
2684 | alias CurlProxy = etc.c.curl.CurlProxy; | |
2685 | ||
2686 | /** Proxy type | |
2687 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) | |
2688 | */ | |
2689 | @property void proxyType(CurlProxy type); | |
2690 | ||
2691 | /// DNS lookup timeout. | |
2692 | @property void dnsTimeout(Duration d); | |
2693 | ||
2694 | /** | |
2695 | * The network interface to use in form of the the IP of the interface. | |
2696 | * | |
2697 | * Example: | |
2698 | * ---- | |
2699 | * theprotocol.netInterface = "192.168.1.32"; | |
2700 | * theprotocol.netInterface = [ 192, 168, 1, 32 ]; | |
2701 | * ---- | |
2702 | * | |
2703 | * See: $(REF InternetAddress, std,socket) | |
2704 | */ | |
2705 | @property void netInterface(const(char)[] i); | |
2706 | ||
2707 | /// ditto | |
2708 | @property void netInterface(const(ubyte)[4] i); | |
2709 | ||
2710 | /// ditto | |
2711 | @property void netInterface(InternetAddress i); | |
2712 | ||
2713 | /** | |
2714 | Set the local outgoing port to use. | |
2715 | Params: | |
2716 | port = the first outgoing port number to try and use | |
2717 | */ | |
2718 | @property void localPort(ushort port); | |
2719 | ||
2720 | /** | |
2721 | Set the local outgoing port range to use. | |
2722 | This can be used together with the localPort property. | |
2723 | Params: | |
2724 | range = if the first port is occupied then try this many | |
2725 | port number forwards | |
2726 | */ | |
2727 | @property void localPortRange(ushort range); | |
2728 | ||
2729 | /** Set the tcp no-delay socket option on or off. | |
2730 | See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) | |
2731 | */ | |
2732 | @property void tcpNoDelay(bool on); | |
2733 | ||
2734 | // Authentication settings | |
2735 | ||
2736 | /** | |
2737 | Set the user name, password and optionally domain for authentication | |
2738 | purposes. | |
2739 | ||
2740 | Some protocols may need authentication in some cases. Use this | |
2741 | function to provide credentials. | |
2742 | ||
2743 | Params: | |
2744 | username = the username | |
2745 | password = the password | |
2746 | domain = used for NTLM authentication only and is set to the NTLM domain | |
2747 | name | |
2748 | */ | |
2749 | void setAuthentication(const(char)[] username, const(char)[] password, | |
2750 | const(char)[] domain = ""); | |
2751 | ||
2752 | /** | |
2753 | Set the user name and password for proxy authentication. | |
2754 | ||
2755 | Params: | |
2756 | username = the username | |
2757 | password = the password | |
2758 | */ | |
2759 | void setProxyAuthentication(const(char)[] username, const(char)[] password); | |
2760 | ||
2761 | /** | |
2762 | * The event handler that gets called when data is needed for sending. The | |
2763 | * length of the $(D void[]) specifies the maximum number of bytes that can | |
2764 | * be sent. | |
2765 | * | |
2766 | * Returns: | |
2767 | * The callback returns the number of elements in the buffer that have been | |
2768 | * filled and are ready to send. | |
2769 | * The special value $(D .abortRequest) can be returned in order to abort the | |
2770 | * current request. | |
2771 | * The special value $(D .pauseRequest) can be returned in order to pause the | |
2772 | * current request. | |
2773 | * | |
2774 | * Example: | |
2775 | * ---- | |
2776 | * import std.net.curl; | |
2777 | * string msg = "Hello world"; | |
2778 | * auto client = HTTP("dlang.org"); | |
2779 | * client.onSend = delegate size_t(void[] data) | |
2780 | * { | |
2781 | * auto m = cast(void[]) msg; | |
2782 | * size_t length = m.length > data.length ? data.length : m.length; | |
2783 | * if (length == 0) return 0; | |
2784 | * data[0 .. length] = m[0 .. length]; | |
2785 | * msg = msg[length..$]; | |
2786 | * return length; | |
2787 | * }; | |
2788 | * client.perform(); | |
2789 | * ---- | |
2790 | */ | |
2791 | @property void onSend(size_t delegate(void[]) callback); | |
2792 | ||
2793 | /** | |
2794 | * The event handler that receives incoming data. Be sure to copy the | |
2795 | * incoming ubyte[] since it is not guaranteed to be valid after the | |
2796 | * callback returns. | |
2797 | * | |
2798 | * Returns: | |
2799 | * The callback returns the incoming bytes read. If not the entire array is | |
2800 | * the request will abort. | |
2801 | * The special value .pauseRequest can be returned in order to pause the | |
2802 | * current request. | |
2803 | * | |
2804 | * Example: | |
2805 | * ---- | |
2806 | * import std.net.curl, std.stdio; | |
2807 | * auto client = HTTP("dlang.org"); | |
2808 | * client.onReceive = (ubyte[] data) | |
2809 | * { | |
2810 | * writeln("Got data", to!(const(char)[])(data)); | |
2811 | * return data.length; | |
2812 | * }; | |
2813 | * client.perform(); | |
2814 | * ---- | |
2815 | */ | |
2816 | @property void onReceive(size_t delegate(ubyte[]) callback); | |
2817 | ||
2818 | /** | |
2819 | * Register an event handler that gets called to inform of | |
2820 | * upload/download progress. | |
2821 | * | |
2822 | * Callback_parameters: | |
2823 | * $(CALLBACK_PARAMS) | |
2824 | * | |
2825 | * Callback_returns: Return 0 to signal success, return non-zero to | |
2826 | * abort transfer. | |
2827 | * | |
2828 | * Example: | |
2829 | * ---- | |
2830 | * import std.net.curl, std.stdio; | |
2831 | * auto client = HTTP("dlang.org"); | |
2832 | * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t ult) | |
2833 | * { | |
2834 | * writeln("Progress: downloaded ", dln, " of ", dl); | |
2835 | * writeln("Progress: uploaded ", uln, " of ", ul); | |
2836 | * }; | |
2837 | * client.perform(); | |
2838 | * ---- | |
2839 | */ | |
2840 | @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, | |
2841 | size_t ulTotal, size_t ulNow) callback); | |
2842 | } | |
2843 | ||
2844 | /** Clear all outgoing headers. | |
2845 | */ | |
2846 | void clearRequestHeaders() | |
2847 | { | |
2848 | if (p.headersOut !is null) | |
2849 | Curl.curl.slist_free_all(p.headersOut); | |
2850 | p.headersOut = null; | |
2851 | p.curl.clear(CurlOption.httpheader); | |
2852 | } | |
2853 | ||
2854 | /** Add a header e.g. "X-CustomField: Something is fishy". | |
2855 | * | |
2856 | * There is no remove header functionality. Do a $(LREF clearRequestHeaders) | |
2857 | * and set the needed headers instead. | |
2858 | * | |
2859 | * Example: | |
2860 | * --- | |
2861 | * import std.net.curl; | |
2862 | * auto client = HTTP(); | |
2863 | * client.addRequestHeader("X-Custom-ABC", "This is the custom value"); | |
2864 | * auto content = get("dlang.org", client); | |
2865 | * --- | |
2866 | */ | |
2867 | void addRequestHeader(const(char)[] name, const(char)[] value) | |
2868 | { | |
2869 | import std.format : format; | |
2870 | import std.uni : icmp; | |
2871 | ||
2872 | if (icmp(name, "User-Agent") == 0) | |
2873 | return setUserAgent(value); | |
2874 | string nv = format("%s: %s", name, value); | |
2875 | p.headersOut = Curl.curl.slist_append(p.headersOut, | |
2876 | nv.tempCString().buffPtr); | |
2877 | p.curl.set(CurlOption.httpheader, p.headersOut); | |
2878 | } | |
2879 | ||
2880 | /** | |
2881 | * The default "User-Agent" value send with a request. | |
2882 | * It has the form "Phobos-std.net.curl/$(I PHOBOS_VERSION) (libcurl/$(I CURL_VERSION))" | |
2883 | */ | |
2884 | static string defaultUserAgent() @property | |
2885 | { | |
2886 | import std.compiler : version_major, version_minor; | |
2887 | import std.format : format, sformat; | |
2888 | ||
2889 | // http://curl.haxx.se/docs/versions.html | |
2890 | enum fmt = "Phobos-std.net.curl/%d.%03d (libcurl/%d.%d.%d)"; | |
2891 | enum maxLen = fmt.length - "%d%03d%d%d%d".length + 10 + 10 + 3 + 3 + 3; | |
2892 | ||
2893 | static char[maxLen] buf = void; | |
2894 | static string userAgent; | |
2895 | ||
2896 | if (!userAgent.length) | |
2897 | { | |
2898 | auto curlVer = Curl.curl.version_info(CURLVERSION_NOW).version_num; | |
2899 | userAgent = cast(immutable) sformat( | |
2900 | buf, fmt, version_major, version_minor, | |
2901 | curlVer >> 16 & 0xFF, curlVer >> 8 & 0xFF, curlVer & 0xFF); | |
2902 | } | |
2903 | return userAgent; | |
2904 | } | |
2905 | ||
2906 | /** Set the value of the user agent request header field. | |
2907 | * | |
2908 | * By default a request has it's "User-Agent" field set to $(LREF | |
2909 | * defaultUserAgent) even if $(D setUserAgent) was never called. Pass | |
2910 | * an empty string to suppress the "User-Agent" field altogether. | |
2911 | */ | |
2912 | void setUserAgent(const(char)[] userAgent) | |
2913 | { | |
2914 | p.curl.set(CurlOption.useragent, userAgent); | |
2915 | } | |
2916 | ||
2917 | /** | |
2918 | * Get various timings defined in $(REF CurlInfo, etc, c, curl). | |
2919 | * The value is usable only if the return value is equal to $(D etc.c.curl.CurlError.ok). | |
2920 | * | |
2921 | * Params: | |
2922 | * timing = one of the timings defined in $(REF CurlInfo, etc, c, curl). | |
2923 | * The values are: | |
2924 | * $(D etc.c.curl.CurlInfo.namelookup_time), | |
2925 | * $(D etc.c.curl.CurlInfo.connect_time), | |
2926 | * $(D etc.c.curl.CurlInfo.pretransfer_time), | |
2927 | * $(D etc.c.curl.CurlInfo.starttransfer_time), | |
2928 | * $(D etc.c.curl.CurlInfo.redirect_time), | |
2929 | * $(D etc.c.curl.CurlInfo.appconnect_time), | |
2930 | * $(D etc.c.curl.CurlInfo.total_time). | |
2931 | * val = the actual value of the inquired timing. | |
2932 | * | |
2933 | * Returns: | |
2934 | * The return code of the operation. The value stored in val | |
2935 | * should be used only if the return value is $(D etc.c.curl.CurlInfo.ok). | |
2936 | * | |
2937 | * Example: | |
2938 | * --- | |
2939 | * import std.net.curl; | |
2940 | * import etc.c.curl : CurlError, CurlInfo; | |
2941 | * | |
2942 | * auto client = HTTP("dlang.org"); | |
2943 | * client.perform(); | |
2944 | * | |
2945 | * double val; | |
2946 | * CurlCode code; | |
2947 | * | |
2948 | * code = http.getTiming(CurlInfo.namelookup_time, val); | |
2949 | * assert(code == CurlError.ok); | |
2950 | * --- | |
2951 | */ | |
2952 | CurlCode getTiming(CurlInfo timing, ref double val) | |
2953 | { | |
2954 | return p.curl.getTiming(timing, val); | |
2955 | } | |
2956 | ||
2957 | /** The headers read from a successful response. | |
2958 | * | |
2959 | */ | |
2960 | @property string[string] responseHeaders() | |
2961 | { | |
2962 | return p.headersIn; | |
2963 | } | |
2964 | ||
2965 | /// HTTP method used. | |
2966 | @property void method(Method m) | |
2967 | { | |
2968 | p.method = m; | |
2969 | } | |
2970 | ||
2971 | /// ditto | |
2972 | @property Method method() | |
2973 | { | |
2974 | return p.method; | |
2975 | } | |
2976 | ||
2977 | /** | |
2978 | HTTP status line of last response. One call to perform may | |
2979 | result in several requests because of redirection. | |
2980 | */ | |
2981 | @property StatusLine statusLine() | |
2982 | { | |
2983 | return p.status; | |
2984 | } | |
2985 | ||
2986 | /// Set the active cookie string e.g. "name1=value1;name2=value2" | |
2987 | void setCookie(const(char)[] cookie) | |
2988 | { | |
2989 | p.curl.set(CurlOption.cookie, cookie); | |
2990 | } | |
2991 | ||
2992 | /// Set a file path to where a cookie jar should be read/stored. | |
2993 | void setCookieJar(const(char)[] path) | |
2994 | { | |
2995 | p.curl.set(CurlOption.cookiefile, path); | |
2996 | if (path.length) | |
2997 | p.curl.set(CurlOption.cookiejar, path); | |
2998 | } | |
2999 | ||
3000 | /// Flush cookie jar to disk. | |
3001 | void flushCookieJar() | |
3002 | { | |
3003 | p.curl.set(CurlOption.cookielist, "FLUSH"); | |
3004 | } | |
3005 | ||
3006 | /// Clear session cookies. | |
3007 | void clearSessionCookies() | |
3008 | { | |
3009 | p.curl.set(CurlOption.cookielist, "SESS"); | |
3010 | } | |
3011 | ||
3012 | /// Clear all cookies. | |
3013 | void clearAllCookies() | |
3014 | { | |
3015 | p.curl.set(CurlOption.cookielist, "ALL"); | |
3016 | } | |
3017 | ||
3018 | /** | |
3019 | Set time condition on the request. | |
3020 | ||
3021 | Params: | |
3022 | cond = $(D CurlTimeCond.{none,ifmodsince,ifunmodsince,lastmod}) | |
3023 | timestamp = Timestamp for the condition | |
3024 | ||
3025 | $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25) | |
3026 | */ | |
3027 | void setTimeCondition(HTTP.TimeCond cond, SysTime timestamp) | |
3028 | { | |
3029 | p.curl.set(CurlOption.timecondition, cond); | |
3030 | p.curl.set(CurlOption.timevalue, timestamp.toUnixTime()); | |
3031 | } | |
3032 | ||
3033 | /** Specifying data to post when not using the onSend callback. | |
3034 | * | |
3035 | * The data is NOT copied by the library. Content-Type will default to | |
3036 | * application/octet-stream. Data is not converted or encoded by this | |
3037 | * method. | |
3038 | * | |
3039 | * Example: | |
3040 | * ---- | |
3041 | * import std.net.curl, std.stdio; | |
3042 | * auto http = HTTP("http://www.mydomain.com"); | |
3043 | * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; }; | |
3044 | * http.postData = [1,2,3,4,5]; | |
3045 | * http.perform(); | |
3046 | * ---- | |
3047 | */ | |
3048 | @property void postData(const(void)[] data) | |
3049 | { | |
3050 | setPostData(data, "application/octet-stream"); | |
3051 | } | |
3052 | ||
3053 | /** Specifying data to post when not using the onSend callback. | |
3054 | * | |
3055 | * The data is NOT copied by the library. Content-Type will default to | |
3056 | * text/plain. Data is not converted or encoded by this method. | |
3057 | * | |
3058 | * Example: | |
3059 | * ---- | |
3060 | * import std.net.curl, std.stdio; | |
3061 | * auto http = HTTP("http://www.mydomain.com"); | |
3062 | * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; }; | |
3063 | * http.postData = "The quick...."; | |
3064 | * http.perform(); | |
3065 | * ---- | |
3066 | */ | |
3067 | @property void postData(const(char)[] data) | |
3068 | { | |
3069 | setPostData(data, "text/plain"); | |
3070 | } | |
3071 | ||
3072 | /** | |
3073 | * Specify data to post when not using the onSend callback, with | |
3074 | * user-specified Content-Type. | |
3075 | * Params: | |
3076 | * data = Data to post. | |
3077 | * contentType = MIME type of the data, for example, "text/plain" or | |
3078 | * "application/octet-stream". See also: | |
3079 | * $(LINK2 http://en.wikipedia.org/wiki/Internet_media_type, | |
3080 | * Internet media type) on Wikipedia. | |
3081 | * ----- | |
3082 | * import std.net.curl; | |
3083 | * auto http = HTTP("http://onlineform.example.com"); | |
3084 | * auto data = "app=login&username=bob&password=s00perS3kret"; | |
3085 | * http.setPostData(data, "application/x-www-form-urlencoded"); | |
3086 | * http.onReceive = (ubyte[] data) { return data.length; }; | |
3087 | * http.perform(); | |
3088 | * ----- | |
3089 | */ | |
3090 | void setPostData(const(void)[] data, string contentType) | |
3091 | { | |
3092 | // cannot use callback when specifying data directly so it is disabled here. | |
3093 | p.curl.clear(CurlOption.readfunction); | |
3094 | addRequestHeader("Content-Type", contentType); | |
3095 | p.curl.set(CurlOption.postfields, cast(void*) data.ptr); | |
3096 | p.curl.set(CurlOption.postfieldsize, data.length); | |
3097 | if (method == Method.undefined) | |
3098 | method = Method.post; | |
3099 | } | |
3100 | ||
3101 | @system unittest | |
3102 | { | |
3103 | import std.algorithm.searching : canFind; | |
3104 | ||
3105 | testServer.handle((s) { | |
3106 | auto req = s.recvReq!ubyte; | |
3107 | assert(req.hdrs.canFind("POST /path")); | |
3108 | assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4])); | |
3109 | assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255])); | |
3110 | s.send(httpOK(cast(ubyte[])[17, 27, 35, 41])); | |
3111 | }); | |
3112 | auto data = new ubyte[](256); | |
3113 | foreach (i, ref ub; data) | |
3114 | ub = cast(ubyte) i; | |
3115 | ||
3116 | auto http = HTTP(testServer.addr~"/path"); | |
3117 | http.postData = data; | |
3118 | ubyte[] res; | |
3119 | http.onReceive = (data) { res ~= data; return data.length; }; | |
3120 | http.perform(); | |
3121 | assert(res == cast(ubyte[])[17, 27, 35, 41]); | |
3122 | } | |
3123 | ||
3124 | /** | |
3125 | * Set the event handler that receives incoming headers. | |
3126 | * | |
3127 | * The callback will receive a header field key, value as parameter. The | |
3128 | * $(D const(char)[]) arrays are not valid after the delegate has returned. | |
3129 | * | |
3130 | * Example: | |
3131 | * ---- | |
3132 | * import std.net.curl, std.stdio; | |
3133 | * auto http = HTTP("dlang.org"); | |
3134 | * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; }; | |
3135 | * http.onReceiveHeader = (in char[] key, in char[] value) { writeln(key, " = ", value); }; | |
3136 | * http.perform(); | |
3137 | * ---- | |
3138 | */ | |
3139 | @property void onReceiveHeader(void delegate(in char[] key, | |
3140 | in char[] value) callback) | |
3141 | { | |
3142 | p.onReceiveHeader = callback; | |
3143 | } | |
3144 | ||
3145 | /** | |
3146 | Callback for each received StatusLine. | |
3147 | ||
3148 | Notice that several callbacks can be done for each call to | |
3149 | $(D perform()) due to redirections. | |
3150 | ||
3151 | See_Also: $(LREF StatusLine) | |
3152 | */ | |
3153 | @property void onReceiveStatusLine(void delegate(StatusLine) callback) | |
3154 | { | |
3155 | p.onReceiveStatusLine = callback; | |
3156 | } | |
3157 | ||
3158 | /** | |
3159 | The content length in bytes when using request that has content | |
3160 | e.g. POST/PUT and not using chunked transfer. Is set as the | |
3161 | "Content-Length" header. Set to ulong.max to reset to chunked transfer. | |
3162 | */ | |
3163 | @property void contentLength(ulong len) | |
3164 | { | |
3165 | import std.conv : to; | |
3166 | ||
3167 | CurlOption lenOpt; | |
3168 | ||
3169 | // Force post if necessary | |
3170 | if (p.method != Method.put && p.method != Method.post && | |
3171 | p.method != Method.patch) | |
3172 | p.method = Method.post; | |
3173 | ||
3174 | if (p.method == Method.post || p.method == Method.patch) | |
3175 | lenOpt = CurlOption.postfieldsize_large; | |
3176 | else | |
3177 | lenOpt = CurlOption.infilesize_large; | |
3178 | ||
3179 | if (size_t.max != ulong.max && len == size_t.max) | |
3180 | len = ulong.max; // check size_t.max for backwards compat, turn into error | |
3181 | ||
3182 | if (len == ulong.max) | |
3183 | { | |
3184 | // HTTP 1.1 supports requests with no length header set. | |
3185 | addRequestHeader("Transfer-Encoding", "chunked"); | |
3186 | addRequestHeader("Expect", "100-continue"); | |
3187 | } | |
3188 | else | |
3189 | { | |
3190 | p.curl.set(lenOpt, to!curl_off_t(len)); | |
3191 | } | |
3192 | } | |
3193 | ||
3194 | /** | |
3195 | Authentication method as specified in $(LREF AuthMethod). | |
3196 | */ | |
3197 | @property void authenticationMethod(AuthMethod authMethod) | |
3198 | { | |
3199 | p.curl.set(CurlOption.httpauth, cast(long) authMethod); | |
3200 | } | |
3201 | ||
3202 | /** | |
3203 | Set max allowed redirections using the location header. | |
3204 | uint.max for infinite. | |
3205 | */ | |
3206 | @property void maxRedirects(uint maxRedirs) | |
3207 | { | |
3208 | if (maxRedirs == uint.max) | |
3209 | { | |
3210 | // Disable | |
3211 | p.curl.set(CurlOption.followlocation, 0); | |
3212 | } | |
3213 | else | |
3214 | { | |
3215 | p.curl.set(CurlOption.followlocation, 1); | |
3216 | p.curl.set(CurlOption.maxredirs, maxRedirs); | |
3217 | } | |
3218 | } | |
3219 | ||
3220 | /** <a name="HTTP.Method"/>The standard HTTP methods : | |
3221 | * $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.1, _RFC2616 Section 5.1.1) | |
3222 | */ | |
3223 | enum Method | |
3224 | { | |
3225 | undefined, | |
3226 | head, /// | |
3227 | get, /// | |
3228 | post, /// | |
3229 | put, /// | |
3230 | del, /// | |
3231 | options, /// | |
3232 | trace, /// | |
3233 | connect, /// | |
3234 | patch, /// | |
3235 | } | |
3236 | ||
3237 | /** | |
3238 | HTTP status line ie. the first line returned in an HTTP response. | |
3239 | ||
3240 | If authentication or redirections are done then the status will be for | |
3241 | the last response received. | |
3242 | */ | |
3243 | struct StatusLine | |
3244 | { | |
3245 | ushort majorVersion; /// Major HTTP version ie. 1 in HTTP/1.0. | |
3246 | ushort minorVersion; /// Minor HTTP version ie. 0 in HTTP/1.0. | |
3247 | ushort code; /// HTTP status line code e.g. 200. | |
3248 | string reason; /// HTTP status line reason string. | |
3249 | ||
3250 | /// Reset this status line | |
3251 | @safe void reset() | |
3252 | { | |
3253 | majorVersion = 0; | |
3254 | minorVersion = 0; | |
3255 | code = 0; | |
3256 | reason = ""; | |
3257 | } | |
3258 | ||
3259 | /// | |
3260 | string toString() const | |
3261 | { | |
3262 | import std.format : format; | |
3263 | return format("%s %s (%s.%s)", | |
3264 | code, reason, majorVersion, minorVersion); | |
3265 | } | |
3266 | } | |
3267 | ||
3268 | } // HTTP | |
3269 | ||
3270 | @system unittest // charset/Charset/CHARSET/... | |
3271 | { | |
3272 | import std.meta : AliasSeq; | |
3273 | ||
3274 | foreach (c; AliasSeq!("charset", "Charset", "CHARSET", "CharSet", "charSet", | |
3275 | "ChArSeT", "cHaRsEt")) | |
3276 | { | |
3277 | testServer.handle((s) { | |
3278 | s.send("HTTP/1.1 200 OK\r\n"~ | |
3279 | "Content-Length: 0\r\n"~ | |
3280 | "Content-Type: text/plain; " ~ c ~ "=foo\r\n" ~ | |
3281 | "\r\n"); | |
3282 | }); | |
3283 | ||
3284 | auto http = HTTP(testServer.addr); | |
3285 | http.perform(); | |
3286 | assert(http.p.charset == "foo"); | |
3287 | ||
3288 | // Bugzilla 16736 | |
3289 | double val; | |
3290 | CurlCode code; | |
3291 | ||
3292 | code = http.getTiming(CurlInfo.total_time, val); | |
3293 | assert(code == CurlError.ok); | |
3294 | code = http.getTiming(CurlInfo.namelookup_time, val); | |
3295 | assert(code == CurlError.ok); | |
3296 | code = http.getTiming(CurlInfo.connect_time, val); | |
3297 | assert(code == CurlError.ok); | |
3298 | code = http.getTiming(CurlInfo.pretransfer_time, val); | |
3299 | assert(code == CurlError.ok); | |
3300 | code = http.getTiming(CurlInfo.starttransfer_time, val); | |
3301 | assert(code == CurlError.ok); | |
3302 | code = http.getTiming(CurlInfo.redirect_time, val); | |
3303 | assert(code == CurlError.ok); | |
3304 | code = http.getTiming(CurlInfo.appconnect_time, val); | |
3305 | assert(code == CurlError.ok); | |
3306 | } | |
3307 | } | |
3308 | ||
3309 | /** | |
3310 | FTP client functionality. | |
3311 | ||
3312 | See_Also: $(HTTP tools.ietf.org/html/rfc959, RFC959) | |
3313 | */ | |
3314 | struct FTP | |
3315 | { | |
3316 | ||
3317 | mixin Protocol; | |
3318 | ||
3319 | private struct Impl | |
3320 | { | |
3321 | ~this() | |
3322 | { | |
3323 | if (commands !is null) | |
3324 | Curl.curl.slist_free_all(commands); | |
3325 | if (curl.handle !is null) // work around RefCounted/emplace bug | |
3326 | curl.shutdown(); | |
3327 | } | |
3328 | curl_slist* commands; | |
3329 | Curl curl; | |
3330 | string encoding; | |
3331 | } | |
3332 | ||
3333 | private RefCounted!Impl p; | |
3334 | ||
3335 | /** | |
3336 | FTP access to the specified url. | |
3337 | */ | |
3338 | static FTP opCall(const(char)[] url) | |
3339 | { | |
3340 | FTP ftp; | |
3341 | ftp.initialize(); | |
3342 | ftp.url = url; | |
3343 | return ftp; | |
3344 | } | |
3345 | ||
3346 | /// | |
3347 | static FTP opCall() | |
3348 | { | |
3349 | FTP ftp; | |
3350 | ftp.initialize(); | |
3351 | return ftp; | |
3352 | } | |
3353 | ||
3354 | /// | |
3355 | FTP dup() | |
3356 | { | |
3357 | FTP copy = FTP(); | |
3358 | copy.initialize(); | |
3359 | copy.p.encoding = p.encoding; | |
3360 | copy.p.curl = p.curl.dup(); | |
3361 | curl_slist* cur = p.commands; | |
3362 | curl_slist* newlist = null; | |
3363 | while (cur) | |
3364 | { | |
3365 | newlist = Curl.curl.slist_append(newlist, cur.data); | |
3366 | cur = cur.next; | |
3367 | } | |
3368 | copy.p.commands = newlist; | |
3369 | copy.p.curl.set(CurlOption.postquote, copy.p.commands); | |
3370 | copy.dataTimeout = _defaultDataTimeout; | |
3371 | return copy; | |
3372 | } | |
3373 | ||
3374 | private void initialize() | |
3375 | { | |
3376 | p.curl.initialize(); | |
3377 | p.encoding = "ISO-8859-1"; | |
3378 | dataTimeout = _defaultDataTimeout; | |
3379 | } | |
3380 | ||
3381 | /** | |
3382 | Performs the ftp request as it has been configured. | |
3383 | ||
3384 | After a FTP client has been setup and possibly assigned callbacks the $(D | |
3385 | perform()) method will start performing the actual communication with the | |
3386 | server. | |
3387 | ||
3388 | Params: | |
3389 | throwOnError = whether to throw an exception or return a CurlCode on error | |
3390 | */ | |
3391 | CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) | |
3392 | { | |
3393 | return p.curl.perform(throwOnError); | |
3394 | } | |
3395 | ||
3396 | /// The URL to specify the location of the resource. | |
3397 | @property void url(const(char)[] url) | |
3398 | { | |
3399 | import std.algorithm.searching : startsWith; | |
3400 | import std.uni : toLower; | |
3401 | ||
3402 | if (!startsWith(url.toLower(), "ftp://", "ftps://")) | |
3403 | url = "ftp://" ~ url; | |
3404 | p.curl.set(CurlOption.url, url); | |
3405 | } | |
3406 | ||
3407 | // This is a workaround for mixed in content not having its | |
3408 | // docs mixed in. | |
3409 | version (StdDdoc) | |
3410 | { | |
3411 | /// Value to return from $(D onSend)/$(D onReceive) delegates in order to | |
3412 | /// pause a request | |
3413 | alias requestPause = CurlReadFunc.pause; | |
3414 | ||
3415 | /// Value to return from onSend delegate in order to abort a request | |
3416 | alias requestAbort = CurlReadFunc.abort; | |
3417 | ||
3418 | /** | |
3419 | True if the instance is stopped. A stopped instance is not usable. | |
3420 | */ | |
3421 | @property bool isStopped(); | |
3422 | ||
3423 | /// Stop and invalidate this instance. | |
3424 | void shutdown(); | |
3425 | ||
3426 | /** Set verbose. | |
3427 | This will print request information to stderr. | |
3428 | */ | |
3429 | @property void verbose(bool on); | |
3430 | ||
3431 | // Connection settings | |
3432 | ||
3433 | /// Set timeout for activity on connection. | |
3434 | @property void dataTimeout(Duration d); | |
3435 | ||
3436 | /** Set maximum time an operation is allowed to take. | |
3437 | This includes dns resolution, connecting, data transfer, etc. | |
3438 | */ | |
3439 | @property void operationTimeout(Duration d); | |
3440 | ||
3441 | /// Set timeout for connecting. | |
3442 | @property void connectTimeout(Duration d); | |
3443 | ||
3444 | // Network settings | |
3445 | ||
3446 | /** Proxy | |
3447 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) | |
3448 | */ | |
3449 | @property void proxy(const(char)[] host); | |
3450 | ||
3451 | /** Proxy port | |
3452 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) | |
3453 | */ | |
3454 | @property void proxyPort(ushort port); | |
3455 | ||
3456 | /// Type of proxy | |
3457 | alias CurlProxy = etc.c.curl.CurlProxy; | |
3458 | ||
3459 | /** Proxy type | |
3460 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) | |
3461 | */ | |
3462 | @property void proxyType(CurlProxy type); | |
3463 | ||
3464 | /// DNS lookup timeout. | |
3465 | @property void dnsTimeout(Duration d); | |
3466 | ||
3467 | /** | |
3468 | * The network interface to use in form of the the IP of the interface. | |
3469 | * | |
3470 | * Example: | |
3471 | * ---- | |
3472 | * theprotocol.netInterface = "192.168.1.32"; | |
3473 | * theprotocol.netInterface = [ 192, 168, 1, 32 ]; | |
3474 | * ---- | |
3475 | * | |
3476 | * See: $(REF InternetAddress, std,socket) | |
3477 | */ | |
3478 | @property void netInterface(const(char)[] i); | |
3479 | ||
3480 | /// ditto | |
3481 | @property void netInterface(const(ubyte)[4] i); | |
3482 | ||
3483 | /// ditto | |
3484 | @property void netInterface(InternetAddress i); | |
3485 | ||
3486 | /** | |
3487 | Set the local outgoing port to use. | |
3488 | Params: | |
3489 | port = the first outgoing port number to try and use | |
3490 | */ | |
3491 | @property void localPort(ushort port); | |
3492 | ||
3493 | /** | |
3494 | Set the local outgoing port range to use. | |
3495 | This can be used together with the localPort property. | |
3496 | Params: | |
3497 | range = if the first port is occupied then try this many | |
3498 | port number forwards | |
3499 | */ | |
3500 | @property void localPortRange(ushort range); | |
3501 | ||
3502 | /** Set the tcp no-delay socket option on or off. | |
3503 | See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) | |
3504 | */ | |
3505 | @property void tcpNoDelay(bool on); | |
3506 | ||
3507 | // Authentication settings | |
3508 | ||
3509 | /** | |
3510 | Set the user name, password and optionally domain for authentication | |
3511 | purposes. | |
3512 | ||
3513 | Some protocols may need authentication in some cases. Use this | |
3514 | function to provide credentials. | |
3515 | ||
3516 | Params: | |
3517 | username = the username | |
3518 | password = the password | |
3519 | domain = used for NTLM authentication only and is set to the NTLM domain | |
3520 | name | |
3521 | */ | |
3522 | void setAuthentication(const(char)[] username, const(char)[] password, | |
3523 | const(char)[] domain = ""); | |
3524 | ||
3525 | /** | |
3526 | Set the user name and password for proxy authentication. | |
3527 | ||
3528 | Params: | |
3529 | username = the username | |
3530 | password = the password | |
3531 | */ | |
3532 | void setProxyAuthentication(const(char)[] username, const(char)[] password); | |
3533 | ||
3534 | /** | |
3535 | * The event handler that gets called when data is needed for sending. The | |
3536 | * length of the $(D void[]) specifies the maximum number of bytes that can | |
3537 | * be sent. | |
3538 | * | |
3539 | * Returns: | |
3540 | * The callback returns the number of elements in the buffer that have been | |
3541 | * filled and are ready to send. | |
3542 | * The special value $(D .abortRequest) can be returned in order to abort the | |
3543 | * current request. | |
3544 | * The special value $(D .pauseRequest) can be returned in order to pause the | |
3545 | * current request. | |
3546 | * | |
3547 | */ | |
3548 | @property void onSend(size_t delegate(void[]) callback); | |
3549 | ||
3550 | /** | |
3551 | * The event handler that receives incoming data. Be sure to copy the | |
3552 | * incoming ubyte[] since it is not guaranteed to be valid after the | |
3553 | * callback returns. | |
3554 | * | |
3555 | * Returns: | |
3556 | * The callback returns the incoming bytes read. If not the entire array is | |
3557 | * the request will abort. | |
3558 | * The special value .pauseRequest can be returned in order to pause the | |
3559 | * current request. | |
3560 | * | |
3561 | */ | |
3562 | @property void onReceive(size_t delegate(ubyte[]) callback); | |
3563 | ||
3564 | /** | |
3565 | * The event handler that gets called to inform of upload/download progress. | |
3566 | * | |
3567 | * Callback_parameters: | |
3568 | * $(CALLBACK_PARAMS) | |
3569 | * | |
3570 | * Callback_returns: | |
3571 | * Return 0 from the callback to signal success, return non-zero to | |
3572 | * abort transfer. | |
3573 | */ | |
3574 | @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, | |
3575 | size_t ulTotal, size_t ulNow) callback); | |
3576 | } | |
3577 | ||
3578 | /** Clear all commands send to ftp server. | |
3579 | */ | |
3580 | void clearCommands() | |
3581 | { | |
3582 | if (p.commands !is null) | |
3583 | Curl.curl.slist_free_all(p.commands); | |
3584 | p.commands = null; | |
3585 | p.curl.clear(CurlOption.postquote); | |
3586 | } | |
3587 | ||
3588 | /** Add a command to send to ftp server. | |
3589 | * | |
3590 | * There is no remove command functionality. Do a $(LREF clearCommands) and | |
3591 | * set the needed commands instead. | |
3592 | * | |
3593 | * Example: | |
3594 | * --- | |
3595 | * import std.net.curl; | |
3596 | * auto client = FTP(); | |
3597 | * client.addCommand("RNFR my_file.txt"); | |
3598 | * client.addCommand("RNTO my_renamed_file.txt"); | |
3599 | * upload("my_file.txt", "ftp.digitalmars.com", client); | |
3600 | * --- | |
3601 | */ | |
3602 | void addCommand(const(char)[] command) | |
3603 | { | |
3604 | p.commands = Curl.curl.slist_append(p.commands, | |
3605 | command.tempCString().buffPtr); | |
3606 | p.curl.set(CurlOption.postquote, p.commands); | |
3607 | } | |
3608 | ||
3609 | /// Connection encoding. Defaults to ISO-8859-1. | |
3610 | @property void encoding(string name) | |
3611 | { | |
3612 | p.encoding = name; | |
3613 | } | |
3614 | ||
3615 | /// ditto | |
3616 | @property string encoding() | |
3617 | { | |
3618 | return p.encoding; | |
3619 | } | |
3620 | ||
3621 | /** | |
3622 | The content length in bytes of the ftp data. | |
3623 | */ | |
3624 | @property void contentLength(ulong len) | |
3625 | { | |
3626 | import std.conv : to; | |
3627 | p.curl.set(CurlOption.infilesize_large, to!curl_off_t(len)); | |
3628 | } | |
3629 | ||
3630 | /** | |
3631 | * Get various timings defined in $(REF CurlInfo, etc, c, curl). | |
3632 | * The value is usable only if the return value is equal to $(D etc.c.curl.CurlError.ok). | |
3633 | * | |
3634 | * Params: | |
3635 | * timing = one of the timings defined in $(REF CurlInfo, etc, c, curl). | |
3636 | * The values are: | |
3637 | * $(D etc.c.curl.CurlInfo.namelookup_time), | |
3638 | * $(D etc.c.curl.CurlInfo.connect_time), | |
3639 | * $(D etc.c.curl.CurlInfo.pretransfer_time), | |
3640 | * $(D etc.c.curl.CurlInfo.starttransfer_time), | |
3641 | * $(D etc.c.curl.CurlInfo.redirect_time), | |
3642 | * $(D etc.c.curl.CurlInfo.appconnect_time), | |
3643 | * $(D etc.c.curl.CurlInfo.total_time). | |
3644 | * val = the actual value of the inquired timing. | |
3645 | * | |
3646 | * Returns: | |
3647 | * The return code of the operation. The value stored in val | |
3648 | * should be used only if the return value is $(D etc.c.curl.CurlInfo.ok). | |
3649 | * | |
3650 | * Example: | |
3651 | * --- | |
3652 | * import std.net.curl; | |
3653 | * import etc.c.curl : CurlError, CurlInfo; | |
3654 | * | |
3655 | * auto client = FTP(); | |
3656 | * client.addCommand("RNFR my_file.txt"); | |
3657 | * client.addCommand("RNTO my_renamed_file.txt"); | |
3658 | * upload("my_file.txt", "ftp.digitalmars.com", client); | |
3659 | * | |
3660 | * double val; | |
3661 | * CurlCode code; | |
3662 | * | |
3663 | * code = http.getTiming(CurlInfo.namelookup_time, val); | |
3664 | * assert(code == CurlError.ok); | |
3665 | * --- | |
3666 | */ | |
3667 | CurlCode getTiming(CurlInfo timing, ref double val) | |
3668 | { | |
3669 | return p.curl.getTiming(timing, val); | |
3670 | } | |
3671 | ||
3672 | @system unittest | |
3673 | { | |
3674 | auto client = FTP(); | |
3675 | ||
3676 | double val; | |
3677 | CurlCode code; | |
3678 | ||
3679 | code = client.getTiming(CurlInfo.total_time, val); | |
3680 | assert(code == CurlError.ok); | |
3681 | code = client.getTiming(CurlInfo.namelookup_time, val); | |
3682 | assert(code == CurlError.ok); | |
3683 | code = client.getTiming(CurlInfo.connect_time, val); | |
3684 | assert(code == CurlError.ok); | |
3685 | code = client.getTiming(CurlInfo.pretransfer_time, val); | |
3686 | assert(code == CurlError.ok); | |
3687 | code = client.getTiming(CurlInfo.starttransfer_time, val); | |
3688 | assert(code == CurlError.ok); | |
3689 | code = client.getTiming(CurlInfo.redirect_time, val); | |
3690 | assert(code == CurlError.ok); | |
3691 | code = client.getTiming(CurlInfo.appconnect_time, val); | |
3692 | assert(code == CurlError.ok); | |
3693 | } | |
3694 | } | |
3695 | ||
3696 | /** | |
3697 | * Basic SMTP protocol support. | |
3698 | * | |
3699 | * Example: | |
3700 | * --- | |
3701 | * import std.net.curl; | |
3702 | * | |
3703 | * // Send an email with SMTPS | |
3704 | * auto smtp = SMTP("smtps://smtp.gmail.com"); | |
3705 | * smtp.setAuthentication("from.addr@gmail.com", "password"); | |
3706 | * smtp.mailTo = ["<to.addr@gmail.com>"]; | |
3707 | * smtp.mailFrom = "<from.addr@gmail.com>"; | |
3708 | * smtp.message = "Example Message"; | |
3709 | * smtp.perform(); | |
3710 | * --- | |
3711 | * | |
3712 | * See_Also: $(HTTP www.ietf.org/rfc/rfc2821.txt, RFC2821) | |
3713 | */ | |
3714 | struct SMTP | |
3715 | { | |
3716 | mixin Protocol; | |
3717 | ||
3718 | private struct Impl | |
3719 | { | |
3720 | ~this() | |
3721 | { | |
3722 | if (curl.handle !is null) // work around RefCounted/emplace bug | |
3723 | curl.shutdown(); | |
3724 | } | |
3725 | Curl curl; | |
3726 | ||
3727 | @property void message(string msg) | |
3728 | { | |
3729 | import std.algorithm.comparison : min; | |
3730 | ||
3731 | auto _message = msg; | |
3732 | /** | |
3733 | This delegate reads the message text and copies it. | |
3734 | */ | |
3735 | curl.onSend = delegate size_t(void[] data) | |
3736 | { | |
3737 | if (!msg.length) return 0; | |
3738 | size_t to_copy = min(data.length, _message.length); | |
3739 | data[0 .. to_copy] = (cast(void[])_message)[0 .. to_copy]; | |
3740 | _message = _message[to_copy..$]; | |
3741 | return to_copy; | |
3742 | }; | |
3743 | } | |
3744 | } | |
3745 | ||
3746 | private RefCounted!Impl p; | |
3747 | ||
3748 | /** | |
3749 | Sets to the URL of the SMTP server. | |
3750 | */ | |
3751 | static SMTP opCall(const(char)[] url) | |
3752 | { | |
3753 | SMTP smtp; | |
3754 | smtp.initialize(); | |
3755 | smtp.url = url; | |
3756 | return smtp; | |
3757 | } | |
3758 | ||
3759 | /// | |
3760 | static SMTP opCall() | |
3761 | { | |
3762 | SMTP smtp; | |
3763 | smtp.initialize(); | |
3764 | return smtp; | |
3765 | } | |
3766 | ||
3767 | /+ TODO: The other structs have this function. | |
3768 | SMTP dup() | |
3769 | { | |
3770 | SMTP copy = SMTP(); | |
3771 | copy.initialize(); | |
3772 | copy.p.encoding = p.encoding; | |
3773 | copy.p.curl = p.curl.dup(); | |
3774 | curl_slist* cur = p.commands; | |
3775 | curl_slist* newlist = null; | |
3776 | while (cur) | |
3777 | { | |
3778 | newlist = Curl.curl.slist_append(newlist, cur.data); | |
3779 | cur = cur.next; | |
3780 | } | |
3781 | copy.p.commands = newlist; | |
3782 | copy.p.curl.set(CurlOption.postquote, copy.p.commands); | |
3783 | copy.dataTimeout = _defaultDataTimeout; | |
3784 | return copy; | |
3785 | } | |
3786 | +/ | |
3787 | ||
3788 | /** | |
3789 | Performs the request as configured. | |
3790 | Params: | |
3791 | throwOnError = whether to throw an exception or return a CurlCode on error | |
3792 | */ | |
3793 | CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) | |
3794 | { | |
3795 | return p.curl.perform(throwOnError); | |
3796 | } | |
3797 | ||
3798 | /// The URL to specify the location of the resource. | |
3799 | @property void url(const(char)[] url) | |
3800 | { | |
3801 | import std.algorithm.searching : startsWith; | |
3802 | import std.uni : toLower; | |
3803 | ||
3804 | auto lowered = url.toLower(); | |
3805 | ||
3806 | if (lowered.startsWith("smtps://")) | |
3807 | { | |
3808 | p.curl.set(CurlOption.use_ssl, CurlUseSSL.all); | |
3809 | } | |
3810 | else | |
3811 | { | |
3812 | enforce!CurlException(lowered.startsWith("smtp://"), | |
3813 | "The url must be for the smtp protocol."); | |
3814 | } | |
3815 | p.curl.set(CurlOption.url, url); | |
3816 | } | |
3817 | ||
3818 | private void initialize() | |
3819 | { | |
3820 | p.curl.initialize(); | |
3821 | p.curl.set(CurlOption.upload, 1L); | |
3822 | dataTimeout = _defaultDataTimeout; | |
3823 | verifyPeer = true; | |
3824 | verifyHost = true; | |
3825 | } | |
3826 | ||
3827 | // This is a workaround for mixed in content not having its | |
3828 | // docs mixed in. | |
3829 | version (StdDdoc) | |
3830 | { | |
3831 | /// Value to return from $(D onSend)/$(D onReceive) delegates in order to | |
3832 | /// pause a request | |
3833 | alias requestPause = CurlReadFunc.pause; | |
3834 | ||
3835 | /// Value to return from onSend delegate in order to abort a request | |
3836 | alias requestAbort = CurlReadFunc.abort; | |
3837 | ||
3838 | /** | |
3839 | True if the instance is stopped. A stopped instance is not usable. | |
3840 | */ | |
3841 | @property bool isStopped(); | |
3842 | ||
3843 | /// Stop and invalidate this instance. | |
3844 | void shutdown(); | |
3845 | ||
3846 | /** Set verbose. | |
3847 | This will print request information to stderr. | |
3848 | */ | |
3849 | @property void verbose(bool on); | |
3850 | ||
3851 | // Connection settings | |
3852 | ||
3853 | /// Set timeout for activity on connection. | |
3854 | @property void dataTimeout(Duration d); | |
3855 | ||
3856 | /** Set maximum time an operation is allowed to take. | |
3857 | This includes dns resolution, connecting, data transfer, etc. | |
3858 | */ | |
3859 | @property void operationTimeout(Duration d); | |
3860 | ||
3861 | /// Set timeout for connecting. | |
3862 | @property void connectTimeout(Duration d); | |
3863 | ||
3864 | // Network settings | |
3865 | ||
3866 | /** Proxy | |
3867 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) | |
3868 | */ | |
3869 | @property void proxy(const(char)[] host); | |
3870 | ||
3871 | /** Proxy port | |
3872 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) | |
3873 | */ | |
3874 | @property void proxyPort(ushort port); | |
3875 | ||
3876 | /// Type of proxy | |
3877 | alias CurlProxy = etc.c.curl.CurlProxy; | |
3878 | ||
3879 | /** Proxy type | |
3880 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) | |
3881 | */ | |
3882 | @property void proxyType(CurlProxy type); | |
3883 | ||
3884 | /// DNS lookup timeout. | |
3885 | @property void dnsTimeout(Duration d); | |
3886 | ||
3887 | /** | |
3888 | * The network interface to use in form of the the IP of the interface. | |
3889 | * | |
3890 | * Example: | |
3891 | * ---- | |
3892 | * theprotocol.netInterface = "192.168.1.32"; | |
3893 | * theprotocol.netInterface = [ 192, 168, 1, 32 ]; | |
3894 | * ---- | |
3895 | * | |
3896 | * See: $(REF InternetAddress, std,socket) | |
3897 | */ | |
3898 | @property void netInterface(const(char)[] i); | |
3899 | ||
3900 | /// ditto | |
3901 | @property void netInterface(const(ubyte)[4] i); | |
3902 | ||
3903 | /// ditto | |
3904 | @property void netInterface(InternetAddress i); | |
3905 | ||
3906 | /** | |
3907 | Set the local outgoing port to use. | |
3908 | Params: | |
3909 | port = the first outgoing port number to try and use | |
3910 | */ | |
3911 | @property void localPort(ushort port); | |
3912 | ||
3913 | /** | |
3914 | Set the local outgoing port range to use. | |
3915 | This can be used together with the localPort property. | |
3916 | Params: | |
3917 | range = if the first port is occupied then try this many | |
3918 | port number forwards | |
3919 | */ | |
3920 | @property void localPortRange(ushort range); | |
3921 | ||
3922 | /** Set the tcp no-delay socket option on or off. | |
3923 | See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) | |
3924 | */ | |
3925 | @property void tcpNoDelay(bool on); | |
3926 | ||
3927 | // Authentication settings | |
3928 | ||
3929 | /** | |
3930 | Set the user name, password and optionally domain for authentication | |
3931 | purposes. | |
3932 | ||
3933 | Some protocols may need authentication in some cases. Use this | |
3934 | function to provide credentials. | |
3935 | ||
3936 | Params: | |
3937 | username = the username | |
3938 | password = the password | |
3939 | domain = used for NTLM authentication only and is set to the NTLM domain | |
3940 | name | |
3941 | */ | |
3942 | void setAuthentication(const(char)[] username, const(char)[] password, | |
3943 | const(char)[] domain = ""); | |
3944 | ||
3945 | /** | |
3946 | Set the user name and password for proxy authentication. | |
3947 | ||
3948 | Params: | |
3949 | username = the username | |
3950 | password = the password | |
3951 | */ | |
3952 | void setProxyAuthentication(const(char)[] username, const(char)[] password); | |
3953 | ||
3954 | /** | |
3955 | * The event handler that gets called when data is needed for sending. The | |
3956 | * length of the $(D void[]) specifies the maximum number of bytes that can | |
3957 | * be sent. | |
3958 | * | |
3959 | * Returns: | |
3960 | * The callback returns the number of elements in the buffer that have been | |
3961 | * filled and are ready to send. | |
3962 | * The special value $(D .abortRequest) can be returned in order to abort the | |
3963 | * current request. | |
3964 | * The special value $(D .pauseRequest) can be returned in order to pause the | |
3965 | * current request. | |
3966 | */ | |
3967 | @property void onSend(size_t delegate(void[]) callback); | |
3968 | ||
3969 | /** | |
3970 | * The event handler that receives incoming data. Be sure to copy the | |
3971 | * incoming ubyte[] since it is not guaranteed to be valid after the | |
3972 | * callback returns. | |
3973 | * | |
3974 | * Returns: | |
3975 | * The callback returns the incoming bytes read. If not the entire array is | |
3976 | * the request will abort. | |
3977 | * The special value .pauseRequest can be returned in order to pause the | |
3978 | * current request. | |
3979 | */ | |
3980 | @property void onReceive(size_t delegate(ubyte[]) callback); | |
3981 | ||
3982 | /** | |
3983 | * The event handler that gets called to inform of upload/download progress. | |
3984 | * | |
3985 | * Callback_parameters: | |
3986 | * $(CALLBACK_PARAMS) | |
3987 | * | |
3988 | * Callback_returns: | |
3989 | * Return 0 from the callback to signal success, return non-zero to | |
3990 | * abort transfer. | |
3991 | */ | |
3992 | @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, | |
3993 | size_t ulTotal, size_t ulNow) callback); | |
3994 | } | |
3995 | ||
3996 | /** | |
3997 | Setter for the sender's email address. | |
3998 | */ | |
3999 | @property void mailFrom()(const(char)[] sender) | |
4000 | { | |
4001 | assert(!sender.empty, "Sender must not be empty"); | |
4002 | p.curl.set(CurlOption.mail_from, sender); | |
4003 | } | |
4004 | ||
4005 | /** | |
4006 | Setter for the recipient email addresses. | |
4007 | */ | |
4008 | void mailTo()(const(char)[][] recipients...) | |
4009 | { | |
4010 | assert(!recipients.empty, "Recipient must not be empty"); | |
4011 | curl_slist* recipients_list = null; | |
4012 | foreach (recipient; recipients) | |
4013 | { | |
4014 | recipients_list = | |
4015 | Curl.curl.slist_append(recipients_list, | |
4016 | recipient.tempCString().buffPtr); | |
4017 | } | |
4018 | p.curl.set(CurlOption.mail_rcpt, recipients_list); | |
4019 | } | |
4020 | ||
4021 | /** | |
4022 | Sets the message body text. | |
4023 | */ | |
4024 | ||
4025 | @property void message(string msg) | |
4026 | { | |
4027 | p.message = msg; | |
4028 | } | |
4029 | } | |
4030 | ||
4031 | /++ | |
4032 | Exception thrown on errors in std.net.curl functions. | |
4033 | +/ | |
4034 | class CurlException : Exception | |
4035 | { | |
4036 | /++ | |
4037 | Params: | |
4038 | msg = The message for the exception. | |
4039 | file = The file where the exception occurred. | |
4040 | line = The line number where the exception occurred. | |
4041 | next = The previous exception in the chain of exceptions, if any. | |
4042 | +/ | |
4043 | @safe pure nothrow | |
4044 | this(string msg, | |
4045 | string file = __FILE__, | |
4046 | size_t line = __LINE__, | |
4047 | Throwable next = null) | |
4048 | { | |
4049 | super(msg, file, line, next); | |
4050 | } | |
4051 | } | |
4052 | ||
4053 | /++ | |
4054 | Exception thrown on timeout errors in std.net.curl functions. | |
4055 | +/ | |
4056 | class CurlTimeoutException : CurlException | |
4057 | { | |
4058 | /++ | |
4059 | Params: | |
4060 | msg = The message for the exception. | |
4061 | file = The file where the exception occurred. | |
4062 | line = The line number where the exception occurred. | |
4063 | next = The previous exception in the chain of exceptions, if any. | |
4064 | +/ | |
4065 | @safe pure nothrow | |
4066 | this(string msg, | |
4067 | string file = __FILE__, | |
4068 | size_t line = __LINE__, | |
4069 | Throwable next = null) | |
4070 | { | |
4071 | super(msg, file, line, next); | |
4072 | } | |
4073 | } | |
4074 | ||
4075 | /++ | |
4076 | Exception thrown on HTTP request failures, e.g. 404 Not Found. | |
4077 | +/ | |
4078 | class HTTPStatusException : CurlException | |
4079 | { | |
4080 | /++ | |
4081 | Params: | |
4082 | status = The HTTP status code. | |
4083 | msg = The message for the exception. | |
4084 | file = The file where the exception occurred. | |
4085 | line = The line number where the exception occurred. | |
4086 | next = The previous exception in the chain of exceptions, if any. | |
4087 | +/ | |
4088 | @safe pure nothrow | |
4089 | this(int status, | |
4090 | string msg, | |
4091 | string file = __FILE__, | |
4092 | size_t line = __LINE__, | |
4093 | Throwable next = null) | |
4094 | { | |
4095 | super(msg, file, line, next); | |
4096 | this.status = status; | |
4097 | } | |
4098 | ||
4099 | immutable int status; /// The HTTP status code | |
4100 | } | |
4101 | ||
4102 | /// Equal to $(REF CURLcode, etc,c,curl) | |
4103 | alias CurlCode = CURLcode; | |
4104 | ||
4105 | import std.typecons : Flag, Yes, No; | |
4106 | /// Flag to specify whether or not an exception is thrown on error. | |
4107 | alias ThrowOnError = Flag!"throwOnError"; | |
4108 | ||
4109 | private struct CurlAPI | |
4110 | { | |
4111 | static struct API | |
4112 | { | |
4113 | extern(C): | |
4114 | import core.stdc.config : c_long; | |
4115 | CURLcode function(c_long flags) global_init; | |
4116 | void function() global_cleanup; | |
4117 | curl_version_info_data * function(CURLversion) version_info; | |
4118 | CURL* function() easy_init; | |
4119 | CURLcode function(CURL *curl, CURLoption option,...) easy_setopt; | |
4120 | CURLcode function(CURL *curl) easy_perform; | |
4121 | CURLcode function(CURL *curl, CURLINFO info,...) easy_getinfo; | |
4122 | CURL* function(CURL *curl) easy_duphandle; | |
4123 | char* function(CURLcode) easy_strerror; | |
4124 | CURLcode function(CURL *handle, int bitmask) easy_pause; | |
4125 | void function(CURL *curl) easy_cleanup; | |
4126 | curl_slist* function(curl_slist *, char *) slist_append; | |
4127 | void function(curl_slist *) slist_free_all; | |
4128 | } | |
4129 | __gshared API _api; | |
4130 | __gshared void* _handle; | |
4131 | ||
4132 | static ref API instance() @property | |
4133 | { | |
4134 | import std.concurrency : initOnce; | |
4135 | initOnce!_handle(loadAPI()); | |
4136 | return _api; | |
4137 | } | |
4138 | ||
4139 | static void* loadAPI() | |
4140 | { | |
4141 | version (Posix) | |
4142 | { | |
4143 | import core.sys.posix.dlfcn : dlsym, dlopen, dlclose, RTLD_LAZY; | |
4144 | alias loadSym = dlsym; | |
4145 | } | |
4146 | else version (Windows) | |
4147 | { | |
4148 | import core.sys.windows.windows : GetProcAddress, GetModuleHandleA, | |
4149 | LoadLibraryA; | |
4150 | alias loadSym = GetProcAddress; | |
4151 | } | |
4152 | else | |
4153 | static assert(0, "unimplemented"); | |
4154 | ||
4155 | void* handle; | |
4156 | version (Posix) | |
4157 | handle = dlopen(null, RTLD_LAZY); | |
4158 | else version (Windows) | |
4159 | handle = GetModuleHandleA(null); | |
4160 | assert(handle !is null); | |
4161 | ||
4162 | // try to load curl from the executable to allow static linking | |
4163 | if (loadSym(handle, "curl_global_init") is null) | |
4164 | { | |
4165 | import std.format : format; | |
4166 | version (Posix) | |
4167 | dlclose(handle); | |
4168 | ||
4169 | version (OSX) | |
4170 | static immutable names = ["libcurl.4.dylib"]; | |
4171 | else version (Posix) | |
4172 | { | |
4173 | static immutable names = ["libcurl.so", "libcurl.so.4", | |
4174 | "libcurl-gnutls.so.4", "libcurl-nss.so.4", "libcurl.so.3"]; | |
4175 | } | |
4176 | else version (Windows) | |
4177 | static immutable names = ["libcurl.dll", "curl.dll"]; | |
4178 | ||
4179 | foreach (name; names) | |
4180 | { | |
4181 | version (Posix) | |
4182 | handle = dlopen(name.ptr, RTLD_LAZY); | |
4183 | else version (Windows) | |
4184 | handle = LoadLibraryA(name.ptr); | |
4185 | if (handle !is null) break; | |
4186 | } | |
4187 | ||
4188 | enforce!CurlException(handle !is null, "Failed to load curl, tried %(%s, %).".format(names)); | |
4189 | } | |
4190 | ||
4191 | foreach (i, FP; typeof(API.tupleof)) | |
4192 | { | |
4193 | enum name = __traits(identifier, _api.tupleof[i]); | |
4194 | auto p = enforce!CurlException(loadSym(handle, "curl_"~name), | |
4195 | "Couldn't load curl_"~name~" from libcurl."); | |
4196 | _api.tupleof[i] = cast(FP) p; | |
4197 | } | |
4198 | ||
4199 | enforce!CurlException(!_api.global_init(CurlGlobal.all), | |
4200 | "Failed to initialize libcurl"); | |
4201 | ||
4202 | static extern(C) void cleanup() | |
4203 | { | |
4204 | if (_handle is null) return; | |
4205 | _api.global_cleanup(); | |
4206 | version (Posix) | |
4207 | { | |
4208 | import core.sys.posix.dlfcn : dlclose; | |
4209 | dlclose(_handle); | |
4210 | } | |
4211 | else version (Windows) | |
4212 | { | |
4213 | import core.sys.windows.windows : FreeLibrary; | |
4214 | FreeLibrary(_handle); | |
4215 | } | |
4216 | else | |
4217 | static assert(0, "unimplemented"); | |
4218 | _api = API.init; | |
4219 | _handle = null; | |
4220 | } | |
4221 | ||
4222 | import core.stdc.stdlib : atexit; | |
4223 | atexit(&cleanup); | |
4224 | ||
4225 | return handle; | |
4226 | } | |
4227 | } | |
4228 | ||
4229 | /** | |
4230 | Wrapper to provide a better interface to libcurl than using the plain C API. | |
4231 | It is recommended to use the $(D HTTP)/$(D FTP) etc. structs instead unless | |
4232 | raw access to libcurl is needed. | |
4233 | ||
4234 | Warning: This struct uses interior pointers for callbacks. Only allocate it | |
4235 | on the stack if you never move or copy it. This also means passing by reference | |
4236 | when passing Curl to other functions. Otherwise always allocate on | |
4237 | the heap. | |
4238 | */ | |
4239 | struct Curl | |
4240 | { | |
4241 | alias OutData = void[]; | |
4242 | alias InData = ubyte[]; | |
4243 | private bool _stopped; | |
4244 | ||
4245 | private static auto ref curl() @property { return CurlAPI.instance; } | |
4246 | ||
4247 | // A handle should not be used by two threads simultaneously | |
4248 | private CURL* handle; | |
4249 | ||
4250 | // May also return $(D CURL_READFUNC_ABORT) or $(D CURL_READFUNC_PAUSE) | |
4251 | private size_t delegate(OutData) _onSend; | |
4252 | private size_t delegate(InData) _onReceive; | |
4253 | private void delegate(in char[]) _onReceiveHeader; | |
4254 | private CurlSeek delegate(long,CurlSeekPos) _onSeek; | |
4255 | private int delegate(curl_socket_t,CurlSockType) _onSocketOption; | |
4256 | private int delegate(size_t dltotal, size_t dlnow, | |
4257 | size_t ultotal, size_t ulnow) _onProgress; | |
4258 | ||
4259 | alias requestPause = CurlReadFunc.pause; | |
4260 | alias requestAbort = CurlReadFunc.abort; | |
4261 | ||
4262 | /** | |
4263 | Initialize the instance by creating a working curl handle. | |
4264 | */ | |
4265 | void initialize() | |
4266 | { | |
4267 | enforce!CurlException(!handle, "Curl instance already initialized"); | |
4268 | handle = curl.easy_init(); | |
4269 | enforce!CurlException(handle, "Curl instance couldn't be initialized"); | |
4270 | _stopped = false; | |
4271 | set(CurlOption.nosignal, 1); | |
4272 | } | |
4273 | ||
4274 | /// | |
4275 | @property bool stopped() const | |
4276 | { | |
4277 | return _stopped; | |
4278 | } | |
4279 | ||
4280 | /** | |
4281 | Duplicate this handle. | |
4282 | ||
4283 | The new handle will have all options set as the one it was duplicated | |
4284 | from. An exception to this is that all options that cannot be shared | |
4285 | across threads are reset thereby making it safe to use the duplicate | |
4286 | in a new thread. | |
4287 | */ | |
4288 | Curl dup() | |
4289 | { | |
4290 | Curl copy; | |
4291 | copy.handle = curl.easy_duphandle(handle); | |
4292 | copy._stopped = false; | |
4293 | ||
4294 | with (CurlOption) { | |
4295 | auto tt = AliasSeq!(file, writefunction, writeheader, | |
4296 | headerfunction, infile, readfunction, ioctldata, ioctlfunction, | |
4297 | seekdata, seekfunction, sockoptdata, sockoptfunction, | |
4298 | opensocketdata, opensocketfunction, progressdata, | |
4299 | progressfunction, debugdata, debugfunction, interleavedata, | |
4300 | interleavefunction, chunk_data, chunk_bgn_function, | |
4301 | chunk_end_function, fnmatch_data, fnmatch_function, cookiejar, postfields); | |
4302 | ||
4303 | foreach (option; tt) | |
4304 | copy.clear(option); | |
4305 | } | |
4306 | ||
4307 | // The options are only supported by libcurl when it has been built | |
4308 | // against certain versions of OpenSSL - if your libcurl uses an old | |
4309 | // OpenSSL, or uses an entirely different SSL engine, attempting to | |
4310 | // clear these normally will raise an exception | |
4311 | copy.clearIfSupported(CurlOption.ssl_ctx_function); | |
4312 | copy.clearIfSupported(CurlOption.ssh_keydata); | |
4313 | ||
4314 | // Enable for curl version > 7.21.7 | |
4315 | static if (LIBCURL_VERSION_MAJOR >= 7 && | |
4316 | LIBCURL_VERSION_MINOR >= 21 && | |
4317 | LIBCURL_VERSION_PATCH >= 7) | |
4318 | { | |
4319 | copy.clear(CurlOption.closesocketdata); | |
4320 | copy.clear(CurlOption.closesocketfunction); | |
4321 | } | |
4322 | ||
4323 | copy.set(CurlOption.nosignal, 1); | |
4324 | ||
4325 | // copy.clear(CurlOption.ssl_ctx_data); Let ssl function be shared | |
4326 | // copy.clear(CurlOption.ssh_keyfunction); Let key function be shared | |
4327 | ||
4328 | /* | |
4329 | Allow sharing of conv functions | |
4330 | copy.clear(CurlOption.conv_to_network_function); | |
4331 | copy.clear(CurlOption.conv_from_network_function); | |
4332 | copy.clear(CurlOption.conv_from_utf8_function); | |
4333 | */ | |
4334 | ||
4335 | return copy; | |
4336 | } | |
4337 | ||
4338 | private void _check(CurlCode code) | |
4339 | { | |
4340 | enforce!CurlTimeoutException(code != CurlError.operation_timedout, | |
4341 | errorString(code)); | |
4342 | ||
4343 | enforce!CurlException(code == CurlError.ok, | |
4344 | errorString(code)); | |
4345 | } | |
4346 | ||
4347 | private string errorString(CurlCode code) | |
4348 | { | |
4349 | import core.stdc.string : strlen; | |
4350 | import std.format : format; | |
4351 | ||
4352 | auto msgZ = curl.easy_strerror(code); | |
4353 | // doing the following (instead of just using std.conv.to!string) avoids 1 allocation | |
4354 | return format("%s on handle %s", msgZ[0 .. strlen(msgZ)], handle); | |
4355 | } | |
4356 | ||
4357 | private void throwOnStopped(string message = null) | |
4358 | { | |
4359 | auto def = "Curl instance called after being cleaned up"; | |
4360 | enforce!CurlException(!stopped, | |
4361 | message == null ? def : message); | |
4362 | } | |
4363 | ||
4364 | /** | |
4365 | Stop and invalidate this curl instance. | |
4366 | Warning: Do not call this from inside a callback handler e.g. $(D onReceive). | |
4367 | */ | |
4368 | void shutdown() | |
4369 | { | |
4370 | throwOnStopped(); | |
4371 | _stopped = true; | |
4372 | curl.easy_cleanup(this.handle); | |
4373 | this.handle = null; | |
4374 | } | |
4375 | ||
4376 | /** | |
4377 | Pausing and continuing transfers. | |
4378 | */ | |
4379 | void pause(bool sendingPaused, bool receivingPaused) | |
4380 | { | |
4381 | throwOnStopped(); | |
4382 | _check(curl.easy_pause(this.handle, | |
4383 | (sendingPaused ? CurlPause.send_cont : CurlPause.send) | | |
4384 | (receivingPaused ? CurlPause.recv_cont : CurlPause.recv))); | |
4385 | } | |
4386 | ||
4387 | /** | |
4388 | Set a string curl option. | |
4389 | Params: | |
4390 | option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation | |
4391 | value = The string | |
4392 | */ | |
4393 | void set(CurlOption option, const(char)[] value) | |
4394 | { | |
4395 | throwOnStopped(); | |
4396 | _check(curl.easy_setopt(this.handle, option, value.tempCString().buffPtr)); | |
4397 | } | |
4398 | ||
4399 | /** | |
4400 | Set a long curl option. | |
4401 | Params: | |
4402 | option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation | |
4403 | value = The long | |
4404 | */ | |
4405 | void set(CurlOption option, long value) | |
4406 | { | |
4407 | throwOnStopped(); | |
4408 | _check(curl.easy_setopt(this.handle, option, value)); | |
4409 | } | |
4410 | ||
4411 | /** | |
4412 | Set a void* curl option. | |
4413 | Params: | |
4414 | option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation | |
4415 | value = The pointer | |
4416 | */ | |
4417 | void set(CurlOption option, void* value) | |
4418 | { | |
4419 | throwOnStopped(); | |
4420 | _check(curl.easy_setopt(this.handle, option, value)); | |
4421 | } | |
4422 | ||
4423 | /** | |
4424 | Clear a pointer option. | |
4425 | Params: | |
4426 | option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation | |
4427 | */ | |
4428 | void clear(CurlOption option) | |
4429 | { | |
4430 | throwOnStopped(); | |
4431 | _check(curl.easy_setopt(this.handle, option, null)); | |
4432 | } | |
4433 | ||
4434 | /** | |
4435 | Clear a pointer option. Does not raise an exception if the underlying | |
4436 | libcurl does not support the option. Use sparingly. | |
4437 | Params: | |
4438 | option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation | |
4439 | */ | |
4440 | void clearIfSupported(CurlOption option) | |
4441 | { | |
4442 | throwOnStopped(); | |
4443 | auto rval = curl.easy_setopt(this.handle, option, null); | |
4444 | if (rval != CurlError.unknown_option && rval != CurlError.not_built_in) | |
4445 | _check(rval); | |
4446 | } | |
4447 | ||
4448 | /** | |
4449 | perform the curl request by doing the HTTP,FTP etc. as it has | |
4450 | been setup beforehand. | |
4451 | ||
4452 | Params: | |
4453 | throwOnError = whether to throw an exception or return a CurlCode on error | |
4454 | */ | |
4455 | CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) | |
4456 | { | |
4457 | throwOnStopped(); | |
4458 | CurlCode code = curl.easy_perform(this.handle); | |
4459 | if (throwOnError) | |
4460 | _check(code); | |
4461 | return code; | |
4462 | } | |
4463 | ||
4464 | /** | |
4465 | Get the various timings like name lookup time, total time, connect time etc. | |
4466 | The timed category is passed through the timing parameter while the timing | |
4467 | value is stored at val. The value is usable only if res is equal to | |
4468 | $(D etc.c.curl.CurlError.ok). | |
4469 | */ | |
4470 | CurlCode getTiming(CurlInfo timing, ref double val) | |
4471 | { | |
4472 | CurlCode code; | |
4473 | code = curl.easy_getinfo(handle, timing, &val); | |
4474 | return code; | |
4475 | } | |
4476 | ||
4477 | /** | |
4478 | * The event handler that receives incoming data. | |
4479 | * | |
4480 | * Params: | |
4481 | * callback = the callback that receives the $(D ubyte[]) data. | |
4482 | * Be sure to copy the incoming data and not store | |
4483 | * a slice. | |
4484 | * | |
4485 | * Returns: | |
4486 | * The callback returns the incoming bytes read. If not the entire array is | |
4487 | * the request will abort. | |
4488 | * The special value HTTP.pauseRequest can be returned in order to pause the | |
4489 | * current request. | |
4490 | * | |
4491 | * Example: | |
4492 | * ---- | |
4493 | * import std.net.curl, std.stdio; | |
4494 | * Curl curl; | |
4495 | * curl.initialize(); | |
4496 | * curl.set(CurlOption.url, "http://dlang.org"); | |
4497 | * curl.onReceive = (ubyte[] data) { writeln("Got data", to!(const(char)[])(data)); return data.length;}; | |
4498 | * curl.perform(); | |
4499 | * ---- | |
4500 | */ | |
4501 | @property void onReceive(size_t delegate(InData) callback) | |
4502 | { | |
4503 | _onReceive = (InData id) | |
4504 | { | |
4505 | throwOnStopped("Receive callback called on cleaned up Curl instance"); | |
4506 | return callback(id); | |
4507 | }; | |
4508 | set(CurlOption.file, cast(void*) &this); | |
4509 | set(CurlOption.writefunction, cast(void*) &Curl._receiveCallback); | |
4510 | } | |
4511 | ||
4512 | /** | |
4513 | * The event handler that receives incoming headers for protocols | |
4514 | * that uses headers. | |
4515 | * | |
4516 | * Params: | |
4517 | * callback = the callback that receives the header string. | |
4518 | * Make sure the callback copies the incoming params if | |
4519 | * it needs to store it because they are references into | |
4520 | * the backend and may very likely change. | |
4521 | * | |
4522 | * Example: | |
4523 | * ---- | |
4524 | * import std.net.curl, std.stdio; | |
4525 | * Curl curl; | |
4526 | * curl.initialize(); | |
4527 | * curl.set(CurlOption.url, "http://dlang.org"); | |
4528 | * curl.onReceiveHeader = (in char[] header) { writeln(header); }; | |
4529 | * curl.perform(); | |
4530 | * ---- | |
4531 | */ | |
4532 | @property void onReceiveHeader(void delegate(in char[]) callback) | |
4533 | { | |
4534 | _onReceiveHeader = (in char[] od) | |
4535 | { | |
4536 | throwOnStopped("Receive header callback called on "~ | |
4537 | "cleaned up Curl instance"); | |
4538 | callback(od); | |
4539 | }; | |
4540 | set(CurlOption.writeheader, cast(void*) &this); | |
4541 | set(CurlOption.headerfunction, | |
4542 | cast(void*) &Curl._receiveHeaderCallback); | |
4543 | } | |
4544 | ||
4545 | /** | |
4546 | * The event handler that gets called when data is needed for sending. | |
4547 | * | |
4548 | * Params: | |
4549 | * callback = the callback that has a $(D void[]) buffer to be filled | |
4550 | * | |
4551 | * Returns: | |
4552 | * The callback returns the number of elements in the buffer that have been | |
4553 | * filled and are ready to send. | |
4554 | * The special value $(D Curl.abortRequest) can be returned in | |
4555 | * order to abort the current request. | |
4556 | * The special value $(D Curl.pauseRequest) can be returned in order to | |
4557 | * pause the current request. | |
4558 | * | |
4559 | * Example: | |
4560 | * ---- | |
4561 | * import std.net.curl; | |
4562 | * Curl curl; | |
4563 | * curl.initialize(); | |
4564 | * curl.set(CurlOption.url, "http://dlang.org"); | |
4565 | * | |
4566 | * string msg = "Hello world"; | |
4567 | * curl.onSend = (void[] data) | |
4568 | * { | |
4569 | * auto m = cast(void[]) msg; | |
4570 | * size_t length = m.length > data.length ? data.length : m.length; | |
4571 | * if (length == 0) return 0; | |
4572 | * data[0 .. length] = m[0 .. length]; | |
4573 | * msg = msg[length..$]; | |
4574 | * return length; | |
4575 | * }; | |
4576 | * curl.perform(); | |
4577 | * ---- | |
4578 | */ | |
4579 | @property void onSend(size_t delegate(OutData) callback) | |
4580 | { | |
4581 | _onSend = (OutData od) | |
4582 | { | |
4583 | throwOnStopped("Send callback called on cleaned up Curl instance"); | |
4584 | return callback(od); | |
4585 | }; | |
4586 | set(CurlOption.infile, cast(void*) &this); | |
4587 | set(CurlOption.readfunction, cast(void*) &Curl._sendCallback); | |
4588 | } | |
4589 | ||
4590 | /** | |
4591 | * The event handler that gets called when the curl backend needs to seek | |
4592 | * the data to be sent. | |
4593 | * | |
4594 | * Params: | |
4595 | * callback = the callback that receives a seek offset and a seek position | |
4596 | * $(REF CurlSeekPos, etc,c,curl) | |
4597 | * | |
4598 | * Returns: | |
4599 | * The callback returns the success state of the seeking | |
4600 | * $(REF CurlSeek, etc,c,curl) | |
4601 | * | |
4602 | * Example: | |
4603 | * ---- | |
4604 | * import std.net.curl; | |
4605 | * Curl curl; | |
4606 | * curl.initialize(); | |
4607 | * curl.set(CurlOption.url, "http://dlang.org"); | |
4608 | * curl.onSeek = (long p, CurlSeekPos sp) | |
4609 | * { | |
4610 | * return CurlSeek.cantseek; | |
4611 | * }; | |
4612 | * curl.perform(); | |
4613 | * ---- | |
4614 | */ | |
4615 | @property void onSeek(CurlSeek delegate(long, CurlSeekPos) callback) | |
4616 | { | |
4617 | _onSeek = (long ofs, CurlSeekPos sp) | |
4618 | { | |
4619 | throwOnStopped("Seek callback called on cleaned up Curl instance"); | |
4620 | return callback(ofs, sp); | |
4621 | }; | |
4622 | set(CurlOption.seekdata, cast(void*) &this); | |
4623 | set(CurlOption.seekfunction, cast(void*) &Curl._seekCallback); | |
4624 | } | |
4625 | ||
4626 | /** | |
4627 | * The event handler that gets called when the net socket has been created | |
4628 | * but a $(D connect()) call has not yet been done. This makes it possible to set | |
4629 | * misc. socket options. | |
4630 | * | |
4631 | * Params: | |
4632 | * callback = the callback that receives the socket and socket type | |
4633 | * $(REF CurlSockType, etc,c,curl) | |
4634 | * | |
4635 | * Returns: | |
4636 | * Return 0 from the callback to signal success, return 1 to signal error | |
4637 | * and make curl close the socket | |
4638 | * | |
4639 | * Example: | |
4640 | * ---- | |
4641 | * import std.net.curl; | |
4642 | * Curl curl; | |
4643 | * curl.initialize(); | |
4644 | * curl.set(CurlOption.url, "http://dlang.org"); | |
4645 | * curl.onSocketOption = delegate int(curl_socket_t s, CurlSockType t) { /+ do stuff +/ }; | |
4646 | * curl.perform(); | |
4647 | * ---- | |
4648 | */ | |
4649 | @property void onSocketOption(int delegate(curl_socket_t, | |
4650 | CurlSockType) callback) | |
4651 | { | |
4652 | _onSocketOption = (curl_socket_t sock, CurlSockType st) | |
4653 | { | |
4654 | throwOnStopped("Socket option callback called on "~ | |
4655 | "cleaned up Curl instance"); | |
4656 | return callback(sock, st); | |
4657 | }; | |
4658 | set(CurlOption.sockoptdata, cast(void*) &this); | |
4659 | set(CurlOption.sockoptfunction, | |
4660 | cast(void*) &Curl._socketOptionCallback); | |
4661 | } | |
4662 | ||
4663 | /** | |
4664 | * The event handler that gets called to inform of upload/download progress. | |
4665 | * | |
4666 | * Params: | |
4667 | * callback = the callback that receives the (total bytes to download, | |
4668 | * currently downloaded bytes, total bytes to upload, currently uploaded | |
4669 | * bytes). | |
4670 | * | |
4671 | * Returns: | |
4672 | * Return 0 from the callback to signal success, return non-zero to abort | |
4673 | * transfer | |
4674 | * | |
4675 | * Example: | |
4676 | * ---- | |
4677 | * import std.net.curl; | |
4678 | * Curl curl; | |
4679 | * curl.initialize(); | |
4680 | * curl.set(CurlOption.url, "http://dlang.org"); | |
4681 | * curl.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t uln) | |
4682 | * { | |
4683 | * writeln("Progress: downloaded bytes ", dlnow, " of ", dltotal); | |
4684 | * writeln("Progress: uploaded bytes ", ulnow, " of ", ultotal); | |
4685 | * curl.perform(); | |
4686 | * }; | |
4687 | * ---- | |
4688 | */ | |
4689 | @property void onProgress(int delegate(size_t dlTotal, | |
4690 | size_t dlNow, | |
4691 | size_t ulTotal, | |
4692 | size_t ulNow) callback) | |
4693 | { | |
4694 | _onProgress = (size_t dlt, size_t dln, size_t ult, size_t uln) | |
4695 | { | |
4696 | throwOnStopped("Progress callback called on cleaned "~ | |
4697 | "up Curl instance"); | |
4698 | return callback(dlt, dln, ult, uln); | |
4699 | }; | |
4700 | set(CurlOption.noprogress, 0); | |
4701 | set(CurlOption.progressdata, cast(void*) &this); | |
4702 | set(CurlOption.progressfunction, cast(void*) &Curl._progressCallback); | |
4703 | } | |
4704 | ||
4705 | // Internal C callbacks to register with libcurl | |
4706 | extern (C) private static | |
4707 | size_t _receiveCallback(const char* str, | |
4708 | size_t size, size_t nmemb, void* ptr) | |
4709 | { | |
4710 | auto b = cast(Curl*) ptr; | |
4711 | if (b._onReceive != null) | |
4712 | return b._onReceive(cast(InData)(str[0 .. size*nmemb])); | |
4713 | return size*nmemb; | |
4714 | } | |
4715 | ||
4716 | extern (C) private static | |
4717 | size_t _receiveHeaderCallback(const char* str, | |
4718 | size_t size, size_t nmemb, void* ptr) | |
4719 | { | |
4720 | import std.string : chomp; | |
4721 | ||
4722 | auto b = cast(Curl*) ptr; | |
4723 | auto s = str[0 .. size*nmemb].chomp(); | |
4724 | if (b._onReceiveHeader != null) | |
4725 | b._onReceiveHeader(s); | |
4726 | ||
4727 | return size*nmemb; | |
4728 | } | |
4729 | ||
4730 | extern (C) private static | |
4731 | size_t _sendCallback(char *str, size_t size, size_t nmemb, void *ptr) | |
4732 | { | |
4733 | Curl* b = cast(Curl*) ptr; | |
4734 | auto a = cast(void[]) str[0 .. size*nmemb]; | |
4735 | if (b._onSend == null) | |
4736 | return 0; | |
4737 | return b._onSend(a); | |
4738 | } | |
4739 | ||
4740 | extern (C) private static | |
4741 | int _seekCallback(void *ptr, curl_off_t offset, int origin) | |
4742 | { | |
4743 | auto b = cast(Curl*) ptr; | |
4744 | if (b._onSeek == null) | |
4745 | return CurlSeek.cantseek; | |
4746 | ||
4747 | // origin: CurlSeekPos.set/current/end | |
4748 | // return: CurlSeek.ok/fail/cantseek | |
4749 | return b._onSeek(cast(long) offset, cast(CurlSeekPos) origin); | |
4750 | } | |
4751 | ||
4752 | extern (C) private static | |
4753 | int _socketOptionCallback(void *ptr, | |
4754 | curl_socket_t curlfd, curlsocktype purpose) | |
4755 | { | |
4756 | auto b = cast(Curl*) ptr; | |
4757 | if (b._onSocketOption == null) | |
4758 | return 0; | |
4759 | ||
4760 | // return: 0 ok, 1 fail | |
4761 | return b._onSocketOption(curlfd, cast(CurlSockType) purpose); | |
4762 | } | |
4763 | ||
4764 | extern (C) private static | |
4765 | int _progressCallback(void *ptr, | |
4766 | double dltotal, double dlnow, | |
4767 | double ultotal, double ulnow) | |
4768 | { | |
4769 | auto b = cast(Curl*) ptr; | |
4770 | if (b._onProgress == null) | |
4771 | return 0; | |
4772 | ||
4773 | // return: 0 ok, 1 fail | |
4774 | return b._onProgress(cast(size_t) dltotal, cast(size_t) dlnow, | |
4775 | cast(size_t) ultotal, cast(size_t) ulnow); | |
4776 | } | |
4777 | ||
4778 | } | |
4779 | ||
4780 | // Internal messages send between threads. | |
4781 | // The data is wrapped in this struct in order to ensure that | |
4782 | // other std.concurrency.receive calls does not pick up our messages | |
4783 | // by accident. | |
4784 | private struct CurlMessage(T) | |
4785 | { | |
4786 | public T data; | |
4787 | } | |
4788 | ||
4789 | private static CurlMessage!T curlMessage(T)(T data) | |
4790 | { | |
4791 | return CurlMessage!T(data); | |
4792 | } | |
4793 | ||
4794 | // Pool of to be used for reusing buffers | |
4795 | private struct Pool(Data) | |
4796 | { | |
4797 | private struct Entry | |
4798 | { | |
4799 | Data data; | |
4800 | Entry* next; | |
4801 | } | |
4802 | private Entry* root; | |
4803 | private Entry* freeList; | |
4804 | ||
4805 | @safe @property bool empty() | |
4806 | { | |
4807 | return root == null; | |
4808 | } | |
4809 | ||
4810 | @safe nothrow void push(Data d) | |
4811 | { | |
4812 | if (freeList == null) | |
4813 | { | |
4814 | // Allocate new Entry since there is no one | |
4815 | // available in the freeList | |
4816 | freeList = new Entry; | |
4817 | } | |
4818 | freeList.data = d; | |
4819 | Entry* oldroot = root; | |
4820 | root = freeList; | |
4821 | freeList = freeList.next; | |
4822 | root.next = oldroot; | |
4823 | } | |
4824 | ||
4825 | @safe Data pop() | |
4826 | { | |
4827 | enforce!Exception(root != null, "pop() called on empty pool"); | |
4828 | auto d = root.data; | |
4829 | auto n = root.next; | |
4830 | root.next = freeList; | |
4831 | freeList = root; | |
4832 | root = n; | |
4833 | return d; | |
4834 | } | |
4835 | } | |
4836 | ||
4837 | // Shared function for reading incoming chunks of data and | |
4838 | // sending the to a parent thread | |
4839 | private static size_t _receiveAsyncChunks(ubyte[] data, ref ubyte[] outdata, | |
4840 | Pool!(ubyte[]) freeBuffers, | |
4841 | ref ubyte[] buffer, Tid fromTid, | |
4842 | ref bool aborted) | |
4843 | { | |
4844 | immutable datalen = data.length; | |
4845 | ||
4846 | // Copy data to fill active buffer | |
4847 | while (!data.empty) | |
4848 | { | |
4849 | ||
4850 | // Make sure a buffer is present | |
4851 | while ( outdata.empty && freeBuffers.empty) | |
4852 | { | |
4853 | // Active buffer is invalid and there are no | |
4854 | // available buffers in the pool. Wait for buffers | |
4855 | // to return from main thread in order to reuse | |
4856 | // them. | |
4857 | receive((immutable(ubyte)[] buf) | |
4858 | { | |
4859 | buffer = cast(ubyte[]) buf; | |
4860 | outdata = buffer[]; | |
4861 | }, | |
4862 | (bool flag) { aborted = true; } | |
4863 | ); | |
4864 | if (aborted) return cast(size_t) 0; | |
4865 | } | |
4866 | if (outdata.empty) | |
4867 | { | |
4868 | buffer = freeBuffers.pop(); | |
4869 | outdata = buffer[]; | |
4870 | } | |
4871 | ||
4872 | // Copy data | |
4873 | auto copyBytes = outdata.length < data.length ? | |
4874 | outdata.length : data.length; | |
4875 | ||
4876 | outdata[0 .. copyBytes] = data[0 .. copyBytes]; | |
4877 | outdata = outdata[copyBytes..$]; | |
4878 | data = data[copyBytes..$]; | |
4879 | ||
4880 | if (outdata.empty) | |
4881 | fromTid.send(thisTid, curlMessage(cast(immutable(ubyte)[])buffer)); | |
4882 | } | |
4883 | ||
4884 | return datalen; | |
4885 | } | |
4886 | ||
4887 | // ditto | |
4888 | private static void _finalizeAsyncChunks(ubyte[] outdata, ref ubyte[] buffer, | |
4889 | Tid fromTid) | |
4890 | { | |
4891 | if (!outdata.empty) | |
4892 | { | |
4893 | // Resize the last buffer | |
4894 | buffer.length = buffer.length - outdata.length; | |
4895 | fromTid.send(thisTid, curlMessage(cast(immutable(ubyte)[])buffer)); | |
4896 | } | |
4897 | } | |
4898 | ||
4899 | ||
4900 | // Shared function for reading incoming lines of data and sending the to a | |
4901 | // parent thread | |
4902 | private static size_t _receiveAsyncLines(Terminator, Unit) | |
4903 | (const(ubyte)[] data, ref EncodingScheme encodingScheme, | |
4904 | bool keepTerminator, Terminator terminator, | |
4905 | ref const(ubyte)[] leftOverBytes, ref bool bufferValid, | |
4906 | ref Pool!(Unit[]) freeBuffers, ref Unit[] buffer, | |
4907 | Tid fromTid, ref bool aborted) | |
4908 | { | |
4909 | import std.format : format; | |
4910 | ||
4911 | immutable datalen = data.length; | |
4912 | ||
4913 | // Terminator is specified and buffers should be resized as determined by | |
4914 | // the terminator | |
4915 | ||
4916 | // Copy data to active buffer until terminator is found. | |
4917 | ||
4918 | // Decode as many lines as possible | |
4919 | while (true) | |
4920 | { | |
4921 | ||
4922 | // Make sure a buffer is present | |
4923 | while (!bufferValid && freeBuffers.empty) | |
4924 | { | |
4925 | // Active buffer is invalid and there are no available buffers in | |
4926 | // the pool. Wait for buffers to return from main thread in order to | |
4927 | // reuse them. | |
4928 | receive((immutable(Unit)[] buf) | |
4929 | { | |
4930 | buffer = cast(Unit[]) buf; | |
4931 | buffer.length = 0; | |
4932 | buffer.assumeSafeAppend(); | |
4933 | bufferValid = true; | |
4934 | }, | |
4935 | (bool flag) { aborted = true; } | |
4936 | ); | |
4937 | if (aborted) return cast(size_t) 0; | |
4938 | } | |
4939 | if (!bufferValid) | |
4940 | { | |
4941 | buffer = freeBuffers.pop(); | |
4942 | bufferValid = true; | |
4943 | } | |
4944 | ||
4945 | // Try to read a line from left over bytes from last onReceive plus the | |
4946 | // newly received bytes. | |
4947 | try | |
4948 | { | |
4949 | if (decodeLineInto(leftOverBytes, data, buffer, | |
4950 | encodingScheme, terminator)) | |
4951 | { | |
4952 | if (keepTerminator) | |
4953 | { | |
4954 | fromTid.send(thisTid, | |
4955 | curlMessage(cast(immutable(Unit)[])buffer)); | |
4956 | } | |
4957 | else | |
4958 | { | |
4959 | static if (isArray!Terminator) | |
4960 | fromTid.send(thisTid, | |
4961 | curlMessage(cast(immutable(Unit)[]) | |
4962 | buffer[0..$-terminator.length])); | |
4963 | else | |
4964 | fromTid.send(thisTid, | |
4965 | curlMessage(cast(immutable(Unit)[]) | |
4966 | buffer[0..$-1])); | |
4967 | } | |
4968 | bufferValid = false; | |
4969 | } | |
4970 | else | |
4971 | { | |
4972 | // Could not decode an entire line. Save | |
4973 | // bytes left in data for next call to | |
4974 | // onReceive. Can be up to a max of 4 bytes. | |
4975 | enforce!CurlException(data.length <= 4, | |
4976 | format( | |
4977 | "Too many bytes left not decoded %s"~ | |
4978 | " > 4. Maybe the charset specified in"~ | |
4979 | " headers does not match "~ | |
4980 | "the actual content downloaded?", | |
4981 | data.length)); | |
4982 | leftOverBytes ~= data; | |
4983 | break; | |
4984 | } | |
4985 | } | |
4986 | catch (CurlException ex) | |
4987 | { | |
4988 | prioritySend(fromTid, cast(immutable(CurlException))ex); | |
4989 | return cast(size_t) 0; | |
4990 | } | |
4991 | } | |
4992 | return datalen; | |
4993 | } | |
4994 | ||
4995 | // ditto | |
4996 | private static | |
4997 | void _finalizeAsyncLines(Unit)(bool bufferValid, Unit[] buffer, Tid fromTid) | |
4998 | { | |
4999 | if (bufferValid && buffer.length != 0) | |
5000 | fromTid.send(thisTid, curlMessage(cast(immutable(Unit)[])buffer[0..$])); | |
5001 | } | |
5002 | ||
5003 | ||
5004 | // Spawn a thread for handling the reading of incoming data in the | |
5005 | // background while the delegate is executing. This will optimize | |
5006 | // throughput by allowing simultaneous input (this struct) and | |
5007 | // output (e.g. AsyncHTTPLineOutputRange). | |
5008 | private static void _spawnAsync(Conn, Unit, Terminator = void)() | |
5009 | { | |
5010 | Tid fromTid = receiveOnly!Tid(); | |
5011 | ||
5012 | // Get buffer to read into | |
5013 | Pool!(Unit[]) freeBuffers; // Free list of buffer objects | |
5014 | ||
5015 | // Number of bytes filled into active buffer | |
5016 | Unit[] buffer; | |
5017 | bool aborted = false; | |
5018 | ||
5019 | EncodingScheme encodingScheme; | |
5020 | static if ( !is(Terminator == void)) | |
5021 | { | |
5022 | // Only lines reading will receive a terminator | |
5023 | const terminator = receiveOnly!Terminator(); | |
5024 | const keepTerminator = receiveOnly!bool(); | |
5025 | ||
5026 | // max number of bytes to carry over from an onReceive | |
5027 | // callback. This is 4 because it is the max code units to | |
5028 | // decode a code point in the supported encodings. | |
5029 | auto leftOverBytes = new const(ubyte)[4]; | |
5030 | leftOverBytes.length = 0; | |
5031 | auto bufferValid = false; | |
5032 | } | |
5033 | else | |
5034 | { | |
5035 | Unit[] outdata; | |
5036 | } | |
5037 | ||
5038 | // no move semantic available in std.concurrency ie. must use casting. | |
5039 | auto connDup = cast(CURL*) receiveOnly!ulong(); | |
5040 | auto client = Conn(); | |
5041 | client.p.curl.handle = connDup; | |
5042 | ||
5043 | // receive a method for both ftp and http but just use it for http | |
5044 | auto method = receiveOnly!(HTTP.Method)(); | |
5045 | ||
5046 | client.onReceive = (ubyte[] data) | |
5047 | { | |
5048 | // If no terminator is specified the chunk size is fixed. | |
5049 | static if ( is(Terminator == void) ) | |
5050 | return _receiveAsyncChunks(data, outdata, freeBuffers, buffer, | |
5051 | fromTid, aborted); | |
5052 | else | |
5053 | return _receiveAsyncLines(data, encodingScheme, | |
5054 | keepTerminator, terminator, leftOverBytes, | |
5055 | bufferValid, freeBuffers, buffer, | |
5056 | fromTid, aborted); | |
5057 | }; | |
5058 | ||
5059 | static if ( is(Conn == HTTP) ) | |
5060 | { | |
5061 | client.method = method; | |
5062 | // register dummy header handler | |
5063 | client.onReceiveHeader = (in char[] key, in char[] value) | |
5064 | { | |
5065 | if (key == "content-type") | |
5066 | encodingScheme = EncodingScheme.create(client.p.charset); | |
5067 | }; | |
5068 | } | |
5069 | else | |
5070 | { | |
5071 | encodingScheme = EncodingScheme.create(client.encoding); | |
5072 | } | |
5073 | ||
5074 | // Start the request | |
5075 | CurlCode code; | |
5076 | try | |
5077 | { | |
5078 | code = client.perform(No.throwOnError); | |
5079 | } | |
5080 | catch (Exception ex) | |
5081 | { | |
5082 | prioritySend(fromTid, cast(immutable(Exception)) ex); | |
5083 | fromTid.send(thisTid, curlMessage(true)); // signal done | |
5084 | return; | |
5085 | } | |
5086 | ||
5087 | if (code != CurlError.ok) | |
5088 | { | |
5089 | if (aborted && (code == CurlError.aborted_by_callback || | |
5090 | code == CurlError.write_error)) | |
5091 | { | |
5092 | fromTid.send(thisTid, curlMessage(true)); // signal done | |
5093 | return; | |
5094 | } | |
5095 | prioritySend(fromTid, cast(immutable(CurlException)) | |
5096 | new CurlException(client.p.curl.errorString(code))); | |
5097 | ||
5098 | fromTid.send(thisTid, curlMessage(true)); // signal done | |
5099 | return; | |
5100 | } | |
5101 | ||
5102 | // Send remaining data that is not a full chunk size | |
5103 | static if ( is(Terminator == void) ) | |
5104 | _finalizeAsyncChunks(outdata, buffer, fromTid); | |
5105 | else | |
5106 | _finalizeAsyncLines(bufferValid, buffer, fromTid); | |
5107 | ||
5108 | fromTid.send(thisTid, curlMessage(true)); // signal done | |
5109 | } |