]>
Commit | Line | Data |
---|---|---|
b4c522fa IB |
1 | // Written in the D programming language. |
2 | ||
3 | /** | |
5fee5ec3 | 4 | Networking client functionality as provided by $(HTTP curl.haxx.se/libcurl, |
b4c522fa IB |
5 | libcurl). The libcurl library must be installed on the system in order to use |
6 | this module. | |
7 | ||
8 | $(SCRIPT inhibitQuickIndex = 1;) | |
9 | ||
10 | $(DIVC quickindex, | |
11 | $(BOOKTABLE , | |
12 | $(TR $(TH Category) $(TH Functions) | |
13 | ) | |
14 | $(TR $(TDNW High level) $(TD $(MYREF download) $(MYREF upload) $(MYREF get) | |
15 | $(MYREF post) $(MYREF put) $(MYREF del) $(MYREF options) $(MYREF trace) | |
16 | $(MYREF connect) $(MYREF byLine) $(MYREF byChunk) | |
17 | $(MYREF byLineAsync) $(MYREF byChunkAsync) ) | |
18 | ) | |
19 | $(TR $(TDNW Low level) $(TD $(MYREF HTTP) $(MYREF FTP) $(MYREF | |
20 | SMTP) ) | |
21 | ) | |
22 | ) | |
23 | ) | |
24 | ||
25 | Note: | |
26 | You may need to link to the $(B curl) library, e.g. by adding $(D "libs": ["curl"]) | |
27 | to your $(B dub.json) file if you are using $(LINK2 http://code.dlang.org, DUB). | |
28 | ||
29 | Windows x86 note: | |
30 | A DMD compatible libcurl static library can be downloaded from the dlang.org | |
7e7ebe3e | 31 | $(LINK2 https://downloads.dlang.org/other/index.html, download archive page). |
5fee5ec3 IB |
32 | |
33 | This module is not available for iOS, tvOS or watchOS. | |
b4c522fa IB |
34 | |
35 | Compared to using libcurl directly this module allows simpler client code for | |
36 | common uses, requires no unsafe operations, and integrates better with the rest | |
5fee5ec3 | 37 | of the language. Futhermore it provides $(MREF_ALTTEXT range, std,range) |
b4c522fa IB |
38 | access to protocols supported by libcurl both synchronously and asynchronously. |
39 | ||
40 | A high level and a low level API are available. The high level API is built | |
41 | entirely on top of the low level one. | |
42 | ||
43 | The high level API is for commonly used functionality such as HTTP/FTP get. The | |
5fee5ec3 IB |
44 | $(LREF byLineAsync) and $(LREF byChunkAsync) provides asynchronous |
45 | $(MREF_ALTTEXT range, std,range) that performs the request in another | |
b4c522fa IB |
46 | thread while handling a line/chunk in the current thread. |
47 | ||
48 | The low level API allows for streaming and other advanced features. | |
49 | ||
50 | $(BOOKTABLE Cheat Sheet, | |
51 | $(TR $(TH Function Name) $(TH Description) | |
52 | ) | |
53 | $(LEADINGROW High level) | |
54 | $(TR $(TDNW $(LREF download)) $(TD $(D | |
55 | download("ftp.digitalmars.com/sieve.ds", "/tmp/downloaded-ftp-file")) | |
56 | downloads file from URL to file system.) | |
57 | ) | |
58 | $(TR $(TDNW $(LREF upload)) $(TD $(D | |
59 | upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds");) | |
60 | uploads file from file system to URL.) | |
61 | ) | |
62 | $(TR $(TDNW $(LREF get)) $(TD $(D | |
63 | get("dlang.org")) returns a char[] containing the dlang.org web page.) | |
64 | ) | |
65 | $(TR $(TDNW $(LREF put)) $(TD $(D | |
66 | put("dlang.org", "Hi")) returns a char[] containing | |
67 | the dlang.org web page. after a HTTP PUT of "hi") | |
68 | ) | |
69 | $(TR $(TDNW $(LREF post)) $(TD $(D | |
70 | post("dlang.org", "Hi")) returns a char[] containing | |
71 | the dlang.org web page. after a HTTP POST of "hi") | |
72 | ) | |
73 | $(TR $(TDNW $(LREF byLine)) $(TD $(D | |
74 | byLine("dlang.org")) returns a range of char[] containing the | |
75 | dlang.org web page.) | |
76 | ) | |
77 | $(TR $(TDNW $(LREF byChunk)) $(TD $(D | |
78 | byChunk("dlang.org", 10)) returns a range of ubyte[10] containing the | |
79 | dlang.org web page.) | |
80 | ) | |
81 | $(TR $(TDNW $(LREF byLineAsync)) $(TD $(D | |
82 | byLineAsync("dlang.org")) returns a range of char[] containing the dlang.org web | |
83 | page asynchronously.) | |
84 | ) | |
85 | $(TR $(TDNW $(LREF byChunkAsync)) $(TD $(D | |
86 | byChunkAsync("dlang.org", 10)) returns a range of ubyte[10] containing the | |
87 | dlang.org web page asynchronously.) | |
88 | ) | |
89 | $(LEADINGROW Low level | |
90 | ) | |
5fee5ec3 IB |
91 | $(TR $(TDNW $(LREF HTTP)) $(TD `HTTP` struct for advanced usage)) |
92 | $(TR $(TDNW $(LREF FTP)) $(TD `FTP` struct for advanced usage)) | |
93 | $(TR $(TDNW $(LREF SMTP)) $(TD `SMTP` struct for advanced usage)) | |
b4c522fa IB |
94 | ) |
95 | ||
96 | ||
97 | Example: | |
98 | --- | |
99 | import std.net.curl, std.stdio; | |
100 | ||
101 | // Return a char[] containing the content specified by a URL | |
102 | auto content = get("dlang.org"); | |
103 | ||
104 | // Post data and return a char[] containing the content specified by a URL | |
105 | auto content = post("mydomain.com/here.cgi", ["name1" : "value1", "name2" : "value2"]); | |
106 | ||
107 | // Get content of file from ftp server | |
108 | auto content = get("ftp.digitalmars.com/sieve.ds"); | |
109 | ||
110 | // Post and print out content line by line. The request is done in another thread. | |
111 | foreach (line; byLineAsync("dlang.org", "Post data")) | |
112 | writeln(line); | |
113 | ||
114 | // Get using a line range and proxy settings | |
115 | auto client = HTTP(); | |
116 | client.proxy = "1.2.3.4"; | |
117 | foreach (line; byLine("dlang.org", client)) | |
118 | writeln(line); | |
119 | --- | |
120 | ||
121 | For more control than the high level functions provide, use the low level API: | |
122 | ||
123 | Example: | |
124 | --- | |
125 | import std.net.curl, std.stdio; | |
126 | ||
127 | // GET with custom data receivers | |
128 | auto http = HTTP("dlang.org"); | |
129 | http.onReceiveHeader = | |
130 | (in char[] key, in char[] value) { writeln(key, ": ", value); }; | |
131 | http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; }; | |
132 | http.perform(); | |
133 | --- | |
134 | ||
135 | First, an instance of the reference-counted HTTP struct is created. Then the | |
136 | custom delegates are set. These will be called whenever the HTTP instance | |
137 | receives a header and a data buffer, respectively. In this simple example, the | |
138 | headers are written to stdout and the data is ignored. If the request should be | |
139 | stopped before it has finished then return something less than data.length from | |
140 | the onReceive callback. See $(LREF onReceiveHeader)/$(LREF onReceive) for more | |
141 | information. Finally the HTTP request is effected by calling perform(), which is | |
142 | synchronous. | |
143 | ||
5fee5ec3 | 144 | Source: $(PHOBOSSRC std/net/curl.d) |
b4c522fa IB |
145 | |
146 | Copyright: Copyright Jonas Drewsen 2011-2012 | |
147 | License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0). | |
148 | Authors: Jonas Drewsen. Some of the SMTP code contributed by Jimmy Cao. | |
149 | ||
5fee5ec3 | 150 | Credits: The functionally is based on $(HTTP curl.haxx.se/libcurl, libcurl). |
b4c522fa IB |
151 | LibCurl is licensed under an MIT/X derivative license. |
152 | */ | |
153 | /* | |
154 | Copyright Jonas Drewsen 2011 - 2012. | |
155 | Distributed under the Boost Software License, Version 1.0. | |
156 | (See accompanying file LICENSE_1_0.txt or copy at | |
157 | http://www.boost.org/LICENSE_1_0.txt) | |
158 | */ | |
159 | module std.net.curl; | |
160 | ||
5fee5ec3 IB |
161 | public import etc.c.curl : CurlOption; |
162 | import core.time : dur; | |
163 | import etc.c.curl : CURLcode; | |
b4c522fa | 164 | import std.range.primitives; |
5fee5ec3 IB |
165 | import std.encoding : EncodingScheme; |
166 | import std.traits : isSomeChar; | |
167 | import std.typecons : Flag, Yes, No, Tuple; | |
b4c522fa | 168 | |
5fee5ec3 IB |
169 | version (iOS) |
170 | version = iOSDerived; | |
171 | else version (TVOS) | |
172 | version = iOSDerived; | |
173 | else version (WatchOS) | |
174 | version = iOSDerived; | |
b4c522fa | 175 | |
5fee5ec3 IB |
176 | version (iOSDerived) {} |
177 | else: | |
b4c522fa | 178 | |
5fee5ec3 | 179 | version (StdUnittest) |
b4c522fa | 180 | { |
5fee5ec3 | 181 | import std.socket : Socket, SocketShutdown; |
b4c522fa IB |
182 | |
183 | private struct TestServer | |
184 | { | |
5fee5ec3 IB |
185 | import std.concurrency : Tid; |
186 | ||
187 | import std.socket : Socket, TcpSocket; | |
188 | ||
b4c522fa IB |
189 | string addr() { return _addr; } |
190 | ||
191 | void handle(void function(Socket s) dg) | |
192 | { | |
5fee5ec3 | 193 | import std.concurrency : send; |
b4c522fa IB |
194 | tid.send(dg); |
195 | } | |
196 | ||
197 | private: | |
198 | string _addr; | |
199 | Tid tid; | |
ab23d6fa | 200 | TcpSocket sock; |
b4c522fa IB |
201 | |
202 | static void loop(shared TcpSocket listener) | |
203 | { | |
5fee5ec3 IB |
204 | import std.concurrency : OwnerTerminated, receiveOnly; |
205 | import std.stdio : stderr; | |
206 | ||
b4c522fa IB |
207 | try while (true) |
208 | { | |
209 | void function(Socket) handler = void; | |
210 | try | |
211 | handler = receiveOnly!(typeof(handler)); | |
212 | catch (OwnerTerminated) | |
213 | return; | |
214 | handler((cast() listener).accept); | |
215 | } | |
216 | catch (Throwable e) | |
217 | { | |
5fee5ec3 IB |
218 | // https://issues.dlang.org/show_bug.cgi?id=7018 |
219 | stderr.writeln(e); | |
b4c522fa IB |
220 | } |
221 | } | |
222 | } | |
223 | ||
224 | private TestServer startServer() | |
225 | { | |
5fee5ec3 IB |
226 | import std.concurrency : spawn; |
227 | import std.socket : INADDR_LOOPBACK, InternetAddress, TcpSocket; | |
228 | ||
ab23d6fa | 229 | tlsInit = true; |
b4c522fa IB |
230 | auto sock = new TcpSocket; |
231 | sock.bind(new InternetAddress(INADDR_LOOPBACK, InternetAddress.PORT_ANY)); | |
232 | sock.listen(1); | |
233 | auto addr = sock.localAddress.toString(); | |
234 | auto tid = spawn(&TestServer.loop, cast(shared) sock); | |
ab23d6fa | 235 | return TestServer(addr, tid, sock); |
b4c522fa IB |
236 | } |
237 | ||
5fee5ec3 | 238 | /** Test server */ |
ab23d6fa | 239 | __gshared TestServer server; |
5fee5ec3 | 240 | /** Thread-local storage init */ |
ab23d6fa IB |
241 | bool tlsInit; |
242 | ||
b4c522fa IB |
243 | private ref TestServer testServer() |
244 | { | |
5fee5ec3 | 245 | import std.concurrency : initOnce; |
b4c522fa IB |
246 | return initOnce!server(startServer()); |
247 | } | |
248 | ||
ab23d6fa IB |
249 | static ~this() |
250 | { | |
251 | // terminate server from a thread local dtor of the thread that started it, | |
252 | // because thread_joinall is called before shared module dtors | |
253 | if (tlsInit && server.sock) | |
254 | { | |
255 | server.sock.shutdown(SocketShutdown.RECEIVE); | |
256 | server.sock.close(); | |
257 | } | |
258 | } | |
259 | ||
b4c522fa IB |
260 | private struct Request(T) |
261 | { | |
262 | string hdrs; | |
263 | immutable(T)[] bdy; | |
264 | } | |
265 | ||
266 | private Request!T recvReq(T=char)(Socket s) | |
267 | { | |
268 | import std.algorithm.comparison : min; | |
269 | import std.algorithm.searching : find, canFind; | |
270 | import std.conv : to; | |
271 | import std.regex : ctRegex, matchFirst; | |
272 | ||
273 | ubyte[1024] tmp=void; | |
274 | ubyte[] buf; | |
275 | ||
276 | while (true) | |
277 | { | |
278 | auto nbytes = s.receive(tmp[]); | |
279 | assert(nbytes >= 0); | |
280 | ||
281 | immutable beg = buf.length > 3 ? buf.length - 3 : 0; | |
282 | buf ~= tmp[0 .. nbytes]; | |
283 | auto bdy = buf[beg .. $].find(cast(ubyte[])"\r\n\r\n"); | |
284 | if (bdy.empty) | |
285 | continue; | |
286 | ||
287 | auto hdrs = cast(string) buf[0 .. $ - bdy.length]; | |
288 | bdy.popFrontN(4); | |
289 | // no support for chunked transfer-encoding | |
290 | if (auto m = hdrs.matchFirst(ctRegex!(`Content-Length: ([0-9]+)`, "i"))) | |
291 | { | |
292 | import std.uni : asUpperCase; | |
293 | if (hdrs.asUpperCase.canFind("EXPECT: 100-CONTINUE")) | |
294 | s.send(httpContinue); | |
295 | ||
296 | size_t remain = m.captures[1].to!size_t - bdy.length; | |
297 | while (remain) | |
298 | { | |
299 | nbytes = s.receive(tmp[0 .. min(remain, $)]); | |
300 | assert(nbytes >= 0); | |
301 | buf ~= tmp[0 .. nbytes]; | |
302 | remain -= nbytes; | |
303 | } | |
304 | } | |
305 | else | |
306 | { | |
307 | assert(bdy.empty); | |
308 | } | |
309 | bdy = buf[hdrs.length + 4 .. $]; | |
310 | return typeof(return)(hdrs, cast(immutable(T)[])bdy); | |
311 | } | |
312 | } | |
313 | ||
314 | private string httpOK(string msg) | |
315 | { | |
316 | import std.conv : to; | |
317 | ||
318 | return "HTTP/1.1 200 OK\r\n"~ | |
319 | "Content-Type: text/plain\r\n"~ | |
320 | "Content-Length: "~msg.length.to!string~"\r\n"~ | |
321 | "\r\n"~ | |
322 | msg; | |
323 | } | |
324 | ||
325 | private string httpOK() | |
326 | { | |
327 | return "HTTP/1.1 200 OK\r\n"~ | |
328 | "Content-Length: 0\r\n"~ | |
329 | "\r\n"; | |
330 | } | |
331 | ||
332 | private string httpNotFound() | |
333 | { | |
334 | return "HTTP/1.1 404 Not Found\r\n"~ | |
335 | "Content-Length: 0\r\n"~ | |
336 | "\r\n"; | |
337 | } | |
338 | ||
339 | private enum httpContinue = "HTTP/1.1 100 Continue\r\n\r\n"; | |
340 | } | |
341 | version (StdDdoc) import std.stdio; | |
342 | ||
343 | // Default data timeout for Protocols | |
344 | private enum _defaultDataTimeout = dur!"minutes"(2); | |
345 | ||
346 | /** | |
347 | Macros: | |
348 | ||
349 | CALLBACK_PARAMS = $(TABLE , | |
350 | $(DDOC_PARAM_ROW | |
351 | $(DDOC_PARAM_ID $(DDOC_PARAM dlTotal)) | |
352 | $(DDOC_PARAM_DESC total bytes to download) | |
353 | ) | |
354 | $(DDOC_PARAM_ROW | |
355 | $(DDOC_PARAM_ID $(DDOC_PARAM dlNow)) | |
356 | $(DDOC_PARAM_DESC currently downloaded bytes) | |
357 | ) | |
358 | $(DDOC_PARAM_ROW | |
359 | $(DDOC_PARAM_ID $(DDOC_PARAM ulTotal)) | |
360 | $(DDOC_PARAM_DESC total bytes to upload) | |
361 | ) | |
362 | $(DDOC_PARAM_ROW | |
363 | $(DDOC_PARAM_ID $(DDOC_PARAM ulNow)) | |
364 | $(DDOC_PARAM_DESC currently uploaded bytes) | |
365 | ) | |
366 | ) | |
367 | */ | |
368 | ||
369 | /** Connection type used when the URL should be used to auto detect the protocol. | |
370 | * | |
371 | * This struct is used as placeholder for the connection parameter when calling | |
372 | * the high level API and the connection type (HTTP/FTP) should be guessed by | |
373 | * inspecting the URL parameter. | |
374 | * | |
375 | * The rules for guessing the protocol are: | |
376 | * 1, if URL starts with ftp://, ftps:// or ftp. then FTP connection is assumed. | |
377 | * 2, HTTP connection otherwise. | |
378 | * | |
379 | * Example: | |
380 | * --- | |
381 | * import std.net.curl; | |
382 | * // Two requests below will do the same. | |
5fee5ec3 | 383 | * char[] content; |
b4c522fa IB |
384 | * |
385 | * // Explicit connection provided | |
386 | * content = get!HTTP("dlang.org"); | |
387 | * | |
388 | * // Guess connection type by looking at the URL | |
389 | * content = get!AutoProtocol("ftp://foo.com/file"); | |
390 | * // and since AutoProtocol is default this is the same as | |
391 | * content = get("ftp://foo.com/file"); | |
392 | * // and will end up detecting FTP from the url and be the same as | |
393 | * content = get!FTP("ftp://foo.com/file"); | |
394 | * --- | |
395 | */ | |
396 | struct AutoProtocol { } | |
397 | ||
398 | // Returns true if the url points to an FTP resource | |
399 | private bool isFTPUrl(const(char)[] url) | |
400 | { | |
401 | import std.algorithm.searching : startsWith; | |
402 | import std.uni : toLower; | |
403 | ||
404 | return startsWith(url.toLower(), "ftp://", "ftps://", "ftp.") != 0; | |
405 | } | |
406 | ||
407 | // Is true if the Conn type is a valid Curl Connection type. | |
408 | private template isCurlConn(Conn) | |
409 | { | |
410 | enum auto isCurlConn = is(Conn : HTTP) || | |
411 | is(Conn : FTP) || is(Conn : AutoProtocol); | |
412 | } | |
413 | ||
414 | /** HTTP/FTP download to local file system. | |
415 | * | |
416 | * Params: | |
417 | * url = resource to download | |
418 | * saveToPath = path to store the downloaded content on local disk | |
419 | * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will | |
420 | * guess connection type and create a new instance for this call only. | |
421 | * | |
422 | * Example: | |
423 | * ---- | |
424 | * import std.net.curl; | |
5fee5ec3 | 425 | * download("https://httpbin.org/get", "/tmp/downloaded-http-file"); |
b4c522fa IB |
426 | * ---- |
427 | */ | |
428 | void download(Conn = AutoProtocol)(const(char)[] url, string saveToPath, Conn conn = Conn()) | |
429 | if (isCurlConn!Conn) | |
430 | { | |
431 | static if (is(Conn : HTTP) || is(Conn : FTP)) | |
432 | { | |
433 | import std.stdio : File; | |
434 | conn.url = url; | |
435 | auto f = File(saveToPath, "wb"); | |
436 | conn.onReceive = (ubyte[] data) { f.rawWrite(data); return data.length; }; | |
437 | conn.perform(); | |
438 | } | |
439 | else | |
440 | { | |
441 | if (isFTPUrl(url)) | |
442 | return download!FTP(url, saveToPath, FTP()); | |
443 | else | |
444 | return download!HTTP(url, saveToPath, HTTP()); | |
445 | } | |
446 | } | |
447 | ||
448 | @system unittest | |
449 | { | |
450 | import std.algorithm.searching : canFind; | |
451 | static import std.file; | |
452 | ||
453 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
454 | { | |
455 | testServer.handle((s) { | |
456 | assert(s.recvReq.hdrs.canFind("GET /")); | |
457 | s.send(httpOK("Hello world")); | |
458 | }); | |
459 | auto fn = std.file.deleteme; | |
ab23d6fa IB |
460 | scope (exit) |
461 | { | |
462 | if (std.file.exists(fn)) | |
463 | std.file.remove(fn); | |
464 | } | |
b4c522fa IB |
465 | download(host, fn); |
466 | assert(std.file.readText(fn) == "Hello world"); | |
467 | } | |
468 | } | |
469 | ||
470 | /** Upload file from local files system using the HTTP or FTP protocol. | |
471 | * | |
472 | * Params: | |
473 | * loadFromPath = path load data from local disk. | |
474 | * url = resource to upload to | |
475 | * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will | |
476 | * guess connection type and create a new instance for this call only. | |
477 | * | |
478 | * Example: | |
479 | * ---- | |
480 | * import std.net.curl; | |
481 | * upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds"); | |
5fee5ec3 | 482 | * upload("/tmp/downloaded-http-file", "https://httpbin.org/post"); |
b4c522fa IB |
483 | * ---- |
484 | */ | |
485 | void upload(Conn = AutoProtocol)(string loadFromPath, const(char)[] url, Conn conn = Conn()) | |
486 | if (isCurlConn!Conn) | |
487 | { | |
488 | static if (is(Conn : HTTP)) | |
489 | { | |
490 | conn.url = url; | |
491 | conn.method = HTTP.Method.put; | |
492 | } | |
493 | else static if (is(Conn : FTP)) | |
494 | { | |
495 | conn.url = url; | |
496 | conn.handle.set(CurlOption.upload, 1L); | |
497 | } | |
498 | else | |
499 | { | |
500 | if (isFTPUrl(url)) | |
501 | return upload!FTP(loadFromPath, url, FTP()); | |
502 | else | |
503 | return upload!HTTP(loadFromPath, url, HTTP()); | |
504 | } | |
505 | ||
506 | static if (is(Conn : HTTP) || is(Conn : FTP)) | |
507 | { | |
508 | import std.stdio : File; | |
509 | auto f = File(loadFromPath, "rb"); | |
510 | conn.onSend = buf => f.rawRead(buf).length; | |
511 | immutable sz = f.size; | |
512 | if (sz != ulong.max) | |
513 | conn.contentLength = sz; | |
514 | conn.perform(); | |
515 | } | |
516 | } | |
517 | ||
518 | @system unittest | |
519 | { | |
520 | import std.algorithm.searching : canFind; | |
521 | static import std.file; | |
522 | ||
523 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
524 | { | |
525 | auto fn = std.file.deleteme; | |
ab23d6fa IB |
526 | scope (exit) |
527 | { | |
528 | if (std.file.exists(fn)) | |
529 | std.file.remove(fn); | |
530 | } | |
b4c522fa IB |
531 | std.file.write(fn, "upload data\n"); |
532 | testServer.handle((s) { | |
533 | auto req = s.recvReq; | |
534 | assert(req.hdrs.canFind("PUT /path")); | |
535 | assert(req.bdy.canFind("upload data")); | |
536 | s.send(httpOK()); | |
537 | }); | |
538 | upload(fn, host ~ "/path"); | |
539 | } | |
540 | } | |
541 | ||
542 | /** HTTP/FTP get content. | |
543 | * | |
544 | * Params: | |
545 | * url = resource to get | |
546 | * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will | |
547 | * guess connection type and create a new instance for this call only. | |
548 | * | |
5fee5ec3 IB |
549 | * The template parameter `T` specifies the type to return. Possible values |
550 | * are `char` and `ubyte` to return `char[]` or `ubyte[]`. If asking | |
551 | * for `char`, content will be converted from the connection character set | |
b4c522fa IB |
552 | * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1 |
553 | * by default) to UTF-8. | |
554 | * | |
555 | * Example: | |
556 | * ---- | |
557 | * import std.net.curl; | |
5fee5ec3 | 558 | * auto content = get("https://httpbin.org/get"); |
b4c522fa IB |
559 | * ---- |
560 | * | |
561 | * Returns: | |
562 | * A T[] range containing the content of the resource pointed to by the URL. | |
563 | * | |
564 | * Throws: | |
565 | * | |
5fee5ec3 | 566 | * `CurlException` on error. |
b4c522fa IB |
567 | * |
568 | * See_Also: $(LREF HTTP.Method) | |
569 | */ | |
570 | T[] get(Conn = AutoProtocol, T = char)(const(char)[] url, Conn conn = Conn()) | |
571 | if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) ) | |
572 | { | |
573 | static if (is(Conn : HTTP)) | |
574 | { | |
575 | conn.method = HTTP.Method.get; | |
576 | return _basicHTTP!(T)(url, "", conn); | |
577 | ||
578 | } | |
579 | else static if (is(Conn : FTP)) | |
580 | { | |
581 | return _basicFTP!(T)(url, "", conn); | |
582 | } | |
583 | else | |
584 | { | |
585 | if (isFTPUrl(url)) | |
586 | return get!(FTP,T)(url, FTP()); | |
587 | else | |
588 | return get!(HTTP,T)(url, HTTP()); | |
589 | } | |
590 | } | |
591 | ||
592 | @system unittest | |
593 | { | |
594 | import std.algorithm.searching : canFind; | |
595 | ||
596 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
597 | { | |
598 | testServer.handle((s) { | |
599 | assert(s.recvReq.hdrs.canFind("GET /path")); | |
600 | s.send(httpOK("GETRESPONSE")); | |
601 | }); | |
602 | auto res = get(host ~ "/path"); | |
603 | assert(res == "GETRESPONSE"); | |
604 | } | |
605 | } | |
606 | ||
607 | ||
608 | /** HTTP post content. | |
609 | * | |
610 | * Params: | |
611 | * url = resource to post to | |
612 | * postDict = data to send as the body of the request. An associative array | |
5fee5ec3 | 613 | * of `string` is accepted and will be encoded using |
b4c522fa IB |
614 | * www-form-urlencoding |
615 | * postData = data to send as the body of the request. An array | |
616 | * of an arbitrary type is accepted and will be cast to ubyte[] | |
617 | * before sending it. | |
618 | * conn = HTTP connection to use | |
5fee5ec3 IB |
619 | * T = The template parameter `T` specifies the type to return. Possible values |
620 | * are `char` and `ubyte` to return `char[]` or `ubyte[]`. If asking | |
621 | * for `char`, content will be converted from the connection character set | |
b4c522fa IB |
622 | * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1 |
623 | * by default) to UTF-8. | |
624 | * | |
625 | * Examples: | |
626 | * ---- | |
627 | * import std.net.curl; | |
628 | * | |
5fee5ec3 IB |
629 | * auto content1 = post("https://httpbin.org/post", ["name1" : "value1", "name2" : "value2"]); |
630 | * auto content2 = post("https://httpbin.org/post", [1,2,3,4]); | |
b4c522fa IB |
631 | * ---- |
632 | * | |
633 | * Returns: | |
634 | * A T[] range containing the content of the resource pointed to by the URL. | |
635 | * | |
636 | * See_Also: $(LREF HTTP.Method) | |
637 | */ | |
638 | T[] post(T = char, PostUnit)(const(char)[] url, const(PostUnit)[] postData, HTTP conn = HTTP()) | |
639 | if (is(T == char) || is(T == ubyte)) | |
640 | { | |
641 | conn.method = HTTP.Method.post; | |
642 | return _basicHTTP!(T)(url, postData, conn); | |
643 | } | |
644 | ||
645 | @system unittest | |
646 | { | |
647 | import std.algorithm.searching : canFind; | |
648 | ||
649 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
650 | { | |
651 | testServer.handle((s) { | |
652 | auto req = s.recvReq; | |
653 | assert(req.hdrs.canFind("POST /path")); | |
654 | assert(req.bdy.canFind("POSTBODY")); | |
655 | s.send(httpOK("POSTRESPONSE")); | |
656 | }); | |
657 | auto res = post(host ~ "/path", "POSTBODY"); | |
658 | assert(res == "POSTRESPONSE"); | |
659 | } | |
660 | } | |
661 | ||
662 | @system unittest | |
663 | { | |
664 | import std.algorithm.searching : canFind; | |
665 | ||
666 | auto data = new ubyte[](256); | |
667 | foreach (i, ref ub; data) | |
668 | ub = cast(ubyte) i; | |
669 | ||
670 | testServer.handle((s) { | |
671 | auto req = s.recvReq!ubyte; | |
672 | assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4])); | |
673 | assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255])); | |
674 | s.send(httpOK(cast(ubyte[])[17, 27, 35, 41])); | |
675 | }); | |
676 | auto res = post!ubyte(testServer.addr, data); | |
677 | assert(res == cast(ubyte[])[17, 27, 35, 41]); | |
678 | } | |
679 | ||
680 | /// ditto | |
681 | T[] post(T = char)(const(char)[] url, string[string] postDict, HTTP conn = HTTP()) | |
682 | if (is(T == char) || is(T == ubyte)) | |
683 | { | |
684 | import std.uri : urlEncode; | |
685 | ||
5fee5ec3 | 686 | return post!T(url, urlEncode(postDict), conn); |
b4c522fa IB |
687 | } |
688 | ||
689 | @system unittest | |
690 | { | |
5fee5ec3 IB |
691 | import std.algorithm.searching : canFind; |
692 | import std.meta : AliasSeq; | |
693 | ||
694 | static immutable expected = ["name1=value1&name2=value2", "name2=value2&name1=value1"]; | |
695 | ||
b4c522fa IB |
696 | foreach (host; [testServer.addr, "http://" ~ testServer.addr]) |
697 | { | |
5fee5ec3 IB |
698 | foreach (T; AliasSeq!(char, ubyte)) |
699 | { | |
700 | testServer.handle((s) { | |
701 | auto req = s.recvReq!char; | |
702 | s.send(httpOK(req.bdy)); | |
703 | }); | |
704 | auto res = post!T(host ~ "/path", ["name1" : "value1", "name2" : "value2"]); | |
705 | assert(canFind(expected, res)); | |
706 | } | |
b4c522fa IB |
707 | } |
708 | } | |
709 | ||
710 | /** HTTP/FTP put content. | |
711 | * | |
712 | * Params: | |
713 | * url = resource to put | |
714 | * putData = data to send as the body of the request. An array | |
715 | * of an arbitrary type is accepted and will be cast to ubyte[] | |
716 | * before sending it. | |
717 | * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will | |
718 | * guess connection type and create a new instance for this call only. | |
719 | * | |
5fee5ec3 IB |
720 | * The template parameter `T` specifies the type to return. Possible values |
721 | * are `char` and `ubyte` to return `char[]` or `ubyte[]`. If asking | |
722 | * for `char`, content will be converted from the connection character set | |
b4c522fa IB |
723 | * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1 |
724 | * by default) to UTF-8. | |
725 | * | |
726 | * Example: | |
727 | * ---- | |
728 | * import std.net.curl; | |
5fee5ec3 | 729 | * auto content = put("https://httpbin.org/put", |
b4c522fa IB |
730 | * "Putting this data"); |
731 | * ---- | |
732 | * | |
733 | * Returns: | |
734 | * A T[] range containing the content of the resource pointed to by the URL. | |
735 | * | |
736 | * See_Also: $(LREF HTTP.Method) | |
737 | */ | |
738 | T[] put(Conn = AutoProtocol, T = char, PutUnit)(const(char)[] url, const(PutUnit)[] putData, | |
739 | Conn conn = Conn()) | |
740 | if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) ) | |
741 | { | |
742 | static if (is(Conn : HTTP)) | |
743 | { | |
744 | conn.method = HTTP.Method.put; | |
745 | return _basicHTTP!(T)(url, putData, conn); | |
746 | } | |
747 | else static if (is(Conn : FTP)) | |
748 | { | |
749 | return _basicFTP!(T)(url, putData, conn); | |
750 | } | |
751 | else | |
752 | { | |
753 | if (isFTPUrl(url)) | |
754 | return put!(FTP,T)(url, putData, FTP()); | |
755 | else | |
756 | return put!(HTTP,T)(url, putData, HTTP()); | |
757 | } | |
758 | } | |
759 | ||
760 | @system unittest | |
761 | { | |
762 | import std.algorithm.searching : canFind; | |
763 | ||
764 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
765 | { | |
766 | testServer.handle((s) { | |
767 | auto req = s.recvReq; | |
768 | assert(req.hdrs.canFind("PUT /path")); | |
769 | assert(req.bdy.canFind("PUTBODY")); | |
770 | s.send(httpOK("PUTRESPONSE")); | |
771 | }); | |
772 | auto res = put(host ~ "/path", "PUTBODY"); | |
773 | assert(res == "PUTRESPONSE"); | |
774 | } | |
775 | } | |
776 | ||
777 | ||
778 | /** HTTP/FTP delete content. | |
779 | * | |
780 | * Params: | |
781 | * url = resource to delete | |
782 | * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will | |
783 | * guess connection type and create a new instance for this call only. | |
784 | * | |
785 | * Example: | |
786 | * ---- | |
787 | * import std.net.curl; | |
5fee5ec3 | 788 | * del("https://httpbin.org/delete"); |
b4c522fa IB |
789 | * ---- |
790 | * | |
791 | * See_Also: $(LREF HTTP.Method) | |
792 | */ | |
793 | void del(Conn = AutoProtocol)(const(char)[] url, Conn conn = Conn()) | |
794 | if (isCurlConn!Conn) | |
795 | { | |
796 | static if (is(Conn : HTTP)) | |
797 | { | |
798 | conn.method = HTTP.Method.del; | |
799 | _basicHTTP!char(url, cast(void[]) null, conn); | |
800 | } | |
801 | else static if (is(Conn : FTP)) | |
802 | { | |
803 | import std.algorithm.searching : findSplitAfter; | |
804 | import std.conv : text; | |
5fee5ec3 | 805 | import std.exception : enforce; |
b4c522fa IB |
806 | |
807 | auto trimmed = url.findSplitAfter("ftp://")[1]; | |
808 | auto t = trimmed.findSplitAfter("/"); | |
809 | enum minDomainNameLength = 3; | |
810 | enforce!CurlException(t[0].length > minDomainNameLength, | |
811 | text("Invalid FTP URL for delete ", url)); | |
812 | conn.url = t[0]; | |
813 | ||
814 | enforce!CurlException(!t[1].empty, | |
815 | text("No filename specified to delete for URL ", url)); | |
816 | conn.addCommand("DELE " ~ t[1]); | |
817 | conn.perform(); | |
818 | } | |
819 | else | |
820 | { | |
821 | if (isFTPUrl(url)) | |
822 | return del!FTP(url, FTP()); | |
823 | else | |
824 | return del!HTTP(url, HTTP()); | |
825 | } | |
826 | } | |
827 | ||
828 | @system unittest | |
829 | { | |
830 | import std.algorithm.searching : canFind; | |
831 | ||
832 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
833 | { | |
834 | testServer.handle((s) { | |
835 | auto req = s.recvReq; | |
836 | assert(req.hdrs.canFind("DELETE /path")); | |
837 | s.send(httpOK()); | |
838 | }); | |
839 | del(host ~ "/path"); | |
840 | } | |
841 | } | |
842 | ||
843 | ||
844 | /** HTTP options request. | |
845 | * | |
846 | * Params: | |
847 | * url = resource make a option call to | |
848 | * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will | |
849 | * guess connection type and create a new instance for this call only. | |
850 | * | |
5fee5ec3 IB |
851 | * The template parameter `T` specifies the type to return. Possible values |
852 | * are `char` and `ubyte` to return `char[]` or `ubyte[]`. | |
b4c522fa IB |
853 | * |
854 | * Example: | |
855 | * ---- | |
856 | * import std.net.curl; | |
857 | * auto http = HTTP(); | |
5fee5ec3 | 858 | * options("https://httpbin.org/headers", http); |
b4c522fa IB |
859 | * writeln("Allow set to " ~ http.responseHeaders["Allow"]); |
860 | * ---- | |
861 | * | |
862 | * Returns: | |
863 | * A T[] range containing the options of the resource pointed to by the URL. | |
864 | * | |
865 | * See_Also: $(LREF HTTP.Method) | |
866 | */ | |
867 | T[] options(T = char)(const(char)[] url, HTTP conn = HTTP()) | |
868 | if (is(T == char) || is(T == ubyte)) | |
869 | { | |
870 | conn.method = HTTP.Method.options; | |
871 | return _basicHTTP!(T)(url, null, conn); | |
872 | } | |
873 | ||
874 | @system unittest | |
875 | { | |
876 | import std.algorithm.searching : canFind; | |
877 | ||
878 | testServer.handle((s) { | |
879 | auto req = s.recvReq; | |
880 | assert(req.hdrs.canFind("OPTIONS /path")); | |
881 | s.send(httpOK("OPTIONSRESPONSE")); | |
882 | }); | |
883 | auto res = options(testServer.addr ~ "/path"); | |
884 | assert(res == "OPTIONSRESPONSE"); | |
885 | } | |
886 | ||
887 | ||
888 | /** HTTP trace request. | |
889 | * | |
890 | * Params: | |
891 | * url = resource make a trace call to | |
892 | * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will | |
893 | * guess connection type and create a new instance for this call only. | |
894 | * | |
5fee5ec3 IB |
895 | * The template parameter `T` specifies the type to return. Possible values |
896 | * are `char` and `ubyte` to return `char[]` or `ubyte[]`. | |
b4c522fa IB |
897 | * |
898 | * Example: | |
899 | * ---- | |
900 | * import std.net.curl; | |
5fee5ec3 | 901 | * trace("https://httpbin.org/headers"); |
b4c522fa IB |
902 | * ---- |
903 | * | |
904 | * Returns: | |
905 | * A T[] range containing the trace info of the resource pointed to by the URL. | |
906 | * | |
907 | * See_Also: $(LREF HTTP.Method) | |
908 | */ | |
909 | T[] trace(T = char)(const(char)[] url, HTTP conn = HTTP()) | |
910 | if (is(T == char) || is(T == ubyte)) | |
911 | { | |
912 | conn.method = HTTP.Method.trace; | |
913 | return _basicHTTP!(T)(url, cast(void[]) null, conn); | |
914 | } | |
915 | ||
916 | @system unittest | |
917 | { | |
918 | import std.algorithm.searching : canFind; | |
919 | ||
920 | testServer.handle((s) { | |
921 | auto req = s.recvReq; | |
922 | assert(req.hdrs.canFind("TRACE /path")); | |
923 | s.send(httpOK("TRACERESPONSE")); | |
924 | }); | |
925 | auto res = trace(testServer.addr ~ "/path"); | |
926 | assert(res == "TRACERESPONSE"); | |
927 | } | |
928 | ||
929 | ||
930 | /** HTTP connect request. | |
931 | * | |
932 | * Params: | |
933 | * url = resource make a connect to | |
934 | * conn = HTTP connection to use | |
935 | * | |
5fee5ec3 IB |
936 | * The template parameter `T` specifies the type to return. Possible values |
937 | * are `char` and `ubyte` to return `char[]` or `ubyte[]`. | |
b4c522fa IB |
938 | * |
939 | * Example: | |
940 | * ---- | |
941 | * import std.net.curl; | |
5fee5ec3 | 942 | * connect("https://httpbin.org/headers"); |
b4c522fa IB |
943 | * ---- |
944 | * | |
945 | * Returns: | |
946 | * A T[] range containing the connect info of the resource pointed to by the URL. | |
947 | * | |
948 | * See_Also: $(LREF HTTP.Method) | |
949 | */ | |
950 | T[] connect(T = char)(const(char)[] url, HTTP conn = HTTP()) | |
951 | if (is(T == char) || is(T == ubyte)) | |
952 | { | |
953 | conn.method = HTTP.Method.connect; | |
954 | return _basicHTTP!(T)(url, cast(void[]) null, conn); | |
955 | } | |
956 | ||
957 | @system unittest | |
958 | { | |
959 | import std.algorithm.searching : canFind; | |
960 | ||
961 | testServer.handle((s) { | |
962 | auto req = s.recvReq; | |
963 | assert(req.hdrs.canFind("CONNECT /path")); | |
964 | s.send(httpOK("CONNECTRESPONSE")); | |
965 | }); | |
966 | auto res = connect(testServer.addr ~ "/path"); | |
967 | assert(res == "CONNECTRESPONSE"); | |
968 | } | |
969 | ||
970 | ||
971 | /** HTTP patch content. | |
972 | * | |
973 | * Params: | |
974 | * url = resource to patch | |
975 | * patchData = data to send as the body of the request. An array | |
976 | * of an arbitrary type is accepted and will be cast to ubyte[] | |
977 | * before sending it. | |
978 | * conn = HTTP connection to use | |
979 | * | |
5fee5ec3 IB |
980 | * The template parameter `T` specifies the type to return. Possible values |
981 | * are `char` and `ubyte` to return `char[]` or `ubyte[]`. | |
b4c522fa IB |
982 | * |
983 | * Example: | |
984 | * ---- | |
985 | * auto http = HTTP(); | |
986 | * http.addRequestHeader("Content-Type", "application/json"); | |
5fee5ec3 | 987 | * auto content = patch("https://httpbin.org/patch", `{"title": "Patched Title"}`, http); |
b4c522fa IB |
988 | * ---- |
989 | * | |
990 | * Returns: | |
991 | * A T[] range containing the content of the resource pointed to by the URL. | |
992 | * | |
993 | * See_Also: $(LREF HTTP.Method) | |
994 | */ | |
995 | T[] patch(T = char, PatchUnit)(const(char)[] url, const(PatchUnit)[] patchData, | |
996 | HTTP conn = HTTP()) | |
997 | if (is(T == char) || is(T == ubyte)) | |
998 | { | |
999 | conn.method = HTTP.Method.patch; | |
1000 | return _basicHTTP!(T)(url, patchData, conn); | |
1001 | } | |
1002 | ||
1003 | @system unittest | |
1004 | { | |
1005 | import std.algorithm.searching : canFind; | |
1006 | ||
1007 | testServer.handle((s) { | |
1008 | auto req = s.recvReq; | |
1009 | assert(req.hdrs.canFind("PATCH /path")); | |
1010 | assert(req.bdy.canFind("PATCHBODY")); | |
1011 | s.send(httpOK("PATCHRESPONSE")); | |
1012 | }); | |
1013 | auto res = patch(testServer.addr ~ "/path", "PATCHBODY"); | |
1014 | assert(res == "PATCHRESPONSE"); | |
1015 | } | |
1016 | ||
1017 | ||
1018 | /* | |
1019 | * Helper function for the high level interface. | |
1020 | * | |
1021 | * It performs an HTTP request using the client which must have | |
1022 | * been setup correctly before calling this function. | |
1023 | */ | |
1024 | private auto _basicHTTP(T)(const(char)[] url, const(void)[] sendData, HTTP client) | |
1025 | { | |
1026 | import std.algorithm.comparison : min; | |
1027 | import std.format : format; | |
5fee5ec3 IB |
1028 | import std.exception : enforce; |
1029 | import etc.c.curl : CurlSeek, CurlSeekPos; | |
b4c522fa IB |
1030 | |
1031 | immutable doSend = sendData !is null && | |
1032 | (client.method == HTTP.Method.post || | |
1033 | client.method == HTTP.Method.put || | |
1034 | client.method == HTTP.Method.patch); | |
1035 | ||
1036 | scope (exit) | |
1037 | { | |
1038 | client.onReceiveHeader = null; | |
1039 | client.onReceiveStatusLine = null; | |
1040 | client.onReceive = null; | |
1041 | ||
1042 | if (doSend) | |
1043 | { | |
1044 | client.onSend = null; | |
1045 | client.handle.onSeek = null; | |
1046 | client.contentLength = 0; | |
1047 | } | |
1048 | } | |
1049 | client.url = url; | |
1050 | HTTP.StatusLine statusLine; | |
1051 | import std.array : appender; | |
1052 | auto content = appender!(ubyte[])(); | |
1053 | client.onReceive = (ubyte[] data) | |
1054 | { | |
1055 | content ~= data; | |
1056 | return data.length; | |
1057 | }; | |
1058 | ||
1059 | if (doSend) | |
1060 | { | |
1061 | client.contentLength = sendData.length; | |
1062 | auto remainingData = sendData; | |
1063 | client.onSend = delegate size_t(void[] buf) | |
1064 | { | |
1065 | size_t minLen = min(buf.length, remainingData.length); | |
1066 | if (minLen == 0) return 0; | |
1067 | buf[0 .. minLen] = remainingData[0 .. minLen]; | |
1068 | remainingData = remainingData[minLen..$]; | |
1069 | return minLen; | |
1070 | }; | |
1071 | client.handle.onSeek = delegate(long offset, CurlSeekPos mode) | |
1072 | { | |
1073 | switch (mode) | |
1074 | { | |
1075 | case CurlSeekPos.set: | |
1076 | remainingData = sendData[cast(size_t) offset..$]; | |
1077 | return CurlSeek.ok; | |
1078 | default: | |
1079 | // As of curl 7.18.0, libcurl will not pass | |
1080 | // anything other than CurlSeekPos.set. | |
1081 | return CurlSeek.cantseek; | |
1082 | } | |
1083 | }; | |
1084 | } | |
1085 | ||
1086 | client.onReceiveHeader = (in char[] key, | |
1087 | in char[] value) | |
1088 | { | |
1089 | if (key == "content-length") | |
1090 | { | |
1091 | import std.conv : to; | |
1092 | content.reserve(value.to!size_t); | |
1093 | } | |
1094 | }; | |
1095 | client.onReceiveStatusLine = (HTTP.StatusLine l) { statusLine = l; }; | |
1096 | client.perform(); | |
1097 | enforce(statusLine.code / 100 == 2, new HTTPStatusException(statusLine.code, | |
1098 | format("HTTP request returned status code %d (%s)", statusLine.code, statusLine.reason))); | |
1099 | ||
1100 | return _decodeContent!T(content.data, client.p.charset); | |
1101 | } | |
1102 | ||
1103 | @system unittest | |
1104 | { | |
1105 | import std.algorithm.searching : canFind; | |
5fee5ec3 | 1106 | import std.exception : collectException; |
b4c522fa IB |
1107 | |
1108 | testServer.handle((s) { | |
1109 | auto req = s.recvReq; | |
1110 | assert(req.hdrs.canFind("GET /path")); | |
1111 | s.send(httpNotFound()); | |
1112 | }); | |
1113 | auto e = collectException!HTTPStatusException(get(testServer.addr ~ "/path")); | |
1114 | assert(e.msg == "HTTP request returned status code 404 (Not Found)"); | |
1115 | assert(e.status == 404); | |
1116 | } | |
1117 | ||
5fee5ec3 IB |
1118 | // Content length must be reset after post |
1119 | // https://issues.dlang.org/show_bug.cgi?id=14760 | |
b4c522fa IB |
1120 | @system unittest |
1121 | { | |
1122 | import std.algorithm.searching : canFind; | |
1123 | ||
1124 | testServer.handle((s) { | |
1125 | auto req = s.recvReq; | |
1126 | assert(req.hdrs.canFind("POST /")); | |
1127 | assert(req.bdy.canFind("POSTBODY")); | |
1128 | s.send(httpOK("POSTRESPONSE")); | |
1129 | ||
1130 | req = s.recvReq; | |
1131 | assert(req.hdrs.canFind("TRACE /")); | |
1132 | assert(req.bdy.empty); | |
1133 | s.blocking = false; | |
1134 | ubyte[6] buf = void; | |
1135 | assert(s.receive(buf[]) < 0); | |
1136 | s.send(httpOK("TRACERESPONSE")); | |
1137 | }); | |
1138 | auto http = HTTP(); | |
1139 | auto res = post(testServer.addr, "POSTBODY", http); | |
1140 | assert(res == "POSTRESPONSE"); | |
1141 | res = trace(testServer.addr, http); | |
1142 | assert(res == "TRACERESPONSE"); | |
1143 | } | |
1144 | ||
1145 | @system unittest // charset detection and transcoding to T | |
1146 | { | |
1147 | testServer.handle((s) { | |
1148 | s.send("HTTP/1.1 200 OK\r\n"~ | |
1149 | "Content-Length: 4\r\n"~ | |
1150 | "Content-Type: text/plain; charset=utf-8\r\n" ~ | |
1151 | "\r\n" ~ | |
1152 | "äbc"); | |
1153 | }); | |
1154 | auto client = HTTP(); | |
1155 | auto result = _basicHTTP!char(testServer.addr, "", client); | |
1156 | assert(result == "äbc"); | |
1157 | ||
1158 | testServer.handle((s) { | |
1159 | s.send("HTTP/1.1 200 OK\r\n"~ | |
1160 | "Content-Length: 3\r\n"~ | |
1161 | "Content-Type: text/plain; charset=iso-8859-1\r\n" ~ | |
1162 | "\r\n" ~ | |
1163 | 0xE4 ~ "bc"); | |
1164 | }); | |
1165 | client = HTTP(); | |
1166 | result = _basicHTTP!char(testServer.addr, "", client); | |
1167 | assert(result == "äbc"); | |
1168 | } | |
1169 | ||
1170 | /* | |
1171 | * Helper function for the high level interface. | |
1172 | * | |
1173 | * It performs an FTP request using the client which must have | |
1174 | * been setup correctly before calling this function. | |
1175 | */ | |
1176 | private auto _basicFTP(T)(const(char)[] url, const(void)[] sendData, FTP client) | |
1177 | { | |
1178 | import std.algorithm.comparison : min; | |
1179 | ||
1180 | scope (exit) | |
1181 | { | |
1182 | client.onReceive = null; | |
1183 | if (!sendData.empty) | |
1184 | client.onSend = null; | |
1185 | } | |
1186 | ||
1187 | ubyte[] content; | |
1188 | ||
1189 | if (client.encoding.empty) | |
1190 | client.encoding = "ISO-8859-1"; | |
1191 | ||
1192 | client.url = url; | |
1193 | client.onReceive = (ubyte[] data) | |
1194 | { | |
1195 | content ~= data; | |
1196 | return data.length; | |
1197 | }; | |
1198 | ||
1199 | if (!sendData.empty) | |
1200 | { | |
1201 | client.handle.set(CurlOption.upload, 1L); | |
1202 | client.onSend = delegate size_t(void[] buf) | |
1203 | { | |
1204 | size_t minLen = min(buf.length, sendData.length); | |
1205 | if (minLen == 0) return 0; | |
1206 | buf[0 .. minLen] = sendData[0 .. minLen]; | |
1207 | sendData = sendData[minLen..$]; | |
1208 | return minLen; | |
1209 | }; | |
1210 | } | |
1211 | ||
1212 | client.perform(); | |
1213 | ||
1214 | return _decodeContent!T(content, client.encoding); | |
1215 | } | |
1216 | ||
1217 | /* Used by _basicHTTP() and _basicFTP() to decode ubyte[] to | |
1218 | * correct string format | |
1219 | */ | |
1220 | private auto _decodeContent(T)(ubyte[] content, string encoding) | |
1221 | { | |
1222 | static if (is(T == ubyte)) | |
1223 | { | |
1224 | return content; | |
1225 | } | |
1226 | else | |
1227 | { | |
5fee5ec3 | 1228 | import std.exception : enforce; |
b4c522fa IB |
1229 | import std.format : format; |
1230 | ||
1231 | // Optimally just return the utf8 encoded content | |
1232 | if (encoding == "UTF-8") | |
1233 | return cast(char[])(content); | |
1234 | ||
1235 | // The content has to be re-encoded to utf8 | |
1236 | auto scheme = EncodingScheme.create(encoding); | |
1237 | enforce!CurlException(scheme !is null, | |
1238 | format("Unknown encoding '%s'", encoding)); | |
1239 | ||
1240 | auto strInfo = decodeString(content, scheme); | |
1241 | enforce!CurlException(strInfo[0] != size_t.max, | |
1242 | format("Invalid encoding sequence for encoding '%s'", | |
1243 | encoding)); | |
1244 | ||
1245 | return strInfo[1]; | |
1246 | } | |
1247 | } | |
1248 | ||
1249 | alias KeepTerminator = Flag!"keepTerminator"; | |
1250 | /+ | |
1251 | struct ByLineBuffer(Char) | |
1252 | { | |
1253 | bool linePresent; | |
1254 | bool EOF; | |
1255 | Char[] buffer; | |
1256 | ubyte[] decodeRemainder; | |
1257 | ||
1258 | bool append(const(ubyte)[] data) | |
1259 | { | |
1260 | byLineBuffer ~= data; | |
1261 | } | |
1262 | ||
1263 | @property bool linePresent() | |
1264 | { | |
1265 | return byLinePresent; | |
1266 | } | |
1267 | ||
1268 | Char[] get() | |
1269 | { | |
1270 | if (!linePresent) | |
1271 | { | |
1272 | // Decode ubyte[] into Char[] until a Terminator is found. | |
1273 | // If not Terminator is found and EOF is false then raise an | |
1274 | // exception. | |
1275 | } | |
1276 | return byLineBuffer; | |
1277 | } | |
1278 | ||
1279 | } | |
1280 | ++/ | |
1281 | /** HTTP/FTP fetch content as a range of lines. | |
1282 | * | |
1283 | * A range of lines is returned when the request is complete. If the method or | |
5fee5ec3 | 1284 | * other request properties is to be customized then set the `conn` parameter |
b4c522fa IB |
1285 | * with a HTTP/FTP instance that has these properties set. |
1286 | * | |
1287 | * Example: | |
1288 | * ---- | |
1289 | * import std.net.curl, std.stdio; | |
1290 | * foreach (line; byLine("dlang.org")) | |
1291 | * writeln(line); | |
1292 | * ---- | |
1293 | * | |
1294 | * Params: | |
1295 | * url = The url to receive content from | |
5fee5ec3 | 1296 | * keepTerminator = `Yes.keepTerminator` signals that the line terminator should be |
b4c522fa IB |
1297 | * returned as part of the lines in the range. |
1298 | * terminator = The character that terminates a line | |
1299 | * conn = The connection to use e.g. HTTP or FTP. | |
1300 | * | |
1301 | * Returns: | |
1302 | * A range of Char[] with the content of the resource pointer to by the URL | |
1303 | */ | |
1304 | auto byLine(Conn = AutoProtocol, Terminator = char, Char = char) | |
1305 | (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator, | |
1306 | Terminator terminator = '\n', Conn conn = Conn()) | |
1307 | if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator) | |
1308 | { | |
1309 | static struct SyncLineInputRange | |
1310 | { | |
1311 | ||
1312 | private Char[] lines; | |
1313 | private Char[] current; | |
1314 | private bool currentValid; | |
1315 | private bool keepTerminator; | |
1316 | private Terminator terminator; | |
1317 | ||
1318 | this(Char[] lines, bool kt, Terminator terminator) | |
1319 | { | |
1320 | this.lines = lines; | |
1321 | this.keepTerminator = kt; | |
1322 | this.terminator = terminator; | |
1323 | currentValid = true; | |
1324 | popFront(); | |
1325 | } | |
1326 | ||
1327 | @property @safe bool empty() | |
1328 | { | |
1329 | return !currentValid; | |
1330 | } | |
1331 | ||
1332 | @property @safe Char[] front() | |
1333 | { | |
5fee5ec3 | 1334 | import std.exception : enforce; |
b4c522fa IB |
1335 | enforce!CurlException(currentValid, "Cannot call front() on empty range"); |
1336 | return current; | |
1337 | } | |
1338 | ||
1339 | void popFront() | |
1340 | { | |
1341 | import std.algorithm.searching : findSplitAfter, findSplit; | |
5fee5ec3 | 1342 | import std.exception : enforce; |
b4c522fa IB |
1343 | |
1344 | enforce!CurlException(currentValid, "Cannot call popFront() on empty range"); | |
1345 | if (lines.empty) | |
1346 | { | |
1347 | currentValid = false; | |
1348 | return; | |
1349 | } | |
1350 | ||
1351 | if (keepTerminator) | |
1352 | { | |
1353 | auto r = findSplitAfter(lines, [ terminator ]); | |
1354 | if (r[0].empty) | |
1355 | { | |
1356 | current = r[1]; | |
1357 | lines = r[0]; | |
1358 | } | |
1359 | else | |
1360 | { | |
1361 | current = r[0]; | |
1362 | lines = r[1]; | |
1363 | } | |
1364 | } | |
1365 | else | |
1366 | { | |
1367 | auto r = findSplit(lines, [ terminator ]); | |
1368 | current = r[0]; | |
1369 | lines = r[2]; | |
1370 | } | |
1371 | } | |
1372 | } | |
1373 | ||
1374 | auto result = _getForRange!Char(url, conn); | |
1375 | return SyncLineInputRange(result, keepTerminator == Yes.keepTerminator, terminator); | |
1376 | } | |
1377 | ||
1378 | @system unittest | |
1379 | { | |
1380 | import std.algorithm.comparison : equal; | |
1381 | ||
1382 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
1383 | { | |
1384 | testServer.handle((s) { | |
1385 | auto req = s.recvReq; | |
1386 | s.send(httpOK("Line1\nLine2\nLine3")); | |
1387 | }); | |
1388 | assert(byLine(host).equal(["Line1", "Line2", "Line3"])); | |
1389 | } | |
1390 | } | |
1391 | ||
1392 | /** HTTP/FTP fetch content as a range of chunks. | |
1393 | * | |
1394 | * A range of chunks is returned when the request is complete. If the method or | |
5fee5ec3 | 1395 | * other request properties is to be customized then set the `conn` parameter |
b4c522fa IB |
1396 | * with a HTTP/FTP instance that has these properties set. |
1397 | * | |
1398 | * Example: | |
1399 | * ---- | |
1400 | * import std.net.curl, std.stdio; | |
1401 | * foreach (chunk; byChunk("dlang.org", 100)) | |
1402 | * writeln(chunk); // chunk is ubyte[100] | |
1403 | * ---- | |
1404 | * | |
1405 | * Params: | |
1406 | * url = The url to receive content from | |
1407 | * chunkSize = The size of each chunk | |
1408 | * conn = The connection to use e.g. HTTP or FTP. | |
1409 | * | |
1410 | * Returns: | |
1411 | * A range of ubyte[chunkSize] with the content of the resource pointer to by the URL | |
1412 | */ | |
1413 | auto byChunk(Conn = AutoProtocol) | |
1414 | (const(char)[] url, size_t chunkSize = 1024, Conn conn = Conn()) | |
1415 | if (isCurlConn!(Conn)) | |
1416 | { | |
1417 | static struct SyncChunkInputRange | |
1418 | { | |
1419 | private size_t chunkSize; | |
1420 | private ubyte[] _bytes; | |
1421 | private size_t offset; | |
1422 | ||
1423 | this(ubyte[] bytes, size_t chunkSize) | |
1424 | { | |
1425 | this._bytes = bytes; | |
1426 | this.chunkSize = chunkSize; | |
1427 | } | |
1428 | ||
1429 | @property @safe auto empty() | |
1430 | { | |
1431 | return offset == _bytes.length; | |
1432 | } | |
1433 | ||
1434 | @property ubyte[] front() | |
1435 | { | |
1436 | size_t nextOffset = offset + chunkSize; | |
1437 | if (nextOffset > _bytes.length) nextOffset = _bytes.length; | |
1438 | return _bytes[offset .. nextOffset]; | |
1439 | } | |
1440 | ||
1441 | @safe void popFront() | |
1442 | { | |
1443 | offset += chunkSize; | |
1444 | if (offset > _bytes.length) offset = _bytes.length; | |
1445 | } | |
1446 | } | |
1447 | ||
1448 | auto result = _getForRange!ubyte(url, conn); | |
1449 | return SyncChunkInputRange(result, chunkSize); | |
1450 | } | |
1451 | ||
1452 | @system unittest | |
1453 | { | |
1454 | import std.algorithm.comparison : equal; | |
1455 | ||
1456 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
1457 | { | |
1458 | testServer.handle((s) { | |
1459 | auto req = s.recvReq; | |
1460 | s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5])); | |
1461 | }); | |
1462 | assert(byChunk(host, 2).equal([[0, 1], [2, 3], [4, 5]])); | |
1463 | } | |
1464 | } | |
1465 | ||
1466 | private T[] _getForRange(T,Conn)(const(char)[] url, Conn conn) | |
1467 | { | |
1468 | static if (is(Conn : HTTP)) | |
1469 | { | |
1470 | conn.method = conn.method == HTTP.Method.undefined ? HTTP.Method.get : conn.method; | |
1471 | return _basicHTTP!(T)(url, null, conn); | |
1472 | } | |
1473 | else static if (is(Conn : FTP)) | |
1474 | { | |
1475 | return _basicFTP!(T)(url, null, conn); | |
1476 | } | |
1477 | else | |
1478 | { | |
1479 | if (isFTPUrl(url)) | |
1480 | return get!(FTP,T)(url, FTP()); | |
1481 | else | |
1482 | return get!(HTTP,T)(url, HTTP()); | |
1483 | } | |
1484 | } | |
1485 | ||
1486 | /* | |
1487 | Main thread part of the message passing protocol used for all async | |
1488 | curl protocols. | |
1489 | */ | |
1490 | private mixin template WorkerThreadProtocol(Unit, alias units) | |
1491 | { | |
5fee5ec3 IB |
1492 | import core.time : Duration; |
1493 | ||
b4c522fa IB |
1494 | @property bool empty() |
1495 | { | |
1496 | tryEnsureUnits(); | |
1497 | return state == State.done; | |
1498 | } | |
1499 | ||
1500 | @property Unit[] front() | |
1501 | { | |
1502 | import std.format : format; | |
1503 | tryEnsureUnits(); | |
1504 | assert(state == State.gotUnits, | |
1505 | format("Expected %s but got $s", | |
1506 | State.gotUnits, state)); | |
1507 | return units; | |
1508 | } | |
1509 | ||
1510 | void popFront() | |
1511 | { | |
5fee5ec3 | 1512 | import std.concurrency : send; |
b4c522fa | 1513 | import std.format : format; |
5fee5ec3 | 1514 | |
b4c522fa IB |
1515 | tryEnsureUnits(); |
1516 | assert(state == State.gotUnits, | |
1517 | format("Expected %s but got $s", | |
1518 | State.gotUnits, state)); | |
1519 | state = State.needUnits; | |
1520 | // Send to worker thread for buffer reuse | |
1521 | workerTid.send(cast(immutable(Unit)[]) units); | |
1522 | units = null; | |
1523 | } | |
1524 | ||
1525 | /** Wait for duration or until data is available and return true if data is | |
1526 | available | |
1527 | */ | |
1528 | bool wait(Duration d) | |
1529 | { | |
5fee5ec3 | 1530 | import core.time : dur; |
b4c522fa | 1531 | import std.datetime.stopwatch : StopWatch; |
5fee5ec3 | 1532 | import std.concurrency : receiveTimeout; |
b4c522fa IB |
1533 | |
1534 | if (state == State.gotUnits) | |
1535 | return true; | |
1536 | ||
1537 | enum noDur = dur!"hnsecs"(0); | |
1538 | StopWatch sw; | |
1539 | sw.start(); | |
1540 | while (state != State.gotUnits && d > noDur) | |
1541 | { | |
1542 | final switch (state) | |
1543 | { | |
1544 | case State.needUnits: | |
1545 | receiveTimeout(d, | |
1546 | (Tid origin, CurlMessage!(immutable(Unit)[]) _data) | |
1547 | { | |
1548 | if (origin != workerTid) | |
1549 | return false; | |
1550 | units = cast(Unit[]) _data.data; | |
1551 | state = State.gotUnits; | |
1552 | return true; | |
1553 | }, | |
1554 | (Tid origin, CurlMessage!bool f) | |
1555 | { | |
1556 | if (origin != workerTid) | |
1557 | return false; | |
1558 | state = state.done; | |
1559 | return true; | |
1560 | } | |
1561 | ); | |
1562 | break; | |
1563 | case State.gotUnits: return true; | |
1564 | case State.done: | |
1565 | return false; | |
1566 | } | |
1567 | d -= sw.peek(); | |
1568 | sw.reset(); | |
1569 | } | |
1570 | return state == State.gotUnits; | |
1571 | } | |
1572 | ||
1573 | enum State | |
1574 | { | |
1575 | needUnits, | |
1576 | gotUnits, | |
1577 | done | |
1578 | } | |
1579 | State state; | |
1580 | ||
1581 | void tryEnsureUnits() | |
1582 | { | |
5fee5ec3 | 1583 | import std.concurrency : receive; |
b4c522fa IB |
1584 | while (true) |
1585 | { | |
1586 | final switch (state) | |
1587 | { | |
1588 | case State.needUnits: | |
1589 | receive( | |
1590 | (Tid origin, CurlMessage!(immutable(Unit)[]) _data) | |
1591 | { | |
1592 | if (origin != workerTid) | |
1593 | return false; | |
1594 | units = cast(Unit[]) _data.data; | |
1595 | state = State.gotUnits; | |
1596 | return true; | |
1597 | }, | |
1598 | (Tid origin, CurlMessage!bool f) | |
1599 | { | |
1600 | if (origin != workerTid) | |
1601 | return false; | |
1602 | state = state.done; | |
1603 | return true; | |
1604 | } | |
1605 | ); | |
1606 | break; | |
1607 | case State.gotUnits: return; | |
1608 | case State.done: | |
1609 | return; | |
1610 | } | |
1611 | } | |
1612 | } | |
1613 | } | |
1614 | ||
b4c522fa IB |
1615 | /** HTTP/FTP fetch content as a range of lines asynchronously. |
1616 | * | |
1617 | * A range of lines is returned immediately and the request that fetches the | |
1618 | * lines is performed in another thread. If the method or other request | |
5fee5ec3 | 1619 | * properties is to be customized then set the `conn` parameter with a |
b4c522fa IB |
1620 | * HTTP/FTP instance that has these properties set. |
1621 | * | |
5fee5ec3 | 1622 | * If `postData` is non-_null the method will be set to `post` for HTTP |
b4c522fa IB |
1623 | * requests. |
1624 | * | |
1625 | * The background thread will buffer up to transmitBuffers number of lines | |
1626 | * before it stops receiving data from network. When the main thread reads the | |
1627 | * lines from the range it frees up buffers and allows for the background thread | |
1628 | * to receive more data from the network. | |
1629 | * | |
1630 | * If no data is available and the main thread accesses the range it will block | |
5fee5ec3 IB |
1631 | * until data becomes available. An exception to this is the `wait(Duration)` method on |
1632 | * the $(LREF LineInputRange). This method will wait at maximum for the | |
b4c522fa IB |
1633 | * specified duration and return true if data is available. |
1634 | * | |
1635 | * Example: | |
1636 | * ---- | |
1637 | * import std.net.curl, std.stdio; | |
1638 | * // Get some pages in the background | |
1639 | * auto range1 = byLineAsync("www.google.com"); | |
1640 | * auto range2 = byLineAsync("www.wikipedia.org"); | |
1641 | * foreach (line; byLineAsync("dlang.org")) | |
1642 | * writeln(line); | |
1643 | * | |
1644 | * // Lines already fetched in the background and ready | |
1645 | * foreach (line; range1) writeln(line); | |
1646 | * foreach (line; range2) writeln(line); | |
1647 | * ---- | |
1648 | * | |
1649 | * ---- | |
1650 | * import std.net.curl, std.stdio; | |
1651 | * // Get a line in a background thread and wait in | |
1652 | * // main thread for 2 seconds for it to arrive. | |
1653 | * auto range3 = byLineAsync("dlang.com"); | |
1654 | * if (range3.wait(dur!"seconds"(2))) | |
1655 | * writeln(range3.front); | |
1656 | * else | |
1657 | * writeln("No line received after 2 seconds!"); | |
1658 | * ---- | |
1659 | * | |
1660 | * Params: | |
1661 | * url = The url to receive content from | |
1662 | * postData = Data to HTTP Post | |
5fee5ec3 | 1663 | * keepTerminator = `Yes.keepTerminator` signals that the line terminator should be |
b4c522fa IB |
1664 | * returned as part of the lines in the range. |
1665 | * terminator = The character that terminates a line | |
1666 | * transmitBuffers = The number of lines buffered asynchronously | |
1667 | * conn = The connection to use e.g. HTTP or FTP. | |
1668 | * | |
1669 | * Returns: | |
1670 | * A range of Char[] with the content of the resource pointer to by the | |
1671 | * URL. | |
1672 | */ | |
1673 | auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char, PostUnit) | |
1674 | (const(char)[] url, const(PostUnit)[] postData, | |
1675 | KeepTerminator keepTerminator = No.keepTerminator, | |
1676 | Terminator terminator = '\n', | |
1677 | size_t transmitBuffers = 10, Conn conn = Conn()) | |
1678 | if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator) | |
1679 | { | |
1680 | static if (is(Conn : AutoProtocol)) | |
1681 | { | |
1682 | if (isFTPUrl(url)) | |
1683 | return byLineAsync(url, postData, keepTerminator, | |
1684 | terminator, transmitBuffers, FTP()); | |
1685 | else | |
1686 | return byLineAsync(url, postData, keepTerminator, | |
1687 | terminator, transmitBuffers, HTTP()); | |
1688 | } | |
1689 | else | |
1690 | { | |
5fee5ec3 | 1691 | import std.concurrency : OnCrowding, send, setMaxMailboxSize, spawn, thisTid, Tid; |
b4c522fa IB |
1692 | // 50 is just an arbitrary number for now |
1693 | setMaxMailboxSize(thisTid, 50, OnCrowding.block); | |
5fee5ec3 | 1694 | auto tid = spawn(&_async!().spawn!(Conn, Char, Terminator)); |
b4c522fa IB |
1695 | tid.send(thisTid); |
1696 | tid.send(terminator); | |
1697 | tid.send(keepTerminator == Yes.keepTerminator); | |
1698 | ||
5fee5ec3 | 1699 | _async!().duplicateConnection(url, conn, postData, tid); |
b4c522fa | 1700 | |
5fee5ec3 IB |
1701 | return _async!().LineInputRange!Char(tid, transmitBuffers, |
1702 | Conn.defaultAsyncStringBufferSize); | |
b4c522fa IB |
1703 | } |
1704 | } | |
1705 | ||
1706 | /// ditto | |
1707 | auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char) | |
1708 | (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator, | |
1709 | Terminator terminator = '\n', | |
1710 | size_t transmitBuffers = 10, Conn conn = Conn()) | |
1711 | { | |
1712 | static if (is(Conn : AutoProtocol)) | |
1713 | { | |
1714 | if (isFTPUrl(url)) | |
1715 | return byLineAsync(url, cast(void[]) null, keepTerminator, | |
1716 | terminator, transmitBuffers, FTP()); | |
1717 | else | |
1718 | return byLineAsync(url, cast(void[]) null, keepTerminator, | |
1719 | terminator, transmitBuffers, HTTP()); | |
1720 | } | |
1721 | else | |
1722 | { | |
1723 | return byLineAsync(url, cast(void[]) null, keepTerminator, | |
1724 | terminator, transmitBuffers, conn); | |
1725 | } | |
1726 | } | |
1727 | ||
1728 | @system unittest | |
1729 | { | |
1730 | import std.algorithm.comparison : equal; | |
1731 | ||
1732 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
1733 | { | |
1734 | testServer.handle((s) { | |
1735 | auto req = s.recvReq; | |
1736 | s.send(httpOK("Line1\nLine2\nLine3")); | |
1737 | }); | |
1738 | assert(byLineAsync(host).equal(["Line1", "Line2", "Line3"])); | |
1739 | } | |
1740 | } | |
1741 | ||
b4c522fa IB |
1742 | /** HTTP/FTP fetch content as a range of chunks asynchronously. |
1743 | * | |
1744 | * A range of chunks is returned immediately and the request that fetches the | |
1745 | * chunks is performed in another thread. If the method or other request | |
5fee5ec3 | 1746 | * properties is to be customized then set the `conn` parameter with a |
b4c522fa IB |
1747 | * HTTP/FTP instance that has these properties set. |
1748 | * | |
5fee5ec3 | 1749 | * If `postData` is non-_null the method will be set to `post` for HTTP |
b4c522fa IB |
1750 | * requests. |
1751 | * | |
1752 | * The background thread will buffer up to transmitBuffers number of chunks | |
1753 | * before is stops receiving data from network. When the main thread reads the | |
1754 | * chunks from the range it frees up buffers and allows for the background | |
1755 | * thread to receive more data from the network. | |
1756 | * | |
1757 | * If no data is available and the main thread access the range it will block | |
5fee5ec3 IB |
1758 | * until data becomes available. An exception to this is the `wait(Duration)` |
1759 | * method on the $(LREF ChunkInputRange). This method will wait at maximum for the specified | |
b4c522fa IB |
1760 | * duration and return true if data is available. |
1761 | * | |
1762 | * Example: | |
1763 | * ---- | |
1764 | * import std.net.curl, std.stdio; | |
1765 | * // Get some pages in the background | |
1766 | * auto range1 = byChunkAsync("www.google.com", 100); | |
1767 | * auto range2 = byChunkAsync("www.wikipedia.org"); | |
1768 | * foreach (chunk; byChunkAsync("dlang.org")) | |
1769 | * writeln(chunk); // chunk is ubyte[100] | |
1770 | * | |
1771 | * // Chunks already fetched in the background and ready | |
1772 | * foreach (chunk; range1) writeln(chunk); | |
1773 | * foreach (chunk; range2) writeln(chunk); | |
1774 | * ---- | |
1775 | * | |
1776 | * ---- | |
1777 | * import std.net.curl, std.stdio; | |
1778 | * // Get a line in a background thread and wait in | |
1779 | * // main thread for 2 seconds for it to arrive. | |
1780 | * auto range3 = byChunkAsync("dlang.com", 10); | |
1781 | * if (range3.wait(dur!"seconds"(2))) | |
1782 | * writeln(range3.front); | |
1783 | * else | |
1784 | * writeln("No chunk received after 2 seconds!"); | |
1785 | * ---- | |
1786 | * | |
1787 | * Params: | |
1788 | * url = The url to receive content from | |
1789 | * postData = Data to HTTP Post | |
1790 | * chunkSize = The size of the chunks | |
1791 | * transmitBuffers = The number of chunks buffered asynchronously | |
1792 | * conn = The connection to use e.g. HTTP or FTP. | |
1793 | * | |
1794 | * Returns: | |
1795 | * A range of ubyte[chunkSize] with the content of the resource pointer to by | |
1796 | * the URL. | |
1797 | */ | |
1798 | auto byChunkAsync(Conn = AutoProtocol, PostUnit) | |
1799 | (const(char)[] url, const(PostUnit)[] postData, | |
1800 | size_t chunkSize = 1024, size_t transmitBuffers = 10, | |
1801 | Conn conn = Conn()) | |
1802 | if (isCurlConn!(Conn)) | |
1803 | { | |
1804 | static if (is(Conn : AutoProtocol)) | |
1805 | { | |
1806 | if (isFTPUrl(url)) | |
1807 | return byChunkAsync(url, postData, chunkSize, | |
1808 | transmitBuffers, FTP()); | |
1809 | else | |
1810 | return byChunkAsync(url, postData, chunkSize, | |
1811 | transmitBuffers, HTTP()); | |
1812 | } | |
1813 | else | |
1814 | { | |
5fee5ec3 | 1815 | import std.concurrency : OnCrowding, send, setMaxMailboxSize, spawn, thisTid, Tid; |
b4c522fa IB |
1816 | // 50 is just an arbitrary number for now |
1817 | setMaxMailboxSize(thisTid, 50, OnCrowding.block); | |
5fee5ec3 | 1818 | auto tid = spawn(&_async!().spawn!(Conn, ubyte)); |
b4c522fa IB |
1819 | tid.send(thisTid); |
1820 | ||
5fee5ec3 | 1821 | _async!().duplicateConnection(url, conn, postData, tid); |
b4c522fa | 1822 | |
5fee5ec3 | 1823 | return _async!().ChunkInputRange(tid, transmitBuffers, chunkSize); |
b4c522fa IB |
1824 | } |
1825 | } | |
1826 | ||
1827 | /// ditto | |
1828 | auto byChunkAsync(Conn = AutoProtocol) | |
1829 | (const(char)[] url, | |
1830 | size_t chunkSize = 1024, size_t transmitBuffers = 10, | |
1831 | Conn conn = Conn()) | |
1832 | if (isCurlConn!(Conn)) | |
1833 | { | |
1834 | static if (is(Conn : AutoProtocol)) | |
1835 | { | |
1836 | if (isFTPUrl(url)) | |
1837 | return byChunkAsync(url, cast(void[]) null, chunkSize, | |
1838 | transmitBuffers, FTP()); | |
1839 | else | |
1840 | return byChunkAsync(url, cast(void[]) null, chunkSize, | |
1841 | transmitBuffers, HTTP()); | |
1842 | } | |
1843 | else | |
1844 | { | |
1845 | return byChunkAsync(url, cast(void[]) null, chunkSize, | |
1846 | transmitBuffers, conn); | |
1847 | } | |
1848 | } | |
1849 | ||
1850 | @system unittest | |
1851 | { | |
1852 | import std.algorithm.comparison : equal; | |
1853 | ||
1854 | foreach (host; [testServer.addr, "http://"~testServer.addr]) | |
1855 | { | |
1856 | testServer.handle((s) { | |
1857 | auto req = s.recvReq; | |
1858 | s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5])); | |
1859 | }); | |
1860 | assert(byChunkAsync(host, 2).equal([[0, 1], [2, 3], [4, 5]])); | |
1861 | } | |
1862 | } | |
1863 | ||
1864 | ||
b4c522fa IB |
1865 | /* |
1866 | Mixin template for all supported curl protocols. This is the commom | |
1867 | functionallity such as timeouts and network interface settings. This should | |
1868 | really be in the HTTP/FTP/SMTP structs but the documentation tool does not | |
1869 | support a mixin to put its doc strings where a mixin is done. Therefore docs | |
1870 | in this template is copied into each of HTTP/FTP/SMTP below. | |
1871 | */ | |
1872 | private mixin template Protocol() | |
1873 | { | |
5fee5ec3 IB |
1874 | import etc.c.curl : CurlReadFunc, RawCurlProxy = CurlProxy; |
1875 | import core.time : Duration; | |
1876 | import std.socket : InternetAddress; | |
b4c522fa | 1877 | |
5fee5ec3 | 1878 | /// Value to return from `onSend`/`onReceive` delegates in order to |
b4c522fa IB |
1879 | /// pause a request |
1880 | alias requestPause = CurlReadFunc.pause; | |
1881 | ||
1882 | /// Value to return from onSend delegate in order to abort a request | |
1883 | alias requestAbort = CurlReadFunc.abort; | |
1884 | ||
1885 | static uint defaultAsyncStringBufferSize = 100; | |
1886 | ||
1887 | /** | |
1888 | The curl handle used by this connection. | |
1889 | */ | |
1890 | @property ref Curl handle() return | |
1891 | { | |
1892 | return p.curl; | |
1893 | } | |
1894 | ||
1895 | /** | |
1896 | True if the instance is stopped. A stopped instance is not usable. | |
1897 | */ | |
1898 | @property bool isStopped() | |
1899 | { | |
1900 | return p.curl.stopped; | |
1901 | } | |
1902 | ||
1903 | /// Stop and invalidate this instance. | |
1904 | void shutdown() | |
1905 | { | |
1906 | p.curl.shutdown(); | |
1907 | } | |
1908 | ||
1909 | /** Set verbose. | |
1910 | This will print request information to stderr. | |
1911 | */ | |
1912 | @property void verbose(bool on) | |
1913 | { | |
1914 | p.curl.set(CurlOption.verbose, on ? 1L : 0L); | |
1915 | } | |
1916 | ||
1917 | // Connection settings | |
1918 | ||
1919 | /// Set timeout for activity on connection. | |
1920 | @property void dataTimeout(Duration d) | |
1921 | { | |
1922 | p.curl.set(CurlOption.low_speed_limit, 1); | |
1923 | p.curl.set(CurlOption.low_speed_time, d.total!"seconds"); | |
1924 | } | |
1925 | ||
1926 | /** Set maximum time an operation is allowed to take. | |
1927 | This includes dns resolution, connecting, data transfer, etc. | |
1928 | */ | |
1929 | @property void operationTimeout(Duration d) | |
1930 | { | |
1931 | p.curl.set(CurlOption.timeout_ms, d.total!"msecs"); | |
1932 | } | |
1933 | ||
1934 | /// Set timeout for connecting. | |
1935 | @property void connectTimeout(Duration d) | |
1936 | { | |
1937 | p.curl.set(CurlOption.connecttimeout_ms, d.total!"msecs"); | |
1938 | } | |
1939 | ||
1940 | // Network settings | |
1941 | ||
1942 | /** Proxy | |
1943 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) | |
1944 | */ | |
1945 | @property void proxy(const(char)[] host) | |
1946 | { | |
1947 | p.curl.set(CurlOption.proxy, host); | |
1948 | } | |
1949 | ||
1950 | /** Proxy port | |
1951 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) | |
1952 | */ | |
1953 | @property void proxyPort(ushort port) | |
1954 | { | |
1955 | p.curl.set(CurlOption.proxyport, cast(long) port); | |
1956 | } | |
1957 | ||
1958 | /// Type of proxy | |
5fee5ec3 | 1959 | alias CurlProxy = RawCurlProxy; |
b4c522fa IB |
1960 | |
1961 | /** Proxy type | |
1962 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) | |
1963 | */ | |
1964 | @property void proxyType(CurlProxy type) | |
1965 | { | |
1966 | p.curl.set(CurlOption.proxytype, cast(long) type); | |
1967 | } | |
1968 | ||
1969 | /// DNS lookup timeout. | |
1970 | @property void dnsTimeout(Duration d) | |
1971 | { | |
1972 | p.curl.set(CurlOption.dns_cache_timeout, d.total!"msecs"); | |
1973 | } | |
1974 | ||
1975 | /** | |
c8dfa79c | 1976 | * The network interface to use in form of the IP of the interface. |
b4c522fa IB |
1977 | * |
1978 | * Example: | |
1979 | * ---- | |
1980 | * theprotocol.netInterface = "192.168.1.32"; | |
1981 | * theprotocol.netInterface = [ 192, 168, 1, 32 ]; | |
1982 | * ---- | |
1983 | * | |
1984 | * See: $(REF InternetAddress, std,socket) | |
1985 | */ | |
1986 | @property void netInterface(const(char)[] i) | |
1987 | { | |
1988 | p.curl.set(CurlOption.intrface, i); | |
1989 | } | |
1990 | ||
1991 | /// ditto | |
1992 | @property void netInterface(const(ubyte)[4] i) | |
1993 | { | |
1994 | import std.format : format; | |
1995 | const str = format("%d.%d.%d.%d", i[0], i[1], i[2], i[3]); | |
1996 | netInterface = str; | |
1997 | } | |
1998 | ||
1999 | /// ditto | |
2000 | @property void netInterface(InternetAddress i) | |
2001 | { | |
2002 | netInterface = i.toAddrString(); | |
2003 | } | |
2004 | ||
2005 | /** | |
2006 | Set the local outgoing port to use. | |
2007 | Params: | |
2008 | port = the first outgoing port number to try and use | |
2009 | */ | |
2010 | @property void localPort(ushort port) | |
2011 | { | |
2012 | p.curl.set(CurlOption.localport, cast(long) port); | |
2013 | } | |
2014 | ||
2015 | /** | |
2016 | Set the no proxy flag for the specified host names. | |
2017 | Params: | |
2018 | test = a list of comma host names that do not require | |
2019 | proxy to get reached | |
2020 | */ | |
2021 | void setNoProxy(string hosts) | |
2022 | { | |
2023 | p.curl.set(CurlOption.noproxy, hosts); | |
2024 | } | |
2025 | ||
2026 | /** | |
2027 | Set the local outgoing port range to use. | |
2028 | This can be used together with the localPort property. | |
2029 | Params: | |
2030 | range = if the first port is occupied then try this many | |
2031 | port number forwards | |
2032 | */ | |
2033 | @property void localPortRange(ushort range) | |
2034 | { | |
2035 | p.curl.set(CurlOption.localportrange, cast(long) range); | |
2036 | } | |
2037 | ||
2038 | /** Set the tcp no-delay socket option on or off. | |
2039 | See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) | |
2040 | */ | |
2041 | @property void tcpNoDelay(bool on) | |
2042 | { | |
2043 | p.curl.set(CurlOption.tcp_nodelay, cast(long) (on ? 1 : 0) ); | |
2044 | } | |
2045 | ||
2046 | /** Sets whether SSL peer certificates should be verified. | |
2047 | See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYPEER, verifypeer) | |
2048 | */ | |
2049 | @property void verifyPeer(bool on) | |
2050 | { | |
2051 | p.curl.set(CurlOption.ssl_verifypeer, on ? 1 : 0); | |
2052 | } | |
2053 | ||
2054 | /** Sets whether the host within an SSL certificate should be verified. | |
2055 | See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYHOST, verifypeer) | |
2056 | */ | |
2057 | @property void verifyHost(bool on) | |
2058 | { | |
2059 | p.curl.set(CurlOption.ssl_verifyhost, on ? 2 : 0); | |
2060 | } | |
2061 | ||
2062 | // Authentication settings | |
2063 | ||
2064 | /** | |
2065 | Set the user name, password and optionally domain for authentication | |
2066 | purposes. | |
2067 | ||
2068 | Some protocols may need authentication in some cases. Use this | |
2069 | function to provide credentials. | |
2070 | ||
2071 | Params: | |
2072 | username = the username | |
2073 | password = the password | |
2074 | domain = used for NTLM authentication only and is set to the NTLM domain | |
2075 | name | |
2076 | */ | |
2077 | void setAuthentication(const(char)[] username, const(char)[] password, | |
2078 | const(char)[] domain = "") | |
2079 | { | |
2080 | import std.format : format; | |
2081 | if (!domain.empty) | |
2082 | username = format("%s/%s", domain, username); | |
2083 | p.curl.set(CurlOption.userpwd, format("%s:%s", username, password)); | |
2084 | } | |
2085 | ||
2086 | @system unittest | |
2087 | { | |
2088 | import std.algorithm.searching : canFind; | |
2089 | ||
2090 | testServer.handle((s) { | |
2091 | auto req = s.recvReq; | |
2092 | assert(req.hdrs.canFind("GET /")); | |
2093 | assert(req.hdrs.canFind("Basic dXNlcjpwYXNz")); | |
2094 | s.send(httpOK()); | |
2095 | }); | |
2096 | ||
2097 | auto http = HTTP(testServer.addr); | |
2098 | http.onReceive = (ubyte[] data) { return data.length; }; | |
2099 | http.setAuthentication("user", "pass"); | |
2100 | http.perform(); | |
2101 | ||
5fee5ec3 | 2102 | // https://issues.dlang.org/show_bug.cgi?id=17540 |
b4c522fa IB |
2103 | http.setNoProxy("www.example.com"); |
2104 | } | |
2105 | ||
2106 | /** | |
2107 | Set the user name and password for proxy authentication. | |
2108 | ||
2109 | Params: | |
2110 | username = the username | |
2111 | password = the password | |
2112 | */ | |
2113 | void setProxyAuthentication(const(char)[] username, const(char)[] password) | |
2114 | { | |
2115 | import std.array : replace; | |
2116 | import std.format : format; | |
2117 | ||
2118 | p.curl.set(CurlOption.proxyuserpwd, | |
2119 | format("%s:%s", | |
2120 | username.replace(":", "%3A"), | |
2121 | password.replace(":", "%3A")) | |
2122 | ); | |
2123 | } | |
2124 | ||
2125 | /** | |
2126 | * The event handler that gets called when data is needed for sending. The | |
5fee5ec3 | 2127 | * length of the `void[]` specifies the maximum number of bytes that can |
b4c522fa IB |
2128 | * be sent. |
2129 | * | |
2130 | * Returns: | |
2131 | * The callback returns the number of elements in the buffer that have been | |
2132 | * filled and are ready to send. | |
5fee5ec3 | 2133 | * The special value `.abortRequest` can be returned in order to abort the |
b4c522fa | 2134 | * current request. |
5fee5ec3 | 2135 | * The special value `.pauseRequest` can be returned in order to pause the |
b4c522fa IB |
2136 | * current request. |
2137 | * | |
2138 | * Example: | |
2139 | * ---- | |
2140 | * import std.net.curl; | |
2141 | * string msg = "Hello world"; | |
2142 | * auto client = HTTP("dlang.org"); | |
2143 | * client.onSend = delegate size_t(void[] data) | |
2144 | * { | |
2145 | * auto m = cast(void[]) msg; | |
2146 | * size_t length = m.length > data.length ? data.length : m.length; | |
2147 | * if (length == 0) return 0; | |
2148 | * data[0 .. length] = m[0 .. length]; | |
2149 | * msg = msg[length..$]; | |
2150 | * return length; | |
2151 | * }; | |
2152 | * client.perform(); | |
2153 | * ---- | |
2154 | */ | |
2155 | @property void onSend(size_t delegate(void[]) callback) | |
2156 | { | |
2157 | p.curl.clear(CurlOption.postfields); // cannot specify data when using callback | |
2158 | p.curl.onSend = callback; | |
2159 | } | |
2160 | ||
2161 | /** | |
2162 | * The event handler that receives incoming data. Be sure to copy the | |
2163 | * incoming ubyte[] since it is not guaranteed to be valid after the | |
2164 | * callback returns. | |
2165 | * | |
2166 | * Returns: | |
2167 | * The callback returns the number of incoming bytes read. If the entire array is | |
2168 | * not read the request will abort. | |
2169 | * The special value .pauseRequest can be returned in order to pause the | |
2170 | * current request. | |
2171 | * | |
2172 | * Example: | |
2173 | * ---- | |
2174 | * import std.net.curl, std.stdio; | |
2175 | * auto client = HTTP("dlang.org"); | |
2176 | * client.onReceive = (ubyte[] data) | |
2177 | * { | |
2178 | * writeln("Got data", to!(const(char)[])(data)); | |
2179 | * return data.length; | |
2180 | * }; | |
2181 | * client.perform(); | |
2182 | * ---- | |
2183 | */ | |
2184 | @property void onReceive(size_t delegate(ubyte[]) callback) | |
2185 | { | |
2186 | p.curl.onReceive = callback; | |
2187 | } | |
2188 | ||
2189 | /** | |
2190 | * The event handler that gets called to inform of upload/download progress. | |
2191 | * | |
2192 | * Params: | |
2193 | * dlTotal = total bytes to download | |
2194 | * dlNow = currently downloaded bytes | |
2195 | * ulTotal = total bytes to upload | |
2196 | * ulNow = currently uploaded bytes | |
2197 | * | |
2198 | * Returns: | |
2199 | * Return 0 from the callback to signal success, return non-zero to abort | |
2200 | * transfer | |
2201 | * | |
2202 | * Example: | |
2203 | * ---- | |
2204 | * import std.net.curl, std.stdio; | |
2205 | * auto client = HTTP("dlang.org"); | |
5fee5ec3 | 2206 | * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t uln) |
b4c522fa IB |
2207 | * { |
2208 | * writeln("Progress: downloaded ", dln, " of ", dl); | |
2209 | * writeln("Progress: uploaded ", uln, " of ", ul); | |
5fee5ec3 | 2210 | * return 0; |
b4c522fa IB |
2211 | * }; |
2212 | * client.perform(); | |
2213 | * ---- | |
2214 | */ | |
2215 | @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, | |
2216 | size_t ulTotal, size_t ulNow) callback) | |
2217 | { | |
2218 | p.curl.onProgress = callback; | |
2219 | } | |
2220 | } | |
2221 | ||
2222 | /* | |
5fee5ec3 IB |
2223 | Decode `ubyte[]` array using the provided EncodingScheme up to maxChars |
2224 | Returns: Tuple of ubytes read and the `Char[]` characters decoded. | |
b4c522fa IB |
2225 | Not all ubytes are guaranteed to be read in case of decoding error. |
2226 | */ | |
2227 | private Tuple!(size_t,Char[]) | |
2228 | decodeString(Char = char)(const(ubyte)[] data, | |
2229 | EncodingScheme scheme, | |
2230 | size_t maxChars = size_t.max) | |
2231 | { | |
5fee5ec3 | 2232 | import std.encoding : INVALID_SEQUENCE; |
b4c522fa IB |
2233 | Char[] res; |
2234 | immutable startLen = data.length; | |
2235 | size_t charsDecoded = 0; | |
2236 | while (data.length && charsDecoded < maxChars) | |
2237 | { | |
2238 | immutable dchar dc = scheme.safeDecode(data); | |
2239 | if (dc == INVALID_SEQUENCE) | |
2240 | { | |
2241 | return typeof(return)(size_t.max, cast(Char[]) null); | |
2242 | } | |
2243 | charsDecoded++; | |
2244 | res ~= dc; | |
2245 | } | |
2246 | return typeof(return)(startLen-data.length, res); | |
2247 | } | |
2248 | ||
2249 | /* | |
5fee5ec3 | 2250 | Decode `ubyte[]` array using the provided `EncodingScheme` until a the |
b4c522fa IB |
2251 | line terminator specified is found. The basesrc parameter is effectively |
2252 | prepended to src as the first thing. | |
2253 | ||
2254 | This function is used for decoding as much of the src buffer as | |
2255 | possible until either the terminator is found or decoding fails. If | |
2256 | it fails as the last data in the src it may mean that the src buffer | |
2257 | were missing some bytes in order to represent a correct code | |
2258 | point. Upon the next call to this function more bytes have been | |
2259 | received from net and the failing bytes should be given as the | |
2260 | basesrc parameter. It is done this way to minimize data copying. | |
2261 | ||
2262 | Returns: true if a terminator was found | |
2263 | Not all ubytes are guaranteed to be read in case of decoding error. | |
2264 | any decoded chars will be inserted into dst. | |
2265 | */ | |
2266 | private bool decodeLineInto(Terminator, Char = char)(ref const(ubyte)[] basesrc, | |
2267 | ref const(ubyte)[] src, | |
2268 | ref Char[] dst, | |
2269 | EncodingScheme scheme, | |
2270 | Terminator terminator) | |
2271 | { | |
2272 | import std.algorithm.searching : endsWith; | |
5fee5ec3 IB |
2273 | import std.encoding : INVALID_SEQUENCE; |
2274 | import std.exception : enforce; | |
b4c522fa IB |
2275 | |
2276 | // if there is anything in the basesrc then try to decode that | |
2277 | // first. | |
2278 | if (basesrc.length != 0) | |
2279 | { | |
2280 | // Try to ensure 4 entries in the basesrc by copying from src. | |
2281 | immutable blen = basesrc.length; | |
2282 | immutable len = (basesrc.length + src.length) >= 4 ? | |
2283 | 4 : basesrc.length + src.length; | |
2284 | basesrc.length = len; | |
2285 | ||
2286 | immutable dchar dc = scheme.safeDecode(basesrc); | |
2287 | if (dc == INVALID_SEQUENCE) | |
2288 | { | |
2289 | enforce!CurlException(len != 4, "Invalid code sequence"); | |
2290 | return false; | |
2291 | } | |
2292 | dst ~= dc; | |
2293 | src = src[len-basesrc.length-blen .. $]; // remove used ubytes from src | |
2294 | basesrc.length = 0; | |
2295 | } | |
2296 | ||
2297 | while (src.length) | |
2298 | { | |
2299 | const lsrc = src; | |
2300 | dchar dc = scheme.safeDecode(src); | |
2301 | if (dc == INVALID_SEQUENCE) | |
2302 | { | |
2303 | if (src.empty) | |
2304 | { | |
2305 | // The invalid sequence was in the end of the src. Maybe there | |
2306 | // just need to be more bytes available so these last bytes are | |
2307 | // put back to src for later use. | |
2308 | src = lsrc; | |
2309 | return false; | |
2310 | } | |
2311 | dc = '?'; | |
2312 | } | |
2313 | dst ~= dc; | |
2314 | ||
2315 | if (dst.endsWith(terminator)) | |
2316 | return true; | |
2317 | } | |
2318 | return false; // no terminator found | |
2319 | } | |
2320 | ||
2321 | /** | |
2322 | * HTTP client functionality. | |
2323 | * | |
2324 | * Example: | |
5fee5ec3 IB |
2325 | * |
2326 | * Get with custom data receivers: | |
2327 | * | |
b4c522fa IB |
2328 | * --- |
2329 | * import std.net.curl, std.stdio; | |
2330 | * | |
5fee5ec3 | 2331 | * auto http = HTTP("https://dlang.org"); |
b4c522fa IB |
2332 | * http.onReceiveHeader = |
2333 | * (in char[] key, in char[] value) { writeln(key ~ ": " ~ value); }; | |
2334 | * http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; }; | |
2335 | * http.perform(); | |
5fee5ec3 | 2336 | * --- |
b4c522fa | 2337 | * |
5fee5ec3 IB |
2338 | */ |
2339 | ||
2340 | /** | |
2341 | * Put with data senders: | |
2342 | * | |
2343 | * --- | |
2344 | * import std.net.curl, std.stdio; | |
2345 | * | |
2346 | * auto http = HTTP("https://dlang.org"); | |
b4c522fa IB |
2347 | * auto msg = "Hello world"; |
2348 | * http.contentLength = msg.length; | |
2349 | * http.onSend = (void[] data) | |
2350 | * { | |
2351 | * auto m = cast(void[]) msg; | |
2352 | * size_t len = m.length > data.length ? data.length : m.length; | |
2353 | * if (len == 0) return len; | |
2354 | * data[0 .. len] = m[0 .. len]; | |
2355 | * msg = msg[len..$]; | |
2356 | * return len; | |
2357 | * }; | |
2358 | * http.perform(); | |
5fee5ec3 | 2359 | * --- |
b4c522fa | 2360 | * |
5fee5ec3 IB |
2361 | */ |
2362 | ||
2363 | /** | |
2364 | * Tracking progress: | |
2365 | * | |
2366 | * --- | |
2367 | * import std.net.curl, std.stdio; | |
2368 | * | |
2369 | * auto http = HTTP(); | |
b4c522fa | 2370 | * http.method = HTTP.Method.get; |
5fee5ec3 | 2371 | * http.url = "http://upload.wikimedia.org/wikipedia/commons/" ~ |
b4c522fa IB |
2372 | * "5/53/Wikipedia-logo-en-big.png"; |
2373 | * http.onReceive = (ubyte[] data) { return data.length; }; | |
2374 | * http.onProgress = (size_t dltotal, size_t dlnow, | |
2375 | * size_t ultotal, size_t ulnow) | |
2376 | * { | |
2377 | * writeln("Progress ", dltotal, ", ", dlnow, ", ", ultotal, ", ", ulnow); | |
2378 | * return 0; | |
2379 | * }; | |
2380 | * http.perform(); | |
2381 | * --- | |
2382 | * | |
5fee5ec3 | 2383 | * See_Also: $(LINK2 http://www.ietf.org/rfc/rfc2616.txt, RFC2616) |
b4c522fa IB |
2384 | * |
2385 | */ | |
2386 | struct HTTP | |
2387 | { | |
2388 | mixin Protocol; | |
2389 | ||
2390 | import std.datetime.systime : SysTime; | |
5fee5ec3 IB |
2391 | import std.typecons : RefCounted; |
2392 | import etc.c.curl : CurlAuth, CurlInfo, curl_slist, CURLVERSION_NOW, curl_off_t; | |
b4c522fa IB |
2393 | |
2394 | /// Authentication method equal to $(REF CurlAuth, etc,c,curl) | |
2395 | alias AuthMethod = CurlAuth; | |
2396 | ||
2397 | static private uint defaultMaxRedirects = 10; | |
2398 | ||
2399 | private struct Impl | |
2400 | { | |
2401 | ~this() | |
2402 | { | |
2403 | if (headersOut !is null) | |
2404 | Curl.curl.slist_free_all(headersOut); | |
2405 | if (curl.handle !is null) // work around RefCounted/emplace bug | |
2406 | curl.shutdown(); | |
2407 | } | |
2408 | Curl curl; | |
2409 | curl_slist* headersOut; | |
2410 | string[string] headersIn; | |
2411 | string charset; | |
2412 | ||
2413 | /// The status line of the final sub-request in a request. | |
2414 | StatusLine status; | |
2415 | private void delegate(StatusLine) onReceiveStatusLine; | |
2416 | ||
2417 | /// The HTTP method to use. | |
2418 | Method method = Method.undefined; | |
2419 | ||
2420 | @system @property void onReceiveHeader(void delegate(in char[] key, | |
2421 | in char[] value) callback) | |
2422 | { | |
2423 | import std.algorithm.searching : startsWith; | |
b4c522fa IB |
2424 | import std.regex : regex, match; |
2425 | import std.uni : toLower; | |
2426 | ||
2427 | // Wrap incoming callback in order to separate http status line from | |
2428 | // http headers. On redirected requests there may be several such | |
2429 | // status lines. The last one is the one recorded. | |
2430 | auto dg = (in char[] header) | |
2431 | { | |
2432 | import std.utf : UTFException; | |
2433 | try | |
2434 | { | |
2435 | if (header.empty) | |
2436 | { | |
2437 | // header delimiter | |
2438 | return; | |
2439 | } | |
2440 | if (header.startsWith("HTTP/")) | |
2441 | { | |
2442 | headersIn.clear(); | |
e977a5df | 2443 | if (parseStatusLine(header, status)) |
b4c522fa | 2444 | { |
b4c522fa IB |
2445 | if (onReceiveStatusLine != null) |
2446 | onReceiveStatusLine(status); | |
2447 | } | |
2448 | return; | |
2449 | } | |
2450 | ||
2451 | // Normal http header | |
2452 | auto m = match(cast(char[]) header, regex("(.*?): (.*)$")); | |
2453 | ||
2454 | auto fieldName = m.captures[1].toLower().idup; | |
2455 | if (fieldName == "content-type") | |
2456 | { | |
2457 | auto mct = match(cast(char[]) m.captures[2], | |
2458 | regex("charset=([^;]*)", "i")); | |
2459 | if (!mct.empty && mct.captures.length > 1) | |
2460 | charset = mct.captures[1].idup; | |
2461 | } | |
2462 | ||
2463 | if (!m.empty && callback !is null) | |
2464 | callback(fieldName, m.captures[2]); | |
2465 | headersIn[fieldName] = m.captures[2].idup; | |
2466 | } | |
2467 | catch (UTFException e) | |
2468 | { | |
2469 | //munch it - a header should be all ASCII, any "wrong UTF" is broken header | |
2470 | } | |
2471 | }; | |
2472 | ||
2473 | curl.onReceiveHeader = dg; | |
2474 | } | |
2475 | } | |
2476 | ||
2477 | private RefCounted!Impl p; | |
5fee5ec3 | 2478 | import etc.c.curl : CurlTimeCond; |
b4c522fa | 2479 | |
e977a5df | 2480 | /// Parse status line, as received from / generated by cURL. |
5fee5ec3 | 2481 | private static bool parseStatusLine(const char[] header, out StatusLine status) @safe |
e977a5df IB |
2482 | { |
2483 | import std.conv : to; | |
2484 | import std.regex : regex, match; | |
2485 | ||
2486 | const m = match(header, regex(r"^HTTP/(\d+)(?:\.(\d+))? (\d+)(?: (.*))?$")); | |
2487 | if (m.empty) | |
2488 | return false; // Invalid status line | |
2489 | else | |
2490 | { | |
2491 | status.majorVersion = to!ushort(m.captures[1]); | |
2492 | status.minorVersion = m.captures[2].length ? to!ushort(m.captures[2]) : 0; | |
2493 | status.code = to!ushort(m.captures[3]); | |
2494 | status.reason = m.captures[4].idup; | |
2495 | return true; | |
2496 | } | |
2497 | } | |
2498 | ||
2499 | @safe unittest | |
2500 | { | |
2501 | StatusLine status; | |
2502 | assert(parseStatusLine("HTTP/1.1 200 OK", status) | |
2503 | && status == StatusLine(1, 1, 200, "OK")); | |
2504 | assert(parseStatusLine("HTTP/1.0 304 Not Modified", status) | |
2505 | && status == StatusLine(1, 0, 304, "Not Modified")); | |
2506 | // The HTTP2 protocol is binary; cURL generates this fake text header. | |
2507 | assert(parseStatusLine("HTTP/2 200", status) | |
2508 | && status == StatusLine(2, 0, 200, null)); | |
2509 | } | |
2510 | ||
b4c522fa IB |
2511 | /** Time condition enumeration as an alias of $(REF CurlTimeCond, etc,c,curl) |
2512 | ||
2513 | $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25) | |
2514 | */ | |
2515 | alias TimeCond = CurlTimeCond; | |
2516 | ||
2517 | /** | |
2518 | Constructor taking the url as parameter. | |
2519 | */ | |
2520 | static HTTP opCall(const(char)[] url) | |
2521 | { | |
2522 | HTTP http; | |
2523 | http.initialize(); | |
2524 | http.url = url; | |
2525 | return http; | |
2526 | } | |
2527 | ||
2528 | /// | |
2529 | static HTTP opCall() | |
2530 | { | |
2531 | HTTP http; | |
2532 | http.initialize(); | |
2533 | return http; | |
2534 | } | |
2535 | ||
2536 | /// | |
2537 | HTTP dup() | |
2538 | { | |
2539 | HTTP copy; | |
2540 | copy.initialize(); | |
2541 | copy.p.method = p.method; | |
2542 | curl_slist* cur = p.headersOut; | |
2543 | curl_slist* newlist = null; | |
2544 | while (cur) | |
2545 | { | |
2546 | newlist = Curl.curl.slist_append(newlist, cur.data); | |
2547 | cur = cur.next; | |
2548 | } | |
2549 | copy.p.headersOut = newlist; | |
2550 | copy.p.curl.set(CurlOption.httpheader, copy.p.headersOut); | |
2551 | copy.p.curl = p.curl.dup(); | |
2552 | copy.dataTimeout = _defaultDataTimeout; | |
2553 | copy.onReceiveHeader = null; | |
2554 | return copy; | |
2555 | } | |
2556 | ||
2557 | private void initialize() | |
2558 | { | |
2559 | p.curl.initialize(); | |
2560 | maxRedirects = HTTP.defaultMaxRedirects; | |
2561 | p.charset = "ISO-8859-1"; // Default charset defined in HTTP RFC | |
2562 | p.method = Method.undefined; | |
2563 | setUserAgent(HTTP.defaultUserAgent); | |
2564 | dataTimeout = _defaultDataTimeout; | |
2565 | onReceiveHeader = null; | |
2566 | verifyPeer = true; | |
2567 | verifyHost = true; | |
2568 | } | |
2569 | ||
2570 | /** | |
2571 | Perform a http request. | |
2572 | ||
2573 | After the HTTP client has been setup and possibly assigned callbacks the | |
5fee5ec3 | 2574 | `perform()` method will start performing the request towards the |
b4c522fa IB |
2575 | specified server. |
2576 | ||
2577 | Params: | |
2578 | throwOnError = whether to throw an exception or return a CurlCode on error | |
2579 | */ | |
2580 | CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) | |
2581 | { | |
2582 | p.status.reset(); | |
2583 | ||
2584 | CurlOption opt; | |
2585 | final switch (p.method) | |
2586 | { | |
2587 | case Method.head: | |
2588 | p.curl.set(CurlOption.nobody, 1L); | |
2589 | opt = CurlOption.nobody; | |
2590 | break; | |
2591 | case Method.undefined: | |
2592 | case Method.get: | |
2593 | p.curl.set(CurlOption.httpget, 1L); | |
2594 | opt = CurlOption.httpget; | |
2595 | break; | |
2596 | case Method.post: | |
2597 | p.curl.set(CurlOption.post, 1L); | |
2598 | opt = CurlOption.post; | |
2599 | break; | |
2600 | case Method.put: | |
2601 | p.curl.set(CurlOption.upload, 1L); | |
2602 | opt = CurlOption.upload; | |
2603 | break; | |
2604 | case Method.del: | |
2605 | p.curl.set(CurlOption.customrequest, "DELETE"); | |
2606 | opt = CurlOption.customrequest; | |
2607 | break; | |
2608 | case Method.options: | |
2609 | p.curl.set(CurlOption.customrequest, "OPTIONS"); | |
2610 | opt = CurlOption.customrequest; | |
2611 | break; | |
2612 | case Method.trace: | |
2613 | p.curl.set(CurlOption.customrequest, "TRACE"); | |
2614 | opt = CurlOption.customrequest; | |
2615 | break; | |
2616 | case Method.connect: | |
2617 | p.curl.set(CurlOption.customrequest, "CONNECT"); | |
2618 | opt = CurlOption.customrequest; | |
2619 | break; | |
2620 | case Method.patch: | |
2621 | p.curl.set(CurlOption.customrequest, "PATCH"); | |
2622 | opt = CurlOption.customrequest; | |
2623 | break; | |
2624 | } | |
2625 | ||
2626 | scope (exit) p.curl.clear(opt); | |
2627 | return p.curl.perform(throwOnError); | |
2628 | } | |
2629 | ||
2630 | /// The URL to specify the location of the resource. | |
2631 | @property void url(const(char)[] url) | |
2632 | { | |
2633 | import std.algorithm.searching : startsWith; | |
2634 | import std.uni : toLower; | |
2635 | if (!startsWith(url.toLower(), "http://", "https://")) | |
2636 | url = "http://" ~ url; | |
2637 | p.curl.set(CurlOption.url, url); | |
2638 | } | |
2639 | ||
2640 | /// Set the CA certificate bundle file to use for SSL peer verification | |
2641 | @property void caInfo(const(char)[] caFile) | |
2642 | { | |
2643 | p.curl.set(CurlOption.cainfo, caFile); | |
2644 | } | |
2645 | ||
2646 | // This is a workaround for mixed in content not having its | |
2647 | // docs mixed in. | |
2648 | version (StdDdoc) | |
2649 | { | |
5fee5ec3 IB |
2650 | static import etc.c.curl; |
2651 | ||
2652 | /// Value to return from `onSend`/`onReceive` delegates in order to | |
b4c522fa IB |
2653 | /// pause a request |
2654 | alias requestPause = CurlReadFunc.pause; | |
2655 | ||
2656 | /// Value to return from onSend delegate in order to abort a request | |
2657 | alias requestAbort = CurlReadFunc.abort; | |
2658 | ||
2659 | /** | |
2660 | True if the instance is stopped. A stopped instance is not usable. | |
2661 | */ | |
2662 | @property bool isStopped(); | |
2663 | ||
2664 | /// Stop and invalidate this instance. | |
2665 | void shutdown(); | |
2666 | ||
2667 | /** Set verbose. | |
2668 | This will print request information to stderr. | |
2669 | */ | |
2670 | @property void verbose(bool on); | |
2671 | ||
2672 | // Connection settings | |
2673 | ||
2674 | /// Set timeout for activity on connection. | |
2675 | @property void dataTimeout(Duration d); | |
2676 | ||
2677 | /** Set maximum time an operation is allowed to take. | |
2678 | This includes dns resolution, connecting, data transfer, etc. | |
2679 | */ | |
2680 | @property void operationTimeout(Duration d); | |
2681 | ||
2682 | /// Set timeout for connecting. | |
2683 | @property void connectTimeout(Duration d); | |
2684 | ||
2685 | // Network settings | |
2686 | ||
2687 | /** Proxy | |
2688 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) | |
2689 | */ | |
2690 | @property void proxy(const(char)[] host); | |
2691 | ||
2692 | /** Proxy port | |
2693 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) | |
2694 | */ | |
2695 | @property void proxyPort(ushort port); | |
2696 | ||
2697 | /// Type of proxy | |
2698 | alias CurlProxy = etc.c.curl.CurlProxy; | |
2699 | ||
2700 | /** Proxy type | |
2701 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) | |
2702 | */ | |
2703 | @property void proxyType(CurlProxy type); | |
2704 | ||
2705 | /// DNS lookup timeout. | |
2706 | @property void dnsTimeout(Duration d); | |
2707 | ||
2708 | /** | |
c8dfa79c | 2709 | * The network interface to use in form of the IP of the interface. |
b4c522fa IB |
2710 | * |
2711 | * Example: | |
2712 | * ---- | |
2713 | * theprotocol.netInterface = "192.168.1.32"; | |
2714 | * theprotocol.netInterface = [ 192, 168, 1, 32 ]; | |
2715 | * ---- | |
2716 | * | |
2717 | * See: $(REF InternetAddress, std,socket) | |
2718 | */ | |
2719 | @property void netInterface(const(char)[] i); | |
2720 | ||
2721 | /// ditto | |
2722 | @property void netInterface(const(ubyte)[4] i); | |
2723 | ||
2724 | /// ditto | |
2725 | @property void netInterface(InternetAddress i); | |
2726 | ||
2727 | /** | |
2728 | Set the local outgoing port to use. | |
2729 | Params: | |
2730 | port = the first outgoing port number to try and use | |
2731 | */ | |
2732 | @property void localPort(ushort port); | |
2733 | ||
2734 | /** | |
2735 | Set the local outgoing port range to use. | |
2736 | This can be used together with the localPort property. | |
2737 | Params: | |
2738 | range = if the first port is occupied then try this many | |
2739 | port number forwards | |
2740 | */ | |
2741 | @property void localPortRange(ushort range); | |
2742 | ||
2743 | /** Set the tcp no-delay socket option on or off. | |
2744 | See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) | |
2745 | */ | |
2746 | @property void tcpNoDelay(bool on); | |
2747 | ||
2748 | // Authentication settings | |
2749 | ||
2750 | /** | |
2751 | Set the user name, password and optionally domain for authentication | |
2752 | purposes. | |
2753 | ||
2754 | Some protocols may need authentication in some cases. Use this | |
2755 | function to provide credentials. | |
2756 | ||
2757 | Params: | |
2758 | username = the username | |
2759 | password = the password | |
2760 | domain = used for NTLM authentication only and is set to the NTLM domain | |
2761 | name | |
2762 | */ | |
2763 | void setAuthentication(const(char)[] username, const(char)[] password, | |
2764 | const(char)[] domain = ""); | |
2765 | ||
2766 | /** | |
2767 | Set the user name and password for proxy authentication. | |
2768 | ||
2769 | Params: | |
2770 | username = the username | |
2771 | password = the password | |
2772 | */ | |
2773 | void setProxyAuthentication(const(char)[] username, const(char)[] password); | |
2774 | ||
2775 | /** | |
2776 | * The event handler that gets called when data is needed for sending. The | |
5fee5ec3 | 2777 | * length of the `void[]` specifies the maximum number of bytes that can |
b4c522fa IB |
2778 | * be sent. |
2779 | * | |
2780 | * Returns: | |
2781 | * The callback returns the number of elements in the buffer that have been | |
2782 | * filled and are ready to send. | |
5fee5ec3 | 2783 | * The special value `.abortRequest` can be returned in order to abort the |
b4c522fa | 2784 | * current request. |
5fee5ec3 | 2785 | * The special value `.pauseRequest` can be returned in order to pause the |
b4c522fa IB |
2786 | * current request. |
2787 | * | |
2788 | * Example: | |
2789 | * ---- | |
2790 | * import std.net.curl; | |
2791 | * string msg = "Hello world"; | |
2792 | * auto client = HTTP("dlang.org"); | |
2793 | * client.onSend = delegate size_t(void[] data) | |
2794 | * { | |
2795 | * auto m = cast(void[]) msg; | |
2796 | * size_t length = m.length > data.length ? data.length : m.length; | |
2797 | * if (length == 0) return 0; | |
2798 | * data[0 .. length] = m[0 .. length]; | |
2799 | * msg = msg[length..$]; | |
2800 | * return length; | |
2801 | * }; | |
2802 | * client.perform(); | |
2803 | * ---- | |
2804 | */ | |
2805 | @property void onSend(size_t delegate(void[]) callback); | |
2806 | ||
2807 | /** | |
2808 | * The event handler that receives incoming data. Be sure to copy the | |
2809 | * incoming ubyte[] since it is not guaranteed to be valid after the | |
2810 | * callback returns. | |
2811 | * | |
2812 | * Returns: | |
2813 | * The callback returns the incoming bytes read. If not the entire array is | |
2814 | * the request will abort. | |
2815 | * The special value .pauseRequest can be returned in order to pause the | |
2816 | * current request. | |
2817 | * | |
2818 | * Example: | |
2819 | * ---- | |
2820 | * import std.net.curl, std.stdio; | |
2821 | * auto client = HTTP("dlang.org"); | |
2822 | * client.onReceive = (ubyte[] data) | |
2823 | * { | |
2824 | * writeln("Got data", to!(const(char)[])(data)); | |
2825 | * return data.length; | |
2826 | * }; | |
2827 | * client.perform(); | |
2828 | * ---- | |
2829 | */ | |
2830 | @property void onReceive(size_t delegate(ubyte[]) callback); | |
2831 | ||
2832 | /** | |
2833 | * Register an event handler that gets called to inform of | |
2834 | * upload/download progress. | |
2835 | * | |
2836 | * Callback_parameters: | |
2837 | * $(CALLBACK_PARAMS) | |
2838 | * | |
2839 | * Callback_returns: Return 0 to signal success, return non-zero to | |
2840 | * abort transfer. | |
2841 | * | |
2842 | * Example: | |
2843 | * ---- | |
2844 | * import std.net.curl, std.stdio; | |
2845 | * auto client = HTTP("dlang.org"); | |
5fee5ec3 | 2846 | * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t uln) |
b4c522fa IB |
2847 | * { |
2848 | * writeln("Progress: downloaded ", dln, " of ", dl); | |
2849 | * writeln("Progress: uploaded ", uln, " of ", ul); | |
5fee5ec3 | 2850 | * return 0; |
b4c522fa IB |
2851 | * }; |
2852 | * client.perform(); | |
2853 | * ---- | |
2854 | */ | |
2855 | @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, | |
2856 | size_t ulTotal, size_t ulNow) callback); | |
2857 | } | |
2858 | ||
2859 | /** Clear all outgoing headers. | |
2860 | */ | |
2861 | void clearRequestHeaders() | |
2862 | { | |
2863 | if (p.headersOut !is null) | |
2864 | Curl.curl.slist_free_all(p.headersOut); | |
2865 | p.headersOut = null; | |
2866 | p.curl.clear(CurlOption.httpheader); | |
2867 | } | |
2868 | ||
2869 | /** Add a header e.g. "X-CustomField: Something is fishy". | |
2870 | * | |
2871 | * There is no remove header functionality. Do a $(LREF clearRequestHeaders) | |
2872 | * and set the needed headers instead. | |
2873 | * | |
2874 | * Example: | |
2875 | * --- | |
2876 | * import std.net.curl; | |
2877 | * auto client = HTTP(); | |
2878 | * client.addRequestHeader("X-Custom-ABC", "This is the custom value"); | |
2879 | * auto content = get("dlang.org", client); | |
2880 | * --- | |
2881 | */ | |
2882 | void addRequestHeader(const(char)[] name, const(char)[] value) | |
2883 | { | |
2884 | import std.format : format; | |
5fee5ec3 | 2885 | import std.internal.cstring : tempCString; |
b4c522fa IB |
2886 | import std.uni : icmp; |
2887 | ||
2888 | if (icmp(name, "User-Agent") == 0) | |
2889 | return setUserAgent(value); | |
2890 | string nv = format("%s: %s", name, value); | |
2891 | p.headersOut = Curl.curl.slist_append(p.headersOut, | |
2892 | nv.tempCString().buffPtr); | |
2893 | p.curl.set(CurlOption.httpheader, p.headersOut); | |
2894 | } | |
2895 | ||
2896 | /** | |
2897 | * The default "User-Agent" value send with a request. | |
2898 | * It has the form "Phobos-std.net.curl/$(I PHOBOS_VERSION) (libcurl/$(I CURL_VERSION))" | |
2899 | */ | |
2900 | static string defaultUserAgent() @property | |
2901 | { | |
2902 | import std.compiler : version_major, version_minor; | |
2903 | import std.format : format, sformat; | |
2904 | ||
2905 | // http://curl.haxx.se/docs/versions.html | |
2906 | enum fmt = "Phobos-std.net.curl/%d.%03d (libcurl/%d.%d.%d)"; | |
2907 | enum maxLen = fmt.length - "%d%03d%d%d%d".length + 10 + 10 + 3 + 3 + 3; | |
2908 | ||
2909 | static char[maxLen] buf = void; | |
2910 | static string userAgent; | |
2911 | ||
2912 | if (!userAgent.length) | |
2913 | { | |
2914 | auto curlVer = Curl.curl.version_info(CURLVERSION_NOW).version_num; | |
2915 | userAgent = cast(immutable) sformat( | |
2916 | buf, fmt, version_major, version_minor, | |
2917 | curlVer >> 16 & 0xFF, curlVer >> 8 & 0xFF, curlVer & 0xFF); | |
2918 | } | |
2919 | return userAgent; | |
2920 | } | |
2921 | ||
2922 | /** Set the value of the user agent request header field. | |
2923 | * | |
2924 | * By default a request has it's "User-Agent" field set to $(LREF | |
5fee5ec3 | 2925 | * defaultUserAgent) even if `setUserAgent` was never called. Pass |
b4c522fa IB |
2926 | * an empty string to suppress the "User-Agent" field altogether. |
2927 | */ | |
2928 | void setUserAgent(const(char)[] userAgent) | |
2929 | { | |
2930 | p.curl.set(CurlOption.useragent, userAgent); | |
2931 | } | |
2932 | ||
2933 | /** | |
2934 | * Get various timings defined in $(REF CurlInfo, etc, c, curl). | |
5fee5ec3 | 2935 | * The value is usable only if the return value is equal to `etc.c.curl.CurlError.ok`. |
b4c522fa IB |
2936 | * |
2937 | * Params: | |
2938 | * timing = one of the timings defined in $(REF CurlInfo, etc, c, curl). | |
2939 | * The values are: | |
5fee5ec3 IB |
2940 | * `etc.c.curl.CurlInfo.namelookup_time`, |
2941 | * `etc.c.curl.CurlInfo.connect_time`, | |
2942 | * `etc.c.curl.CurlInfo.pretransfer_time`, | |
2943 | * `etc.c.curl.CurlInfo.starttransfer_time`, | |
2944 | * `etc.c.curl.CurlInfo.redirect_time`, | |
2945 | * `etc.c.curl.CurlInfo.appconnect_time`, | |
2946 | * `etc.c.curl.CurlInfo.total_time`. | |
b4c522fa IB |
2947 | * val = the actual value of the inquired timing. |
2948 | * | |
2949 | * Returns: | |
2950 | * The return code of the operation. The value stored in val | |
5fee5ec3 | 2951 | * should be used only if the return value is `etc.c.curl.CurlInfo.ok`. |
b4c522fa IB |
2952 | * |
2953 | * Example: | |
2954 | * --- | |
2955 | * import std.net.curl; | |
2956 | * import etc.c.curl : CurlError, CurlInfo; | |
2957 | * | |
2958 | * auto client = HTTP("dlang.org"); | |
2959 | * client.perform(); | |
2960 | * | |
2961 | * double val; | |
2962 | * CurlCode code; | |
2963 | * | |
5fee5ec3 | 2964 | * code = client.getTiming(CurlInfo.namelookup_time, val); |
b4c522fa IB |
2965 | * assert(code == CurlError.ok); |
2966 | * --- | |
2967 | */ | |
2968 | CurlCode getTiming(CurlInfo timing, ref double val) | |
2969 | { | |
2970 | return p.curl.getTiming(timing, val); | |
2971 | } | |
2972 | ||
2973 | /** The headers read from a successful response. | |
2974 | * | |
2975 | */ | |
2976 | @property string[string] responseHeaders() | |
2977 | { | |
2978 | return p.headersIn; | |
2979 | } | |
2980 | ||
2981 | /// HTTP method used. | |
2982 | @property void method(Method m) | |
2983 | { | |
2984 | p.method = m; | |
2985 | } | |
2986 | ||
2987 | /// ditto | |
2988 | @property Method method() | |
2989 | { | |
2990 | return p.method; | |
2991 | } | |
2992 | ||
2993 | /** | |
2994 | HTTP status line of last response. One call to perform may | |
2995 | result in several requests because of redirection. | |
2996 | */ | |
2997 | @property StatusLine statusLine() | |
2998 | { | |
2999 | return p.status; | |
3000 | } | |
3001 | ||
3002 | /// Set the active cookie string e.g. "name1=value1;name2=value2" | |
3003 | void setCookie(const(char)[] cookie) | |
3004 | { | |
3005 | p.curl.set(CurlOption.cookie, cookie); | |
3006 | } | |
3007 | ||
3008 | /// Set a file path to where a cookie jar should be read/stored. | |
3009 | void setCookieJar(const(char)[] path) | |
3010 | { | |
3011 | p.curl.set(CurlOption.cookiefile, path); | |
3012 | if (path.length) | |
3013 | p.curl.set(CurlOption.cookiejar, path); | |
3014 | } | |
3015 | ||
3016 | /// Flush cookie jar to disk. | |
3017 | void flushCookieJar() | |
3018 | { | |
3019 | p.curl.set(CurlOption.cookielist, "FLUSH"); | |
3020 | } | |
3021 | ||
3022 | /// Clear session cookies. | |
3023 | void clearSessionCookies() | |
3024 | { | |
3025 | p.curl.set(CurlOption.cookielist, "SESS"); | |
3026 | } | |
3027 | ||
3028 | /// Clear all cookies. | |
3029 | void clearAllCookies() | |
3030 | { | |
3031 | p.curl.set(CurlOption.cookielist, "ALL"); | |
3032 | } | |
3033 | ||
3034 | /** | |
3035 | Set time condition on the request. | |
3036 | ||
3037 | Params: | |
5fee5ec3 | 3038 | cond = `CurlTimeCond.{none,ifmodsince,ifunmodsince,lastmod}` |
b4c522fa IB |
3039 | timestamp = Timestamp for the condition |
3040 | ||
3041 | $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25) | |
3042 | */ | |
3043 | void setTimeCondition(HTTP.TimeCond cond, SysTime timestamp) | |
3044 | { | |
3045 | p.curl.set(CurlOption.timecondition, cond); | |
3046 | p.curl.set(CurlOption.timevalue, timestamp.toUnixTime()); | |
3047 | } | |
3048 | ||
3049 | /** Specifying data to post when not using the onSend callback. | |
3050 | * | |
3051 | * The data is NOT copied by the library. Content-Type will default to | |
3052 | * application/octet-stream. Data is not converted or encoded by this | |
3053 | * method. | |
3054 | * | |
3055 | * Example: | |
3056 | * ---- | |
3057 | * import std.net.curl, std.stdio; | |
3058 | * auto http = HTTP("http://www.mydomain.com"); | |
3059 | * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; }; | |
3060 | * http.postData = [1,2,3,4,5]; | |
3061 | * http.perform(); | |
3062 | * ---- | |
3063 | */ | |
3064 | @property void postData(const(void)[] data) | |
3065 | { | |
3066 | setPostData(data, "application/octet-stream"); | |
3067 | } | |
3068 | ||
3069 | /** Specifying data to post when not using the onSend callback. | |
3070 | * | |
3071 | * The data is NOT copied by the library. Content-Type will default to | |
3072 | * text/plain. Data is not converted or encoded by this method. | |
3073 | * | |
3074 | * Example: | |
3075 | * ---- | |
3076 | * import std.net.curl, std.stdio; | |
3077 | * auto http = HTTP("http://www.mydomain.com"); | |
3078 | * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; }; | |
3079 | * http.postData = "The quick...."; | |
3080 | * http.perform(); | |
3081 | * ---- | |
3082 | */ | |
3083 | @property void postData(const(char)[] data) | |
3084 | { | |
3085 | setPostData(data, "text/plain"); | |
3086 | } | |
3087 | ||
3088 | /** | |
3089 | * Specify data to post when not using the onSend callback, with | |
3090 | * user-specified Content-Type. | |
3091 | * Params: | |
3092 | * data = Data to post. | |
3093 | * contentType = MIME type of the data, for example, "text/plain" or | |
3094 | * "application/octet-stream". See also: | |
3095 | * $(LINK2 http://en.wikipedia.org/wiki/Internet_media_type, | |
3096 | * Internet media type) on Wikipedia. | |
3097 | * ----- | |
3098 | * import std.net.curl; | |
3099 | * auto http = HTTP("http://onlineform.example.com"); | |
3100 | * auto data = "app=login&username=bob&password=s00perS3kret"; | |
3101 | * http.setPostData(data, "application/x-www-form-urlencoded"); | |
3102 | * http.onReceive = (ubyte[] data) { return data.length; }; | |
3103 | * http.perform(); | |
3104 | * ----- | |
3105 | */ | |
3106 | void setPostData(const(void)[] data, string contentType) | |
3107 | { | |
3108 | // cannot use callback when specifying data directly so it is disabled here. | |
3109 | p.curl.clear(CurlOption.readfunction); | |
3110 | addRequestHeader("Content-Type", contentType); | |
3111 | p.curl.set(CurlOption.postfields, cast(void*) data.ptr); | |
3112 | p.curl.set(CurlOption.postfieldsize, data.length); | |
3113 | if (method == Method.undefined) | |
3114 | method = Method.post; | |
3115 | } | |
3116 | ||
3117 | @system unittest | |
3118 | { | |
3119 | import std.algorithm.searching : canFind; | |
3120 | ||
3121 | testServer.handle((s) { | |
3122 | auto req = s.recvReq!ubyte; | |
3123 | assert(req.hdrs.canFind("POST /path")); | |
3124 | assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4])); | |
3125 | assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255])); | |
3126 | s.send(httpOK(cast(ubyte[])[17, 27, 35, 41])); | |
3127 | }); | |
3128 | auto data = new ubyte[](256); | |
3129 | foreach (i, ref ub; data) | |
3130 | ub = cast(ubyte) i; | |
3131 | ||
3132 | auto http = HTTP(testServer.addr~"/path"); | |
3133 | http.postData = data; | |
3134 | ubyte[] res; | |
3135 | http.onReceive = (data) { res ~= data; return data.length; }; | |
3136 | http.perform(); | |
3137 | assert(res == cast(ubyte[])[17, 27, 35, 41]); | |
3138 | } | |
3139 | ||
3140 | /** | |
3141 | * Set the event handler that receives incoming headers. | |
3142 | * | |
3143 | * The callback will receive a header field key, value as parameter. The | |
5fee5ec3 | 3144 | * `const(char)[]` arrays are not valid after the delegate has returned. |
b4c522fa IB |
3145 | * |
3146 | * Example: | |
3147 | * ---- | |
3148 | * import std.net.curl, std.stdio; | |
3149 | * auto http = HTTP("dlang.org"); | |
3150 | * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; }; | |
3151 | * http.onReceiveHeader = (in char[] key, in char[] value) { writeln(key, " = ", value); }; | |
3152 | * http.perform(); | |
3153 | * ---- | |
3154 | */ | |
3155 | @property void onReceiveHeader(void delegate(in char[] key, | |
3156 | in char[] value) callback) | |
3157 | { | |
3158 | p.onReceiveHeader = callback; | |
3159 | } | |
3160 | ||
3161 | /** | |
3162 | Callback for each received StatusLine. | |
3163 | ||
3164 | Notice that several callbacks can be done for each call to | |
5fee5ec3 | 3165 | `perform()` due to redirections. |
b4c522fa IB |
3166 | |
3167 | See_Also: $(LREF StatusLine) | |
3168 | */ | |
3169 | @property void onReceiveStatusLine(void delegate(StatusLine) callback) | |
3170 | { | |
3171 | p.onReceiveStatusLine = callback; | |
3172 | } | |
3173 | ||
3174 | /** | |
3175 | The content length in bytes when using request that has content | |
3176 | e.g. POST/PUT and not using chunked transfer. Is set as the | |
3177 | "Content-Length" header. Set to ulong.max to reset to chunked transfer. | |
3178 | */ | |
3179 | @property void contentLength(ulong len) | |
3180 | { | |
3181 | import std.conv : to; | |
3182 | ||
3183 | CurlOption lenOpt; | |
3184 | ||
3185 | // Force post if necessary | |
3186 | if (p.method != Method.put && p.method != Method.post && | |
3187 | p.method != Method.patch) | |
3188 | p.method = Method.post; | |
3189 | ||
3190 | if (p.method == Method.post || p.method == Method.patch) | |
3191 | lenOpt = CurlOption.postfieldsize_large; | |
3192 | else | |
3193 | lenOpt = CurlOption.infilesize_large; | |
3194 | ||
3195 | if (size_t.max != ulong.max && len == size_t.max) | |
3196 | len = ulong.max; // check size_t.max for backwards compat, turn into error | |
3197 | ||
3198 | if (len == ulong.max) | |
3199 | { | |
3200 | // HTTP 1.1 supports requests with no length header set. | |
3201 | addRequestHeader("Transfer-Encoding", "chunked"); | |
3202 | addRequestHeader("Expect", "100-continue"); | |
3203 | } | |
3204 | else | |
3205 | { | |
3206 | p.curl.set(lenOpt, to!curl_off_t(len)); | |
3207 | } | |
3208 | } | |
3209 | ||
3210 | /** | |
3211 | Authentication method as specified in $(LREF AuthMethod). | |
3212 | */ | |
3213 | @property void authenticationMethod(AuthMethod authMethod) | |
3214 | { | |
3215 | p.curl.set(CurlOption.httpauth, cast(long) authMethod); | |
3216 | } | |
3217 | ||
3218 | /** | |
3219 | Set max allowed redirections using the location header. | |
3220 | uint.max for infinite. | |
3221 | */ | |
3222 | @property void maxRedirects(uint maxRedirs) | |
3223 | { | |
3224 | if (maxRedirs == uint.max) | |
3225 | { | |
3226 | // Disable | |
3227 | p.curl.set(CurlOption.followlocation, 0); | |
3228 | } | |
3229 | else | |
3230 | { | |
3231 | p.curl.set(CurlOption.followlocation, 1); | |
3232 | p.curl.set(CurlOption.maxredirs, maxRedirs); | |
3233 | } | |
3234 | } | |
3235 | ||
3236 | /** <a name="HTTP.Method"/>The standard HTTP methods : | |
3237 | * $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.1, _RFC2616 Section 5.1.1) | |
3238 | */ | |
3239 | enum Method | |
3240 | { | |
3241 | undefined, | |
3242 | head, /// | |
3243 | get, /// | |
3244 | post, /// | |
3245 | put, /// | |
3246 | del, /// | |
3247 | options, /// | |
3248 | trace, /// | |
3249 | connect, /// | |
3250 | patch, /// | |
3251 | } | |
3252 | ||
3253 | /** | |
3254 | HTTP status line ie. the first line returned in an HTTP response. | |
3255 | ||
3256 | If authentication or redirections are done then the status will be for | |
3257 | the last response received. | |
3258 | */ | |
3259 | struct StatusLine | |
3260 | { | |
3261 | ushort majorVersion; /// Major HTTP version ie. 1 in HTTP/1.0. | |
3262 | ushort minorVersion; /// Minor HTTP version ie. 0 in HTTP/1.0. | |
3263 | ushort code; /// HTTP status line code e.g. 200. | |
3264 | string reason; /// HTTP status line reason string. | |
3265 | ||
3266 | /// Reset this status line | |
3267 | @safe void reset() | |
3268 | { | |
3269 | majorVersion = 0; | |
3270 | minorVersion = 0; | |
3271 | code = 0; | |
3272 | reason = ""; | |
3273 | } | |
3274 | ||
3275 | /// | |
3276 | string toString() const | |
3277 | { | |
3278 | import std.format : format; | |
3279 | return format("%s %s (%s.%s)", | |
3280 | code, reason, majorVersion, minorVersion); | |
3281 | } | |
3282 | } | |
3283 | ||
3284 | } // HTTP | |
3285 | ||
3286 | @system unittest // charset/Charset/CHARSET/... | |
3287 | { | |
5fee5ec3 | 3288 | import etc.c.curl; |
b4c522fa | 3289 | |
5fee5ec3 IB |
3290 | static foreach (c; ["charset", "Charset", "CHARSET", "CharSet", "charSet", |
3291 | "ChArSeT", "cHaRsEt"]) | |
3292 | {{ | |
b4c522fa IB |
3293 | testServer.handle((s) { |
3294 | s.send("HTTP/1.1 200 OK\r\n"~ | |
3295 | "Content-Length: 0\r\n"~ | |
3296 | "Content-Type: text/plain; " ~ c ~ "=foo\r\n" ~ | |
3297 | "\r\n"); | |
3298 | }); | |
3299 | ||
3300 | auto http = HTTP(testServer.addr); | |
3301 | http.perform(); | |
3302 | assert(http.p.charset == "foo"); | |
3303 | ||
5fee5ec3 | 3304 | // https://issues.dlang.org/show_bug.cgi?id=16736 |
b4c522fa IB |
3305 | double val; |
3306 | CurlCode code; | |
3307 | ||
3308 | code = http.getTiming(CurlInfo.total_time, val); | |
3309 | assert(code == CurlError.ok); | |
3310 | code = http.getTiming(CurlInfo.namelookup_time, val); | |
3311 | assert(code == CurlError.ok); | |
3312 | code = http.getTiming(CurlInfo.connect_time, val); | |
3313 | assert(code == CurlError.ok); | |
3314 | code = http.getTiming(CurlInfo.pretransfer_time, val); | |
3315 | assert(code == CurlError.ok); | |
3316 | code = http.getTiming(CurlInfo.starttransfer_time, val); | |
3317 | assert(code == CurlError.ok); | |
3318 | code = http.getTiming(CurlInfo.redirect_time, val); | |
3319 | assert(code == CurlError.ok); | |
3320 | code = http.getTiming(CurlInfo.appconnect_time, val); | |
3321 | assert(code == CurlError.ok); | |
5fee5ec3 | 3322 | }} |
b4c522fa IB |
3323 | } |
3324 | ||
3325 | /** | |
3326 | FTP client functionality. | |
3327 | ||
3328 | See_Also: $(HTTP tools.ietf.org/html/rfc959, RFC959) | |
3329 | */ | |
3330 | struct FTP | |
3331 | { | |
3332 | ||
3333 | mixin Protocol; | |
3334 | ||
5fee5ec3 IB |
3335 | import std.typecons : RefCounted; |
3336 | import etc.c.curl : CurlError, CurlInfo, curl_off_t, curl_slist; | |
3337 | ||
b4c522fa IB |
3338 | private struct Impl |
3339 | { | |
3340 | ~this() | |
3341 | { | |
3342 | if (commands !is null) | |
3343 | Curl.curl.slist_free_all(commands); | |
3344 | if (curl.handle !is null) // work around RefCounted/emplace bug | |
3345 | curl.shutdown(); | |
3346 | } | |
3347 | curl_slist* commands; | |
3348 | Curl curl; | |
3349 | string encoding; | |
3350 | } | |
3351 | ||
3352 | private RefCounted!Impl p; | |
3353 | ||
3354 | /** | |
3355 | FTP access to the specified url. | |
3356 | */ | |
3357 | static FTP opCall(const(char)[] url) | |
3358 | { | |
3359 | FTP ftp; | |
3360 | ftp.initialize(); | |
3361 | ftp.url = url; | |
3362 | return ftp; | |
3363 | } | |
3364 | ||
3365 | /// | |
3366 | static FTP opCall() | |
3367 | { | |
3368 | FTP ftp; | |
3369 | ftp.initialize(); | |
3370 | return ftp; | |
3371 | } | |
3372 | ||
3373 | /// | |
3374 | FTP dup() | |
3375 | { | |
3376 | FTP copy = FTP(); | |
3377 | copy.initialize(); | |
3378 | copy.p.encoding = p.encoding; | |
3379 | copy.p.curl = p.curl.dup(); | |
3380 | curl_slist* cur = p.commands; | |
3381 | curl_slist* newlist = null; | |
3382 | while (cur) | |
3383 | { | |
3384 | newlist = Curl.curl.slist_append(newlist, cur.data); | |
3385 | cur = cur.next; | |
3386 | } | |
3387 | copy.p.commands = newlist; | |
3388 | copy.p.curl.set(CurlOption.postquote, copy.p.commands); | |
3389 | copy.dataTimeout = _defaultDataTimeout; | |
3390 | return copy; | |
3391 | } | |
3392 | ||
3393 | private void initialize() | |
3394 | { | |
3395 | p.curl.initialize(); | |
3396 | p.encoding = "ISO-8859-1"; | |
3397 | dataTimeout = _defaultDataTimeout; | |
3398 | } | |
3399 | ||
3400 | /** | |
3401 | Performs the ftp request as it has been configured. | |
3402 | ||
3403 | After a FTP client has been setup and possibly assigned callbacks the $(D | |
3404 | perform()) method will start performing the actual communication with the | |
3405 | server. | |
3406 | ||
3407 | Params: | |
3408 | throwOnError = whether to throw an exception or return a CurlCode on error | |
3409 | */ | |
3410 | CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) | |
3411 | { | |
3412 | return p.curl.perform(throwOnError); | |
3413 | } | |
3414 | ||
3415 | /// The URL to specify the location of the resource. | |
3416 | @property void url(const(char)[] url) | |
3417 | { | |
3418 | import std.algorithm.searching : startsWith; | |
3419 | import std.uni : toLower; | |
3420 | ||
3421 | if (!startsWith(url.toLower(), "ftp://", "ftps://")) | |
3422 | url = "ftp://" ~ url; | |
3423 | p.curl.set(CurlOption.url, url); | |
3424 | } | |
3425 | ||
3426 | // This is a workaround for mixed in content not having its | |
3427 | // docs mixed in. | |
3428 | version (StdDdoc) | |
3429 | { | |
5fee5ec3 IB |
3430 | static import etc.c.curl; |
3431 | ||
3432 | /// Value to return from `onSend`/`onReceive` delegates in order to | |
b4c522fa IB |
3433 | /// pause a request |
3434 | alias requestPause = CurlReadFunc.pause; | |
3435 | ||
3436 | /// Value to return from onSend delegate in order to abort a request | |
3437 | alias requestAbort = CurlReadFunc.abort; | |
3438 | ||
3439 | /** | |
3440 | True if the instance is stopped. A stopped instance is not usable. | |
3441 | */ | |
3442 | @property bool isStopped(); | |
3443 | ||
3444 | /// Stop and invalidate this instance. | |
3445 | void shutdown(); | |
3446 | ||
3447 | /** Set verbose. | |
3448 | This will print request information to stderr. | |
3449 | */ | |
3450 | @property void verbose(bool on); | |
3451 | ||
3452 | // Connection settings | |
3453 | ||
3454 | /// Set timeout for activity on connection. | |
3455 | @property void dataTimeout(Duration d); | |
3456 | ||
3457 | /** Set maximum time an operation is allowed to take. | |
3458 | This includes dns resolution, connecting, data transfer, etc. | |
3459 | */ | |
3460 | @property void operationTimeout(Duration d); | |
3461 | ||
3462 | /// Set timeout for connecting. | |
3463 | @property void connectTimeout(Duration d); | |
3464 | ||
3465 | // Network settings | |
3466 | ||
3467 | /** Proxy | |
3468 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) | |
3469 | */ | |
3470 | @property void proxy(const(char)[] host); | |
3471 | ||
3472 | /** Proxy port | |
3473 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) | |
3474 | */ | |
3475 | @property void proxyPort(ushort port); | |
3476 | ||
3477 | /// Type of proxy | |
3478 | alias CurlProxy = etc.c.curl.CurlProxy; | |
3479 | ||
3480 | /** Proxy type | |
3481 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) | |
3482 | */ | |
3483 | @property void proxyType(CurlProxy type); | |
3484 | ||
3485 | /// DNS lookup timeout. | |
3486 | @property void dnsTimeout(Duration d); | |
3487 | ||
3488 | /** | |
c8dfa79c | 3489 | * The network interface to use in form of the IP of the interface. |
b4c522fa IB |
3490 | * |
3491 | * Example: | |
3492 | * ---- | |
3493 | * theprotocol.netInterface = "192.168.1.32"; | |
3494 | * theprotocol.netInterface = [ 192, 168, 1, 32 ]; | |
3495 | * ---- | |
3496 | * | |
3497 | * See: $(REF InternetAddress, std,socket) | |
3498 | */ | |
3499 | @property void netInterface(const(char)[] i); | |
3500 | ||
3501 | /// ditto | |
3502 | @property void netInterface(const(ubyte)[4] i); | |
3503 | ||
3504 | /// ditto | |
3505 | @property void netInterface(InternetAddress i); | |
3506 | ||
3507 | /** | |
3508 | Set the local outgoing port to use. | |
3509 | Params: | |
3510 | port = the first outgoing port number to try and use | |
3511 | */ | |
3512 | @property void localPort(ushort port); | |
3513 | ||
3514 | /** | |
3515 | Set the local outgoing port range to use. | |
3516 | This can be used together with the localPort property. | |
3517 | Params: | |
3518 | range = if the first port is occupied then try this many | |
3519 | port number forwards | |
3520 | */ | |
3521 | @property void localPortRange(ushort range); | |
3522 | ||
3523 | /** Set the tcp no-delay socket option on or off. | |
3524 | See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) | |
3525 | */ | |
3526 | @property void tcpNoDelay(bool on); | |
3527 | ||
3528 | // Authentication settings | |
3529 | ||
3530 | /** | |
3531 | Set the user name, password and optionally domain for authentication | |
3532 | purposes. | |
3533 | ||
3534 | Some protocols may need authentication in some cases. Use this | |
3535 | function to provide credentials. | |
3536 | ||
3537 | Params: | |
3538 | username = the username | |
3539 | password = the password | |
3540 | domain = used for NTLM authentication only and is set to the NTLM domain | |
3541 | name | |
3542 | */ | |
3543 | void setAuthentication(const(char)[] username, const(char)[] password, | |
3544 | const(char)[] domain = ""); | |
3545 | ||
3546 | /** | |
3547 | Set the user name and password for proxy authentication. | |
3548 | ||
3549 | Params: | |
3550 | username = the username | |
3551 | password = the password | |
3552 | */ | |
3553 | void setProxyAuthentication(const(char)[] username, const(char)[] password); | |
3554 | ||
3555 | /** | |
3556 | * The event handler that gets called when data is needed for sending. The | |
5fee5ec3 | 3557 | * length of the `void[]` specifies the maximum number of bytes that can |
b4c522fa IB |
3558 | * be sent. |
3559 | * | |
3560 | * Returns: | |
3561 | * The callback returns the number of elements in the buffer that have been | |
3562 | * filled and are ready to send. | |
5fee5ec3 | 3563 | * The special value `.abortRequest` can be returned in order to abort the |
b4c522fa | 3564 | * current request. |
5fee5ec3 | 3565 | * The special value `.pauseRequest` can be returned in order to pause the |
b4c522fa IB |
3566 | * current request. |
3567 | * | |
3568 | */ | |
3569 | @property void onSend(size_t delegate(void[]) callback); | |
3570 | ||
3571 | /** | |
3572 | * The event handler that receives incoming data. Be sure to copy the | |
3573 | * incoming ubyte[] since it is not guaranteed to be valid after the | |
3574 | * callback returns. | |
3575 | * | |
3576 | * Returns: | |
3577 | * The callback returns the incoming bytes read. If not the entire array is | |
3578 | * the request will abort. | |
3579 | * The special value .pauseRequest can be returned in order to pause the | |
3580 | * current request. | |
3581 | * | |
3582 | */ | |
3583 | @property void onReceive(size_t delegate(ubyte[]) callback); | |
3584 | ||
3585 | /** | |
3586 | * The event handler that gets called to inform of upload/download progress. | |
3587 | * | |
3588 | * Callback_parameters: | |
3589 | * $(CALLBACK_PARAMS) | |
3590 | * | |
3591 | * Callback_returns: | |
3592 | * Return 0 from the callback to signal success, return non-zero to | |
3593 | * abort transfer. | |
3594 | */ | |
3595 | @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, | |
3596 | size_t ulTotal, size_t ulNow) callback); | |
3597 | } | |
3598 | ||
3599 | /** Clear all commands send to ftp server. | |
3600 | */ | |
3601 | void clearCommands() | |
3602 | { | |
3603 | if (p.commands !is null) | |
3604 | Curl.curl.slist_free_all(p.commands); | |
3605 | p.commands = null; | |
3606 | p.curl.clear(CurlOption.postquote); | |
3607 | } | |
3608 | ||
3609 | /** Add a command to send to ftp server. | |
3610 | * | |
3611 | * There is no remove command functionality. Do a $(LREF clearCommands) and | |
3612 | * set the needed commands instead. | |
3613 | * | |
3614 | * Example: | |
3615 | * --- | |
3616 | * import std.net.curl; | |
3617 | * auto client = FTP(); | |
3618 | * client.addCommand("RNFR my_file.txt"); | |
3619 | * client.addCommand("RNTO my_renamed_file.txt"); | |
3620 | * upload("my_file.txt", "ftp.digitalmars.com", client); | |
3621 | * --- | |
3622 | */ | |
3623 | void addCommand(const(char)[] command) | |
3624 | { | |
5fee5ec3 | 3625 | import std.internal.cstring : tempCString; |
b4c522fa IB |
3626 | p.commands = Curl.curl.slist_append(p.commands, |
3627 | command.tempCString().buffPtr); | |
3628 | p.curl.set(CurlOption.postquote, p.commands); | |
3629 | } | |
3630 | ||
3631 | /// Connection encoding. Defaults to ISO-8859-1. | |
3632 | @property void encoding(string name) | |
3633 | { | |
3634 | p.encoding = name; | |
3635 | } | |
3636 | ||
3637 | /// ditto | |
3638 | @property string encoding() | |
3639 | { | |
3640 | return p.encoding; | |
3641 | } | |
3642 | ||
3643 | /** | |
3644 | The content length in bytes of the ftp data. | |
3645 | */ | |
3646 | @property void contentLength(ulong len) | |
3647 | { | |
3648 | import std.conv : to; | |
3649 | p.curl.set(CurlOption.infilesize_large, to!curl_off_t(len)); | |
3650 | } | |
3651 | ||
3652 | /** | |
3653 | * Get various timings defined in $(REF CurlInfo, etc, c, curl). | |
5fee5ec3 | 3654 | * The value is usable only if the return value is equal to `etc.c.curl.CurlError.ok`. |
b4c522fa IB |
3655 | * |
3656 | * Params: | |
3657 | * timing = one of the timings defined in $(REF CurlInfo, etc, c, curl). | |
3658 | * The values are: | |
5fee5ec3 IB |
3659 | * `etc.c.curl.CurlInfo.namelookup_time`, |
3660 | * `etc.c.curl.CurlInfo.connect_time`, | |
3661 | * `etc.c.curl.CurlInfo.pretransfer_time`, | |
3662 | * `etc.c.curl.CurlInfo.starttransfer_time`, | |
3663 | * `etc.c.curl.CurlInfo.redirect_time`, | |
3664 | * `etc.c.curl.CurlInfo.appconnect_time`, | |
3665 | * `etc.c.curl.CurlInfo.total_time`. | |
b4c522fa IB |
3666 | * val = the actual value of the inquired timing. |
3667 | * | |
3668 | * Returns: | |
3669 | * The return code of the operation. The value stored in val | |
5fee5ec3 | 3670 | * should be used only if the return value is `etc.c.curl.CurlInfo.ok`. |
b4c522fa IB |
3671 | * |
3672 | * Example: | |
3673 | * --- | |
3674 | * import std.net.curl; | |
3675 | * import etc.c.curl : CurlError, CurlInfo; | |
3676 | * | |
3677 | * auto client = FTP(); | |
3678 | * client.addCommand("RNFR my_file.txt"); | |
3679 | * client.addCommand("RNTO my_renamed_file.txt"); | |
3680 | * upload("my_file.txt", "ftp.digitalmars.com", client); | |
3681 | * | |
3682 | * double val; | |
3683 | * CurlCode code; | |
3684 | * | |
5fee5ec3 | 3685 | * code = client.getTiming(CurlInfo.namelookup_time, val); |
b4c522fa IB |
3686 | * assert(code == CurlError.ok); |
3687 | * --- | |
3688 | */ | |
3689 | CurlCode getTiming(CurlInfo timing, ref double val) | |
3690 | { | |
3691 | return p.curl.getTiming(timing, val); | |
3692 | } | |
3693 | ||
3694 | @system unittest | |
3695 | { | |
3696 | auto client = FTP(); | |
3697 | ||
3698 | double val; | |
3699 | CurlCode code; | |
3700 | ||
3701 | code = client.getTiming(CurlInfo.total_time, val); | |
3702 | assert(code == CurlError.ok); | |
3703 | code = client.getTiming(CurlInfo.namelookup_time, val); | |
3704 | assert(code == CurlError.ok); | |
3705 | code = client.getTiming(CurlInfo.connect_time, val); | |
3706 | assert(code == CurlError.ok); | |
3707 | code = client.getTiming(CurlInfo.pretransfer_time, val); | |
3708 | assert(code == CurlError.ok); | |
3709 | code = client.getTiming(CurlInfo.starttransfer_time, val); | |
3710 | assert(code == CurlError.ok); | |
3711 | code = client.getTiming(CurlInfo.redirect_time, val); | |
3712 | assert(code == CurlError.ok); | |
3713 | code = client.getTiming(CurlInfo.appconnect_time, val); | |
3714 | assert(code == CurlError.ok); | |
3715 | } | |
3716 | } | |
3717 | ||
3718 | /** | |
3719 | * Basic SMTP protocol support. | |
3720 | * | |
3721 | * Example: | |
3722 | * --- | |
3723 | * import std.net.curl; | |
3724 | * | |
3725 | * // Send an email with SMTPS | |
3726 | * auto smtp = SMTP("smtps://smtp.gmail.com"); | |
3727 | * smtp.setAuthentication("from.addr@gmail.com", "password"); | |
3728 | * smtp.mailTo = ["<to.addr@gmail.com>"]; | |
3729 | * smtp.mailFrom = "<from.addr@gmail.com>"; | |
3730 | * smtp.message = "Example Message"; | |
3731 | * smtp.perform(); | |
3732 | * --- | |
3733 | * | |
3734 | * See_Also: $(HTTP www.ietf.org/rfc/rfc2821.txt, RFC2821) | |
3735 | */ | |
3736 | struct SMTP | |
3737 | { | |
3738 | mixin Protocol; | |
5fee5ec3 IB |
3739 | import std.typecons : RefCounted; |
3740 | import etc.c.curl : CurlUseSSL, curl_slist; | |
b4c522fa IB |
3741 | |
3742 | private struct Impl | |
3743 | { | |
3744 | ~this() | |
3745 | { | |
3746 | if (curl.handle !is null) // work around RefCounted/emplace bug | |
3747 | curl.shutdown(); | |
3748 | } | |
3749 | Curl curl; | |
3750 | ||
3751 | @property void message(string msg) | |
3752 | { | |
3753 | import std.algorithm.comparison : min; | |
3754 | ||
3755 | auto _message = msg; | |
3756 | /** | |
3757 | This delegate reads the message text and copies it. | |
3758 | */ | |
3759 | curl.onSend = delegate size_t(void[] data) | |
3760 | { | |
3761 | if (!msg.length) return 0; | |
3762 | size_t to_copy = min(data.length, _message.length); | |
3763 | data[0 .. to_copy] = (cast(void[])_message)[0 .. to_copy]; | |
3764 | _message = _message[to_copy..$]; | |
3765 | return to_copy; | |
3766 | }; | |
3767 | } | |
3768 | } | |
3769 | ||
3770 | private RefCounted!Impl p; | |
3771 | ||
3772 | /** | |
3773 | Sets to the URL of the SMTP server. | |
3774 | */ | |
3775 | static SMTP opCall(const(char)[] url) | |
3776 | { | |
3777 | SMTP smtp; | |
3778 | smtp.initialize(); | |
3779 | smtp.url = url; | |
3780 | return smtp; | |
3781 | } | |
3782 | ||
3783 | /// | |
3784 | static SMTP opCall() | |
3785 | { | |
3786 | SMTP smtp; | |
3787 | smtp.initialize(); | |
3788 | return smtp; | |
3789 | } | |
3790 | ||
3791 | /+ TODO: The other structs have this function. | |
3792 | SMTP dup() | |
3793 | { | |
3794 | SMTP copy = SMTP(); | |
3795 | copy.initialize(); | |
3796 | copy.p.encoding = p.encoding; | |
3797 | copy.p.curl = p.curl.dup(); | |
3798 | curl_slist* cur = p.commands; | |
3799 | curl_slist* newlist = null; | |
3800 | while (cur) | |
3801 | { | |
3802 | newlist = Curl.curl.slist_append(newlist, cur.data); | |
3803 | cur = cur.next; | |
3804 | } | |
3805 | copy.p.commands = newlist; | |
3806 | copy.p.curl.set(CurlOption.postquote, copy.p.commands); | |
3807 | copy.dataTimeout = _defaultDataTimeout; | |
3808 | return copy; | |
3809 | } | |
3810 | +/ | |
3811 | ||
3812 | /** | |
3813 | Performs the request as configured. | |
3814 | Params: | |
3815 | throwOnError = whether to throw an exception or return a CurlCode on error | |
3816 | */ | |
3817 | CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) | |
3818 | { | |
3819 | return p.curl.perform(throwOnError); | |
3820 | } | |
3821 | ||
3822 | /// The URL to specify the location of the resource. | |
3823 | @property void url(const(char)[] url) | |
3824 | { | |
3825 | import std.algorithm.searching : startsWith; | |
5fee5ec3 | 3826 | import std.exception : enforce; |
b4c522fa IB |
3827 | import std.uni : toLower; |
3828 | ||
3829 | auto lowered = url.toLower(); | |
3830 | ||
3831 | if (lowered.startsWith("smtps://")) | |
3832 | { | |
3833 | p.curl.set(CurlOption.use_ssl, CurlUseSSL.all); | |
3834 | } | |
3835 | else | |
3836 | { | |
3837 | enforce!CurlException(lowered.startsWith("smtp://"), | |
3838 | "The url must be for the smtp protocol."); | |
3839 | } | |
3840 | p.curl.set(CurlOption.url, url); | |
3841 | } | |
3842 | ||
3843 | private void initialize() | |
3844 | { | |
3845 | p.curl.initialize(); | |
3846 | p.curl.set(CurlOption.upload, 1L); | |
3847 | dataTimeout = _defaultDataTimeout; | |
3848 | verifyPeer = true; | |
3849 | verifyHost = true; | |
3850 | } | |
3851 | ||
3852 | // This is a workaround for mixed in content not having its | |
3853 | // docs mixed in. | |
3854 | version (StdDdoc) | |
3855 | { | |
5fee5ec3 IB |
3856 | static import etc.c.curl; |
3857 | ||
3858 | /// Value to return from `onSend`/`onReceive` delegates in order to | |
b4c522fa IB |
3859 | /// pause a request |
3860 | alias requestPause = CurlReadFunc.pause; | |
3861 | ||
3862 | /// Value to return from onSend delegate in order to abort a request | |
3863 | alias requestAbort = CurlReadFunc.abort; | |
3864 | ||
3865 | /** | |
3866 | True if the instance is stopped. A stopped instance is not usable. | |
3867 | */ | |
3868 | @property bool isStopped(); | |
3869 | ||
3870 | /// Stop and invalidate this instance. | |
3871 | void shutdown(); | |
3872 | ||
3873 | /** Set verbose. | |
3874 | This will print request information to stderr. | |
3875 | */ | |
3876 | @property void verbose(bool on); | |
3877 | ||
3878 | // Connection settings | |
3879 | ||
3880 | /// Set timeout for activity on connection. | |
3881 | @property void dataTimeout(Duration d); | |
3882 | ||
3883 | /** Set maximum time an operation is allowed to take. | |
3884 | This includes dns resolution, connecting, data transfer, etc. | |
3885 | */ | |
3886 | @property void operationTimeout(Duration d); | |
3887 | ||
3888 | /// Set timeout for connecting. | |
3889 | @property void connectTimeout(Duration d); | |
3890 | ||
3891 | // Network settings | |
3892 | ||
3893 | /** Proxy | |
3894 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) | |
3895 | */ | |
3896 | @property void proxy(const(char)[] host); | |
3897 | ||
3898 | /** Proxy port | |
3899 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) | |
3900 | */ | |
3901 | @property void proxyPort(ushort port); | |
3902 | ||
3903 | /// Type of proxy | |
3904 | alias CurlProxy = etc.c.curl.CurlProxy; | |
3905 | ||
3906 | /** Proxy type | |
3907 | * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) | |
3908 | */ | |
3909 | @property void proxyType(CurlProxy type); | |
3910 | ||
3911 | /// DNS lookup timeout. | |
3912 | @property void dnsTimeout(Duration d); | |
3913 | ||
3914 | /** | |
c8dfa79c | 3915 | * The network interface to use in form of the IP of the interface. |
b4c522fa IB |
3916 | * |
3917 | * Example: | |
3918 | * ---- | |
3919 | * theprotocol.netInterface = "192.168.1.32"; | |
3920 | * theprotocol.netInterface = [ 192, 168, 1, 32 ]; | |
3921 | * ---- | |
3922 | * | |
3923 | * See: $(REF InternetAddress, std,socket) | |
3924 | */ | |
3925 | @property void netInterface(const(char)[] i); | |
3926 | ||
3927 | /// ditto | |
3928 | @property void netInterface(const(ubyte)[4] i); | |
3929 | ||
3930 | /// ditto | |
3931 | @property void netInterface(InternetAddress i); | |
3932 | ||
3933 | /** | |
3934 | Set the local outgoing port to use. | |
3935 | Params: | |
3936 | port = the first outgoing port number to try and use | |
3937 | */ | |
3938 | @property void localPort(ushort port); | |
3939 | ||
3940 | /** | |
3941 | Set the local outgoing port range to use. | |
3942 | This can be used together with the localPort property. | |
3943 | Params: | |
3944 | range = if the first port is occupied then try this many | |
3945 | port number forwards | |
3946 | */ | |
3947 | @property void localPortRange(ushort range); | |
3948 | ||
3949 | /** Set the tcp no-delay socket option on or off. | |
3950 | See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) | |
3951 | */ | |
3952 | @property void tcpNoDelay(bool on); | |
3953 | ||
3954 | // Authentication settings | |
3955 | ||
3956 | /** | |
3957 | Set the user name, password and optionally domain for authentication | |
3958 | purposes. | |
3959 | ||
3960 | Some protocols may need authentication in some cases. Use this | |
3961 | function to provide credentials. | |
3962 | ||
3963 | Params: | |
3964 | username = the username | |
3965 | password = the password | |
3966 | domain = used for NTLM authentication only and is set to the NTLM domain | |
3967 | name | |
3968 | */ | |
3969 | void setAuthentication(const(char)[] username, const(char)[] password, | |
3970 | const(char)[] domain = ""); | |
3971 | ||
3972 | /** | |
3973 | Set the user name and password for proxy authentication. | |
3974 | ||
3975 | Params: | |
3976 | username = the username | |
3977 | password = the password | |
3978 | */ | |
3979 | void setProxyAuthentication(const(char)[] username, const(char)[] password); | |
3980 | ||
3981 | /** | |
3982 | * The event handler that gets called when data is needed for sending. The | |
5fee5ec3 | 3983 | * length of the `void[]` specifies the maximum number of bytes that can |
b4c522fa IB |
3984 | * be sent. |
3985 | * | |
3986 | * Returns: | |
3987 | * The callback returns the number of elements in the buffer that have been | |
3988 | * filled and are ready to send. | |
5fee5ec3 | 3989 | * The special value `.abortRequest` can be returned in order to abort the |
b4c522fa | 3990 | * current request. |
5fee5ec3 | 3991 | * The special value `.pauseRequest` can be returned in order to pause the |
b4c522fa IB |
3992 | * current request. |
3993 | */ | |
3994 | @property void onSend(size_t delegate(void[]) callback); | |
3995 | ||
3996 | /** | |
3997 | * The event handler that receives incoming data. Be sure to copy the | |
3998 | * incoming ubyte[] since it is not guaranteed to be valid after the | |
3999 | * callback returns. | |
4000 | * | |
4001 | * Returns: | |
4002 | * The callback returns the incoming bytes read. If not the entire array is | |
4003 | * the request will abort. | |
4004 | * The special value .pauseRequest can be returned in order to pause the | |
4005 | * current request. | |
4006 | */ | |
4007 | @property void onReceive(size_t delegate(ubyte[]) callback); | |
4008 | ||
4009 | /** | |
4010 | * The event handler that gets called to inform of upload/download progress. | |
4011 | * | |
4012 | * Callback_parameters: | |
4013 | * $(CALLBACK_PARAMS) | |
4014 | * | |
4015 | * Callback_returns: | |
4016 | * Return 0 from the callback to signal success, return non-zero to | |
4017 | * abort transfer. | |
4018 | */ | |
4019 | @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, | |
4020 | size_t ulTotal, size_t ulNow) callback); | |
4021 | } | |
4022 | ||
4023 | /** | |
4024 | Setter for the sender's email address. | |
4025 | */ | |
4026 | @property void mailFrom()(const(char)[] sender) | |
4027 | { | |
4028 | assert(!sender.empty, "Sender must not be empty"); | |
4029 | p.curl.set(CurlOption.mail_from, sender); | |
4030 | } | |
4031 | ||
4032 | /** | |
4033 | Setter for the recipient email addresses. | |
4034 | */ | |
4035 | void mailTo()(const(char)[][] recipients...) | |
4036 | { | |
5fee5ec3 | 4037 | import std.internal.cstring : tempCString; |
b4c522fa IB |
4038 | assert(!recipients.empty, "Recipient must not be empty"); |
4039 | curl_slist* recipients_list = null; | |
4040 | foreach (recipient; recipients) | |
4041 | { | |
4042 | recipients_list = | |
4043 | Curl.curl.slist_append(recipients_list, | |
4044 | recipient.tempCString().buffPtr); | |
4045 | } | |
4046 | p.curl.set(CurlOption.mail_rcpt, recipients_list); | |
4047 | } | |
4048 | ||
4049 | /** | |
4050 | Sets the message body text. | |
4051 | */ | |
4052 | ||
4053 | @property void message(string msg) | |
4054 | { | |
4055 | p.message = msg; | |
4056 | } | |
4057 | } | |
4058 | ||
5fee5ec3 IB |
4059 | @system unittest |
4060 | { | |
4061 | import std.net.curl; | |
4062 | ||
4063 | // Send an email with SMTPS | |
4064 | auto smtp = SMTP("smtps://smtp.gmail.com"); | |
4065 | smtp.setAuthentication("from.addr@gmail.com", "password"); | |
4066 | smtp.mailTo = ["<to.addr@gmail.com>"]; | |
4067 | smtp.mailFrom = "<from.addr@gmail.com>"; | |
4068 | smtp.message = "Example Message"; | |
4069 | //smtp.perform(); | |
4070 | } | |
4071 | ||
4072 | ||
b4c522fa IB |
4073 | /++ |
4074 | Exception thrown on errors in std.net.curl functions. | |
4075 | +/ | |
4076 | class CurlException : Exception | |
4077 | { | |
4078 | /++ | |
4079 | Params: | |
4080 | msg = The message for the exception. | |
4081 | file = The file where the exception occurred. | |
4082 | line = The line number where the exception occurred. | |
4083 | next = The previous exception in the chain of exceptions, if any. | |
4084 | +/ | |
4085 | @safe pure nothrow | |
4086 | this(string msg, | |
4087 | string file = __FILE__, | |
4088 | size_t line = __LINE__, | |
4089 | Throwable next = null) | |
4090 | { | |
4091 | super(msg, file, line, next); | |
4092 | } | |
4093 | } | |
4094 | ||
4095 | /++ | |
4096 | Exception thrown on timeout errors in std.net.curl functions. | |
4097 | +/ | |
4098 | class CurlTimeoutException : CurlException | |
4099 | { | |
4100 | /++ | |
4101 | Params: | |
4102 | msg = The message for the exception. | |
4103 | file = The file where the exception occurred. | |
4104 | line = The line number where the exception occurred. | |
4105 | next = The previous exception in the chain of exceptions, if any. | |
4106 | +/ | |
4107 | @safe pure nothrow | |
4108 | this(string msg, | |
4109 | string file = __FILE__, | |
4110 | size_t line = __LINE__, | |
4111 | Throwable next = null) | |
4112 | { | |
4113 | super(msg, file, line, next); | |
4114 | } | |
4115 | } | |
4116 | ||
4117 | /++ | |
4118 | Exception thrown on HTTP request failures, e.g. 404 Not Found. | |
4119 | +/ | |
4120 | class HTTPStatusException : CurlException | |
4121 | { | |
4122 | /++ | |
4123 | Params: | |
4124 | status = The HTTP status code. | |
4125 | msg = The message for the exception. | |
4126 | file = The file where the exception occurred. | |
4127 | line = The line number where the exception occurred. | |
4128 | next = The previous exception in the chain of exceptions, if any. | |
4129 | +/ | |
4130 | @safe pure nothrow | |
4131 | this(int status, | |
4132 | string msg, | |
4133 | string file = __FILE__, | |
4134 | size_t line = __LINE__, | |
4135 | Throwable next = null) | |
4136 | { | |
4137 | super(msg, file, line, next); | |
4138 | this.status = status; | |
4139 | } | |
4140 | ||
4141 | immutable int status; /// The HTTP status code | |
4142 | } | |
4143 | ||
4144 | /// Equal to $(REF CURLcode, etc,c,curl) | |
4145 | alias CurlCode = CURLcode; | |
4146 | ||
b4c522fa IB |
4147 | /// Flag to specify whether or not an exception is thrown on error. |
4148 | alias ThrowOnError = Flag!"throwOnError"; | |
4149 | ||
4150 | private struct CurlAPI | |
4151 | { | |
5fee5ec3 | 4152 | import etc.c.curl : CurlGlobal; |
b4c522fa IB |
4153 | static struct API |
4154 | { | |
5fee5ec3 IB |
4155 | import etc.c.curl : curl_version_info, curl_version_info_data, |
4156 | CURL, CURLcode, CURLINFO, CURLoption, CURLversion, curl_slist; | |
b4c522fa IB |
4157 | extern(C): |
4158 | import core.stdc.config : c_long; | |
4159 | CURLcode function(c_long flags) global_init; | |
4160 | void function() global_cleanup; | |
4161 | curl_version_info_data * function(CURLversion) version_info; | |
4162 | CURL* function() easy_init; | |
4163 | CURLcode function(CURL *curl, CURLoption option,...) easy_setopt; | |
4164 | CURLcode function(CURL *curl) easy_perform; | |
4165 | CURLcode function(CURL *curl, CURLINFO info,...) easy_getinfo; | |
4166 | CURL* function(CURL *curl) easy_duphandle; | |
4167 | char* function(CURLcode) easy_strerror; | |
4168 | CURLcode function(CURL *handle, int bitmask) easy_pause; | |
4169 | void function(CURL *curl) easy_cleanup; | |
4170 | curl_slist* function(curl_slist *, char *) slist_append; | |
4171 | void function(curl_slist *) slist_free_all; | |
4172 | } | |
4173 | __gshared API _api; | |
4174 | __gshared void* _handle; | |
4175 | ||
4176 | static ref API instance() @property | |
4177 | { | |
4178 | import std.concurrency : initOnce; | |
4179 | initOnce!_handle(loadAPI()); | |
4180 | return _api; | |
4181 | } | |
4182 | ||
4183 | static void* loadAPI() | |
4184 | { | |
5fee5ec3 IB |
4185 | import std.exception : enforce; |
4186 | ||
b4c522fa IB |
4187 | version (Posix) |
4188 | { | |
4189 | import core.sys.posix.dlfcn : dlsym, dlopen, dlclose, RTLD_LAZY; | |
4190 | alias loadSym = dlsym; | |
4191 | } | |
4192 | else version (Windows) | |
4193 | { | |
5fee5ec3 | 4194 | import core.sys.windows.winbase : GetProcAddress, GetModuleHandleA, |
b4c522fa IB |
4195 | LoadLibraryA; |
4196 | alias loadSym = GetProcAddress; | |
4197 | } | |
4198 | else | |
4199 | static assert(0, "unimplemented"); | |
4200 | ||
4201 | void* handle; | |
4202 | version (Posix) | |
4203 | handle = dlopen(null, RTLD_LAZY); | |
4204 | else version (Windows) | |
4205 | handle = GetModuleHandleA(null); | |
4206 | assert(handle !is null); | |
4207 | ||
4208 | // try to load curl from the executable to allow static linking | |
4209 | if (loadSym(handle, "curl_global_init") is null) | |
4210 | { | |
4211 | import std.format : format; | |
4212 | version (Posix) | |
4213 | dlclose(handle); | |
4214 | ||
5fee5ec3 IB |
4215 | version (LibcurlPath) |
4216 | { | |
4217 | import std.string : strip; | |
4218 | static immutable names = [strip(import("LibcurlPathFile"))]; | |
4219 | } | |
4220 | else version (OSX) | |
b4c522fa IB |
4221 | static immutable names = ["libcurl.4.dylib"]; |
4222 | else version (Posix) | |
4223 | { | |
4224 | static immutable names = ["libcurl.so", "libcurl.so.4", | |
4225 | "libcurl-gnutls.so.4", "libcurl-nss.so.4", "libcurl.so.3"]; | |
4226 | } | |
4227 | else version (Windows) | |
4228 | static immutable names = ["libcurl.dll", "curl.dll"]; | |
4229 | ||
4230 | foreach (name; names) | |
4231 | { | |
4232 | version (Posix) | |
4233 | handle = dlopen(name.ptr, RTLD_LAZY); | |
4234 | else version (Windows) | |
4235 | handle = LoadLibraryA(name.ptr); | |
4236 | if (handle !is null) break; | |
4237 | } | |
4238 | ||
4239 | enforce!CurlException(handle !is null, "Failed to load curl, tried %(%s, %).".format(names)); | |
4240 | } | |
4241 | ||
4242 | foreach (i, FP; typeof(API.tupleof)) | |
4243 | { | |
4244 | enum name = __traits(identifier, _api.tupleof[i]); | |
4245 | auto p = enforce!CurlException(loadSym(handle, "curl_"~name), | |
4246 | "Couldn't load curl_"~name~" from libcurl."); | |
4247 | _api.tupleof[i] = cast(FP) p; | |
4248 | } | |
4249 | ||
4250 | enforce!CurlException(!_api.global_init(CurlGlobal.all), | |
4251 | "Failed to initialize libcurl"); | |
4252 | ||
4253 | static extern(C) void cleanup() | |
4254 | { | |
4255 | if (_handle is null) return; | |
4256 | _api.global_cleanup(); | |
4257 | version (Posix) | |
4258 | { | |
4259 | import core.sys.posix.dlfcn : dlclose; | |
4260 | dlclose(_handle); | |
4261 | } | |
4262 | else version (Windows) | |
4263 | { | |
5fee5ec3 | 4264 | import core.sys.windows.winbase : FreeLibrary; |
b4c522fa IB |
4265 | FreeLibrary(_handle); |
4266 | } | |
4267 | else | |
4268 | static assert(0, "unimplemented"); | |
4269 | _api = API.init; | |
4270 | _handle = null; | |
4271 | } | |
4272 | ||
4273 | import core.stdc.stdlib : atexit; | |
4274 | atexit(&cleanup); | |
4275 | ||
4276 | return handle; | |
4277 | } | |
4278 | } | |
4279 | ||
4280 | /** | |
4281 | Wrapper to provide a better interface to libcurl than using the plain C API. | |
5fee5ec3 | 4282 | It is recommended to use the `HTTP`/`FTP` etc. structs instead unless |
b4c522fa IB |
4283 | raw access to libcurl is needed. |
4284 | ||
4285 | Warning: This struct uses interior pointers for callbacks. Only allocate it | |
4286 | on the stack if you never move or copy it. This also means passing by reference | |
4287 | when passing Curl to other functions. Otherwise always allocate on | |
4288 | the heap. | |
4289 | */ | |
4290 | struct Curl | |
4291 | { | |
5fee5ec3 IB |
4292 | import etc.c.curl : CURL, CurlError, CurlPause, CurlSeek, CurlSeekPos, |
4293 | curl_socket_t, CurlSockType, | |
4294 | CurlReadFunc, CurlInfo, curlsocktype, curl_off_t, | |
4295 | LIBCURL_VERSION_MAJOR, LIBCURL_VERSION_MINOR, LIBCURL_VERSION_PATCH; | |
4296 | ||
b4c522fa IB |
4297 | alias OutData = void[]; |
4298 | alias InData = ubyte[]; | |
4299 | private bool _stopped; | |
4300 | ||
4301 | private static auto ref curl() @property { return CurlAPI.instance; } | |
4302 | ||
4303 | // A handle should not be used by two threads simultaneously | |
4304 | private CURL* handle; | |
4305 | ||
5fee5ec3 | 4306 | // May also return `CURL_READFUNC_ABORT` or `CURL_READFUNC_PAUSE` |
b4c522fa IB |
4307 | private size_t delegate(OutData) _onSend; |
4308 | private size_t delegate(InData) _onReceive; | |
4309 | private void delegate(in char[]) _onReceiveHeader; | |
4310 | private CurlSeek delegate(long,CurlSeekPos) _onSeek; | |
4311 | private int delegate(curl_socket_t,CurlSockType) _onSocketOption; | |
4312 | private int delegate(size_t dltotal, size_t dlnow, | |
4313 | size_t ultotal, size_t ulnow) _onProgress; | |
4314 | ||
4315 | alias requestPause = CurlReadFunc.pause; | |
4316 | alias requestAbort = CurlReadFunc.abort; | |
4317 | ||
4318 | /** | |
4319 | Initialize the instance by creating a working curl handle. | |
4320 | */ | |
4321 | void initialize() | |
4322 | { | |
5fee5ec3 | 4323 | import std.exception : enforce; |
b4c522fa IB |
4324 | enforce!CurlException(!handle, "Curl instance already initialized"); |
4325 | handle = curl.easy_init(); | |
4326 | enforce!CurlException(handle, "Curl instance couldn't be initialized"); | |
4327 | _stopped = false; | |
4328 | set(CurlOption.nosignal, 1); | |
4329 | } | |
4330 | ||
4331 | /// | |
4332 | @property bool stopped() const | |
4333 | { | |
4334 | return _stopped; | |
4335 | } | |
4336 | ||
4337 | /** | |
4338 | Duplicate this handle. | |
4339 | ||
4340 | The new handle will have all options set as the one it was duplicated | |
4341 | from. An exception to this is that all options that cannot be shared | |
4342 | across threads are reset thereby making it safe to use the duplicate | |
4343 | in a new thread. | |
4344 | */ | |
4345 | Curl dup() | |
4346 | { | |
5fee5ec3 | 4347 | import std.meta : AliasSeq; |
b4c522fa IB |
4348 | Curl copy; |
4349 | copy.handle = curl.easy_duphandle(handle); | |
4350 | copy._stopped = false; | |
4351 | ||
4352 | with (CurlOption) { | |
4353 | auto tt = AliasSeq!(file, writefunction, writeheader, | |
4354 | headerfunction, infile, readfunction, ioctldata, ioctlfunction, | |
4355 | seekdata, seekfunction, sockoptdata, sockoptfunction, | |
4356 | opensocketdata, opensocketfunction, progressdata, | |
4357 | progressfunction, debugdata, debugfunction, interleavedata, | |
4358 | interleavefunction, chunk_data, chunk_bgn_function, | |
4359 | chunk_end_function, fnmatch_data, fnmatch_function, cookiejar, postfields); | |
4360 | ||
4361 | foreach (option; tt) | |
4362 | copy.clear(option); | |
4363 | } | |
4364 | ||
4365 | // The options are only supported by libcurl when it has been built | |
4366 | // against certain versions of OpenSSL - if your libcurl uses an old | |
4367 | // OpenSSL, or uses an entirely different SSL engine, attempting to | |
4368 | // clear these normally will raise an exception | |
4369 | copy.clearIfSupported(CurlOption.ssl_ctx_function); | |
4370 | copy.clearIfSupported(CurlOption.ssh_keydata); | |
4371 | ||
4372 | // Enable for curl version > 7.21.7 | |
4373 | static if (LIBCURL_VERSION_MAJOR >= 7 && | |
4374 | LIBCURL_VERSION_MINOR >= 21 && | |
4375 | LIBCURL_VERSION_PATCH >= 7) | |
4376 | { | |
4377 | copy.clear(CurlOption.closesocketdata); | |
4378 | copy.clear(CurlOption.closesocketfunction); | |
4379 | } | |
4380 | ||
4381 | copy.set(CurlOption.nosignal, 1); | |
4382 | ||
4383 | // copy.clear(CurlOption.ssl_ctx_data); Let ssl function be shared | |
4384 | // copy.clear(CurlOption.ssh_keyfunction); Let key function be shared | |
4385 | ||
4386 | /* | |
4387 | Allow sharing of conv functions | |
4388 | copy.clear(CurlOption.conv_to_network_function); | |
4389 | copy.clear(CurlOption.conv_from_network_function); | |
4390 | copy.clear(CurlOption.conv_from_utf8_function); | |
4391 | */ | |
4392 | ||
4393 | return copy; | |
4394 | } | |
4395 | ||
4396 | private void _check(CurlCode code) | |
4397 | { | |
5fee5ec3 | 4398 | import std.exception : enforce; |
b4c522fa IB |
4399 | enforce!CurlTimeoutException(code != CurlError.operation_timedout, |
4400 | errorString(code)); | |
4401 | ||
4402 | enforce!CurlException(code == CurlError.ok, | |
4403 | errorString(code)); | |
4404 | } | |
4405 | ||
4406 | private string errorString(CurlCode code) | |
4407 | { | |
4408 | import core.stdc.string : strlen; | |
4409 | import std.format : format; | |
4410 | ||
4411 | auto msgZ = curl.easy_strerror(code); | |
4412 | // doing the following (instead of just using std.conv.to!string) avoids 1 allocation | |
4413 | return format("%s on handle %s", msgZ[0 .. strlen(msgZ)], handle); | |
4414 | } | |
4415 | ||
4416 | private void throwOnStopped(string message = null) | |
4417 | { | |
5fee5ec3 | 4418 | import std.exception : enforce; |
b4c522fa IB |
4419 | auto def = "Curl instance called after being cleaned up"; |
4420 | enforce!CurlException(!stopped, | |
4421 | message == null ? def : message); | |
4422 | } | |
4423 | ||
4424 | /** | |
4425 | Stop and invalidate this curl instance. | |
5fee5ec3 | 4426 | Warning: Do not call this from inside a callback handler e.g. `onReceive`. |
b4c522fa IB |
4427 | */ |
4428 | void shutdown() | |
4429 | { | |
4430 | throwOnStopped(); | |
4431 | _stopped = true; | |
4432 | curl.easy_cleanup(this.handle); | |
4433 | this.handle = null; | |
4434 | } | |
4435 | ||
4436 | /** | |
4437 | Pausing and continuing transfers. | |
4438 | */ | |
4439 | void pause(bool sendingPaused, bool receivingPaused) | |
4440 | { | |
4441 | throwOnStopped(); | |
4442 | _check(curl.easy_pause(this.handle, | |
4443 | (sendingPaused ? CurlPause.send_cont : CurlPause.send) | | |
4444 | (receivingPaused ? CurlPause.recv_cont : CurlPause.recv))); | |
4445 | } | |
4446 | ||
4447 | /** | |
4448 | Set a string curl option. | |
4449 | Params: | |
4450 | option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation | |
4451 | value = The string | |
4452 | */ | |
4453 | void set(CurlOption option, const(char)[] value) | |
4454 | { | |
5fee5ec3 | 4455 | import std.internal.cstring : tempCString; |
b4c522fa IB |
4456 | throwOnStopped(); |
4457 | _check(curl.easy_setopt(this.handle, option, value.tempCString().buffPtr)); | |
4458 | } | |
4459 | ||
4460 | /** | |
4461 | Set a long curl option. | |
4462 | Params: | |
4463 | option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation | |
4464 | value = The long | |
4465 | */ | |
4466 | void set(CurlOption option, long value) | |
4467 | { | |
4468 | throwOnStopped(); | |
4469 | _check(curl.easy_setopt(this.handle, option, value)); | |
4470 | } | |
4471 | ||
4472 | /** | |
4473 | Set a void* curl option. | |
4474 | Params: | |
4475 | option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation | |
4476 | value = The pointer | |
4477 | */ | |
4478 | void set(CurlOption option, void* value) | |
4479 | { | |
4480 | throwOnStopped(); | |
4481 | _check(curl.easy_setopt(this.handle, option, value)); | |
4482 | } | |
4483 | ||
4484 | /** | |
4485 | Clear a pointer option. | |
4486 | Params: | |
4487 | option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation | |
4488 | */ | |
4489 | void clear(CurlOption option) | |
4490 | { | |
4491 | throwOnStopped(); | |
4492 | _check(curl.easy_setopt(this.handle, option, null)); | |
4493 | } | |
4494 | ||
4495 | /** | |
4496 | Clear a pointer option. Does not raise an exception if the underlying | |
4497 | libcurl does not support the option. Use sparingly. | |
4498 | Params: | |
4499 | option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation | |
4500 | */ | |
4501 | void clearIfSupported(CurlOption option) | |
4502 | { | |
4503 | throwOnStopped(); | |
4504 | auto rval = curl.easy_setopt(this.handle, option, null); | |
4505 | if (rval != CurlError.unknown_option && rval != CurlError.not_built_in) | |
4506 | _check(rval); | |
4507 | } | |
4508 | ||
4509 | /** | |
4510 | perform the curl request by doing the HTTP,FTP etc. as it has | |
4511 | been setup beforehand. | |
4512 | ||
4513 | Params: | |
4514 | throwOnError = whether to throw an exception or return a CurlCode on error | |
4515 | */ | |
4516 | CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) | |
4517 | { | |
4518 | throwOnStopped(); | |
4519 | CurlCode code = curl.easy_perform(this.handle); | |
4520 | if (throwOnError) | |
4521 | _check(code); | |
4522 | return code; | |
4523 | } | |
4524 | ||
4525 | /** | |
4526 | Get the various timings like name lookup time, total time, connect time etc. | |
4527 | The timed category is passed through the timing parameter while the timing | |
4528 | value is stored at val. The value is usable only if res is equal to | |
5fee5ec3 | 4529 | `etc.c.curl.CurlError.ok`. |
b4c522fa IB |
4530 | */ |
4531 | CurlCode getTiming(CurlInfo timing, ref double val) | |
4532 | { | |
4533 | CurlCode code; | |
4534 | code = curl.easy_getinfo(handle, timing, &val); | |
4535 | return code; | |
4536 | } | |
4537 | ||
4538 | /** | |
4539 | * The event handler that receives incoming data. | |
4540 | * | |
4541 | * Params: | |
5fee5ec3 | 4542 | * callback = the callback that receives the `ubyte[]` data. |
b4c522fa IB |
4543 | * Be sure to copy the incoming data and not store |
4544 | * a slice. | |
4545 | * | |
4546 | * Returns: | |
4547 | * The callback returns the incoming bytes read. If not the entire array is | |
4548 | * the request will abort. | |
4549 | * The special value HTTP.pauseRequest can be returned in order to pause the | |
4550 | * current request. | |
4551 | * | |
4552 | * Example: | |
4553 | * ---- | |
4554 | * import std.net.curl, std.stdio; | |
4555 | * Curl curl; | |
4556 | * curl.initialize(); | |
4557 | * curl.set(CurlOption.url, "http://dlang.org"); | |
4558 | * curl.onReceive = (ubyte[] data) { writeln("Got data", to!(const(char)[])(data)); return data.length;}; | |
4559 | * curl.perform(); | |
4560 | * ---- | |
4561 | */ | |
4562 | @property void onReceive(size_t delegate(InData) callback) | |
4563 | { | |
4564 | _onReceive = (InData id) | |
4565 | { | |
4566 | throwOnStopped("Receive callback called on cleaned up Curl instance"); | |
4567 | return callback(id); | |
4568 | }; | |
4569 | set(CurlOption.file, cast(void*) &this); | |
4570 | set(CurlOption.writefunction, cast(void*) &Curl._receiveCallback); | |
4571 | } | |
4572 | ||
4573 | /** | |
4574 | * The event handler that receives incoming headers for protocols | |
4575 | * that uses headers. | |
4576 | * | |
4577 | * Params: | |
4578 | * callback = the callback that receives the header string. | |
4579 | * Make sure the callback copies the incoming params if | |
4580 | * it needs to store it because they are references into | |
4581 | * the backend and may very likely change. | |
4582 | * | |
4583 | * Example: | |
4584 | * ---- | |
4585 | * import std.net.curl, std.stdio; | |
4586 | * Curl curl; | |
4587 | * curl.initialize(); | |
4588 | * curl.set(CurlOption.url, "http://dlang.org"); | |
4589 | * curl.onReceiveHeader = (in char[] header) { writeln(header); }; | |
4590 | * curl.perform(); | |
4591 | * ---- | |
4592 | */ | |
4593 | @property void onReceiveHeader(void delegate(in char[]) callback) | |
4594 | { | |
4595 | _onReceiveHeader = (in char[] od) | |
4596 | { | |
4597 | throwOnStopped("Receive header callback called on "~ | |
4598 | "cleaned up Curl instance"); | |
4599 | callback(od); | |
4600 | }; | |
4601 | set(CurlOption.writeheader, cast(void*) &this); | |
4602 | set(CurlOption.headerfunction, | |
4603 | cast(void*) &Curl._receiveHeaderCallback); | |
4604 | } | |
4605 | ||
4606 | /** | |
4607 | * The event handler that gets called when data is needed for sending. | |
4608 | * | |
4609 | * Params: | |
5fee5ec3 | 4610 | * callback = the callback that has a `void[]` buffer to be filled |
b4c522fa IB |
4611 | * |
4612 | * Returns: | |
4613 | * The callback returns the number of elements in the buffer that have been | |
4614 | * filled and are ready to send. | |
5fee5ec3 | 4615 | * The special value `Curl.abortRequest` can be returned in |
b4c522fa | 4616 | * order to abort the current request. |
5fee5ec3 | 4617 | * The special value `Curl.pauseRequest` can be returned in order to |
b4c522fa IB |
4618 | * pause the current request. |
4619 | * | |
4620 | * Example: | |
4621 | * ---- | |
4622 | * import std.net.curl; | |
4623 | * Curl curl; | |
4624 | * curl.initialize(); | |
4625 | * curl.set(CurlOption.url, "http://dlang.org"); | |
4626 | * | |
4627 | * string msg = "Hello world"; | |
4628 | * curl.onSend = (void[] data) | |
4629 | * { | |
4630 | * auto m = cast(void[]) msg; | |
4631 | * size_t length = m.length > data.length ? data.length : m.length; | |
4632 | * if (length == 0) return 0; | |
4633 | * data[0 .. length] = m[0 .. length]; | |
4634 | * msg = msg[length..$]; | |
4635 | * return length; | |
4636 | * }; | |
4637 | * curl.perform(); | |
4638 | * ---- | |
4639 | */ | |
4640 | @property void onSend(size_t delegate(OutData) callback) | |
4641 | { | |
4642 | _onSend = (OutData od) | |
4643 | { | |
4644 | throwOnStopped("Send callback called on cleaned up Curl instance"); | |
4645 | return callback(od); | |
4646 | }; | |
4647 | set(CurlOption.infile, cast(void*) &this); | |
4648 | set(CurlOption.readfunction, cast(void*) &Curl._sendCallback); | |
4649 | } | |
4650 | ||
4651 | /** | |
4652 | * The event handler that gets called when the curl backend needs to seek | |
4653 | * the data to be sent. | |
4654 | * | |
4655 | * Params: | |
4656 | * callback = the callback that receives a seek offset and a seek position | |
4657 | * $(REF CurlSeekPos, etc,c,curl) | |
4658 | * | |
4659 | * Returns: | |
4660 | * The callback returns the success state of the seeking | |
4661 | * $(REF CurlSeek, etc,c,curl) | |
4662 | * | |
4663 | * Example: | |
4664 | * ---- | |
4665 | * import std.net.curl; | |
4666 | * Curl curl; | |
4667 | * curl.initialize(); | |
4668 | * curl.set(CurlOption.url, "http://dlang.org"); | |
4669 | * curl.onSeek = (long p, CurlSeekPos sp) | |
4670 | * { | |
4671 | * return CurlSeek.cantseek; | |
4672 | * }; | |
4673 | * curl.perform(); | |
4674 | * ---- | |
4675 | */ | |
4676 | @property void onSeek(CurlSeek delegate(long, CurlSeekPos) callback) | |
4677 | { | |
4678 | _onSeek = (long ofs, CurlSeekPos sp) | |
4679 | { | |
4680 | throwOnStopped("Seek callback called on cleaned up Curl instance"); | |
4681 | return callback(ofs, sp); | |
4682 | }; | |
4683 | set(CurlOption.seekdata, cast(void*) &this); | |
4684 | set(CurlOption.seekfunction, cast(void*) &Curl._seekCallback); | |
4685 | } | |
4686 | ||
4687 | /** | |
4688 | * The event handler that gets called when the net socket has been created | |
5fee5ec3 | 4689 | * but a `connect()` call has not yet been done. This makes it possible to set |
b4c522fa IB |
4690 | * misc. socket options. |
4691 | * | |
4692 | * Params: | |
4693 | * callback = the callback that receives the socket and socket type | |
4694 | * $(REF CurlSockType, etc,c,curl) | |
4695 | * | |
4696 | * Returns: | |
4697 | * Return 0 from the callback to signal success, return 1 to signal error | |
4698 | * and make curl close the socket | |
4699 | * | |
4700 | * Example: | |
4701 | * ---- | |
4702 | * import std.net.curl; | |
4703 | * Curl curl; | |
4704 | * curl.initialize(); | |
4705 | * curl.set(CurlOption.url, "http://dlang.org"); | |
4706 | * curl.onSocketOption = delegate int(curl_socket_t s, CurlSockType t) { /+ do stuff +/ }; | |
4707 | * curl.perform(); | |
4708 | * ---- | |
4709 | */ | |
4710 | @property void onSocketOption(int delegate(curl_socket_t, | |
4711 | CurlSockType) callback) | |
4712 | { | |
4713 | _onSocketOption = (curl_socket_t sock, CurlSockType st) | |
4714 | { | |
4715 | throwOnStopped("Socket option callback called on "~ | |
4716 | "cleaned up Curl instance"); | |
4717 | return callback(sock, st); | |
4718 | }; | |
4719 | set(CurlOption.sockoptdata, cast(void*) &this); | |
4720 | set(CurlOption.sockoptfunction, | |
4721 | cast(void*) &Curl._socketOptionCallback); | |
4722 | } | |
4723 | ||
4724 | /** | |
4725 | * The event handler that gets called to inform of upload/download progress. | |
4726 | * | |
4727 | * Params: | |
4728 | * callback = the callback that receives the (total bytes to download, | |
4729 | * currently downloaded bytes, total bytes to upload, currently uploaded | |
4730 | * bytes). | |
4731 | * | |
4732 | * Returns: | |
4733 | * Return 0 from the callback to signal success, return non-zero to abort | |
4734 | * transfer | |
4735 | * | |
4736 | * Example: | |
4737 | * ---- | |
5fee5ec3 | 4738 | * import std.net.curl, std.stdio; |
b4c522fa IB |
4739 | * Curl curl; |
4740 | * curl.initialize(); | |
4741 | * curl.set(CurlOption.url, "http://dlang.org"); | |
5fee5ec3 | 4742 | * curl.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) |
b4c522fa IB |
4743 | * { |
4744 | * writeln("Progress: downloaded bytes ", dlnow, " of ", dltotal); | |
4745 | * writeln("Progress: uploaded bytes ", ulnow, " of ", ultotal); | |
5fee5ec3 | 4746 | * return 0; |
b4c522fa | 4747 | * }; |
5fee5ec3 | 4748 | * curl.perform(); |
b4c522fa IB |
4749 | * ---- |
4750 | */ | |
4751 | @property void onProgress(int delegate(size_t dlTotal, | |
4752 | size_t dlNow, | |
4753 | size_t ulTotal, | |
4754 | size_t ulNow) callback) | |
4755 | { | |
4756 | _onProgress = (size_t dlt, size_t dln, size_t ult, size_t uln) | |
4757 | { | |
4758 | throwOnStopped("Progress callback called on cleaned "~ | |
4759 | "up Curl instance"); | |
4760 | return callback(dlt, dln, ult, uln); | |
4761 | }; | |
4762 | set(CurlOption.noprogress, 0); | |
4763 | set(CurlOption.progressdata, cast(void*) &this); | |
4764 | set(CurlOption.progressfunction, cast(void*) &Curl._progressCallback); | |
4765 | } | |
4766 | ||
4767 | // Internal C callbacks to register with libcurl | |
4768 | extern (C) private static | |
4769 | size_t _receiveCallback(const char* str, | |
4770 | size_t size, size_t nmemb, void* ptr) | |
4771 | { | |
4772 | auto b = cast(Curl*) ptr; | |
4773 | if (b._onReceive != null) | |
4774 | return b._onReceive(cast(InData)(str[0 .. size*nmemb])); | |
4775 | return size*nmemb; | |
4776 | } | |
4777 | ||
4778 | extern (C) private static | |
4779 | size_t _receiveHeaderCallback(const char* str, | |
4780 | size_t size, size_t nmemb, void* ptr) | |
4781 | { | |
4782 | import std.string : chomp; | |
4783 | ||
4784 | auto b = cast(Curl*) ptr; | |
4785 | auto s = str[0 .. size*nmemb].chomp(); | |
4786 | if (b._onReceiveHeader != null) | |
4787 | b._onReceiveHeader(s); | |
4788 | ||
4789 | return size*nmemb; | |
4790 | } | |
4791 | ||
4792 | extern (C) private static | |
4793 | size_t _sendCallback(char *str, size_t size, size_t nmemb, void *ptr) | |
4794 | { | |
4795 | Curl* b = cast(Curl*) ptr; | |
4796 | auto a = cast(void[]) str[0 .. size*nmemb]; | |
4797 | if (b._onSend == null) | |
4798 | return 0; | |
4799 | return b._onSend(a); | |
4800 | } | |
4801 | ||
4802 | extern (C) private static | |
4803 | int _seekCallback(void *ptr, curl_off_t offset, int origin) | |
4804 | { | |
4805 | auto b = cast(Curl*) ptr; | |
4806 | if (b._onSeek == null) | |
4807 | return CurlSeek.cantseek; | |
4808 | ||
4809 | // origin: CurlSeekPos.set/current/end | |
4810 | // return: CurlSeek.ok/fail/cantseek | |
4811 | return b._onSeek(cast(long) offset, cast(CurlSeekPos) origin); | |
4812 | } | |
4813 | ||
4814 | extern (C) private static | |
4815 | int _socketOptionCallback(void *ptr, | |
4816 | curl_socket_t curlfd, curlsocktype purpose) | |
4817 | { | |
4818 | auto b = cast(Curl*) ptr; | |
4819 | if (b._onSocketOption == null) | |
4820 | return 0; | |
4821 | ||
4822 | // return: 0 ok, 1 fail | |
4823 | return b._onSocketOption(curlfd, cast(CurlSockType) purpose); | |
4824 | } | |
4825 | ||
4826 | extern (C) private static | |
4827 | int _progressCallback(void *ptr, | |
4828 | double dltotal, double dlnow, | |
4829 | double ultotal, double ulnow) | |
4830 | { | |
4831 | auto b = cast(Curl*) ptr; | |
4832 | if (b._onProgress == null) | |
4833 | return 0; | |
4834 | ||
4835 | // return: 0 ok, 1 fail | |
4836 | return b._onProgress(cast(size_t) dltotal, cast(size_t) dlnow, | |
4837 | cast(size_t) ultotal, cast(size_t) ulnow); | |
4838 | } | |
4839 | ||
4840 | } | |
4841 | ||
4842 | // Internal messages send between threads. | |
4843 | // The data is wrapped in this struct in order to ensure that | |
4844 | // other std.concurrency.receive calls does not pick up our messages | |
4845 | // by accident. | |
4846 | private struct CurlMessage(T) | |
4847 | { | |
4848 | public T data; | |
4849 | } | |
4850 | ||
4851 | private static CurlMessage!T curlMessage(T)(T data) | |
4852 | { | |
4853 | return CurlMessage!T(data); | |
4854 | } | |
4855 | ||
4856 | // Pool of to be used for reusing buffers | |
4857 | private struct Pool(Data) | |
4858 | { | |
4859 | private struct Entry | |
4860 | { | |
4861 | Data data; | |
4862 | Entry* next; | |
4863 | } | |
4864 | private Entry* root; | |
4865 | private Entry* freeList; | |
4866 | ||
4867 | @safe @property bool empty() | |
4868 | { | |
4869 | return root == null; | |
4870 | } | |
4871 | ||
4872 | @safe nothrow void push(Data d) | |
4873 | { | |
4874 | if (freeList == null) | |
4875 | { | |
4876 | // Allocate new Entry since there is no one | |
4877 | // available in the freeList | |
4878 | freeList = new Entry; | |
4879 | } | |
4880 | freeList.data = d; | |
4881 | Entry* oldroot = root; | |
4882 | root = freeList; | |
4883 | freeList = freeList.next; | |
4884 | root.next = oldroot; | |
4885 | } | |
4886 | ||
4887 | @safe Data pop() | |
4888 | { | |
5fee5ec3 | 4889 | import std.exception : enforce; |
b4c522fa IB |
4890 | enforce!Exception(root != null, "pop() called on empty pool"); |
4891 | auto d = root.data; | |
4892 | auto n = root.next; | |
4893 | root.next = freeList; | |
4894 | freeList = root; | |
4895 | root = n; | |
4896 | return d; | |
4897 | } | |
4898 | } | |
4899 | ||
5fee5ec3 IB |
4900 | // Lazily-instantiated namespace to avoid importing std.concurrency until needed. |
4901 | private struct _async() | |
b4c522fa | 4902 | { |
5fee5ec3 IB |
4903 | static: |
4904 | // https://issues.dlang.org/show_bug.cgi?id=15831 | |
4905 | // this should be inside byLineAsync | |
4906 | // Range that reads one chunk at a time asynchronously. | |
4907 | private struct ChunkInputRange | |
b4c522fa | 4908 | { |
5fee5ec3 | 4909 | import std.concurrency : Tid, send; |
b4c522fa | 4910 | |
5fee5ec3 IB |
4911 | private ubyte[] chunk; |
4912 | mixin WorkerThreadProtocol!(ubyte, chunk); | |
b4c522fa | 4913 | |
5fee5ec3 IB |
4914 | private Tid workerTid; |
4915 | private State running; | |
b4c522fa | 4916 | |
5fee5ec3 IB |
4917 | private this(Tid tid, size_t transmitBuffers, size_t chunkSize) |
4918 | { | |
4919 | workerTid = tid; | |
4920 | state = State.needUnits; | |
b4c522fa | 4921 | |
5fee5ec3 IB |
4922 | // Send buffers to other thread for it to use. Since no mechanism is in |
4923 | // place for moving ownership a cast to shared is done here and a cast | |
4924 | // back to non-shared in the receiving end. | |
4925 | foreach (i ; 0 .. transmitBuffers) | |
4926 | { | |
4927 | ubyte[] arr = new ubyte[](chunkSize); | |
4928 | workerTid.send(cast(immutable(ubyte[]))arr); | |
4929 | } | |
4930 | } | |
b4c522fa IB |
4931 | } |
4932 | ||
5fee5ec3 IB |
4933 | // https://issues.dlang.org/show_bug.cgi?id=15831 |
4934 | // this should be inside byLineAsync | |
4935 | // Range that reads one line at a time asynchronously. | |
4936 | private static struct LineInputRange(Char) | |
b4c522fa | 4937 | { |
5fee5ec3 IB |
4938 | private Char[] line; |
4939 | mixin WorkerThreadProtocol!(Char, line); | |
b4c522fa | 4940 | |
5fee5ec3 IB |
4941 | private Tid workerTid; |
4942 | private State running; | |
b4c522fa | 4943 | |
5fee5ec3 IB |
4944 | private this(Tid tid, size_t transmitBuffers, size_t bufferSize) |
4945 | { | |
4946 | import std.concurrency : send; | |
b4c522fa | 4947 | |
5fee5ec3 IB |
4948 | workerTid = tid; |
4949 | state = State.needUnits; | |
b4c522fa | 4950 | |
5fee5ec3 IB |
4951 | // Send buffers to other thread for it to use. Since no mechanism is in |
4952 | // place for moving ownership a cast to shared is done here and casted | |
4953 | // back to non-shared in the receiving end. | |
4954 | foreach (i ; 0 .. transmitBuffers) | |
4955 | { | |
4956 | auto arr = new Char[](bufferSize); | |
4957 | workerTid.send(cast(immutable(Char[]))arr); | |
4958 | } | |
4959 | } | |
4960 | } | |
b4c522fa | 4961 | |
5fee5ec3 | 4962 | import std.concurrency : Tid; |
b4c522fa | 4963 | |
5fee5ec3 IB |
4964 | // Shared function for reading incoming chunks of data and |
4965 | // sending the to a parent thread | |
4966 | private size_t receiveChunks(ubyte[] data, ref ubyte[] outdata, | |
4967 | Pool!(ubyte[]) freeBuffers, | |
4968 | ref ubyte[] buffer, Tid fromTid, | |
4969 | ref bool aborted) | |
b4c522fa | 4970 | { |
5fee5ec3 IB |
4971 | import std.concurrency : receive, send, thisTid; |
4972 | ||
4973 | immutable datalen = data.length; | |
b4c522fa | 4974 | |
5fee5ec3 IB |
4975 | // Copy data to fill active buffer |
4976 | while (!data.empty) | |
b4c522fa | 4977 | { |
5fee5ec3 IB |
4978 | |
4979 | // Make sure a buffer is present | |
4980 | while ( outdata.empty && freeBuffers.empty) | |
4981 | { | |
4982 | // Active buffer is invalid and there are no | |
4983 | // available buffers in the pool. Wait for buffers | |
4984 | // to return from main thread in order to reuse | |
4985 | // them. | |
4986 | receive((immutable(ubyte)[] buf) | |
4987 | { | |
4988 | buffer = cast(ubyte[]) buf; | |
4989 | outdata = buffer[]; | |
4990 | }, | |
4991 | (bool flag) { aborted = true; } | |
4992 | ); | |
4993 | if (aborted) return cast(size_t) 0; | |
4994 | } | |
4995 | if (outdata.empty) | |
4996 | { | |
4997 | buffer = freeBuffers.pop(); | |
4998 | outdata = buffer[]; | |
4999 | } | |
5000 | ||
5001 | // Copy data | |
5002 | auto copyBytes = outdata.length < data.length ? | |
5003 | outdata.length : data.length; | |
5004 | ||
5005 | outdata[0 .. copyBytes] = data[0 .. copyBytes]; | |
5006 | outdata = outdata[copyBytes..$]; | |
5007 | data = data[copyBytes..$]; | |
5008 | ||
5009 | if (outdata.empty) | |
5010 | fromTid.send(thisTid, curlMessage(cast(immutable(ubyte)[])buffer)); | |
b4c522fa | 5011 | } |
5fee5ec3 IB |
5012 | |
5013 | return datalen; | |
5014 | } | |
5015 | ||
5016 | // ditto | |
5017 | private void finalizeChunks(ubyte[] outdata, ref ubyte[] buffer, | |
5018 | Tid fromTid) | |
5019 | { | |
5020 | import std.concurrency : send, thisTid; | |
5021 | if (!outdata.empty) | |
b4c522fa | 5022 | { |
5fee5ec3 IB |
5023 | // Resize the last buffer |
5024 | buffer.length = buffer.length - outdata.length; | |
5025 | fromTid.send(thisTid, curlMessage(cast(immutable(ubyte)[])buffer)); | |
b4c522fa | 5026 | } |
5fee5ec3 | 5027 | } |
b4c522fa | 5028 | |
5fee5ec3 IB |
5029 | |
5030 | // Shared function for reading incoming lines of data and sending the to a | |
5031 | // parent thread | |
5032 | private static size_t receiveLines(Terminator, Unit) | |
5033 | (const(ubyte)[] data, ref EncodingScheme encodingScheme, | |
5034 | bool keepTerminator, Terminator terminator, | |
5035 | ref const(ubyte)[] leftOverBytes, ref bool bufferValid, | |
5036 | ref Pool!(Unit[]) freeBuffers, ref Unit[] buffer, | |
5037 | Tid fromTid, ref bool aborted) | |
5038 | { | |
5039 | import std.concurrency : prioritySend, receive, send, thisTid; | |
5040 | import std.exception : enforce; | |
5041 | import std.format : format; | |
5042 | import std.traits : isArray; | |
5043 | ||
5044 | immutable datalen = data.length; | |
5045 | ||
5046 | // Terminator is specified and buffers should be resized as determined by | |
5047 | // the terminator | |
5048 | ||
5049 | // Copy data to active buffer until terminator is found. | |
5050 | ||
5051 | // Decode as many lines as possible | |
5052 | while (true) | |
b4c522fa | 5053 | { |
5fee5ec3 IB |
5054 | |
5055 | // Make sure a buffer is present | |
5056 | while (!bufferValid && freeBuffers.empty) | |
b4c522fa | 5057 | { |
5fee5ec3 IB |
5058 | // Active buffer is invalid and there are no available buffers in |
5059 | // the pool. Wait for buffers to return from main thread in order to | |
5060 | // reuse them. | |
5061 | receive((immutable(Unit)[] buf) | |
5062 | { | |
5063 | buffer = cast(Unit[]) buf; | |
5064 | buffer.length = 0; | |
5065 | buffer.assumeSafeAppend(); | |
5066 | bufferValid = true; | |
5067 | }, | |
5068 | (bool flag) { aborted = true; } | |
5069 | ); | |
5070 | if (aborted) return cast(size_t) 0; | |
5071 | } | |
5072 | if (!bufferValid) | |
5073 | { | |
5074 | buffer = freeBuffers.pop(); | |
5075 | bufferValid = true; | |
5076 | } | |
5077 | ||
5078 | // Try to read a line from left over bytes from last onReceive plus the | |
5079 | // newly received bytes. | |
5080 | try | |
5081 | { | |
5082 | if (decodeLineInto(leftOverBytes, data, buffer, | |
5083 | encodingScheme, terminator)) | |
b4c522fa | 5084 | { |
5fee5ec3 IB |
5085 | if (keepTerminator) |
5086 | { | |
5087 | fromTid.send(thisTid, | |
5088 | curlMessage(cast(immutable(Unit)[])buffer)); | |
5089 | } | |
5090 | else | |
5091 | { | |
5092 | static if (isArray!Terminator) | |
5093 | fromTid.send(thisTid, | |
5094 | curlMessage(cast(immutable(Unit)[]) | |
5095 | buffer[0..$-terminator.length])); | |
5096 | else | |
5097 | fromTid.send(thisTid, | |
5098 | curlMessage(cast(immutable(Unit)[]) | |
5099 | buffer[0..$-1])); | |
5100 | } | |
5101 | bufferValid = false; | |
b4c522fa IB |
5102 | } |
5103 | else | |
5104 | { | |
5fee5ec3 IB |
5105 | // Could not decode an entire line. Save |
5106 | // bytes left in data for next call to | |
5107 | // onReceive. Can be up to a max of 4 bytes. | |
5108 | enforce!CurlException(data.length <= 4, | |
5109 | format( | |
5110 | "Too many bytes left not decoded %s"~ | |
5111 | " > 4. Maybe the charset specified in"~ | |
5112 | " headers does not match "~ | |
5113 | "the actual content downloaded?", | |
5114 | data.length)); | |
5115 | leftOverBytes ~= data; | |
5116 | break; | |
b4c522fa | 5117 | } |
b4c522fa | 5118 | } |
5fee5ec3 | 5119 | catch (CurlException ex) |
b4c522fa | 5120 | { |
5fee5ec3 IB |
5121 | prioritySend(fromTid, cast(immutable(CurlException))ex); |
5122 | return cast(size_t) 0; | |
b4c522fa IB |
5123 | } |
5124 | } | |
5fee5ec3 | 5125 | return datalen; |
b4c522fa | 5126 | } |
b4c522fa | 5127 | |
5fee5ec3 IB |
5128 | // ditto |
5129 | private static | |
5130 | void finalizeLines(Unit)(bool bufferValid, Unit[] buffer, Tid fromTid) | |
5131 | { | |
5132 | import std.concurrency : send, thisTid; | |
5133 | if (bufferValid && buffer.length != 0) | |
5134 | fromTid.send(thisTid, curlMessage(cast(immutable(Unit)[])buffer[0..$])); | |
5135 | } | |
b4c522fa | 5136 | |
5fee5ec3 IB |
5137 | /* Used by byLineAsync/byChunkAsync to duplicate an existing connection |
5138 | * that can be used exclusively in a spawned thread. | |
5139 | */ | |
5140 | private void duplicateConnection(Conn, PostData) | |
5141 | (const(char)[] url, Conn conn, PostData postData, Tid tid) | |
b4c522fa | 5142 | { |
5fee5ec3 IB |
5143 | import std.concurrency : send; |
5144 | import std.exception : enforce; | |
5145 | ||
5146 | // no move semantic available in std.concurrency ie. must use casting. | |
5147 | auto connDup = conn.dup(); | |
5148 | connDup.url = url; | |
b4c522fa | 5149 | |
5fee5ec3 IB |
5150 | static if ( is(Conn : HTTP) ) |
5151 | { | |
5152 | connDup.p.headersOut = null; | |
5153 | connDup.method = conn.method == HTTP.Method.undefined ? | |
5154 | HTTP.Method.get : conn.method; | |
5155 | if (postData !is null) | |
5156 | { | |
5157 | if (connDup.method == HTTP.Method.put) | |
5158 | { | |
5159 | connDup.handle.set(CurlOption.infilesize_large, | |
5160 | postData.length); | |
5161 | } | |
5162 | else | |
5163 | { | |
5164 | // post | |
5165 | connDup.method = HTTP.Method.post; | |
5166 | connDup.handle.set(CurlOption.postfieldsize_large, | |
5167 | postData.length); | |
5168 | } | |
5169 | connDup.handle.set(CurlOption.copypostfields, | |
5170 | cast(void*) postData.ptr); | |
5171 | } | |
5172 | tid.send(cast(ulong) connDup.handle.handle); | |
5173 | tid.send(connDup.method); | |
5174 | } | |
5175 | else | |
5176 | { | |
5177 | enforce!CurlException(postData is null, | |
5178 | "Cannot put ftp data using byLineAsync()"); | |
5179 | tid.send(cast(ulong) connDup.handle.handle); | |
5180 | tid.send(HTTP.Method.undefined); | |
5181 | } | |
5182 | connDup.p.curl.handle = null; // make sure handle is not freed | |
b4c522fa | 5183 | } |
5fee5ec3 IB |
5184 | |
5185 | // Spawn a thread for handling the reading of incoming data in the | |
5186 | // background while the delegate is executing. This will optimize | |
5187 | // throughput by allowing simultaneous input (this struct) and | |
5188 | // output (e.g. AsyncHTTPLineOutputRange). | |
5189 | private static void spawn(Conn, Unit, Terminator = void)() | |
b4c522fa | 5190 | { |
5fee5ec3 IB |
5191 | import std.concurrency : Tid, prioritySend, receiveOnly, send, thisTid; |
5192 | import etc.c.curl : CURL, CurlError; | |
5193 | Tid fromTid = receiveOnly!Tid(); | |
b4c522fa | 5194 | |
5fee5ec3 IB |
5195 | // Get buffer to read into |
5196 | Pool!(Unit[]) freeBuffers; // Free list of buffer objects | |
b4c522fa | 5197 | |
5fee5ec3 IB |
5198 | // Number of bytes filled into active buffer |
5199 | Unit[] buffer; | |
5200 | bool aborted = false; | |
b4c522fa | 5201 | |
5fee5ec3 IB |
5202 | EncodingScheme encodingScheme; |
5203 | static if ( !is(Terminator == void)) | |
5204 | { | |
5205 | // Only lines reading will receive a terminator | |
5206 | const terminator = receiveOnly!Terminator(); | |
5207 | const keepTerminator = receiveOnly!bool(); | |
5208 | ||
5209 | // max number of bytes to carry over from an onReceive | |
5210 | // callback. This is 4 because it is the max code units to | |
5211 | // decode a code point in the supported encodings. | |
5212 | auto leftOverBytes = new const(ubyte)[4]; | |
5213 | leftOverBytes.length = 0; | |
5214 | auto bufferValid = false; | |
5215 | } | |
b4c522fa | 5216 | else |
5fee5ec3 IB |
5217 | { |
5218 | Unit[] outdata; | |
5219 | } | |
b4c522fa | 5220 | |
5fee5ec3 IB |
5221 | // no move semantic available in std.concurrency ie. must use casting. |
5222 | auto connDup = cast(CURL*) receiveOnly!ulong(); | |
5223 | auto client = Conn(); | |
5224 | client.p.curl.handle = connDup; | |
5225 | ||
5226 | // receive a method for both ftp and http but just use it for http | |
5227 | auto method = receiveOnly!(HTTP.Method)(); | |
5228 | ||
5229 | client.onReceive = (ubyte[] data) | |
b4c522fa | 5230 | { |
5fee5ec3 IB |
5231 | // If no terminator is specified the chunk size is fixed. |
5232 | static if ( is(Terminator == void) ) | |
5233 | return receiveChunks(data, outdata, freeBuffers, buffer, | |
5234 | fromTid, aborted); | |
5235 | else | |
5236 | return receiveLines(data, encodingScheme, | |
5237 | keepTerminator, terminator, leftOverBytes, | |
5238 | bufferValid, freeBuffers, buffer, | |
5239 | fromTid, aborted); | |
b4c522fa | 5240 | }; |
b4c522fa | 5241 | |
5fee5ec3 IB |
5242 | static if ( is(Conn == HTTP) ) |
5243 | { | |
5244 | client.method = method; | |
5245 | // register dummy header handler | |
5246 | client.onReceiveHeader = (in char[] key, in char[] value) | |
5247 | { | |
5248 | if (key == "content-type") | |
5249 | encodingScheme = EncodingScheme.create(client.p.charset); | |
5250 | }; | |
5251 | } | |
5252 | else | |
5253 | { | |
5254 | encodingScheme = EncodingScheme.create(client.encoding); | |
5255 | } | |
b4c522fa | 5256 | |
5fee5ec3 IB |
5257 | // Start the request |
5258 | CurlCode code; | |
5259 | try | |
5260 | { | |
5261 | code = client.perform(No.throwOnError); | |
5262 | } | |
5263 | catch (Exception ex) | |
b4c522fa | 5264 | { |
5fee5ec3 | 5265 | prioritySend(fromTid, cast(immutable(Exception)) ex); |
b4c522fa IB |
5266 | fromTid.send(thisTid, curlMessage(true)); // signal done |
5267 | return; | |
5268 | } | |
b4c522fa | 5269 | |
5fee5ec3 IB |
5270 | if (code != CurlError.ok) |
5271 | { | |
5272 | if (aborted && (code == CurlError.aborted_by_callback || | |
5273 | code == CurlError.write_error)) | |
5274 | { | |
5275 | fromTid.send(thisTid, curlMessage(true)); // signal done | |
5276 | return; | |
5277 | } | |
5278 | prioritySend(fromTid, cast(immutable(CurlException)) | |
5279 | new CurlException(client.p.curl.errorString(code))); | |
b4c522fa | 5280 | |
5fee5ec3 IB |
5281 | fromTid.send(thisTid, curlMessage(true)); // signal done |
5282 | return; | |
5283 | } | |
b4c522fa | 5284 | |
5fee5ec3 IB |
5285 | // Send remaining data that is not a full chunk size |
5286 | static if ( is(Terminator == void) ) | |
5287 | finalizeChunks(outdata, buffer, fromTid); | |
5288 | else | |
5289 | finalizeLines(bufferValid, buffer, fromTid); | |
5290 | ||
5291 | fromTid.send(thisTid, curlMessage(true)); // signal done | |
5292 | } | |
b4c522fa | 5293 | } |