src/Downloader.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 #include "squid.h"
  10 #include "client_side.h"
  11 #include "client_side_request.h"
  12 #include "client_side_reply.h"
  13 #include "ClientRequestContext.h"
  14 #include "Downloader.h"
  15 #include "http/one/RequestParser.h"
  16 #include "http/Stream.h"
  17
  18 CBDATA_CLASS_INIT(Downloader);
  19
  20 /// Used to hold and pass the required info and buffers to the
  21 /// clientStream callbacks
  22 class DownloaderContext: public RefCountable
  23 {
  24     MEMPROXY_CLASS(DownloaderContext);
  25
  26 public:
  27     typedef RefCount<DownloaderContext> Pointer;
  28
  29     DownloaderContext(Downloader *dl, ClientHttpRequest *h);
  30     ~DownloaderContext();
  31     void finished();
  32
  33     CbcPointer<Downloader> downloader;
  34     ClientHttpRequest *http;
  35     char requestBuffer[HTTP_REQBUF_SZ];
  36 };
  37
  38 DownloaderContext::DownloaderContext(Downloader *dl, ClientHttpRequest *h):
  39     downloader(dl),
  40     http(h)
  41 {
  42     debugs(33, 6, "DownloaderContext constructed, this=" << (void*)this);
  43 }
  44
  45 DownloaderContext::~DownloaderContext()
  46 {
  47     debugs(33, 6, "DownloaderContext destructed, this=" << (void*)this);
  48     if (http)
  49         finished();
  50 }
  51
  52 void
  53 DownloaderContext::finished()
  54 {
  55     delete http;
  56     http = nullptr;
  57 }
  58
  59 Downloader::Downloader(SBuf &url, AsyncCall::Pointer &aCallback, unsigned int level):
  60     AsyncJob("Downloader"),
  61     url_(url),
  62     callback_(aCallback),
  63     level_(level)
  64 {
  65 }
  66
  67 Downloader::~Downloader()
  68 {
  69 }
  70
  71 bool
  72 Downloader::doneAll() const
  73 {
  74     return (!callback_ || callback_->canceled()) && AsyncJob::doneAll();
  75 }
  76
  77 static void
  78 downloaderRecipient(clientStreamNode * node, ClientHttpRequest * http,
  79                     HttpReply * rep, StoreIOBuffer receivedData)
  80 {
  81     debugs(33, 6, MYNAME);
  82      /* Test preconditions */
  83     assert(node);
  84
  85     /* TODO: handle this rather than asserting
  86      * - it should only ever happen if we cause an abort and
  87      * the callback chain loops back to here, so we can simply return.
  88      * However, that itself shouldn't happen, so it stays as an assert for now.
  89      */
  90     assert(cbdataReferenceValid(node));
  91     assert(!node->node.next);
  92     DownloaderContext::Pointer context = dynamic_cast<DownloaderContext *>(node->data.getRaw());
  93     assert(context);
  94
  95     if (context->downloader.valid())
  96         context->downloader->handleReply(node, http, rep, receivedData);
  97 }
  98
  99 static void
 100 downloaderDetach(clientStreamNode * node, ClientHttpRequest * http)
 101 {
 102     debugs(33, 5, MYNAME);
 103     clientStreamDetach(node, http);
 104 }
 105
 106 /// Initializes and starts the HTTP GET request to the remote server
 107 bool
 108 Downloader::buildRequest()
 109 {
 110     const HttpRequestMethod method = Http::METHOD_GET;
 111
 112     char *uri = xstrdup(url_.c_str());
 113     HttpRequest *const request = HttpRequest::CreateFromUrl(uri, method);
 114     if (!request) {
 115         debugs(33, 5, "Invalid URI: " << url_);
 116         xfree(uri);
 117         return false; //earlyError(...)
 118     }
 119     request->http_ver = Http::ProtocolVersion();
 120     request->header.putStr(Http::HdrType::HOST, request->url.host());
 121     request->header.putTime(Http::HdrType::DATE, squid_curtime);
 122     request->flags.internalClient = true;
 123     request->client_addr.setNoAddr();
 124 #if FOLLOW_X_FORWARDED_FOR
 125     request->indirect_client_addr.setNoAddr();
 126 #endif /* FOLLOW_X_FORWARDED_FOR */
 127     request->my_addr.setNoAddr();   /* undefined for internal requests */
 128     request->my_addr.port(0);
 129     request->downloader = this;
 130
 131     debugs(11, 2, "HTTP Client Downloader " << this << "/" << id);
 132     debugs(11, 2, "HTTP Client REQUEST:\n---------\n" <<
 133            request->method << " " << url_ << " " << request->http_ver << "\n" <<
 134            "\n----------");
 135
 136     ClientHttpRequest *const http = new ClientHttpRequest(nullptr);
 137     http->request = request;
 138     HTTPMSGLOCK(http->request);
 139     http->req_sz = 0;
 140     http->uri = uri;
 141
 142     context_ = new DownloaderContext(this, http);
 143     StoreIOBuffer tempBuffer;
 144     tempBuffer.data = context_->requestBuffer;
 145     tempBuffer.length = HTTP_REQBUF_SZ;
 146
 147     ClientStreamData newServer = new clientReplyContext(http);
 148     ClientStreamData newClient = context_.getRaw();
 149     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 150                      clientReplyStatus, newServer, downloaderRecipient,
 151                      downloaderDetach, newClient, tempBuffer);
 152
 153     // Build a ClientRequestContext to start doCallouts
 154     http->calloutContext = new ClientRequestContext(http);
 155     http->doCallouts();
 156     return true;
 157 }
 158
 159 void
 160 Downloader::start()
 161 {
 162     if (!buildRequest())
 163         callBack(Http::scInternalServerError);
 164 }
 165
 166 void
 167 Downloader::handleReply(clientStreamNode * node, ClientHttpRequest *http, HttpReply *reply, StoreIOBuffer receivedData)
 168 {
 169     DownloaderContext::Pointer callerContext = dynamic_cast<DownloaderContext *>(node->data.getRaw());
 170     // TODO: remove the following check:
 171     assert(callerContext == context_);
 172
 173     debugs(33, 4, "Received " << receivedData.length <<
 174            " object data, offset: " << receivedData.offset <<
 175            " error flag:" << receivedData.flags.error);
 176
 177     const bool failed = receivedData.flags.error;
 178     if (failed) {
 179         callBack(Http::scInternalServerError);
 180         return;
 181     }
 182
 183     const int64_t existingContent = reply ? reply->content_length : 0;
 184     const size_t maxSize = MaxObjectSize > SBuf::maxSize ? SBuf::maxSize : MaxObjectSize;
 185     const bool tooLarge = (existingContent > -1 && existingContent > static_cast<int64_t>(maxSize)) ||
 186                           (maxSize < object_.length()) ||
 187                           ((maxSize - object_.length()) < receivedData.length);
 188
 189     if (tooLarge) {
 190         callBack(Http::scInternalServerError);
 191         return;
 192     }
 193
 194     if (receivedData.length) {
 195         object_.append(receivedData.data, receivedData.length);
 196         http->out.size += receivedData.length;
 197         http->out.offset += receivedData.length;
 198     }
 199
 200     switch (clientStreamStatus(node, http)) {
 201     case STREAM_NONE: {
 202         debugs(33, 3, "Get more data");
 203         StoreIOBuffer tempBuffer;
 204         tempBuffer.offset = http->out.offset;
 205         tempBuffer.data = context_->requestBuffer;
 206         tempBuffer.length = HTTP_REQBUF_SZ;
 207         clientStreamRead(node, http, tempBuffer);
 208     }
 209         break;
 210     case STREAM_COMPLETE:
 211         debugs(33, 3, "Object data transfer successfully complete");
 212         callBack(Http::scOkay);
 213         break;
 214     case STREAM_UNPLANNED_COMPLETE:
 215         debugs(33, 3, "Object data transfer failed: STREAM_UNPLANNED_COMPLETE");
 216         callBack(Http::scInternalServerError);
 217         break;
 218     case STREAM_FAILED:
 219         debugs(33, 3, "Object data transfer failed: STREAM_FAILED");
 220         callBack(Http::scInternalServerError);
 221         break;
 222     default:
 223         fatal("unreachable code");
 224     }
 225 }
 226
 227 void
 228 Downloader::downloadFinished()
 229 {
 230     debugs(33, 7, this);
 231     // We cannot delay http destruction until refcounting deletes
 232     // DownloaderContext. The http object destruction will cause
 233     // clientStream cleanup and will release the refcount to context_
 234     // object hold by clientStream structures.
 235     context_->finished();
 236     context_ = nullptr;
 237     Must(done());
 238 }
 239
 240 /// Schedules for execution the "callback" with parameters the status
 241 /// and object.
 242 void
 243 Downloader::callBack(Http::StatusCode const statusCode)
 244 {
 245      CbDialer *dialer = dynamic_cast<CbDialer*>(callback_->getDialer());
 246      Must(dialer);
 247      dialer->status = statusCode;
 248      if (statusCode == Http::scOkay)
 249          dialer->object = object_;
 250      ScheduleCallHere(callback_);
 251      callback_ = nullptr;
 252
 253      // Calling deleteThis method here to finish Downloader
 254      // may result to squid crash.
 255      // This method called by handleReply method which maybe called
 256      // by ClientHttpRequest::doCallouts. The doCallouts after this object
 257      // deleted, may operate on non valid objects.
 258      // Schedule an async call here just to force squid to delete this object.
 259      CallJobHere(33, 7, CbcPointer<Downloader>(this), Downloader, downloadFinished);
 260 }
 261