src/Downloader.cc

   1 /*
   2  * Copyright (C) 1996-2023 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 #include "squid.h"
  10 #include "base/Raw.h"
  11 #include "client_side.h"
  12 #include "client_side_reply.h"
  13 #include "client_side_request.h"
  14 #include "ClientRequestContext.h"
  15 #include "Downloader.h"
  16 #include "fatal.h"
  17 #include "http/one/RequestParser.h"
  18 #include "http/Stream.h"
  19
  20 CBDATA_CLASS_INIT(Downloader);
  21
  22 /// Used to hold and pass the required info and buffers to the
  23 /// clientStream callbacks
  24 class DownloaderContext: public RefCountable
  25 {
  26     MEMPROXY_CLASS(DownloaderContext);
  27
  28 public:
  29     typedef RefCount<DownloaderContext> Pointer;
  30
  31     DownloaderContext(Downloader *dl, ClientHttpRequest *h);
  32     ~DownloaderContext() override;
  33     void finished();
  34
  35     CbcPointer<Downloader> downloader;
  36     ClientHttpRequest *http;
  37     char requestBuffer[HTTP_REQBUF_SZ];
  38 };
  39
  40 DownloaderContext::DownloaderContext(Downloader *dl, ClientHttpRequest *h):
  41     downloader(dl),
  42     http(h)
  43 {
  44     debugs(33, 6, "DownloaderContext constructed, this=" << (void*)this);
  45 }
  46
  47 DownloaderContext::~DownloaderContext()
  48 {
  49     debugs(33, 6, "DownloaderContext destructed, this=" << (void*)this);
  50     if (http)
  51         finished();
  52 }
  53
  54 void
  55 DownloaderContext::finished()
  56 {
  57     delete http;
  58     http = nullptr;
  59 }
  60
  61 std::ostream &
  62 operator <<(std::ostream &os, const DownloaderAnswer &answer)
  63 {
  64     os << "outcome=" << answer.outcome;
  65     if (answer.outcome == Http::scOkay)
  66         os << ", resource.size=" << answer.resource.length();
  67     return os;
  68 }
  69
  70 Downloader::Downloader(const SBuf &url, const AsyncCallback<Answer> &cb, const MasterXactionPointer &mx, const unsigned int level):
  71     AsyncJob("Downloader"),
  72     url_(url),
  73     callback_(cb),
  74     level_(level),
  75     masterXaction_(mx)
  76 {
  77 }
  78
  79 Downloader::~Downloader()
  80 {
  81     debugs(33, 6, this);
  82 }
  83
  84 void
  85 Downloader::swanSong()
  86 {
  87     debugs(33, 6, this);
  88
  89     if (callback_) // job-ending emergencies like handleStopRequest() or callException()
  90         callBack(Http::scInternalServerError);
  91
  92     if (context_) {
  93         context_->finished();
  94         context_ = nullptr;
  95     }
  96 }
  97
  98 bool
  99 Downloader::doneAll() const
 100 {
 101     return (!callback_ || callback_->canceled()) && AsyncJob::doneAll();
 102 }
 103
 104 static void
 105 downloaderRecipient(clientStreamNode * node, ClientHttpRequest * http,
 106                     HttpReply * rep, StoreIOBuffer receivedData)
 107 {
 108     debugs(33, 6, MYNAME);
 109     /* Test preconditions */
 110     assert(node);
 111
 112     /* TODO: handle this rather than asserting
 113      * - it should only ever happen if we cause an abort and
 114      * the callback chain loops back to here, so we can simply return.
 115      * However, that itself shouldn't happen, so it stays as an assert for now.
 116      */
 117     assert(cbdataReferenceValid(node));
 118     assert(!node->node.next);
 119     DownloaderContext::Pointer context = dynamic_cast<DownloaderContext *>(node->data.getRaw());
 120     assert(context);
 121
 122     if (context->downloader.valid())
 123         context->downloader->handleReply(node, http, rep, receivedData);
 124 }
 125
 126 static void
 127 downloaderDetach(clientStreamNode * node, ClientHttpRequest * http)
 128 {
 129     debugs(33, 5, MYNAME);
 130     clientStreamDetach(node, http);
 131 }
 132
 133 /// Initializes and starts the HTTP GET request to the remote server
 134 bool
 135 Downloader::buildRequest()
 136 {
 137     const HttpRequestMethod method = Http::METHOD_GET;
 138
 139     const auto request = HttpRequest::FromUrl(url_, masterXaction_, method);
 140     if (!request) {
 141         debugs(33, 5, "Invalid URI: " << url_);
 142         return false; //earlyError(...)
 143     }
 144     request->http_ver = Http::ProtocolVersion();
 145     request->header.putStr(Http::HdrType::HOST, request->url.host());
 146     request->header.putTime(Http::HdrType::DATE, squid_curtime);
 147     request->client_addr.setNoAddr();
 148 #if FOLLOW_X_FORWARDED_FOR
 149     request->indirect_client_addr.setNoAddr();
 150 #endif /* FOLLOW_X_FORWARDED_FOR */
 151     request->my_addr.setNoAddr();   /* undefined for internal requests */
 152     request->my_addr.port(0);
 153     request->downloader = this;
 154
 155     debugs(11, 2, "HTTP Client Downloader " << this << "/" << id);
 156     debugs(11, 2, "HTTP Client REQUEST:\n---------\n" <<
 157            request->method << " " << url_ << " " << request->http_ver << "\n" <<
 158            "\n----------");
 159
 160     ClientHttpRequest *const http = new ClientHttpRequest(nullptr);
 161     http->initRequest(request);
 162     http->req_sz = 0;
 163     // XXX: performance regression. c_str() reallocates
 164     http->uri = xstrdup(url_.c_str());
 165
 166     context_ = new DownloaderContext(this, http);
 167     StoreIOBuffer tempBuffer;
 168     tempBuffer.data = context_->requestBuffer;
 169     tempBuffer.length = HTTP_REQBUF_SZ;
 170
 171     ClientStreamData newServer = new clientReplyContext(http);
 172     ClientStreamData newClient = context_.getRaw();
 173     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 174                      clientReplyStatus, newServer, downloaderRecipient,
 175                      downloaderDetach, newClient, tempBuffer);
 176
 177     // Build a ClientRequestContext to start doCallouts
 178     http->calloutContext = new ClientRequestContext(http);
 179     http->doCallouts();
 180     return true;
 181 }
 182
 183 void
 184 Downloader::start()
 185 {
 186     if (!buildRequest())
 187         callBack(Http::scInternalServerError);
 188 }
 189
 190 void
 191 Downloader::handleReply(clientStreamNode * node, ClientHttpRequest *http, HttpReply *reply, StoreIOBuffer receivedData)
 192 {
 193     DownloaderContext::Pointer callerContext = dynamic_cast<DownloaderContext *>(node->data.getRaw());
 194     // TODO: remove the following check:
 195     assert(callerContext == context_);
 196
 197     debugs(33, 4, "Received " << receivedData.length <<
 198            " object data, offset: " << receivedData.offset <<
 199            " error flag:" << receivedData.flags.error);
 200
 201     const bool failed = receivedData.flags.error;
 202     if (failed) {
 203         callBack(Http::scInternalServerError);
 204         return;
 205     }
 206
 207     const int64_t existingContent = reply ? reply->content_length : 0;
 208     const size_t maxSize = MaxObjectSize > SBuf::maxSize ? SBuf::maxSize : MaxObjectSize;
 209     const bool tooLarge = (existingContent > -1 && existingContent > static_cast<int64_t>(maxSize)) ||
 210                           (maxSize < object_.length()) ||
 211                           ((maxSize - object_.length()) < receivedData.length);
 212
 213     if (tooLarge) {
 214         callBack(Http::scInternalServerError);
 215         return;
 216     }
 217
 218     object_.append(receivedData.data, receivedData.length);
 219     http->out.size += receivedData.length;
 220     http->out.offset += receivedData.length;
 221
 222     switch (clientStreamStatus(node, http)) {
 223     case STREAM_NONE: {
 224         debugs(33, 3, "Get more data");
 225         StoreIOBuffer tempBuffer;
 226         tempBuffer.offset = http->out.offset;
 227         tempBuffer.data = context_->requestBuffer;
 228         tempBuffer.length = HTTP_REQBUF_SZ;
 229         clientStreamRead(node, http, tempBuffer);
 230     }
 231     break;
 232     case STREAM_COMPLETE:
 233         debugs(33, 3, "Object data transfer successfully complete");
 234         callBack(Http::scOkay);
 235         break;
 236     case STREAM_UNPLANNED_COMPLETE:
 237         debugs(33, 3, "Object data transfer failed: STREAM_UNPLANNED_COMPLETE");
 238         callBack(Http::scInternalServerError);
 239         break;
 240     case STREAM_FAILED:
 241         debugs(33, 3, "Object data transfer failed: STREAM_FAILED");
 242         callBack(Http::scInternalServerError);
 243         break;
 244     default:
 245         fatal("unreachable code");
 246     }
 247 }
 248
 249 void
 250 Downloader::downloadFinished()
 251 {
 252     debugs(33, 7, this);
 253     Must(done());
 254 }
 255
 256 /// Schedules for execution the "callback" with parameters the status
 257 /// and object.
 258 void
 259 Downloader::callBack(Http::StatusCode const statusCode)
 260 {
 261     assert(callback_);
 262     auto &answer = callback_.answer();
 263     answer.outcome = statusCode;
 264     if (statusCode == Http::scOkay)
 265         answer.resource = object_;
 266     ScheduleCallHere(callback_.release());
 267
 268     // We cannot deleteThis() because we may be called synchronously from
 269     // doCallouts() via handleReply() (XXX), and doCallouts() may crash if we
 270     // disappear. Instead, schedule an async call now so that later, when the
 271     // call firing code discovers a done() job, it deletes us.
 272     CallJobHere(33, 7, CbcPointer<Downloader>(this), Downloader, downloadFinished);
 273 }
 274