src/redirect.cc

   1 /*
   2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 61    Redirector */
  10
  11 #include "squid.h"
  12 #include "acl/Checklist.h"
  13 #include "cache_cf.h"
  14 #include "client_side.h"
  15 #include "client_side_reply.h"
  16 #include "client_side_request.h"
  17 #include "comm/Connection.h"
  18 #include "fde.h"
  19 #include "format/Format.h"
  20 #include "globals.h"
  21 #include "helper.h"
  22 #include "helper/Reply.h"
  23 #include "http/Stream.h"
  24 #include "HttpRequest.h"
  25 #include "mgr/Registration.h"
  26 #include "redirect.h"
  27 #include "rfc1738.h"
  28 #include "sbuf/SBuf.h"
  29 #include "SquidConfig.h"
  30 #include "Store.h"
  31 #if USE_AUTH
  32 #include "auth/UserRequest.h"
  33 #endif
  34 #if USE_OPENSSL
  35 #include "ssl/support.h"
  36 #endif
  37
  38 /// url maximum lengh + extra informations passed to redirector
  39 #define MAX_REDIRECTOR_REQUEST_STRLEN (MAX_URL + 1024)
  40
  41 class RedirectStateData
  42 {
  43     CBDATA_CLASS(RedirectStateData);
  44
  45 public:
  46     explicit RedirectStateData(const char *url);
  47     ~RedirectStateData();
  48
  49     void *data;
  50     SBuf orig_url;
  51
  52     HLPCB *handler;
  53 };
  54
  55 static HLPCB redirectHandleReply;
  56 static HLPCB storeIdHandleReply;
  57 static helper *redirectors = NULL;
  58 static helper *storeIds = NULL;
  59 static OBJH redirectStats;
  60 static OBJH storeIdStats;
  61 static int redirectorBypassed = 0;
  62 static int storeIdBypassed = 0;
  63 static Format::Format *redirectorExtrasFmt = NULL;
  64 static Format::Format *storeIdExtrasFmt = NULL;
  65
  66 CBDATA_CLASS_INIT(RedirectStateData);
  67
  68 RedirectStateData::RedirectStateData(const char *url) :
  69     data(NULL),
  70     orig_url(url),
  71     handler(NULL)
  72 {
  73 }
  74
  75 RedirectStateData::~RedirectStateData()
  76 {
  77 }
  78
  79 static void
  80 redirectHandleReply(void *data, const Helper::Reply &reply)
  81 {
  82     RedirectStateData *r = static_cast<RedirectStateData *>(data);
  83     debugs(61, 5, HERE << "reply=" << reply);
  84
  85     // XXX: This function is now kept only to check for and display the garbage use-case
  86     // and to map the old helper response format(s) into new format result code and key=value pairs
  87     // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format
  88
  89     if (reply.result == Helper::Unknown) {
  90         // BACKWARD COMPATIBILITY 2012-06-15:
  91         // Some nasty old helpers send back the entire input line including extra format keys.
  92         // This is especially bad for simple perl search-replace filter scripts.
  93         //
  94         // * trim all but the first word off the response.
  95         // * warn once every 50 responses that this will stop being fixed-up soon.
  96         //
  97         if (reply.other().hasContent()) {
  98             const char * res = reply.other().content();
  99             size_t replySize = 0;
 100             if (const char *t = strchr(res, ' ')) {
 101                 static int warn = 0;
 102                 debugs(61, (!(warn++%50)? DBG_CRITICAL:2), "UPGRADE WARNING: URL rewriter reponded with garbage '" << t <<
 103                        "'. Future Squid will treat this as part of the URL.");
 104                 replySize = t - res;
 105             } else
 106                 replySize = reply.other().contentSize();
 107
 108             // if we still have anything in other() after all that
 109             // parse it into status=, url= and rewrite-url= keys
 110             if (replySize) {
 111                 /* 2012-06-28: This cast is due to urlParse() truncating too-long URLs itself.
 112                  * At this point altering the helper buffer in that way is not harmful, but annoying.
 113                  * When Bug 1961 is resolved and urlParse has a const API, this needs to die.
 114                  */
 115                 MemBuf replyBuffer;
 116                 replyBuffer.init(replySize, replySize);
 117                 replyBuffer.append(reply.other().content(), reply.other().contentSize());
 118                 char * result = replyBuffer.content();
 119
 120                 Helper::Reply newReply;
 121                 // BACKWARD COMPATIBILITY 2012-06-15:
 122                 // We got Helper::Unknown reply result but new
 123                 // RedirectStateData handlers require Helper::Okay,
 124                 // else will drop the helper reply
 125                 newReply.result = Helper::Okay;
 126                 newReply.notes.append(&reply.notes);
 127
 128                 // check and parse for obsoleted Squid-2 urlgroup feature
 129                 if (*result == '!') {
 130                     static int urlgroupWarning = 0;
 131                     if (!urlgroupWarning++)
 132                         debugs(85, DBG_IMPORTANT, "UPGRADE WARNING: URL rewriter using obsolete Squid-2 urlgroup feature needs updating.");
 133                     if (char *t = strchr(result+1, '!')) {
 134                         *t = '\0';
 135                         newReply.notes.add("urlgroup", result+1);
 136                         result = t + 1;
 137                     }
 138                 }
 139
 140                 const Http::StatusCode status = static_cast<Http::StatusCode>(atoi(result));
 141
 142                 if (status == Http::scMovedPermanently
 143                         || status == Http::scFound
 144                         || status == Http::scSeeOther
 145                         || status == Http::scPermanentRedirect
 146                         || status == Http::scTemporaryRedirect) {
 147
 148                     if (const char *t = strchr(result, ':')) {
 149                         char statusBuf[4];
 150                         snprintf(statusBuf, sizeof(statusBuf),"%3u",status);
 151                         newReply.notes.add("status", statusBuf);
 152                         ++t;
 153                         // TODO: validate the URL produced here is RFC 2616 compliant URI
 154                         newReply.notes.add("url", t);
 155                     } else {
 156                         debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid " << status << " redirect Location: " << result);
 157                     }
 158                 } else {
 159                     // status code is not a redirect code (or does not exist)
 160                     // treat as a re-write URL request
 161                     // TODO: validate the URL produced here is RFC 2616 compliant URI
 162                     if (*result)
 163                         newReply.notes.add("rewrite-url", result);
 164                 }
 165
 166                 void *cbdata;
 167                 if (cbdataReferenceValidDone(r->data, &cbdata))
 168                     r->handler(cbdata, newReply);
 169
 170                 delete r;
 171                 return;
 172             }
 173         }
 174     }
 175
 176     void *cbdata;
 177     if (cbdataReferenceValidDone(r->data, &cbdata))
 178         r->handler(cbdata, reply);
 179
 180     delete r;
 181 }
 182
 183 static void
 184 storeIdHandleReply(void *data, const Helper::Reply &reply)
 185 {
 186     RedirectStateData *r = static_cast<RedirectStateData *>(data);
 187     debugs(61, 5,"StoreId helper: reply=" << reply);
 188
 189     // XXX: This function is now kept only to check for and display the garbage use-case
 190     // and to map the old helper response format(s) into new format result code and key=value pairs
 191     // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format
 192     void *cbdata;
 193     if (cbdataReferenceValidDone(r->data, &cbdata))
 194         r->handler(cbdata, reply);
 195
 196     delete r;
 197 }
 198
 199 static void
 200 redirectStats(StoreEntry * sentry)
 201 {
 202     if (redirectors == NULL) {
 203         storeAppendPrintf(sentry, "No redirectors defined\n");
 204         return;
 205     }
 206
 207     redirectors->packStatsInto(sentry, "Redirector Statistics");
 208
 209     if (Config.onoff.redirector_bypass)
 210         storeAppendPrintf(sentry, "\nNumber of requests bypassed "
 211                           "because all redirectors were busy: %d\n", redirectorBypassed);
 212 }
 213
 214 static void
 215 storeIdStats(StoreEntry * sentry)
 216 {
 217     if (storeIds == NULL) {
 218         storeAppendPrintf(sentry, "No StoreId helpers defined\n");
 219         return;
 220     }
 221
 222     storeIds->packStatsInto(sentry, "StoreId helper Statistics");
 223
 224     if (Config.onoff.store_id_bypass)
 225         storeAppendPrintf(sentry, "\nNumber of requests bypassed "
 226                           "because all StoreId helpers were busy: %d\n", storeIdBypassed);
 227 }
 228
 229 static void
 230 constructHelperQuery(const char *name, helper *hlp, HLPCB *replyHandler, ClientHttpRequest * http, HLPCB *handler, void *data, Format::Format *requestExtrasFmt)
 231 {
 232     char buf[MAX_REDIRECTOR_REQUEST_STRLEN];
 233     int sz;
 234     Http::StatusCode status;
 235
 236     /** TODO: create a standalone method to initialize
 237      * the RedirectStateData for all the helpers.
 238      */
 239     RedirectStateData *r = new RedirectStateData(http->uri);
 240     r->handler = handler;
 241     r->data = cbdataReference(data);
 242
 243     static MemBuf requestExtras;
 244     requestExtras.reset();
 245     if (requestExtrasFmt)
 246         requestExtrasFmt->assemble(requestExtras, http->al, 0);
 247
 248     sz = snprintf(buf, MAX_REDIRECTOR_REQUEST_STRLEN, "%s%s%s\n",
 249                   r->orig_url.c_str(),
 250                   requestExtras.hasContent() ? " " : "",
 251                   requestExtras.hasContent() ? requestExtras.content() : "");
 252
 253     if ((sz<=0) || (sz>=MAX_REDIRECTOR_REQUEST_STRLEN)) {
 254         if (sz<=0) {
 255             status = Http::scInternalServerError;
 256             debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Can not build request to be passed to " << name << ". Request ABORTED.");
 257         } else {
 258             status = Http::scUriTooLong;
 259             debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Request passed to " << name << " exceeds MAX_REDIRECTOR_REQUEST_STRLEN (" << MAX_REDIRECTOR_REQUEST_STRLEN << "). Request ABORTED.");
 260         }
 261
 262         clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
 263         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 264         assert (repContext);
 265         Ip::Address tmpnoaddr;
 266         tmpnoaddr.setNoAddr();
 267         repContext->setReplyToError(ERR_GATEWAY_FAILURE, status,
 268                                     http->request->method, NULL,
 269                                     http->getConn() != NULL && http->getConn()->clientConnection != NULL ?
 270                                     http->getConn()->clientConnection->remote : tmpnoaddr,
 271                                     http->request,
 272                                     NULL,
 273 #if USE_AUTH
 274                                     http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
 275                                     http->getConn()->getAuth() : http->request->auth_user_request);
 276 #else
 277                                     NULL);
 278 #endif
 279
 280         node = (clientStreamNode *)http->client_stream.tail->data;
 281         clientStreamRead(node, http, node->readBuffer);
 282         return;
 283     }
 284
 285     debugs(61,6, HERE << "sending '" << buf << "' to the " << name << " helper");
 286     helperSubmit(hlp, buf, replyHandler, r);
 287 }
 288
 289 /**** PUBLIC FUNCTIONS ****/
 290
 291 void
 292 redirectStart(ClientHttpRequest * http, HLPCB * handler, void *data)
 293 {
 294     assert(http);
 295     assert(handler);
 296     debugs(61, 5, "redirectStart: '" << http->uri << "'");
 297
 298     // TODO: Deprecate Config.onoff.redirector_bypass in favor of either
 299     // onPersistentOverload or a new onOverload option that applies to all helpers.
 300     if (Config.onoff.redirector_bypass && redirectors->willOverload()) {
 301         /* Skip redirector if the queue is full */
 302         ++redirectorBypassed;
 303         Helper::Reply bypassReply;
 304         bypassReply.result = Helper::Okay;
 305         bypassReply.notes.add("message","URL rewrite/redirect queue too long. Bypassed.");
 306         handler(data, bypassReply);
 307         return;
 308     }
 309
 310     constructHelperQuery("redirector", redirectors, redirectHandleReply, http, handler, data, redirectorExtrasFmt);
 311 }
 312
 313 /**
 314  * Handles the StoreID feature helper starting.
 315  * For now it cannot be done using the redirectStart method.
 316  */
 317 void
 318 storeIdStart(ClientHttpRequest * http, HLPCB * handler, void *data)
 319 {
 320     assert(http);
 321     assert(handler);
 322     debugs(61, 5, "storeIdStart: '" << http->uri << "'");
 323
 324     if (Config.onoff.store_id_bypass && storeIds->willOverload()) {
 325         /* Skip StoreID Helper if the queue is full */
 326         ++storeIdBypassed;
 327         Helper::Reply bypassReply;
 328
 329         bypassReply.result = Helper::Okay;
 330
 331         bypassReply.notes.add("message","StoreId helper queue too long. Bypassed.");
 332         handler(data, bypassReply);
 333         return;
 334     }
 335
 336     constructHelperQuery("storeId helper", storeIds, storeIdHandleReply, http, handler, data, storeIdExtrasFmt);
 337 }
 338
 339 void
 340 redirectInit(void)
 341 {
 342     static bool init = false;
 343
 344     if (!init) {
 345         Mgr::RegisterAction("redirector", "URL Redirector Stats", redirectStats, 0, 1);
 346         Mgr::RegisterAction("store_id", "StoreId helper Stats", storeIdStats, 0, 1);
 347     }
 348
 349     if (Config.Program.redirect) {
 350
 351         if (redirectors == NULL)
 352             redirectors = new helper("redirector");
 353
 354         redirectors->cmdline = Config.Program.redirect;
 355
 356         // BACKWARD COMPATIBILITY:
 357         // if redirectot_bypass is set then use queue_size=0 as default size
 358         if (Config.onoff.redirector_bypass && Config.redirectChildren.defaultQueueSize)
 359             Config.redirectChildren.queue_size = 0;
 360
 361         redirectors->childs.updateLimits(Config.redirectChildren);
 362
 363         redirectors->ipc_type = IPC_STREAM;
 364
 365         redirectors->timeout = Config.Timeout.urlRewrite;
 366
 367         redirectors->retryTimedOut = (Config.onUrlRewriteTimeout.action == toutActRetry);
 368         redirectors->retryBrokenHelper = true; // XXX: make this configurable ?
 369         redirectors->onTimedOutResponse.clear();
 370         if (Config.onUrlRewriteTimeout.action == toutActUseConfiguredResponse)
 371             redirectors->onTimedOutResponse.assign(Config.onUrlRewriteTimeout.response);
 372
 373         helperOpenServers(redirectors);
 374     }
 375
 376     if (Config.Program.store_id) {
 377
 378         if (storeIds == NULL)
 379             storeIds = new helper("store_id");
 380
 381         storeIds->cmdline = Config.Program.store_id;
 382
 383         // BACKWARD COMPATIBILITY:
 384         // if store_id_bypass is set then use queue_size=0 as default size
 385         if (Config.onoff.store_id_bypass && Config.storeIdChildren.defaultQueueSize)
 386             Config.storeIdChildren.queue_size = 0;
 387
 388         storeIds->childs.updateLimits(Config.storeIdChildren);
 389
 390         storeIds->ipc_type = IPC_STREAM;
 391
 392         storeIds->retryBrokenHelper = true; // XXX: make this configurable ?
 393
 394         helperOpenServers(storeIds);
 395     }
 396
 397     if (Config.redirector_extras) {
 398         delete redirectorExtrasFmt;
 399         redirectorExtrasFmt = new ::Format::Format("url_rewrite_extras");
 400         (void)redirectorExtrasFmt->parse(Config.redirector_extras);
 401     }
 402
 403     if (Config.storeId_extras) {
 404         delete storeIdExtrasFmt;
 405         storeIdExtrasFmt = new ::Format::Format("store_id_extras");
 406         (void)storeIdExtrasFmt->parse(Config.storeId_extras);
 407     }
 408
 409     init = true;
 410 }
 411
 412 void
 413 redirectShutdown(void)
 414 {
 415     /** FIXME: Temporary unified helpers Shutdown
 416      * When and if needed for more helpers a separated shutdown
 417      * method will be added for each of them.
 418      */
 419     if (redirectors)
 420         helperShutdown(redirectors);
 421
 422     if (storeIds)
 423         helperShutdown(storeIds);
 424
 425     if (!shutting_down)
 426         return;
 427
 428     delete redirectors;
 429     redirectors = NULL;
 430
 431     delete storeIds;
 432     storeIds = NULL;
 433
 434     delete redirectorExtrasFmt;
 435     redirectorExtrasFmt = NULL;
 436
 437     delete storeIdExtrasFmt;
 438     storeIdExtrasFmt = NULL;
 439 }
 440