src/redirect.cc

   1 /*
   2  * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 61    Redirector */
  10
  11 #include "squid.h"
  12 #include "acl/Checklist.h"
  13 #include "cache_cf.h"
  14 #include "client_side.h"
  15 #include "client_side_reply.h"
  16 #include "client_side_request.h"
  17 #include "comm/Connection.h"
  18 #include "fde.h"
  19 #include "format/Format.h"
  20 #include "globals.h"
  21 #include "helper.h"
  22 #include "helper/Reply.h"
  23 #include "http/Stream.h"
  24 #include "HttpRequest.h"
  25 #include "mgr/Registration.h"
  26 #include "redirect.h"
  27 #include "rfc1738.h"
  28 #include "sbuf/SBuf.h"
  29 #include "SquidConfig.h"
  30 #include "Store.h"
  31 #if USE_AUTH
  32 #include "auth/UserRequest.h"
  33 #endif
  34 #if USE_OPENSSL
  35 #include "ssl/support.h"
  36 #endif
  37
  38 /// url maximum length + extra informations passed to redirector
  39 #define MAX_REDIRECTOR_REQUEST_STRLEN (MAX_URL + 1024)
  40
  41 class RedirectStateData
  42 {
  43     CBDATA_CLASS(RedirectStateData);
  44
  45 public:
  46     explicit RedirectStateData(const char *url);
  47     ~RedirectStateData();
  48
  49     void *data;
  50     SBuf orig_url;
  51
  52     HLPCB *handler;
  53 };
  54
  55 static HLPCB redirectHandleReply;
  56 static HLPCB storeIdHandleReply;
  57 static helper *redirectors = NULL;
  58 static helper *storeIds = NULL;
  59 static OBJH redirectStats;
  60 static OBJH storeIdStats;
  61 static int redirectorBypassed = 0;
  62 static int storeIdBypassed = 0;
  63 static Format::Format *redirectorExtrasFmt = NULL;
  64 static Format::Format *storeIdExtrasFmt = NULL;
  65
  66 CBDATA_CLASS_INIT(RedirectStateData);
  67
  68 RedirectStateData::RedirectStateData(const char *url) :
  69     data(NULL),
  70     orig_url(url),
  71     handler(NULL)
  72 {
  73 }
  74
  75 RedirectStateData::~RedirectStateData()
  76 {
  77 }
  78
  79 static void
  80 redirectHandleReply(void *data, const Helper::Reply &reply)
  81 {
  82     RedirectStateData *r = static_cast<RedirectStateData *>(data);
  83     debugs(61, 5, HERE << "reply=" << reply);
  84
  85     // XXX: This function is now kept only to check for and display the garbage use-case
  86     // and to map the old helper response format(s) into new format result code and key=value pairs
  87     // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format
  88
  89     if (reply.result == Helper::Unknown) {
  90         // BACKWARD COMPATIBILITY 2012-06-15:
  91         // Some nasty old helpers send back the entire input line including extra format keys.
  92         // This is especially bad for simple perl search-replace filter scripts.
  93         //
  94         // * trim all but the first word off the response.
  95         // * warn once every 50 responses that this will stop being fixed-up soon.
  96         //
  97         if (reply.other().hasContent()) {
  98             const char * res = reply.other().content();
  99             size_t replySize = 0;
 100             if (const char *t = strchr(res, ' ')) {
 101                 static int warn = 0;
 102                 debugs(61, (!(warn++%50)? DBG_CRITICAL:2), "UPGRADE WARNING: URL rewriter reponded with garbage '" << t <<
 103                        "'. Future Squid will treat this as part of the URL.");
 104                 replySize = t - res;
 105             } else
 106                 replySize = reply.other().contentSize();
 107
 108             // if we still have anything in other() after all that
 109             // parse it into status=, url= and rewrite-url= keys
 110             if (replySize) {
 111                 MemBuf replyBuffer;
 112                 replyBuffer.init(replySize, replySize);
 113                 replyBuffer.append(reply.other().content(), reply.other().contentSize());
 114                 char * result = replyBuffer.content();
 115
 116                 Helper::Reply newReply;
 117                 // BACKWARD COMPATIBILITY 2012-06-15:
 118                 // We got Helper::Unknown reply result but new
 119                 // RedirectStateData handlers require Helper::Okay,
 120                 // else will drop the helper reply
 121                 newReply.result = Helper::Okay;
 122                 newReply.notes.append(&reply.notes);
 123
 124                 // check and parse for obsoleted Squid-2 urlgroup feature
 125                 if (*result == '!') {
 126                     static int urlgroupWarning = 0;
 127                     if (!urlgroupWarning++)
 128                         debugs(85, DBG_IMPORTANT, "UPGRADE WARNING: URL rewriter using obsolete Squid-2 urlgroup feature needs updating.");
 129                     if (char *t = strchr(result+1, '!')) {
 130                         *t = '\0';
 131                         newReply.notes.add("urlgroup", result+1);
 132                         result = t + 1;
 133                     }
 134                 }
 135
 136                 const Http::StatusCode status = static_cast<Http::StatusCode>(atoi(result));
 137
 138                 if (status == Http::scMovedPermanently
 139                         || status == Http::scFound
 140                         || status == Http::scSeeOther
 141                         || status == Http::scPermanentRedirect
 142                         || status == Http::scTemporaryRedirect) {
 143
 144                     if (const char *t = strchr(result, ':')) {
 145                         char statusBuf[4];
 146                         snprintf(statusBuf, sizeof(statusBuf),"%3u",status);
 147                         newReply.notes.add("status", statusBuf);
 148                         ++t;
 149                         // TODO: validate the URL produced here is RFC 2616 compliant URI
 150                         newReply.notes.add("url", t);
 151                     } else {
 152                         debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid " << status << " redirect Location: " << result);
 153                     }
 154                 } else {
 155                     // status code is not a redirect code (or does not exist)
 156                     // treat as a re-write URL request
 157                     // TODO: validate the URL produced here is RFC 2616 compliant URI
 158                     if (*result)
 159                         newReply.notes.add("rewrite-url", result);
 160                 }
 161
 162                 void *cbdata;
 163                 if (cbdataReferenceValidDone(r->data, &cbdata))
 164                     r->handler(cbdata, newReply);
 165
 166                 delete r;
 167                 return;
 168             }
 169         }
 170     }
 171
 172     void *cbdata;
 173     if (cbdataReferenceValidDone(r->data, &cbdata))
 174         r->handler(cbdata, reply);
 175
 176     delete r;
 177 }
 178
 179 static void
 180 storeIdHandleReply(void *data, const Helper::Reply &reply)
 181 {
 182     RedirectStateData *r = static_cast<RedirectStateData *>(data);
 183     debugs(61, 5,"StoreId helper: reply=" << reply);
 184
 185     // XXX: This function is now kept only to check for and display the garbage use-case
 186     // and to map the old helper response format(s) into new format result code and key=value pairs
 187     // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format
 188     void *cbdata;
 189     if (cbdataReferenceValidDone(r->data, &cbdata))
 190         r->handler(cbdata, reply);
 191
 192     delete r;
 193 }
 194
 195 static void
 196 redirectStats(StoreEntry * sentry)
 197 {
 198     if (redirectors == NULL) {
 199         storeAppendPrintf(sentry, "No redirectors defined\n");
 200         return;
 201     }
 202
 203     redirectors->packStatsInto(sentry, "Redirector Statistics");
 204
 205     if (Config.onoff.redirector_bypass)
 206         storeAppendPrintf(sentry, "\nNumber of requests bypassed "
 207                           "because all redirectors were busy: %d\n", redirectorBypassed);
 208 }
 209
 210 static void
 211 storeIdStats(StoreEntry * sentry)
 212 {
 213     if (storeIds == NULL) {
 214         storeAppendPrintf(sentry, "No StoreId helpers defined\n");
 215         return;
 216     }
 217
 218     storeIds->packStatsInto(sentry, "StoreId helper Statistics");
 219
 220     if (Config.onoff.store_id_bypass)
 221         storeAppendPrintf(sentry, "\nNumber of requests bypassed "
 222                           "because all StoreId helpers were busy: %d\n", storeIdBypassed);
 223 }
 224
 225 static void
 226 constructHelperQuery(const char *name, helper *hlp, HLPCB *replyHandler, ClientHttpRequest * http, HLPCB *handler, void *data, Format::Format *requestExtrasFmt)
 227 {
 228     char buf[MAX_REDIRECTOR_REQUEST_STRLEN];
 229     int sz;
 230     Http::StatusCode status;
 231
 232     /** TODO: create a standalone method to initialize
 233      * the RedirectStateData for all the helpers.
 234      */
 235     RedirectStateData *r = new RedirectStateData(http->uri);
 236     r->handler = handler;
 237     r->data = cbdataReference(data);
 238
 239     static MemBuf requestExtras;
 240     requestExtras.reset();
 241     if (requestExtrasFmt)
 242         requestExtrasFmt->assemble(requestExtras, http->al, 0);
 243
 244     sz = snprintf(buf, MAX_REDIRECTOR_REQUEST_STRLEN, "%s%s%s\n",
 245                   r->orig_url.c_str(),
 246                   requestExtras.hasContent() ? " " : "",
 247                   requestExtras.hasContent() ? requestExtras.content() : "");
 248
 249     if ((sz<=0) || (sz>=MAX_REDIRECTOR_REQUEST_STRLEN)) {
 250         if (sz<=0) {
 251             status = Http::scInternalServerError;
 252             debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Can not build request to be passed to " << name << ". Request ABORTED.");
 253         } else {
 254             status = Http::scUriTooLong;
 255             debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Request passed to " << name << " exceeds MAX_REDIRECTOR_REQUEST_STRLEN (" << MAX_REDIRECTOR_REQUEST_STRLEN << "). Request ABORTED.");
 256         }
 257
 258         clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
 259         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 260         assert (repContext);
 261         Ip::Address tmpnoaddr;
 262         tmpnoaddr.setNoAddr();
 263         repContext->setReplyToError(ERR_GATEWAY_FAILURE, status,
 264                                     http->request->method, NULL,
 265                                     http->getConn() != NULL && http->getConn()->clientConnection != NULL ?
 266                                     http->getConn()->clientConnection->remote : tmpnoaddr,
 267                                     http->request,
 268                                     NULL,
 269 #if USE_AUTH
 270                                     http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
 271                                     http->getConn()->getAuth() : http->request->auth_user_request);
 272 #else
 273                                     NULL);
 274 #endif
 275
 276         node = (clientStreamNode *)http->client_stream.tail->data;
 277         clientStreamRead(node, http, node->readBuffer);
 278         return;
 279     }
 280
 281     debugs(61,6, HERE << "sending '" << buf << "' to the " << name << " helper");
 282     helperSubmit(hlp, buf, replyHandler, r);
 283 }
 284
 285 /**** PUBLIC FUNCTIONS ****/
 286
 287 void
 288 redirectStart(ClientHttpRequest * http, HLPCB * handler, void *data)
 289 {
 290     assert(http);
 291     assert(handler);
 292     debugs(61, 5, "redirectStart: '" << http->uri << "'");
 293
 294     // TODO: Deprecate Config.onoff.redirector_bypass in favor of either
 295     // onPersistentOverload or a new onOverload option that applies to all helpers.
 296     if (Config.onoff.redirector_bypass && redirectors->willOverload()) {
 297         /* Skip redirector if the queue is full */
 298         ++redirectorBypassed;
 299         Helper::Reply bypassReply;
 300         bypassReply.result = Helper::Okay;
 301         bypassReply.notes.add("message","URL rewrite/redirect queue too long. Bypassed.");
 302         handler(data, bypassReply);
 303         return;
 304     }
 305
 306     constructHelperQuery("redirector", redirectors, redirectHandleReply, http, handler, data, redirectorExtrasFmt);
 307 }
 308
 309 /**
 310  * Handles the StoreID feature helper starting.
 311  * For now it cannot be done using the redirectStart method.
 312  */
 313 void
 314 storeIdStart(ClientHttpRequest * http, HLPCB * handler, void *data)
 315 {
 316     assert(http);
 317     assert(handler);
 318     debugs(61, 5, "storeIdStart: '" << http->uri << "'");
 319
 320     if (Config.onoff.store_id_bypass && storeIds->willOverload()) {
 321         /* Skip StoreID Helper if the queue is full */
 322         ++storeIdBypassed;
 323         Helper::Reply bypassReply;
 324
 325         bypassReply.result = Helper::Okay;
 326
 327         bypassReply.notes.add("message","StoreId helper queue too long. Bypassed.");
 328         handler(data, bypassReply);
 329         return;
 330     }
 331
 332     constructHelperQuery("storeId helper", storeIds, storeIdHandleReply, http, handler, data, storeIdExtrasFmt);
 333 }
 334
 335 void
 336 redirectInit(void)
 337 {
 338     static bool init = false;
 339
 340     if (!init) {
 341         Mgr::RegisterAction("redirector", "URL Redirector Stats", redirectStats, 0, 1);
 342         Mgr::RegisterAction("store_id", "StoreId helper Stats", storeIdStats, 0, 1);
 343     }
 344
 345     if (Config.Program.redirect) {
 346
 347         if (redirectors == NULL)
 348             redirectors = new helper("redirector");
 349
 350         redirectors->cmdline = Config.Program.redirect;
 351
 352         // BACKWARD COMPATIBILITY:
 353         // if redirectot_bypass is set then use queue_size=0 as default size
 354         if (Config.onoff.redirector_bypass && Config.redirectChildren.defaultQueueSize)
 355             Config.redirectChildren.queue_size = 0;
 356
 357         redirectors->childs.updateLimits(Config.redirectChildren);
 358
 359         redirectors->ipc_type = IPC_STREAM;
 360
 361         redirectors->timeout = Config.Timeout.urlRewrite;
 362
 363         redirectors->retryTimedOut = (Config.onUrlRewriteTimeout.action == toutActRetry);
 364         redirectors->retryBrokenHelper = true; // XXX: make this configurable ?
 365         redirectors->onTimedOutResponse.clear();
 366         if (Config.onUrlRewriteTimeout.action == toutActUseConfiguredResponse)
 367             redirectors->onTimedOutResponse.assign(Config.onUrlRewriteTimeout.response);
 368
 369         helperOpenServers(redirectors);
 370     }
 371
 372     if (Config.Program.store_id) {
 373
 374         if (storeIds == NULL)
 375             storeIds = new helper("store_id");
 376
 377         storeIds->cmdline = Config.Program.store_id;
 378
 379         // BACKWARD COMPATIBILITY:
 380         // if store_id_bypass is set then use queue_size=0 as default size
 381         if (Config.onoff.store_id_bypass && Config.storeIdChildren.defaultQueueSize)
 382             Config.storeIdChildren.queue_size = 0;
 383
 384         storeIds->childs.updateLimits(Config.storeIdChildren);
 385
 386         storeIds->ipc_type = IPC_STREAM;
 387
 388         storeIds->retryBrokenHelper = true; // XXX: make this configurable ?
 389
 390         helperOpenServers(storeIds);
 391     }
 392
 393     if (Config.redirector_extras) {
 394         delete redirectorExtrasFmt;
 395         redirectorExtrasFmt = new ::Format::Format("url_rewrite_extras");
 396         (void)redirectorExtrasFmt->parse(Config.redirector_extras);
 397     }
 398
 399     if (Config.storeId_extras) {
 400         delete storeIdExtrasFmt;
 401         storeIdExtrasFmt = new ::Format::Format("store_id_extras");
 402         (void)storeIdExtrasFmt->parse(Config.storeId_extras);
 403     }
 404
 405     init = true;
 406 }
 407
 408 void
 409 redirectShutdown(void)
 410 {
 411     /** FIXME: Temporary unified helpers Shutdown
 412      * When and if needed for more helpers a separated shutdown
 413      * method will be added for each of them.
 414      */
 415     if (redirectors)
 416         helperShutdown(redirectors);
 417
 418     if (storeIds)
 419         helperShutdown(storeIds);
 420
 421     if (!shutting_down)
 422         return;
 423
 424     delete redirectors;
 425     redirectors = NULL;
 426
 427     delete storeIds;
 428     storeIds = NULL;
 429
 430     delete redirectorExtrasFmt;
 431     redirectorExtrasFmt = NULL;
 432
 433     delete storeIdExtrasFmt;
 434     storeIdExtrasFmt = NULL;
 435 }
 436
 437 void
 438 redirectReconfigure()
 439 {
 440     redirectShutdown();
 441     redirectInit();
 442 }
 443