src/redirect.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 61    Redirector */
  10
  11 #include "squid.h"
  12 #include "acl/Checklist.h"
  13 #include "cache_cf.h"
  14 #include "client_side.h"
  15 #include "client_side_reply.h"
  16 #include "client_side_request.h"
  17 #include "comm/Connection.h"
  18 #include "fde.h"
  19 #include "format/Format.h"
  20 #include "globals.h"
  21 #include "helper.h"
  22 #include "helper/Reply.h"
  23 #include "http/Stream.h"
  24 #include "HttpRequest.h"
  25 #include "mgr/Registration.h"
  26 #include "redirect.h"
  27 #include "rfc1738.h"
  28 #include "sbuf/SBuf.h"
  29 #include "SquidConfig.h"
  30 #include "Store.h"
  31 #if USE_AUTH
  32 #include "auth/UserRequest.h"
  33 #endif
  34 #if USE_OPENSSL
  35 #include "ssl/support.h"
  36 #endif
  37
  38 /// url maximum lengh + extra informations passed to redirector
  39 #define MAX_REDIRECTOR_REQUEST_STRLEN (MAX_URL + 1024)
  40
  41 class RedirectStateData
  42 {
  43     CBDATA_CLASS(RedirectStateData);
  44
  45 public:
  46     explicit RedirectStateData(const char *url);
  47     ~RedirectStateData();
  48
  49     void *data;
  50     SBuf orig_url;
  51
  52     HLPCB *handler;
  53 };
  54
  55 static HLPCB redirectHandleReply;
  56 static HLPCB storeIdHandleReply;
  57 static helper *redirectors = NULL;
  58 static helper *storeIds = NULL;
  59 static OBJH redirectStats;
  60 static OBJH storeIdStats;
  61 static int redirectorBypassed = 0;
  62 static int storeIdBypassed = 0;
  63 static Format::Format *redirectorExtrasFmt = NULL;
  64 static Format::Format *storeIdExtrasFmt = NULL;
  65
  66 CBDATA_CLASS_INIT(RedirectStateData);
  67
  68 RedirectStateData::RedirectStateData(const char *url) :
  69     data(NULL),
  70     orig_url(url),
  71     handler(NULL)
  72 {
  73 }
  74
  75 RedirectStateData::~RedirectStateData()
  76 {
  77 }
  78
  79 static void
  80 redirectHandleReply(void *data, const Helper::Reply &reply)
  81 {
  82     RedirectStateData *r = static_cast<RedirectStateData *>(data);
  83     debugs(61, 5, HERE << "reply=" << reply);
  84
  85     // XXX: This function is now kept only to check for and display the garbage use-case
  86     // and to map the old helper response format(s) into new format result code and key=value pairs
  87     // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format
  88
  89     if (reply.result == Helper::Unknown) {
  90         // BACKWARD COMPATIBILITY 2012-06-15:
  91         // Some nasty old helpers send back the entire input line including extra format keys.
  92         // This is especially bad for simple perl search-replace filter scripts.
  93         //
  94         // * trim all but the first word off the response.
  95         // * warn once every 50 responses that this will stop being fixed-up soon.
  96         //
  97         if (const char * res = reply.other().content()) {
  98             if (const char *t = strchr(res, ' ')) {
  99                 static int warn = 0;
 100                 debugs(61, (!(warn++%50)? DBG_CRITICAL:2), "UPGRADE WARNING: URL rewriter reponded with garbage '" << t <<
 101                        "'. Future Squid will treat this as part of the URL.");
 102                 const mb_size_t garbageLength = reply.other().contentSize() - (t-res);
 103                 reply.modifiableOther().truncate(garbageLength);
 104             }
 105             if (reply.other().hasContent() && *res == '\0')
 106                 reply.modifiableOther().clean(); // drop the whole buffer of garbage.
 107
 108             // if we still have anything in other() after all that
 109             // parse it into status=, url= and rewrite-url= keys
 110             if (reply.other().hasContent()) {
 111                 /* 2012-06-28: This cast is due to urlParse() truncating too-long URLs itself.
 112                  * At this point altering the helper buffer in that way is not harmful, but annoying.
 113                  * When Bug 1961 is resolved and urlParse has a const API, this needs to die.
 114                  */
 115                 char * result = reply.modifiableOther().content();
 116
 117                 Helper::Reply newReply;
 118                 // BACKWARD COMPATIBILITY 2012-06-15:
 119                 // We got Helper::Unknown reply result but new
 120                 // RedirectStateData handlers require Helper::Okay,
 121                 // else will drop the helper reply
 122                 newReply.result = Helper::Okay;
 123                 newReply.notes.append(&reply.notes);
 124
 125                 // check and parse for obsoleted Squid-2 urlgroup feature
 126                 if (*result == '!') {
 127                     static int urlgroupWarning = 0;
 128                     if (!urlgroupWarning++)
 129                         debugs(85, DBG_IMPORTANT, "UPGRADE WARNING: URL rewriter using obsolete Squid-2 urlgroup feature needs updating.");
 130                     if (char *t = strchr(result+1, '!')) {
 131                         *t = '\0';
 132                         newReply.notes.add("urlgroup", result+1);
 133                         result = t + 1;
 134                     }
 135                 }
 136
 137                 const Http::StatusCode status = static_cast<Http::StatusCode>(atoi(result));
 138
 139                 if (status == Http::scMovedPermanently
 140                         || status == Http::scFound
 141                         || status == Http::scSeeOther
 142                         || status == Http::scPermanentRedirect
 143                         || status == Http::scTemporaryRedirect) {
 144
 145                     if (const char *t = strchr(result, ':')) {
 146                         char statusBuf[4];
 147                         snprintf(statusBuf, sizeof(statusBuf),"%3u",status);
 148                         newReply.notes.add("status", statusBuf);
 149                         ++t;
 150                         // TODO: validate the URL produced here is RFC 2616 compliant URI
 151                         newReply.notes.add("url", t);
 152                     } else {
 153                         debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid " << status << " redirect Location: " << result);
 154                     }
 155                 } else {
 156                     // status code is not a redirect code (or does not exist)
 157                     // treat as a re-write URL request
 158                     // TODO: validate the URL produced here is RFC 2616 compliant URI
 159                     if (*result)
 160                         newReply.notes.add("rewrite-url", result);
 161                 }
 162
 163                 void *cbdata;
 164                 if (cbdataReferenceValidDone(r->data, &cbdata))
 165                     r->handler(cbdata, newReply);
 166
 167                 delete r;
 168                 return;
 169             }
 170         }
 171     }
 172
 173     void *cbdata;
 174     if (cbdataReferenceValidDone(r->data, &cbdata))
 175         r->handler(cbdata, reply);
 176
 177     delete r;
 178 }
 179
 180 static void
 181 storeIdHandleReply(void *data, const Helper::Reply &reply)
 182 {
 183     RedirectStateData *r = static_cast<RedirectStateData *>(data);
 184     debugs(61, 5,"StoreId helper: reply=" << reply);
 185
 186     // XXX: This function is now kept only to check for and display the garbage use-case
 187     // and to map the old helper response format(s) into new format result code and key=value pairs
 188     // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format
 189     void *cbdata;
 190     if (cbdataReferenceValidDone(r->data, &cbdata))
 191         r->handler(cbdata, reply);
 192
 193     delete r;
 194 }
 195
 196 static void
 197 redirectStats(StoreEntry * sentry)
 198 {
 199     if (redirectors == NULL) {
 200         storeAppendPrintf(sentry, "No redirectors defined\n");
 201         return;
 202     }
 203
 204     redirectors->packStatsInto(sentry, "Redirector Statistics");
 205
 206     if (Config.onoff.redirector_bypass)
 207         storeAppendPrintf(sentry, "\nNumber of requests bypassed "
 208                           "because all redirectors were busy: %d\n", redirectorBypassed);
 209 }
 210
 211 static void
 212 storeIdStats(StoreEntry * sentry)
 213 {
 214     if (storeIds == NULL) {
 215         storeAppendPrintf(sentry, "No StoreId helpers defined\n");
 216         return;
 217     }
 218
 219     storeIds->packStatsInto(sentry, "StoreId helper Statistics");
 220
 221     if (Config.onoff.store_id_bypass)
 222         storeAppendPrintf(sentry, "\nNumber of requests bypassed "
 223                           "because all StoreId helpers were busy: %d\n", storeIdBypassed);
 224 }
 225
 226 static void
 227 constructHelperQuery(const char *name, helper *hlp, HLPCB *replyHandler, ClientHttpRequest * http, HLPCB *handler, void *data, Format::Format *requestExtrasFmt)
 228 {
 229     char buf[MAX_REDIRECTOR_REQUEST_STRLEN];
 230     int sz;
 231     Http::StatusCode status;
 232
 233     /** TODO: create a standalone method to initialize
 234      * the RedirectStateData for all the helpers.
 235      */
 236     RedirectStateData *r = new RedirectStateData(http->uri);
 237     r->handler = handler;
 238     r->data = cbdataReference(data);
 239
 240     static MemBuf requestExtras;
 241     requestExtras.reset();
 242     if (requestExtrasFmt)
 243         requestExtrasFmt->assemble(requestExtras, http->al, 0);
 244
 245     sz = snprintf(buf, MAX_REDIRECTOR_REQUEST_STRLEN, "%s%s%s\n",
 246                   r->orig_url.c_str(),
 247                   requestExtras.hasContent() ? " " : "",
 248                   requestExtras.hasContent() ? requestExtras.content() : "");
 249
 250     if ((sz<=0) || (sz>=MAX_REDIRECTOR_REQUEST_STRLEN)) {
 251         if (sz<=0) {
 252             status = Http::scInternalServerError;
 253             debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Can not build request to be passed to " << name << ". Request ABORTED.");
 254         } else {
 255             status = Http::scUriTooLong;
 256             debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Request passed to " << name << " exceeds MAX_REDIRECTOR_REQUEST_STRLEN (" << MAX_REDIRECTOR_REQUEST_STRLEN << "). Request ABORTED.");
 257         }
 258
 259         clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
 260         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 261         assert (repContext);
 262         Ip::Address tmpnoaddr;
 263         tmpnoaddr.setNoAddr();
 264         repContext->setReplyToError(ERR_GATEWAY_FAILURE, status,
 265                                     http->request->method, NULL,
 266                                     http->getConn() != NULL && http->getConn()->clientConnection != NULL ?
 267                                     http->getConn()->clientConnection->remote : tmpnoaddr,
 268                                     http->request,
 269                                     NULL,
 270 #if USE_AUTH
 271                                     http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
 272                                     http->getConn()->getAuth() : http->request->auth_user_request);
 273 #else
 274                                     NULL);
 275 #endif
 276
 277         node = (clientStreamNode *)http->client_stream.tail->data;
 278         clientStreamRead(node, http, node->readBuffer);
 279         return;
 280     }
 281
 282     debugs(61,6, HERE << "sending '" << buf << "' to the " << name << " helper");
 283     helperSubmit(hlp, buf, replyHandler, r);
 284 }
 285
 286 /**** PUBLIC FUNCTIONS ****/
 287
 288 void
 289 redirectStart(ClientHttpRequest * http, HLPCB * handler, void *data)
 290 {
 291     assert(http);
 292     assert(handler);
 293     debugs(61, 5, "redirectStart: '" << http->uri << "'");
 294
 295     if (Config.onoff.redirector_bypass && redirectors->queueFull()) {
 296         /* Skip redirector if the queue is full */
 297         ++redirectorBypassed;
 298         Helper::Reply bypassReply;
 299         bypassReply.result = Helper::Okay;
 300         bypassReply.notes.add("message","URL rewrite/redirect queue too long. Bypassed.");
 301         handler(data, bypassReply);
 302         return;
 303     }
 304
 305     constructHelperQuery("redirector", redirectors, redirectHandleReply, http, handler, data, redirectorExtrasFmt);
 306 }
 307
 308 /**
 309  * Handles the StoreID feature helper starting.
 310  * For now it cannot be done using the redirectStart method.
 311  */
 312 void
 313 storeIdStart(ClientHttpRequest * http, HLPCB * handler, void *data)
 314 {
 315     assert(http);
 316     assert(handler);
 317     debugs(61, 5, "storeIdStart: '" << http->uri << "'");
 318
 319     if (Config.onoff.store_id_bypass && storeIds->queueFull()) {
 320         /* Skip StoreID Helper if the queue is full */
 321         ++storeIdBypassed;
 322         Helper::Reply bypassReply;
 323
 324         bypassReply.result = Helper::Okay;
 325
 326         bypassReply.notes.add("message","StoreId helper queue too long. Bypassed.");
 327         handler(data, bypassReply);
 328         return;
 329     }
 330
 331     constructHelperQuery("storeId helper", storeIds, storeIdHandleReply, http, handler, data, storeIdExtrasFmt);
 332 }
 333
 334 void
 335 redirectInit(void)
 336 {
 337     static bool init = false;
 338
 339     if (!init) {
 340         Mgr::RegisterAction("redirector", "URL Redirector Stats", redirectStats, 0, 1);
 341         Mgr::RegisterAction("store_id", "StoreId helper Stats", storeIdStats, 0, 1);
 342     }
 343
 344     if (Config.Program.redirect) {
 345
 346         if (redirectors == NULL)
 347             redirectors = new helper("redirector");
 348
 349         redirectors->cmdline = Config.Program.redirect;
 350
 351         // BACKWARD COMPATIBILITY:
 352         // if redirectot_bypass is set then use queue_size=0 as default size
 353         if (Config.onoff.redirector_bypass && Config.redirectChildren.defaultQueueSize)
 354             Config.redirectChildren.queue_size = 0;
 355
 356         redirectors->childs.updateLimits(Config.redirectChildren);
 357
 358         redirectors->ipc_type = IPC_STREAM;
 359
 360         redirectors->timeout = Config.Timeout.urlRewrite;
 361
 362         redirectors->retryTimedOut = (Config.onUrlRewriteTimeout.action == toutActRetry);
 363         redirectors->retryBrokenHelper = true; // XXX: make this configurable ?
 364         redirectors->onTimedOutResponse.clear();
 365         if (Config.onUrlRewriteTimeout.action == toutActUseConfiguredResponse)
 366             redirectors->onTimedOutResponse.assign(Config.onUrlRewriteTimeout.response);
 367
 368         helperOpenServers(redirectors);
 369     }
 370
 371     if (Config.Program.store_id) {
 372
 373         if (storeIds == NULL)
 374             storeIds = new helper("store_id");
 375
 376         storeIds->cmdline = Config.Program.store_id;
 377
 378         // BACKWARD COMPATIBILITY:
 379         // if store_id_bypass is set then use queue_size=0 as default size
 380         if (Config.onoff.store_id_bypass && Config.storeIdChildren.defaultQueueSize)
 381             Config.storeIdChildren.queue_size = 0;
 382
 383         storeIds->childs.updateLimits(Config.storeIdChildren);
 384
 385         storeIds->ipc_type = IPC_STREAM;
 386
 387         storeIds->retryBrokenHelper = true; // XXX: make this configurable ?
 388
 389         helperOpenServers(storeIds);
 390     }
 391
 392     if (Config.redirector_extras) {
 393         redirectorExtrasFmt = new ::Format::Format("url_rewrite_extras");
 394         (void)redirectorExtrasFmt->parse(Config.redirector_extras);
 395     }
 396
 397     if (Config.storeId_extras) {
 398         storeIdExtrasFmt = new ::Format::Format("store_id_extras");
 399         (void)storeIdExtrasFmt->parse(Config.storeId_extras);
 400     }
 401
 402     init = true;
 403 }
 404
 405 void
 406 redirectShutdown(void)
 407 {
 408     /** FIXME: Temporary unified helpers Shutdown
 409      * When and if needed for more helpers a separated shutdown
 410      * method will be added for each of them.
 411      */
 412     if (!storeIds && !redirectors)
 413         return;
 414
 415     if (redirectors)
 416         helperShutdown(redirectors);
 417
 418     if (storeIds)
 419         helperShutdown(storeIds);
 420
 421     if (!shutting_down)
 422         return;
 423
 424     delete redirectors;
 425     redirectors = NULL;
 426
 427     delete storeIds;
 428     storeIds = NULL;
 429
 430     delete redirectorExtrasFmt;
 431     redirectorExtrasFmt = NULL;
 432
 433     delete storeIdExtrasFmt;
 434     storeIdExtrasFmt = NULL;
 435 }
 436