2 * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
9 /* DEBUG: section 61 Redirector */
12 #include "acl/Checklist.h"
14 #include "client_side.h"
15 #include "client_side_reply.h"
16 #include "client_side_request.h"
17 #include "comm/Connection.h"
19 #include "format/Format.h"
22 #include "helper/Reply.h"
23 #include "http/Stream.h"
24 #include "HttpRequest.h"
25 #include "mgr/Registration.h"
28 #include "sbuf/SBuf.h"
29 #include "SquidConfig.h"
32 #include "auth/UserRequest.h"
35 #include "ssl/support.h"
38 /// url maximum length + extra information passed to redirector
39 #define MAX_REDIRECTOR_REQUEST_STRLEN (MAX_URL + 1024)
41 class RedirectStateData
43 CBDATA_CLASS(RedirectStateData
);
46 explicit RedirectStateData(const char *url
);
55 static HLPCB redirectHandleReply
;
56 static HLPCB storeIdHandleReply
;
57 static helper
*redirectors
= NULL
;
58 static helper
*storeIds
= NULL
;
59 static OBJH redirectStats
;
60 static OBJH storeIdStats
;
61 static int redirectorBypassed
= 0;
62 static int storeIdBypassed
= 0;
63 static Format::Format
*redirectorExtrasFmt
= NULL
;
64 static Format::Format
*storeIdExtrasFmt
= NULL
;
66 CBDATA_CLASS_INIT(RedirectStateData
);
68 RedirectStateData::RedirectStateData(const char *url
) :
75 RedirectStateData::~RedirectStateData()
80 redirectHandleReply(void *data
, const Helper::Reply
&reply
)
82 RedirectStateData
*r
= static_cast<RedirectStateData
*>(data
);
83 debugs(61, 5, HERE
<< "reply=" << reply
);
85 // XXX: This function is now kept only to check for and display the garbage use-case
86 // and to map the old helper response format(s) into new format result code and key=value pairs
87 // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format
89 if (reply
.result
== Helper::Unknown
) {
90 // BACKWARD COMPATIBILITY 2012-06-15:
91 // Some nasty old helpers send back the entire input line including extra format keys.
92 // This is especially bad for simple perl search-replace filter scripts.
94 // * trim all but the first word off the response.
95 // * warn once every 50 responses that this will stop being fixed-up soon.
97 if (reply
.other().hasContent()) {
98 const char * res
= reply
.other().content();
100 if (const char *t
= strchr(res
, ' ')) {
102 debugs(61, (!(warn
++%50)? DBG_CRITICAL
:2), "UPGRADE WARNING: URL rewriter reponded with garbage '" << t
<<
103 "'. Future Squid will treat this as part of the URL.");
106 replySize
= reply
.other().contentSize();
108 // if we still have anything in other() after all that
109 // parse it into status=, url= and rewrite-url= keys
112 replyBuffer
.init(replySize
, replySize
);
113 replyBuffer
.append(reply
.other().content(), reply
.other().contentSize());
114 char * result
= replyBuffer
.content();
116 Helper::Reply newReply
;
117 // BACKWARD COMPATIBILITY 2012-06-15:
118 // We got Helper::Unknown reply result but new
119 // RedirectStateData handlers require Helper::Okay,
120 // else will drop the helper reply
121 newReply
.result
= Helper::Okay
;
122 newReply
.notes
.append(&reply
.notes
);
124 // check and parse for obsoleted Squid-2 urlgroup feature
125 if (*result
== '!') {
126 static int urlgroupWarning
= 0;
127 if (!urlgroupWarning
++)
128 debugs(85, DBG_IMPORTANT
, "UPGRADE WARNING: URL rewriter using obsolete Squid-2 urlgroup feature needs updating.");
129 if (char *t
= strchr(result
+1, '!')) {
131 newReply
.notes
.add("urlgroup", result
+1);
136 const Http::StatusCode status
= static_cast<Http::StatusCode
>(atoi(result
));
138 if (status
== Http::scMovedPermanently
139 || status
== Http::scFound
140 || status
== Http::scSeeOther
141 || status
== Http::scPermanentRedirect
142 || status
== Http::scTemporaryRedirect
) {
144 if (const char *t
= strchr(result
, ':')) {
146 snprintf(statusBuf
, sizeof(statusBuf
),"%3u",status
);
147 newReply
.notes
.add("status", statusBuf
);
149 // TODO: validate the URL produced here is RFC 2616 compliant URI
150 newReply
.notes
.add("url", t
);
152 debugs(85, DBG_CRITICAL
, "ERROR: URL-rewrite produces invalid " << status
<< " redirect Location: " << result
);
155 // status code is not a redirect code (or does not exist)
156 // treat as a re-write URL request
157 // TODO: validate the URL produced here is RFC 2616 compliant URI
159 newReply
.notes
.add("rewrite-url", result
);
163 if (cbdataReferenceValidDone(r
->data
, &cbdata
))
164 r
->handler(cbdata
, newReply
);
173 if (cbdataReferenceValidDone(r
->data
, &cbdata
))
174 r
->handler(cbdata
, reply
);
180 storeIdHandleReply(void *data
, const Helper::Reply
&reply
)
182 RedirectStateData
*r
= static_cast<RedirectStateData
*>(data
);
183 debugs(61, 5,"StoreId helper: reply=" << reply
);
185 // XXX: This function is now kept only to check for and display the garbage use-case
186 // and to map the old helper response format(s) into new format result code and key=value pairs
187 // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format
189 if (cbdataReferenceValidDone(r
->data
, &cbdata
))
190 r
->handler(cbdata
, reply
);
196 redirectStats(StoreEntry
* sentry
)
198 if (redirectors
== NULL
) {
199 storeAppendPrintf(sentry
, "No redirectors defined\n");
203 redirectors
->packStatsInto(sentry
, "Redirector Statistics");
205 if (Config
.onoff
.redirector_bypass
)
206 storeAppendPrintf(sentry
, "\nNumber of requests bypassed "
207 "because all redirectors were busy: %d\n", redirectorBypassed
);
211 storeIdStats(StoreEntry
* sentry
)
213 if (storeIds
== NULL
) {
214 storeAppendPrintf(sentry
, "No StoreId helpers defined\n");
218 storeIds
->packStatsInto(sentry
, "StoreId helper Statistics");
220 if (Config
.onoff
.store_id_bypass
)
221 storeAppendPrintf(sentry
, "\nNumber of requests bypassed "
222 "because all StoreId helpers were busy: %d\n", storeIdBypassed
);
226 constructHelperQuery(const char *name
, helper
*hlp
, HLPCB
*replyHandler
, ClientHttpRequest
* http
, HLPCB
*handler
, void *data
, Format::Format
*requestExtrasFmt
)
228 char buf
[MAX_REDIRECTOR_REQUEST_STRLEN
];
230 Http::StatusCode status
;
232 /** TODO: create a standalone method to initialize
233 * the RedirectStateData for all the helpers.
235 RedirectStateData
*r
= new RedirectStateData(http
->uri
);
236 r
->handler
= handler
;
237 r
->data
= cbdataReference(data
);
239 static MemBuf requestExtras
;
240 requestExtras
.reset();
241 if (requestExtrasFmt
)
242 requestExtrasFmt
->assemble(requestExtras
, http
->al
, 0);
244 sz
= snprintf(buf
, MAX_REDIRECTOR_REQUEST_STRLEN
, "%s%s%s\n",
246 requestExtras
.hasContent() ? " " : "",
247 requestExtras
.hasContent() ? requestExtras
.content() : "");
249 if ((sz
<=0) || (sz
>=MAX_REDIRECTOR_REQUEST_STRLEN
)) {
251 status
= Http::scInternalServerError
;
252 debugs(61, DBG_CRITICAL
, "ERROR: Gateway Failure. Can not build request to be passed to " << name
<< ". Request ABORTED.");
254 status
= Http::scUriTooLong
;
255 debugs(61, DBG_CRITICAL
, "ERROR: Gateway Failure. Request passed to " << name
<< " exceeds MAX_REDIRECTOR_REQUEST_STRLEN (" << MAX_REDIRECTOR_REQUEST_STRLEN
<< "). Request ABORTED.");
258 clientStreamNode
*node
= (clientStreamNode
*)http
->client_stream
.tail
->prev
->data
;
259 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
261 Ip::Address tmpnoaddr
;
262 tmpnoaddr
.setNoAddr();
263 repContext
->setReplyToError(ERR_GATEWAY_FAILURE
, status
,
264 http
->request
->method
, NULL
,
265 http
->getConn() != NULL
&& http
->getConn()->clientConnection
!= NULL
?
266 http
->getConn()->clientConnection
->remote
: tmpnoaddr
,
270 http
->getConn() != NULL
&& http
->getConn()->getAuth() != NULL
?
271 http
->getConn()->getAuth() : http
->request
->auth_user_request
);
276 node
= (clientStreamNode
*)http
->client_stream
.tail
->data
;
277 clientStreamRead(node
, http
, node
->readBuffer
);
281 debugs(61,6, HERE
<< "sending '" << buf
<< "' to the " << name
<< " helper");
282 helperSubmit(hlp
, buf
, replyHandler
, r
);
285 /**** PUBLIC FUNCTIONS ****/
288 redirectStart(ClientHttpRequest
* http
, HLPCB
* handler
, void *data
)
292 debugs(61, 5, "redirectStart: '" << http
->uri
<< "'");
294 // TODO: Deprecate Config.onoff.redirector_bypass in favor of either
295 // onPersistentOverload or a new onOverload option that applies to all helpers.
296 if (Config
.onoff
.redirector_bypass
&& redirectors
->willOverload()) {
297 /* Skip redirector if the queue is full */
298 ++redirectorBypassed
;
299 Helper::Reply bypassReply
;
300 bypassReply
.result
= Helper::Okay
;
301 bypassReply
.notes
.add("message","URL rewrite/redirect queue too long. Bypassed.");
302 handler(data
, bypassReply
);
306 constructHelperQuery("redirector", redirectors
, redirectHandleReply
, http
, handler
, data
, redirectorExtrasFmt
);
310 * Handles the StoreID feature helper starting.
311 * For now it cannot be done using the redirectStart method.
314 storeIdStart(ClientHttpRequest
* http
, HLPCB
* handler
, void *data
)
318 debugs(61, 5, "storeIdStart: '" << http
->uri
<< "'");
320 if (Config
.onoff
.store_id_bypass
&& storeIds
->willOverload()) {
321 /* Skip StoreID Helper if the queue is full */
323 Helper::Reply bypassReply
;
325 bypassReply
.result
= Helper::Okay
;
327 bypassReply
.notes
.add("message","StoreId helper queue too long. Bypassed.");
328 handler(data
, bypassReply
);
332 constructHelperQuery("storeId helper", storeIds
, storeIdHandleReply
, http
, handler
, data
, storeIdExtrasFmt
);
338 static bool init
= false;
341 Mgr::RegisterAction("redirector", "URL Redirector Stats", redirectStats
, 0, 1);
342 Mgr::RegisterAction("store_id", "StoreId helper Stats", storeIdStats
, 0, 1);
345 if (Config
.Program
.redirect
) {
347 if (redirectors
== NULL
)
348 redirectors
= new helper("redirector");
350 redirectors
->cmdline
= Config
.Program
.redirect
;
352 // BACKWARD COMPATIBILITY:
353 // if redirectot_bypass is set then use queue_size=0 as default size
354 if (Config
.onoff
.redirector_bypass
&& Config
.redirectChildren
.defaultQueueSize
)
355 Config
.redirectChildren
.queue_size
= 0;
357 redirectors
->childs
.updateLimits(Config
.redirectChildren
);
359 redirectors
->ipc_type
= IPC_STREAM
;
361 redirectors
->timeout
= Config
.Timeout
.urlRewrite
;
363 redirectors
->retryTimedOut
= (Config
.onUrlRewriteTimeout
.action
== toutActRetry
);
364 redirectors
->retryBrokenHelper
= true; // XXX: make this configurable ?
365 redirectors
->onTimedOutResponse
.clear();
366 if (Config
.onUrlRewriteTimeout
.action
== toutActUseConfiguredResponse
)
367 redirectors
->onTimedOutResponse
.assign(Config
.onUrlRewriteTimeout
.response
);
369 helperOpenServers(redirectors
);
372 if (Config
.Program
.store_id
) {
374 if (storeIds
== NULL
)
375 storeIds
= new helper("store_id");
377 storeIds
->cmdline
= Config
.Program
.store_id
;
379 // BACKWARD COMPATIBILITY:
380 // if store_id_bypass is set then use queue_size=0 as default size
381 if (Config
.onoff
.store_id_bypass
&& Config
.storeIdChildren
.defaultQueueSize
)
382 Config
.storeIdChildren
.queue_size
= 0;
384 storeIds
->childs
.updateLimits(Config
.storeIdChildren
);
386 storeIds
->ipc_type
= IPC_STREAM
;
388 storeIds
->retryBrokenHelper
= true; // XXX: make this configurable ?
390 helperOpenServers(storeIds
);
393 if (Config
.redirector_extras
) {
394 delete redirectorExtrasFmt
;
395 redirectorExtrasFmt
= new ::Format::Format("url_rewrite_extras");
396 (void)redirectorExtrasFmt
->parse(Config
.redirector_extras
);
399 if (Config
.storeId_extras
) {
400 delete storeIdExtrasFmt
;
401 storeIdExtrasFmt
= new ::Format::Format("store_id_extras");
402 (void)storeIdExtrasFmt
->parse(Config
.storeId_extras
);
409 redirectShutdown(void)
411 /** TODO: Temporary unified helpers Shutdown
412 * When and if needed for more helpers a separated shutdown
413 * method will be added for each of them.
416 helperShutdown(redirectors
);
419 helperShutdown(storeIds
);
430 delete redirectorExtrasFmt
;
431 redirectorExtrasFmt
= NULL
;
433 delete storeIdExtrasFmt
;
434 storeIdExtrasFmt
= NULL
;
438 redirectReconfigure()