2 * Copyright (C) 1996-2023 The Squid Software Foundation and contributors
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
9 /* DEBUG: section 61 Redirector */
12 #include "acl/Checklist.h"
14 #include "client_side.h"
15 #include "client_side_reply.h"
16 #include "client_side_request.h"
17 #include "comm/Connection.h"
19 #include "format/Format.h"
22 #include "helper/Reply.h"
23 #include "http/Stream.h"
24 #include "HttpRequest.h"
25 #include "mgr/Registration.h"
28 #include "sbuf/SBuf.h"
29 #include "SquidConfig.h"
32 #include "auth/UserRequest.h"
35 #include "ssl/support.h"
38 /// url maximum length + extra information passed to redirector
39 #define MAX_REDIRECTOR_REQUEST_STRLEN (MAX_URL + 1024)
41 class RedirectStateData
43 CBDATA_CLASS(RedirectStateData
);
46 explicit RedirectStateData(const char *url
);
55 static HLPCB redirectHandleReply
;
56 static HLPCB storeIdHandleReply
;
57 static Helper::Client::Pointer redirectors
;
58 static Helper::Client::Pointer storeIds
;
59 static OBJH redirectStats
;
60 static OBJH storeIdStats
;
61 static int redirectorBypassed
= 0;
62 static int storeIdBypassed
= 0;
63 static Format::Format
*redirectorExtrasFmt
= nullptr;
64 static Format::Format
*storeIdExtrasFmt
= nullptr;
66 CBDATA_CLASS_INIT(RedirectStateData
);
68 RedirectStateData::RedirectStateData(const char *url
) :
75 RedirectStateData::~RedirectStateData()
80 redirectHandleReply(void *data
, const Helper::Reply
&reply
)
82 RedirectStateData
*r
= static_cast<RedirectStateData
*>(data
);
83 debugs(61, 5, "reply=" << reply
);
85 // XXX: This function is now kept only to check for and display the garbage use-case
86 // and to map the old helper response format(s) into new format result code and key=value pairs
87 // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format
89 if (reply
.result
== Helper::Unknown
) {
90 // BACKWARD COMPATIBILITY 2012-06-15:
91 // Some nasty old helpers send back the entire input line including extra format keys.
92 // This is especially bad for simple perl search-replace filter scripts.
94 // * trim all but the first word off the response.
95 // * warn once every 50 responses that this will stop being fixed-up soon.
97 if (reply
.other().hasContent()) {
98 const char * res
= reply
.other().content();
100 if (const char *t
= strchr(res
, ' ')) {
102 debugs(61, (!(warn
++%50)? DBG_CRITICAL
:2), "WARNING: UPGRADE: URL rewriter reponded with garbage '" << t
<<
103 "'. Future Squid will treat this as part of the URL.");
106 replySize
= reply
.other().contentSize();
108 // if we still have anything in other() after all that
109 // parse it into status=, url= and rewrite-url= keys
112 replyBuffer
.init(replySize
, replySize
);
113 replyBuffer
.append(reply
.other().content(), reply
.other().contentSize());
114 char * result
= replyBuffer
.content();
116 Helper::Reply newReply
;
117 // BACKWARD COMPATIBILITY 2012-06-15:
118 // We got Helper::Unknown reply result but new
119 // RedirectStateData handlers require Helper::Okay,
120 // else will drop the helper reply
121 newReply
.result
= Helper::Okay
;
122 newReply
.notes
.append(&reply
.notes
);
124 // check and parse for obsoleted Squid-2 urlgroup feature
125 if (*result
== '!') {
126 static int urlgroupWarning
= 0;
127 if (!urlgroupWarning
++)
128 debugs(85, DBG_IMPORTANT
, "WARNING: UPGRADE: URL rewriter using obsolete Squid-2 urlgroup feature needs updating.");
129 if (char *t
= strchr(result
+1, '!')) {
131 newReply
.notes
.add("urlgroup", result
+1);
136 const Http::StatusCode status
= static_cast<Http::StatusCode
>(atoi(result
));
138 if (status
== Http::scMovedPermanently
139 || status
== Http::scFound
140 || status
== Http::scSeeOther
141 || status
== Http::scPermanentRedirect
142 || status
== Http::scTemporaryRedirect
) {
144 if (const char *t
= strchr(result
, ':')) {
146 snprintf(statusBuf
, sizeof(statusBuf
),"%3u",status
);
147 newReply
.notes
.add("status", statusBuf
);
149 // TODO: validate the URL produced here is RFC 2616 compliant URI
150 newReply
.notes
.add("url", t
);
152 debugs(85, DBG_CRITICAL
, "ERROR: URL-rewrite produces invalid " << status
<< " redirect Location: " << result
);
155 // status code is not a redirect code (or does not exist)
156 // treat as a re-write URL request
157 // TODO: validate the URL produced here is RFC 2616 compliant URI
159 newReply
.notes
.add("rewrite-url", result
);
163 if (cbdataReferenceValidDone(r
->data
, &cbdata
))
164 r
->handler(cbdata
, newReply
);
173 if (cbdataReferenceValidDone(r
->data
, &cbdata
))
174 r
->handler(cbdata
, reply
);
180 storeIdHandleReply(void *data
, const Helper::Reply
&reply
)
182 RedirectStateData
*r
= static_cast<RedirectStateData
*>(data
);
183 debugs(61, 5,"StoreId helper: reply=" << reply
);
185 // XXX: This function is now kept only to check for and display the garbage use-case
186 // and to map the old helper response format(s) into new format result code and key=value pairs
187 // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format
189 if (cbdataReferenceValidDone(r
->data
, &cbdata
))
190 r
->handler(cbdata
, reply
);
196 redirectStats(StoreEntry
* sentry
)
198 if (redirectors
== nullptr) {
199 storeAppendPrintf(sentry
, "No redirectors defined\n");
203 redirectors
->packStatsInto(sentry
, "Redirector Statistics");
205 if (Config
.onoff
.redirector_bypass
)
206 storeAppendPrintf(sentry
, "\nNumber of requests bypassed "
207 "because all redirectors were busy: %d\n", redirectorBypassed
);
211 storeIdStats(StoreEntry
* sentry
)
213 if (storeIds
== nullptr) {
214 storeAppendPrintf(sentry
, "No StoreId helpers defined\n");
218 storeIds
->packStatsInto(sentry
, "StoreId helper Statistics");
220 if (Config
.onoff
.store_id_bypass
)
221 storeAppendPrintf(sentry
, "\nNumber of requests bypassed "
222 "because all StoreId helpers were busy: %d\n", storeIdBypassed
);
226 constructHelperQuery(const char * const name
, const Helper::Client::Pointer
&hlp
, HLPCB
* const replyHandler
, ClientHttpRequest
* const http
, HLPCB
* const handler
, void * const data
, Format::Format
* const requestExtrasFmt
)
228 char buf
[MAX_REDIRECTOR_REQUEST_STRLEN
];
230 Http::StatusCode status
;
232 /** TODO: create a standalone method to initialize
233 * the RedirectStateData for all the helpers.
235 RedirectStateData
*r
= new RedirectStateData(http
->uri
);
236 r
->handler
= handler
;
237 r
->data
= cbdataReference(data
);
239 static MemBuf requestExtras
;
240 requestExtras
.reset();
241 if (requestExtrasFmt
)
242 requestExtrasFmt
->assemble(requestExtras
, http
->al
, 0);
244 sz
= snprintf(buf
, MAX_REDIRECTOR_REQUEST_STRLEN
, "%s%s%s\n",
246 requestExtras
.hasContent() ? " " : "",
247 requestExtras
.hasContent() ? requestExtras
.content() : "");
249 if ((sz
<=0) || (sz
>=MAX_REDIRECTOR_REQUEST_STRLEN
)) {
251 status
= Http::scInternalServerError
;
252 debugs(61, DBG_CRITICAL
, "ERROR: Gateway Failure. Can not build request to be passed to " << name
<< ". Request ABORTED.");
254 status
= Http::scUriTooLong
;
255 debugs(61, DBG_CRITICAL
, "ERROR: Gateway Failure. Request passed to " << name
<< " exceeds MAX_REDIRECTOR_REQUEST_STRLEN (" << MAX_REDIRECTOR_REQUEST_STRLEN
<< "). Request ABORTED.");
258 clientStreamNode
*node
= (clientStreamNode
*)http
->client_stream
.tail
->prev
->data
;
259 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
261 repContext
->setReplyToError(ERR_GATEWAY_FAILURE
, status
,
267 http
->getConn() != nullptr && http
->getConn()->getAuth() != nullptr ?
268 http
->getConn()->getAuth() : http
->request
->auth_user_request
);
273 node
= (clientStreamNode
*)http
->client_stream
.tail
->data
;
274 clientStreamRead(node
, http
, node
->readBuffer
);
278 debugs(61,6, "sending '" << buf
<< "' to the " << name
<< " helper");
279 helperSubmit(hlp
, buf
, replyHandler
, r
);
282 /**** PUBLIC FUNCTIONS ****/
285 redirectStart(ClientHttpRequest
* http
, HLPCB
* handler
, void *data
)
289 debugs(61, 5, "redirectStart: '" << http
->uri
<< "'");
291 // TODO: Deprecate Config.onoff.redirector_bypass in favor of either
292 // onPersistentOverload or a new onOverload option that applies to all helpers.
293 if (Config
.onoff
.redirector_bypass
&& redirectors
->willOverload()) {
294 /* Skip redirector if the queue is full */
295 ++redirectorBypassed
;
296 Helper::Reply bypassReply
;
297 bypassReply
.result
= Helper::Okay
;
298 bypassReply
.notes
.add("message","URL rewrite/redirect queue too long. Bypassed.");
299 handler(data
, bypassReply
);
303 constructHelperQuery("redirector", redirectors
, redirectHandleReply
, http
, handler
, data
, redirectorExtrasFmt
);
307 * Handles the StoreID feature helper starting.
308 * For now it cannot be done using the redirectStart method.
311 storeIdStart(ClientHttpRequest
* http
, HLPCB
* handler
, void *data
)
315 debugs(61, 5, "storeIdStart: '" << http
->uri
<< "'");
317 if (Config
.onoff
.store_id_bypass
&& storeIds
->willOverload()) {
318 /* Skip StoreID Helper if the queue is full */
320 Helper::Reply bypassReply
;
322 bypassReply
.result
= Helper::Okay
;
324 bypassReply
.notes
.add("message","StoreId helper queue too long. Bypassed.");
325 handler(data
, bypassReply
);
329 constructHelperQuery("storeId helper", storeIds
, storeIdHandleReply
, http
, handler
, data
, storeIdExtrasFmt
);
335 static bool init
= false;
338 Mgr::RegisterAction("redirector", "URL Redirector Stats", redirectStats
, 0, 1);
339 Mgr::RegisterAction("store_id", "StoreId helper Stats", storeIdStats
, 0, 1);
342 if (Config
.Program
.redirect
) {
344 if (redirectors
== nullptr)
345 redirectors
= Helper::Client::Make("redirector");
347 redirectors
->cmdline
= Config
.Program
.redirect
;
349 // BACKWARD COMPATIBILITY:
350 // if redirectot_bypass is set then use queue_size=0 as default size
351 if (Config
.onoff
.redirector_bypass
&& Config
.redirectChildren
.defaultQueueSize
)
352 Config
.redirectChildren
.queue_size
= 0;
354 redirectors
->childs
.updateLimits(Config
.redirectChildren
);
356 redirectors
->ipc_type
= IPC_STREAM
;
358 redirectors
->timeout
= Config
.Timeout
.urlRewrite
;
360 redirectors
->retryTimedOut
= (Config
.onUrlRewriteTimeout
.action
== toutActRetry
);
361 redirectors
->retryBrokenHelper
= true; // XXX: make this configurable ?
362 redirectors
->onTimedOutResponse
.clear();
363 if (Config
.onUrlRewriteTimeout
.action
== toutActUseConfiguredResponse
)
364 redirectors
->onTimedOutResponse
.assign(Config
.onUrlRewriteTimeout
.response
);
366 redirectors
->openSessions();
369 if (Config
.Program
.store_id
) {
371 if (storeIds
== nullptr)
372 storeIds
= Helper::Client::Make("store_id");
374 storeIds
->cmdline
= Config
.Program
.store_id
;
376 // BACKWARD COMPATIBILITY:
377 // if store_id_bypass is set then use queue_size=0 as default size
378 if (Config
.onoff
.store_id_bypass
&& Config
.storeIdChildren
.defaultQueueSize
)
379 Config
.storeIdChildren
.queue_size
= 0;
381 storeIds
->childs
.updateLimits(Config
.storeIdChildren
);
383 storeIds
->ipc_type
= IPC_STREAM
;
385 storeIds
->retryBrokenHelper
= true; // XXX: make this configurable ?
387 storeIds
->openSessions();
390 if (Config
.redirector_extras
) {
391 delete redirectorExtrasFmt
;
392 redirectorExtrasFmt
= new ::Format::Format("url_rewrite_extras");
393 (void)redirectorExtrasFmt
->parse(Config
.redirector_extras
);
396 if (Config
.storeId_extras
) {
397 delete storeIdExtrasFmt
;
398 storeIdExtrasFmt
= new ::Format::Format("store_id_extras");
399 (void)storeIdExtrasFmt
->parse(Config
.storeId_extras
);
406 redirectShutdown(void)
408 /** TODO: Temporary unified helpers Shutdown
409 * When and if needed for more helpers a separated shutdown
410 * method will be added for each of them.
413 helperShutdown(redirectors
);
416 helperShutdown(storeIds
);
421 redirectors
= nullptr;
425 delete redirectorExtrasFmt
;
426 redirectorExtrasFmt
= nullptr;
428 delete storeIdExtrasFmt
;
429 storeIdExtrasFmt
= nullptr;
433 redirectReconfigure()