src/helper.h

   1 /*
   2  * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 84    Helper process maintenance */
  10
  11 #ifndef SQUID_HELPER_H
  12 #define SQUID_HELPER_H
  13
  14 #include "base/AsyncCall.h"
  15 #include "base/InstanceId.h"
  16 #include "cbdata.h"
  17 #include "comm/forward.h"
  18 #include "dlink.h"
  19 #include "helper/ChildConfig.h"
  20 #include "helper/forward.h"
  21 #include "helper/Reply.h"
  22 #include "helper/Request.h"
  23 #include "ip/Address.h"
  24 #include "sbuf/SBuf.h"
  25
  26 #include <list>
  27 #include <map>
  28 #include <queue>
  29
  30 class Packable;
  31 class wordlist;
  32
  33 namespace Helper
  34 {
  35 /// Holds the  required data to serve a helper request.
  36 class Xaction {
  37     MEMPROXY_CLASS(Helper::Xaction);
  38 public:
  39     Xaction(HLPCB *c, void *d, const char *b): request(c, d, b) {}
  40     Helper::Request request;
  41     Helper::Reply reply;
  42 };
  43 }
  44
  45 /**
  46  * Managers a set of individual helper processes with a common queue of requests.
  47  *
  48  * With respect to load, a helper goes through these states (roughly):
  49  *   idle:   no processes are working on requests (and no requests are queued);
  50  *   normal: some, but not all processes are working (and no requests are queued);
  51  *   busy:   all processes are working (and some requests are possibly queued);
  52  *   overloaded: a busy helper with more than queue-size requests in the queue.
  53  *
  54  * A busy helper queues new requests and issues a WARNING every 10 minutes or so.
  55  * An overloaded helper either drops new requests or keeps queuing them, depending on
  56  *   whether the caller can handle dropped requests (trySubmit vs helperSubmit APIs).
  57  * If an overloaded helper has been overloaded for 3+ minutes, an attempt to use
  58  *   it results in on-persistent-overload action, which may kill worker.
  59  */
  60 class helper
  61 {
  62     CBDATA_CLASS(helper);
  63
  64 public:
  65     inline helper(const char *name) :
  66         cmdline(NULL),
  67         id_name(name),
  68         ipc_type(0),
  69         droppedRequests(0),
  70         overloadStart(0),
  71         last_queue_warn(0),
  72         last_restart(0),
  73         timeout(0),
  74         retryTimedOut(false),
  75         retryBrokenHelper(false),
  76         eom('\n') {
  77         memset(&stats, 0, sizeof(stats));
  78     }
  79     ~helper();
  80
  81     /// \returns next request in the queue, or nil.
  82     Helper::Xaction *nextRequest();
  83
  84     /// If possible, submit request. Otherwise, either kill Squid or return false.
  85     bool trySubmit(const char *buf, HLPCB * callback, void *data);
  86
  87     /// Submits a request to the helper or add it to the queue if none of
  88     /// the servers is available.
  89     void submitRequest(Helper::Xaction *r);
  90
  91     /// Dump some stats about the helper state to a Packable object
  92     void packStatsInto(Packable *p, const char *label = NULL) const;
  93     /// whether the helper will be in "overloaded" state after one more request
  94     /// already overloaded helpers return true
  95     bool willOverload() const;
  96
  97 public:
  98     wordlist *cmdline;
  99     dlink_list servers;
 100     std::queue<Helper::Xaction *> queue;
 101     const char *id_name;
 102     Helper::ChildConfig childs;    ///< Configuration settings for number running.
 103     int ipc_type;
 104     Ip::Address addr;
 105     unsigned int droppedRequests; ///< requests not sent during helper overload
 106     time_t overloadStart; ///< when the helper became overloaded (zero if it is not)
 107     time_t last_queue_warn;
 108     time_t last_restart;
 109     time_t timeout; ///< Requests timeout
 110     bool retryTimedOut; ///< Whether the timed-out requests must retried
 111     bool retryBrokenHelper; ///< Whether the requests must retried on BH replies
 112     SBuf onTimedOutResponse; ///< The response to use when helper response timedout
 113     char eom;   ///< The char which marks the end of (response) message, normally '\n'
 114
 115     struct _stats {
 116         int requests;
 117         int replies;
 118         int timedout;
 119         int queue_size;
 120         int avg_svc_time;
 121     } stats;
 122
 123 protected:
 124     friend void helperSubmit(helper * hlp, const char *buf, HLPCB * callback, void *data);
 125     bool queueFull() const;
 126     bool overloaded() const;
 127     void syncQueueStats();
 128     bool prepSubmit();
 129     void submit(const char *buf, HLPCB * callback, void *data);
 130 };
 131
 132 class statefulhelper : public helper
 133 {
 134     CBDATA_CLASS(statefulhelper);
 135
 136 public:
 137     inline statefulhelper(const char *name) : helper(name) {}
 138     inline ~statefulhelper() {}
 139
 140 private:
 141     friend void helperStatefulSubmit(statefulhelper * hlp, const char *buf, HLPCB * callback, void *data, helper_stateful_server * lastserver);
 142     void submit(const char *buf, HLPCB * callback, void *data, helper_stateful_server *lastserver);
 143     bool trySubmit(const char *buf, HLPCB * callback, void *data, helper_stateful_server *lastserver);
 144 };
 145
 146 /**
 147  * Fields shared between stateless and stateful helper servers.
 148  */
 149 class HelperServerBase
 150 {
 151 public:
 152     /** Closes pipes to the helper safely.
 153      * Handles the case where the read and write pipes are the same FD.
 154      *
 155      * \param name displayed for the helper being shutdown if logging an error
 156      */
 157     void closePipesSafely(const char *name);
 158
 159     /** Closes the reading pipe.
 160      * If the read and write sockets are the same the write pipe will
 161      * also be closed. Otherwise its left open for later handling.
 162      *
 163      * \param name displayed for the helper being shutdown if logging an error
 164      */
 165     void closeWritePipeSafely(const char *name);
 166
 167 public:
 168     /// Helper program identifier; does not change when contents do,
 169     ///   including during assignment
 170     const InstanceId<HelperServerBase> index;
 171     int pid;
 172     Ip::Address addr;
 173     Comm::ConnectionPointer readPipe;
 174     Comm::ConnectionPointer writePipe;
 175     void *hIpc;
 176
 177     char *rbuf;
 178     size_t rbuf_sz;
 179     size_t roffset;
 180
 181     struct timeval dispatch_time;
 182     struct timeval answer_time;
 183
 184     dlink_node link;
 185
 186     struct _helper_flags {
 187         bool writing;
 188         bool closing;
 189         bool shutdown;
 190         bool reserved;
 191     } flags;
 192
 193     typedef std::list<Helper::Xaction *> Requests;
 194     Requests requests; ///< requests in order of submission/expiration
 195
 196     struct {
 197         uint64_t uses;     //< requests sent to this helper
 198         uint64_t replies;  //< replies received from this helper
 199         uint64_t pending;  //< queued lookups waiting to be sent to this helper
 200         uint64_t releases; //< times release() has been called on this helper (if stateful)
 201         uint64_t timedout; //< requests which timed-out
 202     } stats;
 203     void initStats();
 204 };
 205
 206 class MemBuf;
 207 class CommTimeoutCbParams;
 208
 209 class helper_server : public HelperServerBase
 210 {
 211     CBDATA_CLASS(helper_server);
 212
 213 public:
 214     uint64_t nextRequestId;
 215
 216     MemBuf *wqueue;
 217     MemBuf *writebuf;
 218
 219     helper *parent;
 220
 221     /// The helper request Xaction object for the current reply .
 222     /// A helper reply may be distributed to more than one of the retrieved
 223     /// packets from helper. This member stores the Xaction object as long as
 224     /// the end-of-message for current reply is not retrieved.
 225     Helper::Xaction *replyXaction;
 226
 227     /// Whether to ignore current message, because it is timed-out or other reason
 228     bool ignoreToEom;
 229
 230     // STL says storing std::list iterators is safe when changing the list
 231     typedef std::map<uint64_t, Requests::iterator> RequestIndex;
 232     RequestIndex requestsIndex; ///< maps request IDs to requests
 233
 234     /// Search in queue for the request with requestId, return the related
 235     /// Xaction object and remove it from queue.
 236     /// If concurrency is disabled then the requestId is ignored and the
 237     /// Xaction of the next request in queue is retrieved.
 238     Helper::Xaction *popRequest(int requestId);
 239
 240     /// Run over the active requests lists and forces a retry, or timedout reply
 241     /// or the configured "on timeout response" for timedout requests.
 242     void checkForTimedOutRequests(bool const retry);
 243
 244     /// Read timeout handler
 245     static void requestTimeout(const CommTimeoutCbParams &io);
 246 };
 247
 248 class helper_stateful_server : public HelperServerBase
 249 {
 250     CBDATA_CLASS(helper_stateful_server);
 251
 252 public:
 253     statefulhelper *parent;
 254 };
 255
 256 /* helper.c */
 257 void helperOpenServers(helper * hlp);
 258 void helperStatefulOpenServers(statefulhelper * hlp);
 259 void helperSubmit(helper * hlp, const char *buf, HLPCB * callback, void *data);
 260 void helperStatefulSubmit(statefulhelper * hlp, const char *buf, HLPCB * callback, void *data, helper_stateful_server * lastserver);
 261 void helperShutdown(helper * hlp);
 262 void helperStatefulShutdown(statefulhelper * hlp);
 263 void helperStatefulReleaseServer(helper_stateful_server * srv);
 264
 265 #endif /* SQUID_HELPER_H */
 266