src/helper.h

   1 /*
   2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 84    Helper process maintenance */
  10
  11 #ifndef SQUID_HELPER_H
  12 #define SQUID_HELPER_H
  13
  14 #include "base/AsyncCall.h"
  15 #include "base/InstanceId.h"
  16 #include "cbdata.h"
  17 #include "comm/forward.h"
  18 #include "dlink.h"
  19 #include "helper/ChildConfig.h"
  20 #include "helper/forward.h"
  21 #include "helper/Reply.h"
  22 #include "helper/Request.h"
  23 #include "ip/Address.h"
  24 #include "sbuf/SBuf.h"
  25
  26 #include <list>
  27 #include <map>
  28 #include <queue>
  29
  30 class Packable;
  31 class wordlist;
  32
  33 namespace Helper
  34 {
  35 /// Holds the  required data to serve a helper request.
  36 class Xaction {
  37     MEMPROXY_CLASS(Helper::Xaction);
  38 public:
  39     Xaction(HLPCB *c, void *d, const char *b): request(c, d, b) {}
  40     Helper::Request request;
  41     Helper::Reply reply;
  42 };
  43 }
  44
  45 /**
  46  * Managers a set of individual helper processes with a common queue of requests.
  47  *
  48  * With respect to load, a helper goes through these states (roughly):
  49  *   idle:   no processes are working on requests (and no requests are queued);
  50  *   normal: some, but not all processes are working (and no requests are queued);
  51  *   busy:   all processes are working (and some requests are possibly queued);
  52  *   overloaded: a busy helper with more than queue-size requests in the queue.
  53  *
  54  * A busy helper queues new requests and issues a WARNING every 10 minutes or so.
  55  * An overloaded helper either drops new requests or keeps queuing them, depending on
  56  *   whether the caller can handle dropped requests (trySubmit vs helperSubmit APIs).
  57  * If an overloaded helper has been overloaded for 3+ minutes, an attempt to use
  58  *   it results in on-persistent-overload action, which may kill worker.
  59  */
  60 class helper
  61 {
  62     CBDATA_CLASS(helper);
  63
  64 public:
  65     inline helper(const char *name) :
  66         cmdline(NULL),
  67         id_name(name),
  68         ipc_type(0),
  69         droppedRequests(0),
  70         overloadStart(0),
  71         last_queue_warn(0),
  72         last_restart(0),
  73         timeout(0),
  74         retryTimedOut(false),
  75         retryBrokenHelper(false),
  76         eom('\n') {
  77         memset(&stats, 0, sizeof(stats));
  78     }
  79     ~helper();
  80
  81     /// \returns next request in the queue, or nil.
  82     Helper::Xaction *nextRequest();
  83
  84     /// If possible, submit request. Otherwise, either kill Squid or return false.
  85     bool trySubmit(const char *buf, HLPCB * callback, void *data);
  86
  87     /// Submits a request to the helper or add it to the queue if none of
  88     /// the servers is available.
  89     void submitRequest(Helper::Xaction *r);
  90
  91     /// Dump some stats about the helper state to a Packable object
  92     void packStatsInto(Packable *p, const char *label = NULL) const;
  93     /// whether the helper will be in "overloaded" state after one more request
  94     /// already overloaded helpers return true
  95     bool willOverload() const;
  96
  97 public:
  98     wordlist *cmdline;
  99     dlink_list servers;
 100     std::queue<Helper::Xaction *> queue;
 101     const char *id_name;
 102     Helper::ChildConfig childs;    ///< Configuration settings for number running.
 103     int ipc_type;
 104     Ip::Address addr;
 105     unsigned int droppedRequests; ///< requests not sent during helper overload
 106     time_t overloadStart; ///< when the helper became overloaded (zero if it is not)
 107     time_t last_queue_warn;
 108     time_t last_restart;
 109     time_t timeout; ///< Requests timeout
 110     bool retryTimedOut; ///< Whether the timed-out requests must retried
 111     bool retryBrokenHelper; ///< Whether the requests must retried on BH replies
 112     SBuf onTimedOutResponse; ///< The response to use when helper response timedout
 113     char eom;   ///< The char which marks the end of (response) message, normally '\n'
 114
 115     struct _stats {
 116         int requests;
 117         int replies;
 118         int timedout;
 119         int queue_size;
 120         int avg_svc_time;
 121     } stats;
 122
 123 protected:
 124     friend void helperSubmit(helper * hlp, const char *buf, HLPCB * callback, void *data);
 125     bool queueFull() const;
 126     bool overloaded() const;
 127     void syncQueueStats();
 128     bool prepSubmit();
 129     void submit(const char *buf, HLPCB * callback, void *data);
 130 };
 131
 132 class statefulhelper : public helper
 133 {
 134     CBDATA_CLASS(statefulhelper);
 135
 136 public:
 137     inline statefulhelper(const char *name) : helper(name), datapool(NULL) {}
 138     inline ~statefulhelper() {}
 139
 140 public:
 141     MemAllocator *datapool;
 142
 143 private:
 144     friend void helperStatefulSubmit(statefulhelper * hlp, const char *buf, HLPCB * callback, void *data, helper_stateful_server * lastserver);
 145     void submit(const char *buf, HLPCB * callback, void *data, helper_stateful_server *lastserver);
 146     bool trySubmit(const char *buf, HLPCB * callback, void *data, helper_stateful_server *lastserver);
 147 };
 148
 149 /**
 150  * Fields shared between stateless and stateful helper servers.
 151  */
 152 class HelperServerBase
 153 {
 154 public:
 155     /** Closes pipes to the helper safely.
 156      * Handles the case where the read and write pipes are the same FD.
 157      *
 158      * \param name displayed for the helper being shutdown if logging an error
 159      */
 160     void closePipesSafely(const char *name);
 161
 162     /** Closes the reading pipe.
 163      * If the read and write sockets are the same the write pipe will
 164      * also be closed. Otherwise its left open for later handling.
 165      *
 166      * \param name displayed for the helper being shutdown if logging an error
 167      */
 168     void closeWritePipeSafely(const char *name);
 169
 170 public:
 171     /// Helper program identifier; does not change when contents do,
 172     ///   including during assignment
 173     const InstanceId<HelperServerBase> index;
 174     int pid;
 175     Ip::Address addr;
 176     Comm::ConnectionPointer readPipe;
 177     Comm::ConnectionPointer writePipe;
 178     void *hIpc;
 179
 180     char *rbuf;
 181     size_t rbuf_sz;
 182     size_t roffset;
 183
 184     struct timeval dispatch_time;
 185     struct timeval answer_time;
 186
 187     dlink_node link;
 188
 189     struct _helper_flags {
 190         bool writing;
 191         bool closing;
 192         bool shutdown;
 193         bool reserved;
 194     } flags;
 195
 196     typedef std::list<Helper::Xaction *> Requests;
 197     Requests requests; ///< requests in order of submission/expiration
 198
 199     struct {
 200         uint64_t uses;     //< requests sent to this helper
 201         uint64_t replies;  //< replies received from this helper
 202         uint64_t pending;  //< queued lookups waiting to be sent to this helper
 203         uint64_t releases; //< times release() has been called on this helper (if stateful)
 204         uint64_t timedout; //< requests which timed-out
 205     } stats;
 206     void initStats();
 207 };
 208
 209 class MemBuf;
 210 class CommTimeoutCbParams;
 211
 212 class helper_server : public HelperServerBase
 213 {
 214     CBDATA_CLASS(helper_server);
 215
 216 public:
 217     uint64_t nextRequestId;
 218
 219     MemBuf *wqueue;
 220     MemBuf *writebuf;
 221
 222     helper *parent;
 223
 224     /// The helper request Xaction object for the current reply .
 225     /// A helper reply may be distributed to more than one of the retrieved
 226     /// packets from helper. This member stores the Xaction object as long as
 227     /// the end-of-message for current reply is not retrieved.
 228     Helper::Xaction *replyXaction;
 229
 230     /// Whether to ignore current message, because it is timed-out or other reason
 231     bool ignoreToEom;
 232
 233     // STL says storing std::list iterators is safe when changing the list
 234     typedef std::map<uint64_t, Requests::iterator> RequestIndex;
 235     RequestIndex requestsIndex; ///< maps request IDs to requests
 236
 237     /// Search in queue for the request with requestId, return the related
 238     /// Xaction object and remove it from queue.
 239     /// If concurrency is disabled then the requestId is ignored and the
 240     /// Xaction of the next request in queue is retrieved.
 241     Helper::Xaction *popRequest(int requestId);
 242
 243     /// Run over the active requests lists and forces a retry, or timedout reply
 244     /// or the configured "on timeout response" for timedout requests.
 245     void checkForTimedOutRequests(bool const retry);
 246
 247     /// Read timeout handler
 248     static void requestTimeout(const CommTimeoutCbParams &io);
 249 };
 250
 251 class helper_stateful_server : public HelperServerBase
 252 {
 253     CBDATA_CLASS(helper_stateful_server);
 254
 255 public:
 256     /* MemBuf wqueue; */
 257     /* MemBuf writebuf; */
 258
 259     statefulhelper *parent;
 260
 261     void *data;         /* State data used by the calling routines */
 262 };
 263
 264 /* helper.c */
 265 void helperOpenServers(helper * hlp);
 266 void helperStatefulOpenServers(statefulhelper * hlp);
 267 void helperSubmit(helper * hlp, const char *buf, HLPCB * callback, void *data);
 268 void helperStatefulSubmit(statefulhelper * hlp, const char *buf, HLPCB * callback, void *data, helper_stateful_server * lastserver);
 269 void helperShutdown(helper * hlp);
 270 void helperStatefulShutdown(statefulhelper * hlp);
 271 void helperStatefulReleaseServer(helper_stateful_server * srv);
 272 void *helperStatefulServerGetData(helper_stateful_server * srv);
 273
 274 #endif /* SQUID_HELPER_H */
 275