src/client_side.h

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 33    Client-side Routines */
  10
  11 #ifndef SQUID_CLIENTSIDE_H
  12 #define SQUID_CLIENTSIDE_H
  13
  14 #include "base/RunnersRegistry.h"
  15 #include "clientStreamForward.h"
  16 #include "comm.h"
  17 #include "helper/forward.h"
  18 #include "http/forward.h"
  19 #include "HttpControlMsg.h"
  20 #include "ipc/FdNotes.h"
  21 #include "SBuf.h"
  22 #if USE_AUTH
  23 #include "auth/UserRequest.h"
  24 #endif
  25 #if USE_OPENSSL
  26 #include "ssl/support.h"
  27 #endif
  28
  29 class ConnStateData;
  30 class ClientHttpRequest;
  31 class clientStreamNode;
  32 namespace AnyP
  33 {
  34 class PortCfg;
  35 } // namespace Anyp
  36
  37 /**
  38  * Badly named.
  39  * This is in fact the processing context for a single HTTP request.
  40  *
  41  * Managing what has been done, and what happens next to the data buffer
  42  * holding what we hope is an HTTP request.
  43  *
  44  * Parsing is still a mess of global functions done in conjunction with the
  45  * real socket controller which generated ClientHttpRequest.
  46  * It also generates one of us and passes us control from there based on
  47  * the results of the parse.
  48  *
  49  * After that all the request interpretation and adaptation is in our scope.
  50  * Then finally the reply fetcher is created by this and we get the result
  51  * back. Which we then have to manage writing of it to the ConnStateData.
  52  *
  53  * The socket level management is done by a ConnStateData which owns us.
  54  * The scope of this objects control over a socket consists of the data
  55  * buffer received from ConnStateData with an initially unknown length.
  56  * When that length is known it sets the end bounary of our acces to the
  57  * buffer.
  58  *
  59  * The individual processing actions are done by other Jobs which we
  60  * kick off as needed.
  61  *
  62  * XXX: If an async call ends the ClientHttpRequest job, ClientSocketContext
  63  * (and ConnStateData) may not know about it, leading to segfaults and
  64  * assertions like areAllContextsForThisConnection(). This is difficult to fix
  65  * because ClientHttpRequest lacks a good way to communicate its ongoing
  66  * destruction back to the ClientSocketContext which pretends to "own" *http.
  67  */
  68 class ClientSocketContext : public RefCountable
  69 {
  70     CBDATA_CLASS(ClientSocketContext);
  71
  72 public:
  73     typedef RefCount<ClientSocketContext> Pointer;
  74     ClientSocketContext(const Comm::ConnectionPointer &aConn, ClientHttpRequest *aReq);
  75     ~ClientSocketContext();
  76     bool startOfOutput() const;
  77     void writeComplete(const Comm::ConnectionPointer &conn, char *bufnotused, size_t size, Comm::Flag errflag);
  78     void keepaliveNextRequest();
  79
  80     Comm::ConnectionPointer clientConnection; /// details about the client connection socket.
  81     ClientHttpRequest *http;    /* we pretend to own that job */
  82     HttpReply *reply;
  83     char reqbuf[HTTP_REQBUF_SZ];
  84     Pointer next;
  85
  86     struct {
  87
  88         unsigned deferred:1; /* This is a pipelined request waiting for the current object to complete */
  89
  90         unsigned parsed_ok:1; /* Was this parsed correctly? */
  91     } flags;
  92     bool mayUseConnection() const {return mayUseConnection_;}
  93
  94     void mayUseConnection(bool aBool) {
  95         mayUseConnection_ = aBool;
  96         debugs(33,3, HERE << "This " << this << " marked " << aBool);
  97     }
  98
  99     class DeferredParams
 100     {
 101
 102     public:
 103         clientStreamNode *node;
 104         HttpReply *rep;
 105         StoreIOBuffer queuedBuffer;
 106     };
 107
 108     DeferredParams deferredparams;
 109     int64_t writtenToSocket;
 110     void pullData();
 111     int64_t getNextRangeOffset() const;
 112     bool canPackMoreRanges() const;
 113     clientStream_status_t socketState();
 114     void sendBody(HttpReply * rep, StoreIOBuffer bodyData);
 115     void sendStartOfMessage(HttpReply * rep, StoreIOBuffer bodyData);
 116     size_t lengthToSend(Range<int64_t> const &available);
 117     void noteSentBodyBytes(size_t);
 118     void buildRangeHeader(HttpReply * rep);
 119     clientStreamNode * getTail() const;
 120     clientStreamNode * getClientReplyContext() const;
 121     ConnStateData *getConn() const;
 122     void connIsFinished();
 123     void removeFromConnectionList(ConnStateData * conn);
 124     void deferRecipientForLater(clientStreamNode * node, HttpReply * rep, StoreIOBuffer receivedData);
 125     bool multipartRangeRequest() const;
 126     void registerWithConn();
 127     void noteIoError(const int xerrno); ///< update state to reflect I/O error
 128
 129     /// starts writing 1xx control message to the client
 130     void writeControlMsg(HttpControlMsg &msg);
 131
 132 protected:
 133     static IOCB WroteControlMsg;
 134     void wroteControlMsg(const Comm::ConnectionPointer &conn, char *bufnotused, size_t size, Comm::Flag errflag, int xerrno);
 135
 136 private:
 137     void prepareReply(HttpReply * rep);
 138     void packChunk(const StoreIOBuffer &bodyData, MemBuf &mb);
 139     void packRange(StoreIOBuffer const &, MemBuf * mb);
 140     void deRegisterWithConn();
 141     void doClose();
 142     void initiateClose(const char *reason);
 143
 144     AsyncCall::Pointer cbControlMsgSent; ///< notifies HttpControlMsg Source
 145
 146     bool mayUseConnection_; /* This request may use the connection. Don't read anymore requests for now */
 147     bool connRegistered_;
 148 };
 149
 150 class ConnectionDetail;
 151 #if USE_OPENSSL
 152 namespace Ssl
 153 {
 154 class ServerBump;
 155 }
 156 #endif
 157 /**
 158  * Manages a connection to a client.
 159  *
 160  * Multiple requests (up to pipeline_prefetch) can be pipelined. This object is responsible for managing
 161  * which one is currently being fulfilled and what happens to the queue if the current one
 162  * causes the client connection to be closed early.
 163  *
 164  * Act as a manager for the connection and passes data in buffer to the current parser.
 165  * the parser has ambiguous scope at present due to being made from global functions
 166  * I believe this object uses the parser to identify boundaries and kick off the
 167  * actual HTTP request handling objects (ClientSocketContext, ClientHttpRequest, HttpRequest)
 168  *
 169  * If the above can be confirmed accurate we can call this object PipelineManager or similar
 170  */
 171 class ConnStateData : public BodyProducer, public HttpControlMsgSink, public RegisteredRunner
 172 {
 173
 174 public:
 175     explicit ConnStateData(const MasterXaction::Pointer &xact);
 176     virtual ~ConnStateData();
 177
 178     void readSomeData();
 179     bool areAllContextsForThisConnection() const;
 180     void freeAllContexts();
 181     void notifyAllContexts(const int xerrno); ///< tell everybody about the err
 182     /// Traffic parsing
 183     bool clientParseRequests();
 184     void readNextRequest();
 185     ClientSocketContext::Pointer getCurrentContext() const;
 186     void addContextToQueue(ClientSocketContext * context);
 187     int getConcurrentRequestCount() const;
 188     bool isOpen() const;
 189
 190     /// Update flags and timeout after the first byte received
 191     void receivedFirstByte();
 192
 193     // HttpControlMsgSink API
 194     virtual void sendControlMsg(HttpControlMsg msg);
 195
 196     // Client TCP connection details from comm layer.
 197     Comm::ConnectionPointer clientConnection;
 198
 199     /**
 200      * The transfer protocol currently being spoken on this connection.
 201      * HTTP/1 CONNECT and HTTP/2 SETTINGS offers the ability to change
 202      * protocols on the fly.
 203      */
 204     AnyP::ProtocolVersion transferProtocol;
 205
 206     struct In {
 207         In();
 208         ~In();
 209         bool maybeMakeSpaceAvailable();
 210
 211         Http1::TeChunkedParser *bodyParser; ///< parses chunked request body
 212         SBuf buf;
 213     } in;
 214
 215     /** number of body bytes we need to comm_read for the "current" request
 216      *
 217      * \retval 0         We do not need to read any [more] body bytes
 218      * \retval negative  May need more but do not know how many; could be zero!
 219      * \retval positive  Need to read exactly that many more body bytes
 220      */
 221     int64_t mayNeedToReadMoreBody() const;
 222
 223 #if USE_AUTH
 224     /**
 225      * Fetch the user details for connection based authentication
 226      * NOTE: this is ONLY connection based because NTLM and Negotiate is against HTTP spec.
 227      */
 228     const Auth::UserRequest::Pointer &getAuth() const { return auth_; }
 229
 230     /**
 231      * Set the user details for connection-based authentication to use from now until connection closure.
 232      *
 233      * Any change to existing credentials shows that something invalid has happened. Such as:
 234      * - NTLM/Negotiate auth was violated by the per-request headers missing a revalidation token
 235      * - NTLM/Negotiate auth was violated by the per-request headers being for another user
 236      * - SSL-Bump CONNECT tunnel with persistent credentials has ended
 237      */
 238     void setAuth(const Auth::UserRequest::Pointer &aur, const char *cause);
 239 #endif
 240
 241     /**
 242      * used by the owner of the connection, opaque otherwise
 243      * TODO: generalise the connection owner concept.
 244      */
 245     ClientSocketContext::Pointer currentobject;
 246
 247     Ip::Address log_addr;
 248     int nrequests;
 249
 250     struct {
 251         bool readMore; ///< needs comm_read (for this request or new requests)
 252         bool swanSang; // XXX: temporary flag to check proper cleanup
 253     } flags;
 254     struct {
 255         Comm::ConnectionPointer serverConnection; /* pinned server side connection */
 256         char *host;             /* host name of pinned connection */
 257         int port;               /* port of pinned connection */
 258         bool pinned;             /* this connection was pinned */
 259         bool auth;               /* pinned for www authentication */
 260         bool reading;   ///< we are monitoring for peer connection closure
 261         bool zeroReply; ///< server closed w/o response (ERR_ZERO_SIZE_OBJECT)
 262         CachePeer *peer;             /* CachePeer the connection goes via */
 263         AsyncCall::Pointer readHandler; ///< detects serverConnection closure
 264         AsyncCall::Pointer closeHandler; /*The close handler for pinned server side connection*/
 265     } pinning;
 266
 267     /// Squid listening port details where this connection arrived.
 268     AnyP::PortCfgPointer port;
 269
 270     bool transparent() const;
 271     bool reading() const;
 272     void stopReading(); ///< cancels comm_read if it is scheduled
 273
 274     /// true if we stopped receiving the request
 275     const char *stoppedReceiving() const { return stoppedReceiving_; }
 276     /// true if we stopped sending the response
 277     const char *stoppedSending() const { return stoppedSending_; }
 278     /// note request receiving error and close as soon as we write the response
 279     void stopReceiving(const char *error);
 280     /// note response sending error and close as soon as we read the request
 281     void stopSending(const char *error);
 282
 283     void expectNoForwarding(); ///< cleans up virgin request [body] forwarding state
 284
 285     /* BodyPipe API */
 286     BodyPipe::Pointer expectRequestBody(int64_t size);
 287     virtual void noteMoreBodySpaceAvailable(BodyPipe::Pointer) = 0;
 288     virtual void noteBodyConsumerAborted(BodyPipe::Pointer) = 0;
 289
 290     bool handleReadData();
 291     bool handleRequestBodyData();
 292
 293     /// Forward future client requests using the given server connection.
 294     /// Optionally, monitor pinned server connection for remote-end closures.
 295     void pinConnection(const Comm::ConnectionPointer &pinServerConn, HttpRequest *request, CachePeer *peer, bool auth, bool monitor = true);
 296     /// Undo pinConnection() and, optionally, close the pinned connection.
 297     void unpinConnection(const bool andClose);
 298     /// Returns validated pinnned server connection (and stops its monitoring).
 299     Comm::ConnectionPointer borrowPinnedConnection(HttpRequest *request, const CachePeer *aPeer);
 300     /**
 301      * Checks if there is pinning info if it is valid. It can close the server side connection
 302      * if pinned info is not valid.
 303      \param request   if it is not NULL also checks if the pinning info refers to the request client side HttpRequest
 304      \param CachePeer      if it is not NULL also check if the CachePeer is the pinning CachePeer
 305      \return          The details of the server side connection (may be closed if failures were present).
 306      */
 307     const Comm::ConnectionPointer validatePinnedConnection(HttpRequest *request, const CachePeer *peer);
 308     /**
 309      * returts the pinned CachePeer if exists, NULL otherwise
 310      */
 311     CachePeer *pinnedPeer() const {return pinning.peer;}
 312     bool pinnedAuth() const {return pinning.auth;}
 313
 314     /// called just before a FwdState-dispatched job starts using connection
 315     virtual void notePeerConnection(Comm::ConnectionPointer) {}
 316
 317     // pining related comm callbacks
 318     virtual void clientPinnedConnectionClosed(const CommCloseCbParams &io);
 319
 320     // comm callbacks
 321     void clientReadRequest(const CommIoCbParams &io);
 322     void clientReadFtpData(const CommIoCbParams &io);
 323     void connStateClosed(const CommCloseCbParams &io);
 324     void requestTimeout(const CommTimeoutCbParams &params);
 325
 326     // AsyncJob API
 327     virtual void start();
 328     virtual bool doneAll() const { return BodyProducer::doneAll() && false;}
 329     virtual void swanSong();
 330
 331     /// Changes state so that we close the connection and quit after serving
 332     /// the client-side-detected error response instead of getting stuck.
 333     void quitAfterError(HttpRequest *request); // meant to be private
 334
 335     /// The caller assumes responsibility for connection closure detection.
 336     void stopPinnedConnectionMonitoring();
 337
 338 #if USE_OPENSSL
 339     /// the second part of old httpsAccept, waiting for future HttpsServer home
 340     void postHttpsAccept();
 341
 342     /// Initializes and starts a peek-and-splice negotiation with the SSL client
 343     void startPeekAndSplice();
 344     /// Called when the initialization of peek-and-splice negotiation finidhed
 345     void startPeekAndSpliceDone();
 346     /// Called when a peek-and-splice step finished. For example after
 347     /// server SSL certificates received and fake server SSL certificates
 348     /// generated
 349     void doPeekAndSpliceStep();
 350     /// called by FwdState when it is done bumping the server
 351     void httpsPeeked(Comm::ConnectionPointer serverConnection);
 352
 353     /// Splice a bumped client connection on peek-and-splice mode
 354     void splice();
 355
 356     /// Check on_unsupported_protocol access list and splice if required
 357     /// \retval true on splice
 358     /// \retval false otherwise
 359     bool spliceOnError(const err_type err);
 360
 361     /// Start to create dynamic SSL_CTX for host or uses static port SSL context.
 362     void getSslContextStart();
 363     /**
 364      * Done create dynamic ssl certificate.
 365      *
 366      * \param[in] isNew if generated certificate is new, so we need to add this certificate to storage.
 367      */
 368     void getSslContextDone(Security::ContextPointer sslContext, bool isNew = false);
 369     /// Callback function. It is called when squid receive message from ssl_crtd.
 370     static void sslCrtdHandleReplyWrapper(void *data, const Helper::Reply &reply);
 371     /// Proccess response from ssl_crtd.
 372     void sslCrtdHandleReply(const Helper::Reply &reply);
 373
 374     void switchToHttps(HttpRequest *request, Ssl::BumpMode bumpServerMode);
 375     bool switchedToHttps() const { return switchedToHttps_; }
 376     Ssl::ServerBump *serverBump() {return sslServerBump;}
 377     inline void setServerBump(Ssl::ServerBump *srvBump) {
 378         if (!sslServerBump)
 379             sslServerBump = srvBump;
 380         else
 381             assert(sslServerBump == srvBump);
 382     }
 383     const SBuf &sslCommonName() const {return sslCommonName_;}
 384     void resetSslCommonName(const char *name) {sslCommonName_ = name;}
 385     /// Fill the certAdaptParams with the required data for certificate adaptation
 386     /// and create the key for storing/retrieve the certificate to/from the cache
 387     void buildSslCertGenerationParams(Ssl::CertificateProperties &certProperties);
 388     /// Called when the client sends the first request on a bumped connection.
 389     /// Returns false if no [delayed] error should be written to the client.
 390     /// Otherwise, writes the error to the client and returns true. Also checks
 391     /// for SQUID_X509_V_ERR_DOMAIN_MISMATCH on bumped requests.
 392     bool serveDelayedError(ClientSocketContext *context);
 393
 394     Ssl::BumpMode sslBumpMode; ///< ssl_bump decision (Ssl::bumpEnd if n/a).
 395
 396 #else
 397     bool switchedToHttps() const { return false; }
 398 #endif
 399
 400     /* clt_conn_tag=tag annotation access */
 401     const SBuf &connectionTag() const { return connectionTag_; }
 402     void connectionTag(const char *aTag) { connectionTag_ = aTag; }
 403
 404     /// handle a control message received by context from a peer and call back
 405     virtual void writeControlMsgAndCall(ClientSocketContext *context, HttpReply *rep, AsyncCall::Pointer &call) = 0;
 406
 407     /// ClientStream calls this to supply response header (once) and data
 408     /// for the current ClientSocketContext.
 409     virtual void handleReply(HttpReply *header, StoreIOBuffer receivedData) = 0;
 410
 411     /// remove no longer needed leading bytes from the input buffer
 412     void consumeInput(const size_t byteCount);
 413
 414     /* TODO: Make the methods below (at least) non-public when possible. */
 415
 416     /// stop parsing the request and create context for relaying error info
 417     ClientSocketContext *abortRequestParsing(const char *const errUri);
 418
 419     /// client data which may need to forward as-is to server after an
 420     /// on_unsupported_protocol tunnel decision.
 421     SBuf preservedClientData;
 422
 423     /* Registered Runner API */
 424     virtual void startShutdown();
 425     virtual void endingShutdown();
 426
 427 protected:
 428     void startDechunkingRequest();
 429     void finishDechunkingRequest(bool withSuccess);
 430     void abortChunkedRequestBody(const err_type error);
 431     err_type handleChunkedRequestBody();
 432
 433     void startPinnedConnectionMonitoring();
 434     void clientPinnedConnectionRead(const CommIoCbParams &io);
 435
 436     /// parse input buffer prefix into a single transfer protocol request
 437     /// return NULL to request more header bytes (after checking any limits)
 438     /// use abortRequestParsing() to handle parsing errors w/o creating request
 439     virtual ClientSocketContext *parseOneRequest() = 0;
 440
 441     /// start processing a freshly parsed request
 442     virtual void processParsedRequest(ClientSocketContext *context) = 0;
 443
 444     /// returning N allows a pipeline of 1+N requests (see pipeline_prefetch)
 445     virtual int pipelinePrefetchMax() const;
 446
 447     /// timeout to use when waiting for the next request
 448     virtual time_t idleTimeout() const = 0;
 449
 450     BodyPipe::Pointer bodyPipe; ///< set when we are reading request body
 451
 452 private:
 453     int connFinishedWithConn(int size);
 454     void clientAfterReadingRequests();
 455     bool concurrentRequestQueueFilled() const;
 456
 457     void pinNewConnection(const Comm::ConnectionPointer &pinServer, HttpRequest *request, CachePeer *aPeer, bool auth);
 458
 459     /* PROXY protocol functionality */
 460     bool proxyProtocolValidateClient();
 461     bool parseProxyProtocolHeader();
 462     bool parseProxy1p0();
 463     bool parseProxy2p0();
 464     bool proxyProtocolError(const char *reason);
 465
 466     /// whether PROXY protocol header is still expected
 467     bool needProxyProtocolHeader_;
 468
 469 #if USE_AUTH
 470     /// some user details that can be used to perform authentication on this connection
 471     Auth::UserRequest::Pointer auth_;
 472 #endif
 473
 474     /// the parser state for current HTTP/1.x input buffer processing
 475     Http1::RequestParserPointer parser_;
 476
 477 #if USE_OPENSSL
 478     bool switchedToHttps_;
 479     /// The SSL server host name appears in CONNECT request or the server ip address for the intercepted requests
 480     String sslConnectHostOrIp; ///< The SSL server host name as passed in the CONNECT request
 481     SBuf sslCommonName_; ///< CN name for SSL certificate generation
 482     String sslBumpCertKey; ///< Key to use to store/retrieve generated certificate
 483
 484     /// HTTPS server cert. fetching state for bump-ssl-server-first
 485     Ssl::ServerBump *sslServerBump;
 486     Ssl::CertSignAlgorithm signAlgorithm; ///< The signing algorithm to use
 487 #endif
 488
 489     /// the reason why we no longer write the response or nil
 490     const char *stoppedSending_;
 491     /// the reason why we no longer read the request or nil
 492     const char *stoppedReceiving_;
 493
 494     AsyncCall::Pointer reader; ///< set when we are reading
 495
 496     bool receivedFirstByte_; ///< true if at least one byte received on this connection
 497     SBuf connectionTag_; ///< clt_conn_tag=Tag annotation for client connection
 498 };
 499
 500 void setLogUri(ClientHttpRequest * http, char const *uri, bool cleanUrl = false);
 501
 502 const char *findTrailingHTTPVersion(const char *uriAndHTTPVersion, const char *end = NULL);
 503
 504 int varyEvaluateMatch(StoreEntry * entry, HttpRequest * req);
 505
 506 /// accept requests to a given port and inform subCall about them
 507 void clientStartListeningOn(AnyP::PortCfgPointer &port, const RefCount< CommCbFunPtrCallT<CommAcceptCbPtrFun> > &subCall, const Ipc::FdNoteId noteId);
 508
 509 void clientOpenListenSockets(void);
 510 void clientConnectionsClose(void);
 511 void httpRequestFree(void *);
 512
 513 /// decide whether to expect multiple requests on the corresponding connection
 514 void clientSetKeepaliveFlag(ClientHttpRequest *http);
 515
 516 /* misplaced declaratrions of Stream callbacks provided/used by client side */
 517 SQUIDCEXTERN CSR clientGetMoreData;
 518 SQUIDCEXTERN CSS clientReplyStatus;
 519 SQUIDCEXTERN CSD clientReplyDetach;
 520 CSCB clientSocketRecipient;
 521 CSD clientSocketDetach;
 522
 523 /* TODO: Move to HttpServer. Warning: Move requires large code nonchanges! */
 524 ClientSocketContext *parseHttpRequest(ConnStateData *, const Http1::RequestParserPointer &);
 525 void clientProcessRequest(ConnStateData *, const Http1::RequestParserPointer &, ClientSocketContext *);
 526 void clientPostHttpsAccept(ConnStateData *);
 527
 528 #endif /* SQUID_CLIENTSIDE_H */
 529