src/gopher.cc

   1 /*
   2  * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 10    Gopher */
  10
  11 #include "squid.h"
  12 #include "comm.h"
  13 #include "comm/Read.h"
  14 #include "comm/Write.h"
  15 #include "errorpage.h"
  16 #include "fd.h"
  17 #include "FwdState.h"
  18 #include "globals.h"
  19 #include "html_quote.h"
  20 #include "HttpReply.h"
  21 #include "HttpRequest.h"
  22 #include "MemBuf.h"
  23 #include "mime.h"
  24 #include "parser/Tokenizer.h"
  25 #include "rfc1738.h"
  26 #include "SquidConfig.h"
  27 #include "SquidTime.h"
  28 #include "StatCounters.h"
  29 #include "Store.h"
  30 #include "tools.h"
  31
  32 #if USE_DELAY_POOLS
  33 #include "DelayPools.h"
  34 #include "MemObject.h"
  35 #endif
  36
  37 /* gopher type code from rfc. Anawat. */
  38 #define GOPHER_FILE         '0'
  39 #define GOPHER_DIRECTORY    '1'
  40 #define GOPHER_CSO          '2'
  41 #define GOPHER_ERROR        '3'
  42 #define GOPHER_MACBINHEX    '4'
  43 #define GOPHER_DOSBIN       '5'
  44 #define GOPHER_UUENCODED    '6'
  45 #define GOPHER_INDEX        '7'
  46 #define GOPHER_TELNET       '8'
  47 #define GOPHER_BIN          '9'
  48 #define GOPHER_REDUNT       '+'
  49 #define GOPHER_3270         'T'
  50 #define GOPHER_GIF          'g'
  51 #define GOPHER_IMAGE        'I'
  52
  53 #define GOPHER_HTML         'h'
  54 #define GOPHER_INFO         'i'
  55
  56 ///  W3 address
  57 #define GOPHER_WWW          'w'
  58 #define GOPHER_SOUND        's'
  59
  60 #define GOPHER_PLUS_IMAGE   ':'
  61 #define GOPHER_PLUS_MOVIE   ';'
  62 #define GOPHER_PLUS_SOUND   '<'
  63
  64 #define GOPHER_PORT         70
  65
  66 #define TAB                 '\t'
  67
  68 // TODO CODE: should this be a protocol-specific thing?
  69 #define TEMP_BUF_SIZE       4096
  70
  71 #define MAX_CSO_RESULT      1024
  72
  73 /**
  74  * Gopher Gateway Internals
  75  *
  76  * Gopher is somewhat complex and gross because it must convert from
  77  * the Gopher protocol to HTTP.
  78  */
  79 class GopherStateData
  80 {
  81     CBDATA_CLASS(GopherStateData);
  82
  83 public:
  84     GopherStateData(FwdState *aFwd) :
  85         entry(aFwd->entry),
  86         conversion(NORMAL),
  87         HTML_header_added(0),
  88         HTML_pre(0),
  89         type_id(GOPHER_FILE /* '0' */),
  90         cso_recno(0),
  91         len(0),
  92         buf(NULL),
  93         fwd(aFwd)
  94     {
  95         *request = 0;
  96         buf = (char *)memAllocate(MEM_4K_BUF);
  97         entry->lock("gopherState");
  98         *replybuf = 0;
  99     }
 100     ~GopherStateData() {if(buf) swanSong();}
 101
 102     /* AsyncJob API emulated */
 103     void deleteThis(const char *aReason);
 104     void swanSong();
 105
 106 public:
 107     StoreEntry *entry;
 108     enum {
 109         NORMAL,
 110         HTML_DIR,
 111         HTML_INDEX_RESULT,
 112         HTML_CSO_RESULT,
 113         HTML_INDEX_PAGE,
 114         HTML_CSO_PAGE
 115     } conversion;
 116     int HTML_header_added;
 117     int HTML_pre;
 118     char type_id;
 119     char request[MAX_URL];
 120     int cso_recno;
 121     int len;
 122     char *buf;          /* pts to a 4k page */
 123     Comm::ConnectionPointer serverConn;
 124     FwdState::Pointer fwd;
 125     HttpReply::Pointer reply_;
 126     char replybuf[BUFSIZ];
 127 };
 128
 129 CBDATA_CLASS_INIT(GopherStateData);
 130
 131 static CLCB gopherStateFree;
 132 static void gopherMimeCreate(GopherStateData *);
 133 static void gopher_request_parse(const HttpRequest * req,
 134                                  char *type_id,
 135                                  char *request);
 136 static void gopherEndHTML(GopherStateData *);
 137 static void gopherToHTML(GopherStateData *, char *inbuf, int len);
 138 static CTCB gopherTimeout;
 139 static IOCB gopherReadReply;
 140 static IOCB gopherSendComplete;
 141 static PF gopherSendRequest;
 142
 143 static char def_gopher_bin[] = "www/unknown";
 144
 145 static char def_gopher_text[] = "text/plain";
 146
 147 static void
 148 gopherStateFree(const CommCloseCbParams &params)
 149 {
 150     GopherStateData *gopherState = (GopherStateData *)params.data;
 151
 152     if (gopherState == NULL)
 153         return;
 154
 155     gopherState->deleteThis("gopherStateFree");
 156 }
 157
 158 void
 159 GopherStateData::deleteThis(const char *)
 160 {
 161     swanSong();
 162     delete this;
 163 }
 164
 165 void
 166 GopherStateData::swanSong()
 167 {
 168     if (entry)
 169         entry->unlock("gopherState");
 170
 171     if (buf) {
 172         memFree(buf, MEM_4K_BUF);
 173         buf = nullptr;
 174     }
 175 }
 176
 177 /**
 178  * Create MIME Header for Gopher Data
 179  */
 180 static void
 181 gopherMimeCreate(GopherStateData * gopherState)
 182 {
 183     StoreEntry *entry = gopherState->entry;
 184     const char *mime_type = NULL;
 185     const char *mime_enc = NULL;
 186
 187     switch (gopherState->type_id) {
 188
 189     case GOPHER_DIRECTORY:
 190
 191     case GOPHER_INDEX:
 192
 193     case GOPHER_HTML:
 194
 195     case GOPHER_WWW:
 196
 197     case GOPHER_CSO:
 198         mime_type = "text/html";
 199         break;
 200
 201     case GOPHER_GIF:
 202
 203     case GOPHER_IMAGE:
 204
 205     case GOPHER_PLUS_IMAGE:
 206         mime_type = "image/gif";
 207         break;
 208
 209     case GOPHER_SOUND:
 210
 211     case GOPHER_PLUS_SOUND:
 212         mime_type = "audio/basic";
 213         break;
 214
 215     case GOPHER_PLUS_MOVIE:
 216         mime_type = "video/mpeg";
 217         break;
 218
 219     case GOPHER_MACBINHEX:
 220
 221     case GOPHER_DOSBIN:
 222
 223     case GOPHER_UUENCODED:
 224
 225     case GOPHER_BIN:
 226         /* Rightnow We have no idea what it is. */
 227         mime_enc = mimeGetContentEncoding(gopherState->request);
 228         mime_type = mimeGetContentType(gopherState->request);
 229         if (!mime_type)
 230             mime_type = def_gopher_bin;
 231         break;
 232
 233     case GOPHER_FILE:
 234
 235     default:
 236         mime_enc = mimeGetContentEncoding(gopherState->request);
 237         mime_type = mimeGetContentType(gopherState->request);
 238         if (!mime_type)
 239             mime_type = def_gopher_text;
 240         break;
 241     }
 242
 243     assert(entry->isEmpty());
 244
 245     HttpReply *reply = new HttpReply;
 246     entry->buffer();
 247     reply->setHeaders(Http::scOkay, "Gatewaying", mime_type, -1, -1, -2);
 248     if (mime_enc)
 249         reply->header.putStr(Http::HdrType::CONTENT_ENCODING, mime_enc);
 250
 251     entry->replaceHttpReply(reply);
 252     gopherState->reply_ = reply;
 253 }
 254
 255 /**
 256  * Parse a gopher request into components.  By Anawat.
 257  */
 258 static void
 259 gopher_request_parse(const HttpRequest * req, char *type_id, char *request)
 260 {
 261     ::Parser::Tokenizer tok(req->url.path());
 262
 263     if (request)
 264         *request = 0;
 265
 266     tok.skip('/'); // ignore failures? path could be ab-empty
 267
 268     if (tok.atEnd()) {
 269         *type_id = GOPHER_DIRECTORY;
 270         return;
 271     }
 272
 273     static const CharacterSet anyByte("UTF-8",0x00, 0xFF);
 274
 275     SBuf typeId;
 276     (void)tok.prefix(typeId, anyByte, 1); // never fails since !atEnd()
 277     *type_id = typeId[0];
 278
 279     if (request) {
 280         SBufToCstring(request, tok.remaining().substr(0, MAX_URL-1));
 281         /* convert %xx to char */
 282         rfc1738_unescape(request);
 283     }
 284 }
 285
 286 /**
 287  * Parse the request to determine whether it is cachable.
 288  *
 289  * \param req   Request data.
 290  * \retval 0    Not cachable.
 291  * \retval 1    Cachable.
 292  */
 293 int
 294 gopherCachable(const HttpRequest * req)
 295 {
 296     int cachable = 1;
 297     char type_id;
 298     /* parse to see type */
 299     gopher_request_parse(req,
 300                          &type_id,
 301                          NULL);
 302
 303     switch (type_id) {
 304
 305     case GOPHER_INDEX:
 306
 307     case GOPHER_CSO:
 308
 309     case GOPHER_TELNET:
 310
 311     case GOPHER_3270:
 312         cachable = 0;
 313         break;
 314
 315     default:
 316         cachable = 1;
 317     }
 318
 319     return cachable;
 320 }
 321
 322 static void
 323 gopherHTMLHeader(StoreEntry * e, const char *title, const char *substring)
 324 {
 325     storeAppendPrintf(e, "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">\n");
 326     storeAppendPrintf(e, "<HTML><HEAD><TITLE>");
 327     storeAppendPrintf(e, title, substring);
 328     storeAppendPrintf(e, "</TITLE>");
 329     storeAppendPrintf(e, "<STYLE type=\"text/css\"><!--BODY{background-color:#ffffff;font-family:verdana,sans-serif}--></STYLE>\n");
 330     storeAppendPrintf(e, "</HEAD>\n<BODY><H1>");
 331     storeAppendPrintf(e, title, substring);
 332     storeAppendPrintf(e, "</H1>\n");
 333 }
 334
 335 static void
 336 gopherHTMLFooter(StoreEntry * e)
 337 {
 338     storeAppendPrintf(e, "<HR noshade size=\"1px\">\n");
 339     storeAppendPrintf(e, "<ADDRESS>\n");
 340     storeAppendPrintf(e, "Generated %s by %s (%s)\n",
 341                       mkrfc1123(squid_curtime),
 342                       getMyHostname(),
 343                       visible_appname_string);
 344     storeAppendPrintf(e, "</ADDRESS></BODY></HTML>\n");
 345 }
 346
 347 static void
 348 gopherEndHTML(GopherStateData * gopherState)
 349 {
 350     StoreEntry *e = gopherState->entry;
 351
 352     if (!gopherState->HTML_header_added) {
 353         gopherHTMLHeader(e, "Server Return Nothing", NULL);
 354         storeAppendPrintf(e, "<P>The Gopher query resulted in a blank response</P>");
 355     } else if (gopherState->HTML_pre) {
 356         storeAppendPrintf(e, "</PRE>\n");
 357     }
 358
 359     gopherHTMLFooter(e);
 360 }
 361
 362 /**
 363  * Convert Gopher to HTML.
 364  *
 365  * Borrow part of code from libwww2 came with Mosaic distribution.
 366  */
 367 static void
 368 gopherToHTML(GopherStateData * gopherState, char *inbuf, int len)
 369 {
 370     char *pos = inbuf;
 371     char *lpos = NULL;
 372     char *tline = NULL;
 373     LOCAL_ARRAY(char, line, TEMP_BUF_SIZE);
 374     LOCAL_ARRAY(char, tmpbuf, TEMP_BUF_SIZE);
 375     char *name = NULL;
 376     char *selector = NULL;
 377     char *host = NULL;
 378     char *port = NULL;
 379     char *escaped_selector = NULL;
 380     const char *icon_url = NULL;
 381     char gtype;
 382     StoreEntry *entry = NULL;
 383
 384     memset(tmpbuf, '\0', TEMP_BUF_SIZE);
 385     memset(line, '\0', TEMP_BUF_SIZE);
 386
 387     entry = gopherState->entry;
 388
 389     if (gopherState->conversion == GopherStateData::HTML_INDEX_PAGE) {
 390         char *html_url = html_quote(entry->url());
 391         gopherHTMLHeader(entry, "Gopher Index %s", html_url);
 392         storeAppendPrintf(entry,
 393                           "<p>This is a searchable Gopher index. Use the search\n"
 394                           "function of your browser to enter search terms.\n"
 395                           "<ISINDEX>\n");
 396         gopherHTMLFooter(entry);
 397         /* now let start sending stuff to client */
 398         entry->flush();
 399         gopherState->HTML_header_added = 1;
 400
 401         return;
 402     }
 403
 404     if (gopherState->conversion == GopherStateData::HTML_CSO_PAGE) {
 405         char *html_url = html_quote(entry->url());
 406         gopherHTMLHeader(entry, "CSO Search of %s", html_url);
 407         storeAppendPrintf(entry,
 408                           "<P>A CSO database usually contains a phonebook or\n"
 409                           "directory.  Use the search function of your browser to enter\n"
 410                           "search terms.</P><ISINDEX>\n");
 411         gopherHTMLFooter(entry);
 412         /* now let start sending stuff to client */
 413         entry->flush();
 414         gopherState->HTML_header_added = 1;
 415
 416         return;
 417     }
 418
 419     String outbuf;
 420
 421     if (!gopherState->HTML_header_added) {
 422         if (gopherState->conversion == GopherStateData::HTML_CSO_RESULT)
 423             gopherHTMLHeader(entry, "CSO Search Result", NULL);
 424         else
 425             gopherHTMLHeader(entry, "Gopher Menu", NULL);
 426
 427         outbuf.append ("<PRE>");
 428
 429         gopherState->HTML_header_added = 1;
 430
 431         gopherState->HTML_pre = 1;
 432     }
 433
 434     while (pos < inbuf + len) {
 435         int llen;
 436         int left = len - (pos - inbuf);
 437         lpos = (char *)memchr(pos, '\n', left);
 438         if (lpos) {
 439             ++lpos;             /* Next line is after \n */
 440             llen = lpos - pos;
 441         } else {
 442             llen = left;
 443         }
 444         if (gopherState->len + llen >= TEMP_BUF_SIZE) {
 445             debugs(10, DBG_IMPORTANT, "GopherHTML: Buffer overflow. Lost some data on URL: " << entry->url()  );
 446             llen = TEMP_BUF_SIZE - gopherState->len - 1;
 447         }
 448         if (!lpos) {
 449             /* there is no complete line in inbuf */
 450             /* copy it to temp buffer */
 451             /* note: llen is adjusted above */
 452             memcpy(gopherState->buf + gopherState->len, pos, llen);
 453             gopherState->len += llen;
 454             break;
 455         }
 456         if (gopherState->len != 0) {
 457             /* there is something left from last tx. */
 458             memcpy(line, gopherState->buf, gopherState->len);
 459             memcpy(line + gopherState->len, pos, llen);
 460             llen += gopherState->len;
 461             gopherState->len = 0;
 462         } else {
 463             memcpy(line, pos, llen);
 464         }
 465         line[llen + 1] = '\0';
 466         /* move input to next line */
 467         pos = lpos;
 468
 469         /* at this point. We should have one line in buffer to process */
 470
 471         if (*line == '.') {
 472             /* skip it */
 473             memset(line, '\0', TEMP_BUF_SIZE);
 474             continue;
 475         }
 476
 477         switch (gopherState->conversion) {
 478
 479         case GopherStateData::HTML_INDEX_RESULT:
 480
 481         case GopherStateData::HTML_DIR: {
 482             tline = line;
 483             gtype = *tline;
 484             ++tline;
 485             name = tline;
 486             selector = strchr(tline, TAB);
 487
 488             if (selector) {
 489                 *selector = '\0';
 490                 ++selector;
 491                 host = strchr(selector, TAB);
 492
 493                 if (host) {
 494                     *host = '\0';
 495                     ++host;
 496                     port = strchr(host, TAB);
 497
 498                     if (port) {
 499                         char *junk;
 500                         port[0] = ':';
 501                         junk = strchr(host, TAB);
 502
 503                         if (junk)
 504                             *junk++ = 0;    /* Chop port */
 505                         else {
 506                             junk = strchr(host, '\r');
 507
 508                             if (junk)
 509                                 *junk++ = 0;    /* Chop port */
 510                             else {
 511                                 junk = strchr(host, '\n');
 512
 513                                 if (junk)
 514                                     *junk++ = 0;    /* Chop port */
 515                             }
 516                         }
 517
 518                         if ((port[1] == '0') && (!port[2]))
 519                             port[0] = 0;    /* 0 means none */
 520                     }
 521
 522                     /* escape a selector here */
 523                     escaped_selector = xstrdup(rfc1738_escape_part(selector));
 524
 525                     switch (gtype) {
 526
 527                     case GOPHER_DIRECTORY:
 528                         icon_url = mimeGetIconURL("internal-menu");
 529                         break;
 530
 531                     case GOPHER_HTML:
 532
 533                     case GOPHER_FILE:
 534                         icon_url = mimeGetIconURL("internal-text");
 535                         break;
 536
 537                     case GOPHER_INDEX:
 538
 539                     case GOPHER_CSO:
 540                         icon_url = mimeGetIconURL("internal-index");
 541                         break;
 542
 543                     case GOPHER_IMAGE:
 544
 545                     case GOPHER_GIF:
 546
 547                     case GOPHER_PLUS_IMAGE:
 548                         icon_url = mimeGetIconURL("internal-image");
 549                         break;
 550
 551                     case GOPHER_SOUND:
 552
 553                     case GOPHER_PLUS_SOUND:
 554                         icon_url = mimeGetIconURL("internal-sound");
 555                         break;
 556
 557                     case GOPHER_PLUS_MOVIE:
 558                         icon_url = mimeGetIconURL("internal-movie");
 559                         break;
 560
 561                     case GOPHER_TELNET:
 562
 563                     case GOPHER_3270:
 564                         icon_url = mimeGetIconURL("internal-telnet");
 565                         break;
 566
 567                     case GOPHER_BIN:
 568
 569                     case GOPHER_MACBINHEX:
 570
 571                     case GOPHER_DOSBIN:
 572
 573                     case GOPHER_UUENCODED:
 574                         icon_url = mimeGetIconURL("internal-binary");
 575                         break;
 576
 577                     case GOPHER_INFO:
 578                         icon_url = NULL;
 579                         break;
 580
 581                     default:
 582                         icon_url = mimeGetIconURL("internal-unknown");
 583                         break;
 584                     }
 585
 586                     memset(tmpbuf, '\0', TEMP_BUF_SIZE);
 587
 588                     if ((gtype == GOPHER_TELNET) || (gtype == GOPHER_3270)) {
 589                         if (strlen(escaped_selector) != 0)
 590                             snprintf(tmpbuf, TEMP_BUF_SIZE, "<IMG border=\"0\" SRC=\"%s\"> <A HREF=\"telnet://%s@%s%s%s/\">%s</A>\n",
 591                                      icon_url, escaped_selector, rfc1738_escape_part(host),
 592                                      *port ? ":" : "", port, html_quote(name));
 593                         else
 594                             snprintf(tmpbuf, TEMP_BUF_SIZE, "<IMG border=\"0\" SRC=\"%s\"> <A HREF=\"telnet://%s%s%s/\">%s</A>\n",
 595                                      icon_url, rfc1738_escape_part(host), *port ? ":" : "",
 596                                      port, html_quote(name));
 597
 598                     } else if (gtype == GOPHER_INFO) {
 599                         snprintf(tmpbuf, TEMP_BUF_SIZE, "\t%s\n", html_quote(name));
 600                     } else {
 601                         if (strncmp(selector, "GET /", 5) == 0) {
 602                             /* WWW link */
 603                             snprintf(tmpbuf, TEMP_BUF_SIZE, "<IMG border=\"0\" SRC=\"%s\"> <A HREF=\"http://%s/%s\">%s</A>\n",
 604                                      icon_url, host, rfc1738_escape_unescaped(selector + 5), html_quote(name));
 605                         } else {
 606                             /* Standard link */
 607                             snprintf(tmpbuf, TEMP_BUF_SIZE, "<IMG border=\"0\" SRC=\"%s\"> <A HREF=\"gopher://%s/%c%s\">%s</A>\n",
 608                                      icon_url, host, gtype, escaped_selector, html_quote(name));
 609                         }
 610                     }
 611
 612                     safe_free(escaped_selector);
 613                     outbuf.append(tmpbuf);
 614                 } else {
 615                     memset(line, '\0', TEMP_BUF_SIZE);
 616                     continue;
 617                 }
 618             } else {
 619                 memset(line, '\0', TEMP_BUF_SIZE);
 620                 continue;
 621             }
 622
 623             break;
 624             }           /* HTML_DIR, HTML_INDEX_RESULT */
 625
 626         case GopherStateData::HTML_CSO_RESULT: {
 627             if (line[0] == '-') {
 628                 int code, recno;
 629                 char *s_code, *s_recno, *result;
 630
 631                 s_code = strtok(line + 1, ":\n");
 632                 s_recno = strtok(NULL, ":\n");
 633                 result = strtok(NULL, "\n");
 634
 635                 if (!result)
 636                     break;
 637
 638                 code = atoi(s_code);
 639
 640                 recno = atoi(s_recno);
 641
 642                 if (code != 200)
 643                     break;
 644
 645                 if (gopherState->cso_recno != recno) {
 646                     snprintf(tmpbuf, TEMP_BUF_SIZE, "</PRE><HR noshade size=\"1px\"><H2>Record# %d<br><i>%s</i></H2>\n<PRE>", recno, html_quote(result));
 647                     gopherState->cso_recno = recno;
 648                 } else {
 649                     snprintf(tmpbuf, TEMP_BUF_SIZE, "%s\n", html_quote(result));
 650                 }
 651
 652                 outbuf.append(tmpbuf);
 653                 break;
 654             } else {
 655                 int code;
 656                 char *s_code, *result;
 657
 658                 s_code = strtok(line, ":");
 659                 result = strtok(NULL, "\n");
 660
 661                 if (!result)
 662                     break;
 663
 664                 code = atoi(s_code);
 665
 666                 switch (code) {
 667
 668                 case 200: {
 669                     /* OK */
 670                     /* Do nothing here */
 671                     break;
 672                 }
 673
 674                 case 102:   /* Number of matches */
 675
 676                 case 501:   /* No Match */
 677
 678                 case 502: { /* Too Many Matches */
 679                     /* Print the message the server returns */
 680                     snprintf(tmpbuf, TEMP_BUF_SIZE, "</PRE><HR noshade size=\"1px\"><H2>%s</H2>\n<PRE>", html_quote(result));
 681                     outbuf.append(tmpbuf);
 682                     break;
 683                 }
 684
 685                 }
 686             }
 687
 688             }           /* HTML_CSO_RESULT */
 689
 690         default:
 691             break;      /* do nothing */
 692
 693         }           /* switch */
 694
 695     }               /* while loop */
 696
 697     if (outbuf.size() > 0) {
 698         entry->append(outbuf.rawBuf(), outbuf.size());
 699         /* now let start sending stuff to client */
 700         entry->flush();
 701     }
 702
 703     outbuf.clean();
 704     return;
 705 }
 706
 707 static void
 708 gopherTimeout(const CommTimeoutCbParams &io)
 709 {
 710     GopherStateData *gopherState = static_cast<GopherStateData *>(io.data);
 711     debugs(10, 4, HERE << io.conn << ": '" << gopherState->entry->url() << "'" );
 712
 713     gopherState->fwd->fail(new ErrorState(ERR_READ_TIMEOUT, Http::scGatewayTimeout, gopherState->fwd->request, gopherState->fwd->al));
 714
 715     if (Comm::IsConnOpen(io.conn))
 716         io.conn->close();
 717 }
 718
 719 /**
 720  * This will be called when data is ready to be read from fd.
 721  * Read until error or connection closed.
 722  */
 723 static void
 724 gopherReadReply(const Comm::ConnectionPointer &conn, char *buf, size_t len, Comm::Flag flag, int xerrno, void *data)
 725 {
 726     GopherStateData *gopherState = (GopherStateData *)data;
 727     StoreEntry *entry = gopherState->entry;
 728     int clen;
 729     int bin;
 730     size_t read_sz = BUFSIZ;
 731 #if USE_DELAY_POOLS
 732     DelayId delayId = entry->mem_obj->mostBytesAllowed();
 733 #endif
 734
 735     /* Bail out early on Comm::ERR_CLOSING - close handlers will tidy up for us */
 736
 737     if (flag == Comm::ERR_CLOSING) {
 738         return;
 739     }
 740
 741     assert(buf == gopherState->replybuf);
 742
 743     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 744         gopherState->serverConn->close();
 745         return;
 746     }
 747
 748 #if USE_DELAY_POOLS
 749     read_sz = delayId.bytesWanted(1, read_sz);
 750 #endif
 751
 752     /* leave one space for \0 in gopherToHTML */
 753
 754     if (flag == Comm::OK && len > 0) {
 755 #if USE_DELAY_POOLS
 756         delayId.bytesIn(len);
 757 #endif
 758
 759         statCounter.server.all.kbytes_in += len;
 760         statCounter.server.other.kbytes_in += len;
 761     }
 762
 763     debugs(10, 5, HERE << conn << " read len=" << len);
 764
 765     if (flag == Comm::OK && len > 0) {
 766         AsyncCall::Pointer nil;
 767         commSetConnTimeout(conn, Config.Timeout.read, nil);
 768         ++IOStats.Gopher.reads;
 769
 770         for (clen = len - 1, bin = 0; clen; ++bin)
 771             clen >>= 1;
 772
 773         ++IOStats.Gopher.read_hist[bin];
 774
 775         HttpRequest *req = gopherState->fwd->request;
 776         if (req->hier.bodyBytesRead < 0) {
 777             req->hier.bodyBytesRead = 0;
 778             // first bytes read, update Reply flags:
 779             gopherState->reply_->sources |= Http::Message::srcGopher;
 780         }
 781
 782         req->hier.bodyBytesRead += len;
 783     }
 784
 785     if (flag != Comm::OK) {
 786         debugs(50, DBG_IMPORTANT, MYNAME << "error reading: " << xstrerr(xerrno));
 787
 788         if (ignoreErrno(xerrno)) {
 789             AsyncCall::Pointer call = commCbCall(5,4, "gopherReadReply",
 790                                                  CommIoCbPtrFun(gopherReadReply, gopherState));
 791             comm_read(conn, buf, read_sz, call);
 792         } else {
 793             const auto err = new ErrorState(ERR_READ_ERROR, Http::scInternalServerError, gopherState->fwd->request, gopherState->fwd->al);
 794             err->xerrno = xerrno;
 795             gopherState->fwd->fail(err);
 796             gopherState->serverConn->close();
 797         }
 798     } else if (len == 0 && entry->isEmpty()) {
 799         gopherState->fwd->fail(new ErrorState(ERR_ZERO_SIZE_OBJECT, Http::scServiceUnavailable, gopherState->fwd->request, gopherState->fwd->al));
 800         gopherState->serverConn->close();
 801     } else if (len == 0) {
 802         /* Connection closed; retrieval done. */
 803         /* flush the rest of data in temp buf if there is one. */
 804
 805         if (gopherState->conversion != GopherStateData::NORMAL)
 806             gopherEndHTML(gopherState);
 807
 808         entry->timestampsSet();
 809         entry->flush();
 810         gopherState->fwd->complete();
 811         gopherState->serverConn->close();
 812     } else {
 813         if (gopherState->conversion != GopherStateData::NORMAL) {
 814             gopherToHTML(gopherState, buf, len);
 815         } else {
 816             entry->append(buf, len);
 817         }
 818         AsyncCall::Pointer call = commCbCall(5,4, "gopherReadReply",
 819                                              CommIoCbPtrFun(gopherReadReply, gopherState));
 820         comm_read(conn, buf, read_sz, call);
 821     }
 822 }
 823
 824 /**
 825  * This will be called when request write is complete. Schedule read of reply.
 826  */
 827 static void
 828 gopherSendComplete(const Comm::ConnectionPointer &conn, char *, size_t size, Comm::Flag errflag, int xerrno, void *data)
 829 {
 830     GopherStateData *gopherState = (GopherStateData *) data;
 831     StoreEntry *entry = gopherState->entry;
 832     debugs(10, 5, HERE << conn << " size: " << size << " errflag: " << errflag);
 833
 834     if (size > 0) {
 835         fd_bytes(conn->fd, size, FD_WRITE);
 836         statCounter.server.all.kbytes_out += size;
 837         statCounter.server.other.kbytes_out += size;
 838     }
 839
 840     if (errflag) {
 841         const auto err = new ErrorState(ERR_WRITE_ERROR, Http::scServiceUnavailable, gopherState->fwd->request, gopherState->fwd->al);
 842         err->xerrno = xerrno;
 843         err->port = gopherState->fwd->request->url.port();
 844         err->url = xstrdup(entry->url());
 845         gopherState->fwd->fail(err);
 846         gopherState->serverConn->close();
 847         return;
 848     }
 849
 850     /*
 851      * OK. We successfully reach remote site.  Start MIME typing
 852      * stuff.  Do it anyway even though request is not HTML type.
 853      */
 854     entry->buffer();
 855
 856     gopherMimeCreate(gopherState);
 857
 858     switch (gopherState->type_id) {
 859
 860     case GOPHER_DIRECTORY:
 861         /* we got to convert it first */
 862         gopherState->conversion = GopherStateData::HTML_DIR;
 863         gopherState->HTML_header_added = 0;
 864         break;
 865
 866     case GOPHER_INDEX:
 867         /* we got to convert it first */
 868         gopherState->conversion = GopherStateData::HTML_INDEX_RESULT;
 869         gopherState->HTML_header_added = 0;
 870         break;
 871
 872     case GOPHER_CSO:
 873         /* we got to convert it first */
 874         gopherState->conversion = GopherStateData::HTML_CSO_RESULT;
 875         gopherState->cso_recno = 0;
 876         gopherState->HTML_header_added = 0;
 877         break;
 878
 879     default:
 880         gopherState->conversion = GopherStateData::NORMAL;
 881         entry->flush();
 882     }
 883
 884     /* Schedule read reply. */
 885     AsyncCall::Pointer call =  commCbCall(5,5, "gopherReadReply",
 886                                           CommIoCbPtrFun(gopherReadReply, gopherState));
 887     entry->delayAwareRead(conn, gopherState->replybuf, BUFSIZ, call);
 888 }
 889
 890 /**
 891  * This will be called when connect completes. Write request.
 892  */
 893 static void
 894 gopherSendRequest(int, void *data)
 895 {
 896     GopherStateData *gopherState = (GopherStateData *)data;
 897     MemBuf mb;
 898     mb.init();
 899
 900     if (gopherState->type_id == GOPHER_CSO) {
 901         const char *t = strchr(gopherState->request, '?');
 902
 903         if (t)
 904             ++t;        /* skip the ? */
 905         else
 906             t = "";
 907
 908         mb.appendf("query %s\r\nquit", t);
 909     } else {
 910         if (gopherState->type_id == GOPHER_INDEX) {
 911             if (char *t = strchr(gopherState->request, '?'))
 912                 *t = '\t';
 913         }
 914         mb.append(gopherState->request, strlen(gopherState->request));
 915     }
 916     mb.append("\r\n", 2);
 917
 918     debugs(10, 5, gopherState->serverConn);
 919     AsyncCall::Pointer call = commCbCall(5,5, "gopherSendComplete",
 920                                          CommIoCbPtrFun(gopherSendComplete, gopherState));
 921     Comm::Write(gopherState->serverConn, &mb, call);
 922
 923     if (!gopherState->entry->makePublic())
 924         gopherState->entry->makePrivate(true);
 925 }
 926
 927 void
 928 gopherStart(FwdState * fwd)
 929 {
 930     GopherStateData *gopherState = new GopherStateData(fwd);
 931
 932     debugs(10, 3, gopherState->entry->url());
 933
 934     ++ statCounter.server.all.requests;
 935
 936     ++ statCounter.server.other.requests;
 937
 938     /* Parse url. */
 939     gopher_request_parse(fwd->request,
 940                          &gopherState->type_id, gopherState->request);
 941
 942     comm_add_close_handler(fwd->serverConnection()->fd, gopherStateFree, gopherState);
 943
 944     if (((gopherState->type_id == GOPHER_INDEX) || (gopherState->type_id == GOPHER_CSO))
 945             && (strchr(gopherState->request, '?') == NULL)) {
 946         /* Index URL without query word */
 947         /* We have to generate search page back to client. No need for connection */
 948         gopherMimeCreate(gopherState);
 949
 950         if (gopherState->type_id == GOPHER_INDEX) {
 951             gopherState->conversion = GopherStateData::HTML_INDEX_PAGE;
 952         } else {
 953             if (gopherState->type_id == GOPHER_CSO) {
 954                 gopherState->conversion = GopherStateData::HTML_CSO_PAGE;
 955             } else {
 956                 gopherState->conversion = GopherStateData::HTML_INDEX_PAGE;
 957             }
 958         }
 959
 960         gopherToHTML(gopherState, (char *) NULL, 0);
 961         fwd->complete();
 962         return;
 963     }
 964
 965     gopherState->serverConn = fwd->serverConnection();
 966     gopherSendRequest(fwd->serverConnection()->fd, gopherState);
 967     AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "gopherTimeout",
 968                                      CommTimeoutCbPtrFun(gopherTimeout, gopherState));
 969     commSetConnTimeout(fwd->serverConnection(), Config.Timeout.read, timeoutCall);
 970 }
 971