src/fs/ufs/store_io_ufs.cc

   1
   2 /*
   3  * $Id$
   4  *
   5  * DEBUG: section 79    Storage Manager UFS Interface
   6  * AUTHOR: Duane Wessels
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "Store.h"
  38 #include "ufscommon.h"
  39 #include "Generic.h"
  40 #include "DiskIO/DiskFile.h"
  41 #include "DiskIO/DiskIOStrategy.h"
  42 #include "DiskIO/ReadRequest.h"
  43 #include "DiskIO/WriteRequest.h"
  44
  45 #include "SwapDir.h"
  46
  47 bool
  48 UFSStrategy::shedLoad()
  49 {
  50     return io->shedLoad();
  51 }
  52
  53 int
  54 UFSStrategy::load()
  55 {
  56     return io->load();
  57 }
  58
  59 UFSStrategy::UFSStrategy (DiskIOStrategy *anIO) : io(anIO)
  60 {}
  61
  62 UFSStrategy::~UFSStrategy ()
  63 {
  64     delete io;
  65 }
  66
  67 StoreIOState::Pointer
  68 UFSStrategy::createState(SwapDir *SD, StoreEntry *e, StoreIOState::STIOCB * callback, void *callback_data) const
  69 {
  70     return new UFSStoreState (SD, e, callback, callback_data);
  71 }
  72
  73 DiskFile::Pointer
  74 UFSStrategy::newFile (char const *path)
  75 {
  76     return io->newFile(path);
  77 }
  78
  79
  80 void
  81 UFSStrategy::unlinkFile(char const *path)
  82 {
  83     io->unlinkFile(path);
  84 }
  85
  86 CBDATA_CLASS_INIT(UFSStoreState);
  87
  88 void *
  89 UFSStoreState::operator new (size_t)
  90 {
  91     CBDATA_INIT_TYPE(UFSStoreState);
  92     return cbdataAlloc(UFSStoreState);
  93 }
  94
  95 void
  96 UFSStoreState::operator delete (void *address)
  97 {
  98     cbdataFree(address);
  99 }
 100
 101 void
 102 UFSStoreState::ioCompletedNotification()
 103 {
 104     if (opening) {
 105         opening = false;
 106         debugs(79, 3, "UFSStoreState::ioCompletedNotification: dirno " <<
 107                swap_dirn  << ", fileno "<< std::setfill('0') << std::hex <<
 108                std::setw(8) << swap_filen  << " status "<< std::setfill(' ') <<
 109                std::dec << theFile->error());
 110
 111         assert (FILE_MODE(mode) == O_RDONLY);
 112         openDone();
 113
 114         return;
 115     }
 116
 117     if (creating) {
 118         creating = false;
 119         debugs(79, 3, "UFSStoreState::ioCompletedNotification: dirno " <<
 120                swap_dirn  << ", fileno "<< std::setfill('0') << std::hex <<
 121                std::setw(8) << swap_filen  << " status "<< std::setfill(' ') <<
 122                std::dec << theFile->error());
 123
 124         openDone();
 125
 126         return;
 127     }
 128
 129     assert (!(closing ||opening));
 130     debugs(79, 3, "diskd::ioCompleted: dirno " << swap_dirn  << ", fileno "<<
 131            std::setfill('0') << std::hex << std::setw(8) << swap_filen  <<
 132            " status "<< std::setfill(' ') << std::dec << theFile->error());
 133
 134     /* Ok, notification past open means an error has occured */
 135     assert (theFile->error());
 136     tryClosing();
 137 }
 138
 139 void
 140 UFSStoreState::openDone()
 141 {
 142     if (closing)
 143         debugs(0,0,HERE << "already closing in openDone()!?");
 144
 145     if (theFile->error()) {
 146         tryClosing();
 147         return;
 148     }
 149
 150     if (FILE_MODE(mode) == O_WRONLY) {
 151         drainWriteQueue();
 152
 153     } else if ((FILE_MODE(mode) == O_RDONLY) && !closing) {
 154         if (kickReadQueue())
 155             return;
 156     }
 157
 158     if (flags.try_closing)
 159         tryClosing();
 160
 161     debugs(79, 3, "UFSStoreState::openDone: exiting");
 162 }
 163
 164 void
 165 UFSStoreState::closeCompleted()
 166 {
 167     assert (closing);
 168     debugs(79, 3, "UFSStoreState::closeCompleted: dirno " << swap_dirn  <<
 169            ", fileno "<< std::setfill('0') << std::hex << std::setw(8) <<
 170            swap_filen  << " status "<< std::setfill(' ') << std::dec <<
 171            theFile->error());
 172
 173     if (theFile->error()) {
 174         debugs(79,3,HERE<< "theFile->error() ret " << theFile->error());
 175         doCloseCallback(DISK_ERROR);
 176     } else {
 177         doCloseCallback(DISK_OK);
 178     }
 179
 180     closing = false;
 181 }
 182
 183 /*
 184  * DPW 2006-05-24
 185  * This close function is called by the higher layer when it has finished
 186  * reading/writing everything, or otherwise wants to close the swap
 187  * file.  In the case of writing and using aufs storage, close() might
 188  * be called before any/all data is written, and even before the open
 189  * callback occurs.  Thus, we use our tryClosing() method, which knows
 190  * when it is safe to actually signal the lower layer for closing.
 191  */
 192 void
 193 UFSStoreState::close()
 194 {
 195     debugs(79, 3, "UFSStoreState::close: dirno " << swap_dirn  << ", fileno "<<
 196            std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swap_filen);
 197     tryClosing();
 198 }
 199
 200 void
 201 UFSStoreState::read_(char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data)
 202 {
 203     assert(read.callback == NULL);
 204     assert(read.callback_data == NULL);
 205     assert(!reading);
 206     assert(!closing);
 207     assert (callback);
 208
 209     if (!theFile->canRead()) {
 210         debugs(79, 3, "UFSStoreState::read_: queueing read because theFile can't read");
 211         queueRead (buf, size, offset, callback, callback_data);
 212         return;
 213     }
 214
 215     read.callback = callback;
 216     read.callback_data = cbdataReference(callback_data);
 217     debugs(79, 3, "UFSStoreState::read_: dirno " << swap_dirn  << ", fileno "<<
 218            std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swap_filen);
 219     offset_ = offset;
 220     read_buf = buf;
 221     reading = true;
 222     theFile->read(new ReadRequest(buf,offset,size));
 223 }
 224
 225
 226 /*
 227  * DPW 2006-05-24
 228  * This, the public write interface, places the write request at the end
 229  * of the pending_writes queue to ensure correct ordering of writes.
 230  * We could optimize things a little if there are no other pending
 231  * writes and just do the write directly.  But for now we'll keep the
 232  * code simpler and always go through the pending_writes queue.
 233  */
 234 void
 235 UFSStoreState::write(char const *buf, size_t size, off_t offset, FREE * free_func)
 236 {
 237     debugs(79, 3, "UFSStoreState::write: dirn " << swap_dirn  << ", fileno "<<
 238            std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swap_filen);
 239
 240     if (theFile->error()) {
 241         debugs(79,1,HERE << "avoid write on theFile with error");
 242         debugs(79,1,HERE << "calling free_func for " << (void*) buf);
 243         free_func((void*)buf);
 244         return;
 245     }
 246
 247     queueWrite(buf, size, offset, free_func);
 248     drainWriteQueue();
 249 }
 250
 251
 252 /*
 253  * DPW 2006-05-24
 254  * This, the private write method, calls the lower level write for the
 255  * first write request in the pending_writes queue.  doWrite() is only
 256  * called by drainWriteQueue().
 257  */
 258 void
 259 UFSStoreState::doWrite()
 260 {
 261     debugs(79, 3, HERE << this << " UFSStoreState::doWrite");
 262
 263     assert(theFile->canWrite());
 264
 265     _queued_write *q = (_queued_write *)linklistShift(&pending_writes);
 266
 267     if (q == NULL) {
 268         debugs(79, 3, HERE << this << " UFSStoreState::doWrite queue is empty");
 269         return;
 270     }
 271
 272     if (theFile->error()) {
 273         debugs(79,1,HERE << "avoid write on theFile with error");
 274         debugs(79,3,HERE << "calling free_func for " << (void*) q->buf);
 275         /*
 276          * DPW 2006-05-24
 277          * Note "free_func" is memNodeWriteComplete(), which doesn't
 278          * really free the memory.  Instead it clears the node's
 279          * write_pending flag.
 280          */
 281         q->free_func((void*)q->buf);
 282         delete q;
 283         return;
 284     }
 285
 286     /*
 287      * DPW 2006-05-24
 288      * UFSStoreState has a 'writing' flag that we used to set here,
 289      * but it wasn't really used anywhere.  In fact, some lower
 290      * layers such as DISKD allow multiple outstanding writes, which
 291      * makes the boolean writing flag meaningless.  We would need
 292      * a counter to keep track of writes going out and write callbacks
 293      * coming in.  For now let's just not use the writing flag at
 294      * all.
 295      */
 296     debugs(79, 3, HERE << this << " calling theFile->write(" << q->size << ")");
 297
 298     theFile->write(new WriteRequest(q->buf, q->offset, q->size, q->free_func));
 299     delete q;
 300 }
 301
 302 void
 303 UFSStoreState::readCompleted(const char *buf, int len, int errflag, RefCount<ReadRequest> result)
 304 {
 305     assert (result.getRaw());
 306     reading = false;
 307     debugs(79, 3, "UFSStoreState::readCompleted: dirno " << swap_dirn  <<
 308            ", fileno "<< std::setfill('0') << std::hex << std::setw(8) <<
 309            swap_filen  << " len "<< std::setfill(' ') << std::dec << len);
 310
 311     if (len > 0)
 312         offset_ += len;
 313
 314     STRCB *callback = read.callback;
 315
 316     assert(callback);
 317
 318     read.callback = NULL;
 319
 320     void *cbdata;
 321
 322     /* A note:
 323      * diskd IO queues closes via the diskd queue. So close callbacks
 324      * occur strictly after reads and writes.
 325      * ufs doesn't queue, it simply completes, so close callbacks occur
 326      * strictly after reads and writes.
 327      * aufs performs closes syncronously, so close events must be managed
 328      * to force strict ordering.
 329      * The below does this:
 330      * closing is set when theFile->close() has been called, and close only triggers
 331      * when no io's are pending.
 332      * writeCompleted likewise.
 333      */
 334     if (!closing && cbdataReferenceValidDone(read.callback_data, &cbdata)) {
 335         if (len > 0 && read_buf != buf)
 336             memcpy(read_buf, buf, len);
 337
 338         callback(cbdata, read_buf, len, this);
 339     }
 340
 341     if (flags.try_closing || (theFile != NULL && theFile->error()) )
 342         tryClosing();
 343 }
 344
 345 void
 346 UFSStoreState::writeCompleted(int errflag, size_t len, RefCount<WriteRequest> writeRequest)
 347 {
 348     debugs(79, 3, "UFSStoreState::writeCompleted: dirno " << swap_dirn << ", fileno " <<
 349            std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swap_filen <<
 350            ", len " << len );
 351     /*
 352      * DPW 2006-05-24
 353      * See doWrites() for why we don't update UFSStoreState::writing
 354      * here anymore.
 355      */
 356
 357     offset_ += len;
 358
 359     if (theFile->error()) {
 360         debugs(79,2,HERE << "UFSStoreState::writeCompleted" <<
 361                " detected an error, will try to close");
 362         tryClosing();
 363     }
 364
 365     /*
 366      * DPW 2007-04-12
 367      * I'm seeing disk files remain open under vanilla UFS storage
 368      * because storeClose() gets called before the last write is
 369      * complete.  I guess we have to check for the try_closing
 370      * flag here.
 371      */
 372     if (flags.try_closing) {
 373         debugs(72, 2, HERE << "UFSStoreState::writeCompleted" <<
 374                " flags.try_closing is set");
 375         tryClosing();
 376     }
 377 }
 378
 379 void
 380 UFSStoreState::doCloseCallback(int errflag)
 381 {
 382     debugs(79, 3, "storeUfsIOCallback: errflag=" << errflag);
 383     /*
 384      * DPW 2006-05-24
 385      * When we signal the higher layer with this callback, it might unlock
 386      * the StoreEntry and its associated data.  We must "free" any queued
 387      * I/Os (especially writes) now, otherwise the StoreEntry's mem_node's
 388      * will have their write_pending flag set, and we'll get an assertion.
 389      */
 390     freePending();
 391     STIOCB *theCallback = callback;
 392     callback = NULL;
 393
 394     void *cbdata;
 395
 396     if (cbdataReferenceValidDone(callback_data, &cbdata) && theCallback)
 397         theCallback(cbdata, errflag, this);
 398
 399     /*
 400      * We are finished with theFile since the lower layer signalled
 401      * us that the file has been closed.  This must be the last line,
 402      * as theFile may be the only object holding us in memory.
 403      */
 404     theFile = NULL;     // refcounted
 405 }
 406
 407 /* ============= THE REAL UFS CODE ================ */
 408
 409 UFSStoreState::UFSStoreState(SwapDir * SD, StoreEntry * anEntry, STIOCB * callback_, void *callback_data_) : opening (false), creating (false), closing (false), reading(false), writing(false), pending_reads(NULL), pending_writes (NULL)
 410 {
 411     swap_filen = anEntry->swap_filen;
 412     swap_dirn = SD->index;
 413     mode = O_BINARY;
 414     callback = callback_;
 415     callback_data = cbdataReference(callback_data_);
 416     e = anEntry;
 417     flags.write_draining = false;
 418     flags.try_closing = false;
 419 }
 420
 421 UFSStoreState::~UFSStoreState()
 422 {
 423     assert(pending_reads == NULL);
 424     assert(pending_writes == NULL);
 425 }
 426
 427 void
 428 UFSStoreState::freePending()
 429 {
 430     _queued_read *qr;
 431
 432     while ((qr = (_queued_read *)linklistShift(&pending_reads))) {
 433         cbdataReferenceDone(qr->callback_data);
 434         delete qr;
 435     }
 436
 437     debugs(79,3,HERE << "UFSStoreState::freePending: freed pending reads");
 438
 439     _queued_write *qw;
 440
 441     while ((qw = (_queued_write *)linklistShift(&pending_writes))) {
 442         if (qw->free_func)
 443             qw->free_func(const_cast<char *>(qw->buf));
 444         delete qw;
 445     }
 446
 447     debugs(79,3,HERE << "UFSStoreState::freePending: freed pending writes");
 448 }
 449
 450 bool
 451 UFSStoreState::kickReadQueue()
 452 {
 453     _queued_read *q = (_queued_read *)linklistShift(&pending_reads);
 454
 455     if (NULL == q)
 456         return false;
 457
 458     debugs(79, 3, "UFSStoreState::kickReadQueue: reading queued request of " << q->size << " bytes");
 459
 460     void *cbdata;
 461
 462     if (cbdataReferenceValidDone(q->callback_data, &cbdata))
 463         read_(q->buf, q->size, q->offset, q->callback, cbdata);
 464
 465     delete q;
 466
 467     return true;
 468 }
 469
 470 void
 471 UFSStoreState::queueRead(char *buf, size_t size, off_t offset, STRCB *callback, void *callback_data)
 472 {
 473     debugs(79, 3, "UFSStoreState::queueRead: queueing read");
 474     assert(opening);
 475     assert (pending_reads == NULL);
 476     _queued_read *q = new _queued_read;
 477     q->buf = buf;
 478     q->size = size;
 479     q->offset = offset;
 480     q->callback = callback;
 481     q->callback_data = cbdataReference(callback_data);
 482     linklistPush(&pending_reads, q);
 483 }
 484
 485 /*
 486  * DPW 2006-05-24
 487  * drainWriteQueue() is a loop around doWrite().
 488  */
 489 void
 490 UFSStoreState::drainWriteQueue()
 491 {
 492     /*
 493      * DPW 2007-04-12
 494      * We might find that flags.write_draining is already set
 495      * because schemes like diskd can process I/O acks
 496      * before sending another I/O request.    e.g. the following
 497      * sequence of events: open request -> write request ->
 498      * drainWriteQueue() -> queue full -> callbacks -> openDone() ->
 499      * drainWriteQueue().
 500      */
 501     if (flags.write_draining)
 502         return;
 503
 504     if (!theFile->canWrite())
 505         return;
 506
 507     flags.write_draining = true;
 508
 509     while (pending_writes != NULL) {
 510         doWrite();
 511     }
 512
 513     flags.write_draining = false;
 514
 515     if (flags.try_closing)
 516         tryClosing();
 517 }
 518
 519 /*
 520  * DPW 2006-05-24
 521  * This blows.  DiskThreadsDiskFile::close() won't actually do the close
 522  * if ioInProgress() is true.  So we have to check it here.  Maybe someday
 523  * DiskThreadsDiskFile::close() will be modified to have a return value,
 524  * or will remember to do the close for us.
 525  */
 526 void
 527 UFSStoreState::tryClosing()
 528 {
 529     debugs(79,3,HERE << this << " tryClosing()" <<
 530            " closing = " << closing <<
 531            " flags.try_closing = " << flags.try_closing <<
 532            " ioInProgress = " << theFile->ioInProgress());
 533
 534     if (theFile->ioInProgress()) {
 535         debugs(79, 3, HERE << this <<
 536                " won't close since ioInProgress is true, bailing");
 537         flags.try_closing = true;
 538         return;
 539     }
 540
 541     closing = true;
 542     flags.try_closing = false;
 543     theFile->close();
 544 }
 545
 546 void
 547 UFSStoreState::queueWrite(char const *buf, size_t size, off_t offset, FREE * free_func)
 548 {
 549     debugs(79, 3, HERE << this << " UFSStoreState::queueWrite: queueing write of size " << size);
 550
 551     _queued_write *q;
 552     q = new _queued_write;
 553     q->buf = buf;
 554     q->size = size;
 555     q->offset = offset;
 556     q->free_func = free_func;
 557     linklistPush(&pending_writes, q);
 558 }
 559
 560 StoreIOState::Pointer
 561 UFSStrategy::open(SwapDir * SD, StoreEntry * e, StoreIOState::STFNCB * file_callback,
 562                   StoreIOState::STIOCB * callback, void *callback_data)
 563 {
 564     assert (((UFSSwapDir *)SD)->IO == this);
 565     debugs(79, 3, "UFSStrategy::open: fileno "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e->swap_filen);
 566
 567     /* to consider: make createstate a private UFSStrategy call */
 568     StoreIOState::Pointer sio = createState (SD, e, callback, callback_data);
 569
 570     sio->mode |= O_RDONLY;
 571
 572     UFSStoreState *state = dynamic_cast <UFSStoreState *>(sio.getRaw());
 573
 574     assert (state);
 575
 576     char *path = ((UFSSwapDir *)SD)->fullPath(e->swap_filen, NULL);
 577
 578     DiskFile::Pointer myFile = newFile (path);
 579
 580     if (myFile.getRaw() == NULL)
 581         return NULL;
 582
 583     state->theFile = myFile;
 584
 585     state->opening = true;
 586
 587     myFile->open (sio->mode, 0644, state);
 588
 589     if (myFile->error())
 590         return NULL;
 591
 592     return sio;
 593 }
 594
 595 StoreIOState::Pointer
 596 UFSStrategy::create(SwapDir * SD, StoreEntry * e, StoreIOState::STFNCB * file_callback,
 597                     StoreIOState::STIOCB * callback, void *callback_data)
 598 {
 599     assert (((UFSSwapDir *)SD)->IO == this);
 600     /* Allocate a number */
 601     sfileno filn = ((UFSSwapDir *)SD)->mapBitAllocate();
 602     debugs(79, 3, "UFSStrategy::create: fileno "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << filn);
 603
 604     /* Shouldn't we handle a 'bitmap full' error here? */
 605
 606     StoreIOState::Pointer sio = createState (SD, e, callback, callback_data);
 607
 608     sio->mode |= O_WRONLY | O_CREAT | O_TRUNC;
 609
 610     sio->swap_filen = filn;
 611
 612     UFSStoreState *state = dynamic_cast <UFSStoreState *>(sio.getRaw());
 613
 614     assert (state);
 615
 616     char *path = ((UFSSwapDir *)SD)->fullPath(filn, NULL);
 617
 618     DiskFile::Pointer myFile = newFile (path);
 619
 620     if (myFile.getRaw() == NULL) {
 621         ((UFSSwapDir *)SD)->mapBitReset (filn);
 622         return NULL;
 623     }
 624
 625     state->theFile = myFile;
 626
 627     state->creating = true;
 628
 629     myFile->create (state->mode, 0644, state);
 630
 631     if (myFile->error()) {
 632         ((UFSSwapDir *)SD)->mapBitReset (filn);
 633         return NULL;
 634     }
 635
 636     /* now insert into the replacement policy */
 637     ((UFSSwapDir *)SD)->replacementAdd(e);
 638
 639     return sio;
 640 }
 641
 642 int
 643 UFSStrategy::callback()
 644 {
 645     return io->callback();
 646 }
 647
 648 void
 649 UFSStrategy::init()
 650 {
 651     io->init();
 652 }
 653
 654 void
 655 UFSStrategy::sync()
 656 {
 657     io->sync();
 658 }
 659
 660 void
 661 UFSStrategy::statfs(StoreEntry & sentry)const
 662 {
 663     io->statfs(sentry);
 664 }
 665