src/fs/coss/store_io_coss.cc

   1
   2 /*
   3  * $Id: store_io_coss.cc,v 1.21 2003/01/23 00:38:13 robertc Exp $
   4  *
   5  * DEBUG: section 79    Storage Manager COSS Interface
   6  * AUTHOR: Eric Stern
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "Store.h"
  38 #include <aio.h>
  39 #include "async_io.h"
  40 #include "store_coss.h"
  41 #include "MemObject.h"
  42 #include "fde.h"
  43 #include "SwapDir.h"
  44
  45 static DWCB storeCossWriteMemBufDone;
  46 static DRCB storeCossReadDone;
  47 static void storeCossIOCallback(storeIOState * sio, int errflag);
  48 static char *storeCossMemPointerFromDiskOffset(CossSwapDir * SD, size_t offset, CossMemBuf ** mb);
  49 static void storeCossMemBufLock(CossSwapDir * SD, storeIOState * e);
  50 static void storeCossMemBufUnlock(CossSwapDir * SD, storeIOState * e);
  51 static void storeCossWriteMemBuf(CossSwapDir * SD, CossMemBuf * t);
  52 static void storeCossWriteMemBufDone(int, int errflag, size_t len, void *my_data);
  53 static CossMemBuf *storeCossCreateMemBuf(CossSwapDir * SD, size_t start,
  54     sfileno curfn, int *collision);
  55
  56 CBDATA_TYPE(CossMemBuf);
  57
  58 /* === PUBLIC =========================================================== */
  59
  60 MemPool *CossState::Pool = NULL;
  61
  62 void *
  63 CossState::operator new (size_t)
  64 {
  65     if (!Pool)
  66         Pool = memPoolCreate("Squid COSS State Data", sizeof (CossState));
  67     return memPoolAlloc(Pool);
  68 }
  69
  70 void
  71 CossState::operator delete (void *address)
  72 {
  73     memPoolFree (Pool, address);
  74 }
  75
  76 CossState::CossState(CossSwapDir *aCSD):SD (aCSD)
  77 {
  78 }
  79
  80
  81 /*
  82  * This routine sucks. I want to rewrite it when possible, and I also think
  83  * that we should check after creatmembuf() to see if the object has a
  84  * RELEASE_REQUEST set on it (thanks Eric!) rather than this way which seems
  85  * to work..
  86  * -- Adrian
  87  */
  88 off_t
  89 storeCossAllocate(CossSwapDir * SD, const StoreEntry * e, int which)
  90 {
  91     CossMemBuf *newmb;
  92     off_t retofs;
  93     size_t allocsize;
  94     int coll = 0;
  95     sfileno checkf;
  96
  97     /* Make sure we chcek collisions if reallocating */
  98     if (which == COSS_ALLOC_REALLOC)
  99         checkf = e->swap_filen;
 100     else
 101         checkf = -1;
 102
 103     retofs = e->swap_filen;     /* Just for defaults, or while rebuilding */
 104
 105     if (e->swap_file_sz > 0)
 106         allocsize = e->swap_file_sz;
 107     else
 108         allocsize = objectLen(e) + e->mem_obj->swap_hdr_sz;
 109
 110     /* Since we're not supporting NOTIFY anymore, lets fail */
 111     assert(which != COSS_ALLOC_NOTIFY);
 112
 113     /* Check if we have overflowed the disk .. */
 114     if ((SD->current_offset + allocsize) > (size_t)(SD->max_size << 10)) {
 115         /*
 116          * tried to allocate past the end of the disk, so wrap
 117          * back to the beginning
 118          */
 119         SD->current_membuf->flags.full = 1;
 120         SD->current_membuf->diskend = SD->current_offset - 1;
 121         SD->current_offset = 0; /* wrap back to beginning */
 122         debug(79, 2) ("storeCossAllocate: wrap to 0\n");
 123
 124         newmb = storeCossCreateMemBuf(SD, 0, checkf, &coll);
 125         SD->current_membuf = newmb;
 126
 127         /* Check if we have overflowed the MemBuf */
 128     } else if ((SD->current_offset + allocsize) > SD->current_membuf->diskend) {
 129         /*
 130          * Skip the blank space at the end of the stripe. start over.
 131          */
 132         SD->current_membuf->flags.full = 1;
 133         SD->current_offset = SD->current_membuf->diskend + 1;
 134         debug(79, 2) ("storeCossAllocate: New offset - %ld\n",
 135             (long int) SD->current_offset);
 136         newmb = storeCossCreateMemBuf(SD, SD->current_offset, checkf, &coll);
 137         SD->current_membuf = newmb;
 138     }
 139     /* If we didn't get a collision, then update the current offset and return it */
 140     if (coll == 0) {
 141         retofs = SD->current_offset;
 142         SD->current_offset = retofs + allocsize;
 143         return retofs;
 144     } else {
 145         debug(79, 3) ("storeCossAllocate: Collision\n");
 146         return -1;
 147     }
 148 }
 149
 150 void
 151 CossSwapDir::unlink(StoreEntry & e)
 152 {
 153     debug(79, 3) ("storeCossUnlink: offset %d\n", e.swap_filen);
 154     storeCossRemove(this, &e);
 155 }
 156
 157 StoreIOState::Pointer
 158 CossSwapDir::createStoreIO(StoreEntry &e, STFNCB * file_callback, STIOCB * callback, void *callback_data)
 159 {
 160     CossState *cstate;
 161     StoreIOState::Pointer sio = new CossState(this);
 162     cstate = dynamic_cast<CossState *>(sio.getRaw());
 163     sio->offset_ = 0;
 164     sio->mode = O_WRONLY | O_BINARY;
 165
 166     /*
 167      * If we get handed an object with a size of -1,
 168      * the squid code is broken
 169      */
 170     assert(e.mem_obj->object_sz != -1);
 171
 172     /*
 173      * this one is kinda strange - Eric called storeCossAllocate(), then
 174      * storeCossOpen(O_RDONLY) .. weird. Anyway, I'm allocating this now.
 175      */
 176     cstate->st_size = objectLen(&e) + e.mem_obj->swap_hdr_sz;
 177     sio->swap_dirn = index;
 178     sio->swap_filen = storeCossAllocate(this, &e, COSS_ALLOC_ALLOCATE);
 179     debug(79, 3) ("storeCossCreate: offset %d, size %ld, end %ld\n", sio->swap_filen, (long int) cstate->st_size, (long int) (sio->swap_filen + cstate->st_size));
 180
 181     sio->callback = callback;
 182     sio->file_callback = file_callback;
 183     sio->callback_data = cbdataReference(callback_data);
 184     sio->e = &e;
 185
 186     cstate->flags.writing = 0;
 187     cstate->flags.reading = 0;
 188     cstate->readbuffer = NULL;
 189     cstate->reqdiskoffset = -1;
 190
 191     /* Now add it into the index list */
 192     storeCossAdd(this, &e);
 193
 194     storeCossMemBufLock(this, sio.getRaw());
 195     return sio;
 196 }
 197
 198 StoreIOState::Pointer
 199 CossSwapDir::openStoreIO(StoreEntry & e, STFNCB * file_callback,
 200     STIOCB * callback, void *callback_data)
 201 {
 202     char *p;
 203     CossState *cstate;
 204     sfileno f = e.swap_filen;
 205
 206     debug(79, 3) ("storeCossOpen: offset %d\n", f);
 207
 208     StoreIOState::Pointer sio = new CossState (this);
 209     cstate = dynamic_cast<CossState *>(sio.getRaw());
 210
 211     sio->swap_filen = f;
 212     sio->swap_dirn = index;
 213     sio->offset_ = 0;
 214     sio->mode = O_RDONLY | O_BINARY;
 215     sio->callback = callback;
 216     sio->file_callback = file_callback;
 217     sio->callback_data = cbdataReference(callback_data);
 218     cstate->st_size = e.swap_file_sz;
 219     sio->e = &e;
 220
 221     cstate->flags.writing = 0;
 222     cstate->flags.reading = 0;
 223     cstate->readbuffer = NULL;
 224     cstate->reqdiskoffset = -1;
 225     p = storeCossMemPointerFromDiskOffset(this, f, NULL);
 226     /* make local copy so we don't have to lock membuf */
 227     if (p) {
 228         cstate->readbuffer = (char *)xmalloc(cstate->st_size);
 229         xmemcpy(cstate->readbuffer, p, cstate->st_size);
 230     } else {
 231         /* Do the allocation */
 232         /* this is the first time we've been called on a new sio
 233          * read the whole object into memory, then return the
 234          * requested amount
 235          */
 236         /*
 237          * This bit of code actually does the LRU disk thing - we realloc
 238          * a place for the object here, and the file_read() reads the object
 239          * into the cossmembuf for later writing ..
 240          */
 241         cstate->reqdiskoffset = sio->swap_filen;
 242         sio->swap_filen = -1;
 243         sio->swap_filen = storeCossAllocate(this, &e, COSS_ALLOC_REALLOC);
 244         if (sio->swap_filen == -1) {
 245             /* We have to clean up neatly .. */
 246             numcollisions++;
 247             debug(79, 2) ("storeCossOpen: Reallocation of %d/%d failed\n", e.swap_dirn, e.swap_filen);
 248             /* XXX XXX XXX Will squid call storeUnlink for this object? */
 249             return NULL;
 250         }
 251         /* Notify the upper levels that we've changed file number */
 252         sio->file_callback(sio->callback_data, 0, sio.getRaw());
 253
 254         /*
 255          * lock the buffer so it doesn't get swapped out on us
 256          * this will get unlocked in storeCossReadDone
 257          */
 258         storeCossMemBufLock(this, sio.getRaw());
 259
 260         /*
 261          * Do the index magic to keep the disk and memory LRUs identical
 262          */
 263         storeCossRemove(this, &e);
 264         storeCossAdd(this, &e);
 265
 266         /*
 267          * Since we've reallocated a spot for this object, we need to
 268          * write it to the cossmembuf *and* return it in the read ..
 269          */
 270         cstate->readbuffer = NULL;
 271     }
 272     return sio;
 273 }
 274
 275 void
 276 CossState::close()
 277 {
 278     debug(79, 3) ("storeCossClose: offset %d\n", swap_filen);
 279     if (FILE_MODE(mode) == O_WRONLY)
 280         storeCossMemBufUnlock(SD, this);
 281     storeCossIOCallback(this, 0);
 282 }
 283
 284 void
 285 CossState::read_(char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data)
 286 {
 287     char *p;
 288     CossSwapDir *SD = (CossSwapDir *)INDEXSD(swap_dirn);
 289
 290     assert(read.callback == NULL);
 291     assert(read.callback_data == NULL);
 292     read.callback = callback;
 293     read.callback_data = cbdataReference(callback_data);
 294     debug(79, 3) ("storeCossRead: offset %ld\n", (long int) offset);
 295     offset_ = offset;
 296     flags.reading = 1;
 297     if ((offset + size) > st_size)
 298         size = st_size - offset;
 299     requestlen = size;
 300     requestbuf = buf;
 301     requestoffset = offset;
 302     if (readbuffer == NULL) {
 303         p = storeCossMemPointerFromDiskOffset(SD, swap_filen, NULL);
 304         /* Remember we need to translate the block offset to a disk offset! */
 305         a_file_read(&SD->aq, SD->fd,
 306             p,
 307             st_size,
 308             reqdiskoffset,
 309             storeCossReadDone,
 310             this);
 311         reqdiskoffset = 0;      /* XXX */
 312     } else {
 313         storeCossReadDone(SD->fd,
 314             readbuffer,
 315             st_size,
 316             0,
 317             this);
 318     }
 319 }
 320
 321 void
 322 CossState::write(char const *buf, size_t size, off_t offset, FREE * free_func)
 323 {
 324     char *dest;
 325     CossMemBuf *membuf;
 326     off_t diskoffset;
 327
 328     /*
 329      * If we get handed an object with a size of -1,
 330      * the squid code is broken
 331      */
 332     assert(e->mem_obj->object_sz != -1);
 333
 334     debug(79, 3) ("storeCossWrite: offset %ld, len %lu\n", (long int) offset_, (unsigned long int) size);
 335     diskoffset = swap_filen + offset_;
 336     CossSwapDir *SD = (CossSwapDir *)INDEXSD(swap_dirn);
 337     dest = storeCossMemPointerFromDiskOffset(SD, diskoffset, &membuf);
 338     assert(dest != NULL);
 339     xmemcpy(dest, buf, size);
 340     offset_ += size;
 341     if (free_func)
 342         (free_func) ((char *)buf);
 343 }
 344
 345
 346 /*  === STATIC =========================================================== */
 347
 348 static void
 349 storeCossReadDone(int rvfd, const char *buf, int len, int errflag, void *my_data)
 350 {
 351     storeIOState *sio = (storeIOState *)my_data;
 352     char *p;
 353     STRCB *callback = sio->read.callback;
 354     void *cbdata;
 355     CossSwapDir *SD = (CossSwapDir *)INDEXSD(sio->swap_dirn);
 356     CossState *cstate = dynamic_cast<CossState *>(sio);
 357     ssize_t rlen;
 358
 359     debug(79, 3) ("storeCossReadDone: fileno %d, FD %d, len %d\n",
 360         sio->swap_filen, rvfd, len);
 361     cstate->flags.reading = 0;
 362     if (errflag) {
 363         debug(79, 3) ("storeCossReadDone: got failure (%d)\n", errflag);
 364         rlen = -1;
 365     } else {
 366         if (cstate->readbuffer == NULL) {
 367             cstate->readbuffer = (char *)xmalloc(cstate->st_size);
 368             p = storeCossMemPointerFromDiskOffset(SD, sio->swap_filen, NULL);
 369             xmemcpy(cstate->readbuffer, p, cstate->st_size);
 370             storeCossMemBufUnlock(SD, sio);
 371         }
 372         sio->offset_ += len;
 373         xmemcpy(cstate->requestbuf, &cstate->readbuffer[cstate->requestoffset],
 374             cstate->requestlen);
 375         rlen = (size_t) cstate->requestlen;
 376     }
 377     assert(callback);
 378     sio->read.callback = NULL;
 379     if (cbdataReferenceValidDone(sio->read.callback_data, &cbdata))
 380         callback(cbdata, cstate->requestbuf, rlen);
 381 }
 382
 383 static void
 384 storeCossIOCallback(storeIOState * sio, int errflag)
 385 {
 386     CossState *cstate = dynamic_cast<CossState *>(sio);
 387     STIOCB *callback = sio->callback;
 388     void *cbdata;
 389     debug(79, 3) ("storeCossIOCallback: errflag=%d\n", errflag);
 390     xfree(cstate->readbuffer);
 391     sio->callback = NULL;
 392     if (cbdataReferenceValidDone(sio->callback_data, &cbdata))
 393         callback(cbdata, errflag, sio);
 394     cbdataFree(sio);
 395 }
 396
 397 static char *
 398 storeCossMemPointerFromDiskOffset(CossSwapDir * SD, size_t offset, CossMemBuf ** mb)
 399 {
 400     CossMemBuf *t;
 401     dlink_node *m;
 402
 403     for (m = SD->membufs.head; m; m = m->next) {
 404         t = (CossMemBuf *)m->data;
 405         if ((offset >= t->diskstart) && (offset <= t->diskend)) {
 406             if (mb)
 407                 *mb = t;
 408             return &t->buffer[offset - t->diskstart];
 409         }
 410     }
 411
 412     if (mb)
 413         *mb = NULL;
 414     return NULL;
 415 }
 416
 417 static void
 418 storeCossMemBufLock(CossSwapDir * SD, storeIOState * e)
 419 {
 420     CossMemBuf *t;
 421     dlink_node *m;
 422
 423     for (m = SD->membufs.head; m; m = m->next) {
 424         t = (CossMemBuf *)m->data;
 425         if (((size_t)e->swap_filen >= t->diskstart) && ((size_t)e->swap_filen <= t->diskend)) {
 426             debug(79, 3) ("storeCossMemBufLock: locking %p, lockcount %d\n", t, t->lockcount);
 427             t->lockcount++;
 428             return;
 429         }
 430     }
 431     debug(79, 3) ("storeCossMemBufLock: FAILED to lock %p\n", e);
 432 }
 433
 434 static void
 435 storeCossMemBufUnlock(CossSwapDir * SD, storeIOState * e)
 436 {
 437     CossMemBuf *t;
 438     dlink_node *m, *n;
 439
 440     for (m = SD->membufs.head; m; m = n) {
 441         /*
 442          * Note that storeCossWriteMemBuf() might call storeCossWriteMemBufDone
 443          * immediately (if the write finishes immediately, of course!) which
 444          * will make m = m->next kinda unworkable. So, get the next pointer.
 445          */
 446         n = m->next;
 447         t = (CossMemBuf *)m->data;
 448         if (((size_t)e->swap_filen >= t->diskstart) && ((size_t)e->swap_filen <= t->diskend)) {
 449             t->lockcount--;
 450             debug(79, 3) ("storeCossMemBufUnlock: unlocking %p, lockcount %d\n", t, t->lockcount);
 451         }
 452         if (t->flags.full && !t->flags.writing && !t->lockcount)
 453             storeCossWriteMemBuf(SD, t);
 454     }
 455 }
 456
 457 void
 458 CossSwapDir::sync()
 459 {
 460     CossMemBuf *t;
 461     dlink_node *m;
 462     int end;
 463
 464     /* First, flush pending IO ops */
 465     a_file_syncqueue(&aq);
 466
 467     /* Then, flush any in-memory partial membufs */
 468     if (!membufs.head)
 469         return;
 470     for (m = membufs.head; m; m = m->next) {
 471         t = (CossMemBuf *)m->data;
 472         if (t->flags.writing)
 473             sleep(5);           /* XXX EEEWWW! */
 474         lseek(fd, t->diskstart, SEEK_SET);
 475         end = (t == current_membuf) ? current_offset : t->diskend;
 476         FD_WRITE_METHOD(fd, t->buffer, end - t->diskstart);
 477     }
 478 }
 479
 480 static void
 481 storeCossWriteMemBuf(CossSwapDir * SD, CossMemBuf * t)
 482 {
 483     debug(79, 3) ("storeCossWriteMemBuf: offset %ld, len %ld\n",
 484         (long int) t->diskstart, (long int) (t->diskend - t->diskstart));
 485     t->flags.writing = 1;
 486     /* Remember that diskstart/diskend are block offsets! */
 487     a_file_write(&SD->aq, SD->fd, t->diskstart, &t->buffer,
 488         t->diskend - t->diskstart, storeCossWriteMemBufDone, t, NULL);
 489 }
 490
 491
 492 static void
 493 storeCossWriteMemBufDone(int rvfd, int errflag, size_t len, void *my_data)
 494 {
 495     CossMemBuf *t = (CossMemBuf *)my_data;
 496
 497     debug(79, 3) ("storeCossWriteMemBufDone: buf %p, len %ld\n", t, (long int) len);
 498     if (errflag)
 499         debug(79, 0) ("storeCossMemBufWriteDone: got failure (%d)\n", errflag);
 500
 501     dlinkDelete(&t->node, &t->SD->membufs);
 502     cbdataFree(t);
 503 }
 504
 505 static CossMemBuf *
 506 storeCossCreateMemBuf(CossSwapDir * SD, size_t start,
 507     sfileno curfn, int *collision)
 508 {
 509     CossMemBuf *newmb, *t;
 510     StoreEntry *e;
 511     dlink_node *m, *prev;
 512     int numreleased = 0;
 513
 514     CBDATA_INIT_TYPE_FREECB(CossMemBuf, NULL);
 515     newmb = cbdataAlloc(CossMemBuf);
 516     newmb->diskstart = start;
 517     debug(79, 3) ("storeCossCreateMemBuf: creating new membuf at %ld\n", (long int) newmb->diskstart);
 518     debug(79, 3) ("storeCossCreateMemBuf: at %p\n", newmb);
 519     newmb->diskend = newmb->diskstart + COSS_MEMBUF_SZ - 1;
 520     newmb->flags.full = 0;
 521     newmb->flags.writing = 0;
 522     newmb->lockcount = 0;
 523     newmb->SD = SD;
 524     /* XXX This should be reversed, with the new buffer last in the chain */
 525     dlinkAdd(newmb, &newmb->node, &SD->membufs);
 526
 527     /* Print out the list of membufs */
 528     for (m = SD->membufs.head; m; m = m->next) {
 529         t = (CossMemBuf *)m->data;
 530         debug(79, 3) ("storeCossCreateMemBuf: membuflist %ld lockcount %d\n", (long int) t->diskstart, t->lockcount);
 531     }
 532
 533     /*
 534      * Kill objects from the tail to make space for a new chunk
 535      */
 536     for (m = SD->cossindex.tail; m; m = prev) {
 537         prev = m->prev;
 538         e = (StoreEntry *)m->data;
 539         if (curfn == e->swap_filen)
 540             *collision = 1;     /* Mark an object alloc collision */
 541         if (((size_t)e->swap_filen >= newmb->diskstart) &&
 542             ((size_t)e->swap_filen <= newmb->diskend)) {
 543             storeRelease(e);
 544             numreleased++;
 545         } else
 546             break;
 547     }
 548     if (numreleased > 0)
 549         debug(79, 3) ("storeCossCreateMemBuf: this allocation released %d storeEntries\n", numreleased);
 550     return newmb;
 551 }
 552
 553 /*
 554  * Creates the initial membuf after rebuild
 555  */
 556 void
 557 storeCossStartMembuf(CossSwapDir * sd)
 558 {
 559     CossMemBuf *newmb = storeCossCreateMemBuf(sd, sd->current_offset, -1, NULL);
 560     assert(!sd->current_membuf);
 561     sd->current_membuf = newmb;
 562 }
 563
 564 /*
 565  * Clean up any references from the SIO before it get's released.
 566  */
 567 CossState::~CossState()
 568 {
 569 }