From: hno <> Date: Mon, 13 Aug 2007 23:20:50 +0000 (+0000) Subject: Author: wessels & Christos Tsantilas X-Git-Tag: SQUID_3_0_PRE7~83 X-Git-Url: http://git.ipfire.org/gitweb/?a=commitdiff_plain;h=47f6e231eb0dcf02f69a97a3adebc10ae744fefb;p=thirdparty%2Fsquid.git Author: wessels & Christos Tsantilas Import squid3-largeobj branch, fixing proxying and caching of >2GB objects this patch converts all variables referring to object sizes & offsets to 64-bits, and gets rid of a lot of related casts. The cache format is also updated to use 64-bit offsets, using a format compatible with Squid-2.6. --- diff --git a/include/Range.h b/include/Range.h index e5693f004a..2023bafff1 100644 --- a/include/Range.h +++ b/include/Range.h @@ -1,6 +1,6 @@ /* - * $Id: Range.h,v 1.7 2006/04/22 05:28:55 robertc Exp $ + * $Id: Range.h,v 1.8 2007/08/13 17:20:50 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -50,7 +50,7 @@ public: C start; C end; Range intersection (Range const &) const; - size_t size() const; + C size() const; }; template @@ -75,7 +75,7 @@ Range::intersection (Range const &rhs) const } template -size_t +C Range::size() const { return end > start ? end - start : 0; diff --git a/include/util.h b/include/util.h index 9345f37b0e..688b14408a 100644 --- a/include/util.h +++ b/include/util.h @@ -1,5 +1,5 @@ /* - * $Id: util.h,v 1.74 2006/10/14 13:43:18 serassio Exp $ + * $Id: util.h,v 1.75 2007/08/13 17:20:50 hno Exp $ * * AUTHOR: Harvest Derived * @@ -135,6 +135,7 @@ SQUIDCEXTERN int xpercentInt(double part, double whole); SQUIDCEXTERN double xdiv(double nom, double denom); SQUIDCEXTERN const char *xitoa(int num); +SQUIDCEXTERN const char *xint64toa(int64_t num); #if !HAVE_DRAND48 SQUIDCEXTERN double drand48(void); diff --git a/lib/util.c b/lib/util.c index a4b35ede21..953cf9db33 100644 --- a/lib/util.c +++ b/lib/util.c @@ -1,6 +1,6 @@ /* - * $Id: util.c,v 1.95 2007/04/06 12:15:51 serassio Exp $ + * $Id: util.c,v 1.96 2007/08/13 17:20:50 hno Exp $ * * DEBUG: * AUTHOR: Harvest Derived @@ -919,6 +919,15 @@ xitoa(int num) return buf; } +/* int64_t to string */ +const char * +xint64toa(int64_t num) +{ + static char buf[24]; /* 2^64 = 18446744073709551616 */ + snprintf(buf, sizeof(buf), "%"PRId64, num); + return buf; +} + /* A default failure notifier when the main program hasn't installed any */ void default_failure_notify(const char *message) diff --git a/src/AccessLogEntry.h b/src/AccessLogEntry.h index 661c139e47..bb4408294e 100644 --- a/src/AccessLogEntry.h +++ b/src/AccessLogEntry.h @@ -1,6 +1,6 @@ /* - * $Id: AccessLogEntry.h,v 1.5 2006/05/29 00:14:59 robertc Exp $ + * $Id: AccessLogEntry.h,v 1.6 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -95,9 +95,9 @@ public: } struct IN_ADDR caddr; - size_t size; - off_t highOffset; - size_t objectSize; + int64_t size; + int64_t highOffset; + int64_t objectSize; log_type code; int msec; const char *rfc931; diff --git a/src/BodyPipe.cc b/src/BodyPipe.cc index 9997115b5c..a1e547a175 100644 --- a/src/BodyPipe.cc +++ b/src/BodyPipe.cc @@ -44,7 +44,7 @@ BodyPipe::~BodyPipe() theBuf.clean(); } -void BodyPipe::setBodySize(size_t aBodySize) +void BodyPipe::setBodySize(uint64_t aBodySize) { assert(!bodySizeKnown()); assert(aBodySize >= 0); @@ -59,13 +59,13 @@ void BodyPipe::setBodySize(size_t aBodySize) debugs(91,7, HERE << "set body size" << status()); } -size_t BodyPipe::bodySize() const +uint64_t BodyPipe::bodySize() const { assert(bodySizeKnown()); - return static_cast(theBodySize); + return static_cast(theBodySize); } -bool BodyPipe::expectMoreAfter(size_t offset) const +bool BodyPipe::expectMoreAfter(uint64_t offset) const { assert(theGetSize <= offset); return offset < thePutSize || // buffer has more now or @@ -333,9 +333,9 @@ const char *BodyPipe::status() const buf.append(" [", 2); - buf.Printf("%d<=%d", (int)theGetSize, (int)thePutSize); + buf.Printf("%"PRIu64"<=%"PRIu64, theGetSize, thePutSize); if (theBodySize >= 0) - buf.Printf("<=%d", (int)theBodySize); + buf.Printf("<=%"PRId64, theBodySize); else buf.append("<=?", 3); diff --git a/src/BodyPipe.h b/src/BodyPipe.h index 0a336ff0b0..25a3fa29d4 100644 --- a/src/BodyPipe.h +++ b/src/BodyPipe.h @@ -52,7 +52,7 @@ class BodyPipeCheckout { public: BodyPipe &pipe; MemBuf &buf; - const size_t offset; // of current content, relative to the body start + const uint64_t offset; // of current content, relative to the body start protected: const size_t checkedOutSize; @@ -81,10 +81,10 @@ class BodyPipe: public RefCountable { BodyPipe(Producer *aProducer); ~BodyPipe(); // asserts that producer and consumer are cleared - void setBodySize(size_t aSize); // set body size + void setBodySize(uint64_t aSize); // set body size bool bodySizeKnown() const { return theBodySize >= 0; } - size_t bodySize() const; - size_t consumedSize() const { return theGetSize; } + uint64_t bodySize() const; + uint64_t consumedSize() const { return theGetSize; } bool productionEnded() const { return !theProducer; } // called by producers @@ -99,7 +99,7 @@ class BodyPipe: public RefCountable { void clearConsumer(); // aborts if still piping size_t getMoreData(MemBuf &buf); void consume(size_t size); - bool expectMoreAfter(size_t offset) const; + bool expectMoreAfter(uint64_t offset) const; bool exhausted() const; // saw eof/abort and all data consumed // start or continue consuming when there is no consumer @@ -143,12 +143,12 @@ class BodyPipe: public RefCountable { AsyncCallWrapper(91,5, BodyPipe, tellBodyProducerAborted); private: - ssize_t theBodySize; // expected total content length, if known + int64_t theBodySize; // expected total content length, if known Producer *theProducer; // content producer, if any Consumer *theConsumer; // content consumer, if any - size_t thePutSize; // ever-increasing total - size_t theGetSize; // ever-increasing total + uint64_t thePutSize; // ever-increasing total + uint64_t theGetSize; // ever-increasing total int theCCallsPending; // outstanding calls to the consumer int theCCallsToSkip; // how many calls to the consumer we should skip diff --git a/src/HttpHdrContRange.cc b/src/HttpHdrContRange.cc index caade261b2..5e3ed914c2 100644 --- a/src/HttpHdrContRange.cc +++ b/src/HttpHdrContRange.cc @@ -1,6 +1,6 @@ /* - * $Id: HttpHdrContRange.cc,v 1.20 2007/05/04 20:30:16 wessels Exp $ + * $Id: HttpHdrContRange.cc,v 1.21 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 68 HTTP Content-Range Header * AUTHOR: Alex Rousskov @@ -81,16 +81,16 @@ httpHdrRangeRespSpecParseInit(HttpHdrRangeSpec * spec, const char *field, int fl } /* parse offset */ - if (!httpHeaderParseSize(field, &spec->offset)) + if (!httpHeaderParseOffset(field, &spec->offset)) return 0; p++; /* do we have last-pos ? */ if (p - field < flen) { - ssize_t last_pos; + int64_t last_pos; - if (!httpHeaderParseSize(p, &last_pos)) + if (!httpHeaderParseOffset(p, &last_pos)) return 0; spec->length = size_diff(last_pos + 1, spec->offset); @@ -100,7 +100,7 @@ httpHdrRangeRespSpecParseInit(HttpHdrRangeSpec * spec, const char *field, int fl assert (spec->length >= 0); /* we managed to parse, check if the result makes sence */ - if (known_spec((size_t)spec->length) && spec->length == 0) { + if (known_spec(spec->length) && spec->length == 0) { debugs(68, 2, "invalid range (" << spec->offset << " += " << (long int) spec->length << ") in resp-range-spec near: '" << field << "'"); return 0; @@ -115,11 +115,11 @@ httpHdrRangeRespSpecPackInto(const HttpHdrRangeSpec * spec, Packer * p) /* Ensure typecast is safe */ assert (spec->length >= 0); - if (!known_spec((size_t)spec->offset) || !known_spec((size_t)spec->length)) + if (!known_spec(spec->offset) || !known_spec(spec->length)) packerPrintf(p, "*"); else - packerPrintf(p, "bytes %ld-%ld", - (long int) spec->offset, (long int) spec->offset + spec->length - 1); + packerPrintf(p, "bytes %"PRId64"-%"PRId64, + spec->offset, spec->offset + spec->length - 1); } /* @@ -175,7 +175,7 @@ httpHdrContRangeParseInit(HttpHdrContRange * range, const char *str) if (*p == '*') range->elength = range_spec_unknown; - else if (!httpHeaderParseSize(p, &range->elength)) + else if (!httpHeaderParseOffset(p, &range->elength)) return 0; debugs(68, 8, "parsed content-range field: " << @@ -211,14 +211,14 @@ httpHdrContRangePackInto(const HttpHdrContRange * range, Packer * p) /* Ensure typecast is safe */ assert (range->elength >= 0); - if (!known_spec((size_t)range->elength)) + if (!known_spec(range->elength)) packerPrintf(p, "/*"); else - packerPrintf(p, "/%ld", (long int) range->elength); + packerPrintf(p, "/%"PRId64, range->elength); } void -httpHdrContRangeSet(HttpHdrContRange * cr, HttpHdrRangeSpec spec, ssize_t ent_len) +httpHdrContRangeSet(HttpHdrContRange * cr, HttpHdrRangeSpec spec, int64_t ent_len) { assert(cr && ent_len >= 0); cr->spec = spec; diff --git a/src/HttpHdrContRange.h b/src/HttpHdrContRange.h index f423a9bbe1..a49f6e9ce8 100644 --- a/src/HttpHdrContRange.h +++ b/src/HttpHdrContRange.h @@ -1,6 +1,6 @@ /* - * $Id: HttpHdrContRange.h,v 1.3 2006/04/22 05:29:17 robertc Exp $ + * $Id: HttpHdrContRange.h,v 1.4 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -43,7 +43,7 @@ class HttpHdrContRange public: HttpHdrRangeSpec spec; - ssize_t elength; /* entity length, not content length */ + int64_t elength; /* entity length, not content length */ }; /* Http Content Range Header Field */ @@ -55,9 +55,9 @@ SQUIDCEXTERN void httpHdrContRangeDestroy(HttpHdrContRange * crange); SQUIDCEXTERN HttpHdrContRange *httpHdrContRangeDup(const HttpHdrContRange * crange); SQUIDCEXTERN void httpHdrContRangePackInto(const HttpHdrContRange * crange, Packer * p); /* inits with given spec */ -SQUIDCEXTERN void httpHdrContRangeSet(HttpHdrContRange *, HttpHdrRangeSpec, ssize_t); +SQUIDCEXTERN void httpHdrContRangeSet(HttpHdrContRange *, HttpHdrRangeSpec, int64_t); ; -SQUIDCEXTERN void httpHeaderAddContRange(HttpHeader *, HttpHdrRangeSpec, ssize_t); +SQUIDCEXTERN void httpHeaderAddContRange(HttpHeader *, HttpHdrRangeSpec, int64_t); #endif /* SQUID_HTTPHDRCONTRANGE_H */ diff --git a/src/HttpHdrRange.cc b/src/HttpHdrRange.cc index 2aafc49f9c..0cb86ff902 100644 --- a/src/HttpHdrRange.cc +++ b/src/HttpHdrRange.cc @@ -1,6 +1,6 @@ /* - * $Id: HttpHdrRange.cc,v 1.44 2007/05/29 13:31:37 amosjeffries Exp $ + * $Id: HttpHdrRange.cc,v 1.45 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 64 HTTP Range Header * AUTHOR: Alex Rousskov @@ -63,7 +63,7 @@ /* globals */ size_t HttpHdrRange::ParsedCount = 0; -ssize_t const HttpHdrRangeSpec::UnknownPosition = -1; +int64_t const HttpHdrRangeSpec::UnknownPosition = -1; /* * Range-Spec @@ -93,7 +93,7 @@ HttpHdrRangeSpec::parseInit(const char *field, int flen) /* is it a suffix-byte-range-spec ? */ if (*field == '-') { - if (!httpHeaderParseSize(field + 1, &length)) + if (!httpHeaderParseOffset(field + 1, &length)) return false; } else /* must have a '-' somewhere in _this_ field */ @@ -101,16 +101,16 @@ HttpHdrRangeSpec::parseInit(const char *field, int flen) debugs(64, 2, "ignoring invalid (missing '-') range-spec near: '" << field << "'"); return false; } else { - if (!httpHeaderParseSize(field, &offset)) + if (!httpHeaderParseOffset(field, &offset)) return false; p++; /* do we have last-pos ? */ if (p - field < flen) { - ssize_t last_pos; + int64_t last_pos; - if (!httpHeaderParseSize(p, &last_pos)) + if (!httpHeaderParseOffset(p, &last_pos)) return false; HttpHdrRangeSpec::HttpRange aSpec (offset, last_pos + 1); @@ -132,20 +132,20 @@ void HttpHdrRangeSpec::packInto(Packer * packer) const { if (!known_spec(offset)) /* suffix */ - packerPrintf(packer, "-%ld", (long int) length); + packerPrintf(packer, "-%"PRId64, length); else if (!known_spec(length)) /* trailer */ - packerPrintf(packer, "%ld-", (long int) offset); + packerPrintf(packer, "%"PRId64"-", offset); else /* range */ - packerPrintf(packer, "%ld-%ld", - (long int) offset, (long int) offset + length - 1); + packerPrintf(packer, "%"PRId64"-%"PRId64, + offset, offset + length - 1); } void HttpHdrRangeSpec::outputInfo( char const *note) const { debugs(64, 5, "HttpHdrRangeSpec::canonize: " << note << ": [" << - (long int) offset << ", " << (long int) offset + length << - ") len: " << (long int) length); + offset << ", " << offset + length << + ") len: " << length); } /* fills "absent" positions in range specification based on response body size @@ -153,7 +153,7 @@ HttpHdrRangeSpec::outputInfo( char const *note) const * range is valid if its intersection with [0,length-1] is not empty */ int -HttpHdrRangeSpec::canonize(size_t clen) +HttpHdrRangeSpec::canonize(int64_t clen) { outputInfo ("have"); HttpRange object(0, clen); @@ -190,8 +190,8 @@ HttpHdrRangeSpec::mergeWith(const HttpHdrRangeSpec * donor) bool merged (false); #if MERGING_BREAKS_NOTHING /* Note: this code works, but some clients may not like its effects */ - size_t rhs = offset + length; /* no -1 ! */ - const size_t donor_rhs = donor->offset + donor->length; /* no -1 ! */ + int64_t rhs = offset + length; /* no -1 ! */ + const int64_t donor_rhs = donor->offset + donor->length; /* no -1 ! */ assert(known_spec(offset)); assert(known_spec(donor->offset)); assert(length > 0); @@ -402,7 +402,7 @@ HttpHdrRange::canonize(HttpReply *rep) } int -HttpHdrRange::canonize (size_t newClen) +HttpHdrRange::canonize (int64_t newClen) { clen = newClen; debugs(64, 3, "HttpHdrRange::canonize: started with " << specs.count << @@ -420,7 +420,7 @@ HttpHdrRange::canonize (size_t newClen) bool HttpHdrRange::isComplex() const { - size_t offset = 0; + int64_t offset = 0; assert(this); /* check that all rangers are in "strong" order */ const_iterator pos (begin()); @@ -429,7 +429,7 @@ HttpHdrRange::isComplex() const /* Ensure typecasts is safe */ assert ((*pos)->offset >= 0); - if ((unsigned int)(*pos)->offset < offset) + if ((*pos)->offset < offset) return 1; offset = (*pos)->offset + (*pos)->length; @@ -450,7 +450,7 @@ HttpHdrRange::willBeComplex() const assert(this); /* check that all rangers are in "strong" order, */ /* as far as we can tell without the content length */ - size_t offset = 0; + int64_t offset = 0; for (const_iterator pos (begin()); pos != end(); ++pos) { if (!known_spec((*pos)->offset)) /* ignore unknowns */ @@ -459,7 +459,7 @@ HttpHdrRange::willBeComplex() const /* Ensure typecasts is safe */ assert ((*pos)->offset >= 0); - if ((size_t) (*pos)->offset < offset) + if ((*pos)->offset < offset) return true; offset = (*pos)->offset; @@ -476,10 +476,10 @@ HttpHdrRange::willBeComplex() const * or HttpHdrRangeSpec::UnknownPosition * this is used for size limiting */ -ssize_t +int64_t HttpHdrRange::firstOffset() const { - ssize_t offset = HttpHdrRangeSpec::UnknownPosition; + int64_t offset = HttpHdrRangeSpec::UnknownPosition; assert(this); const_iterator pos = begin(); @@ -499,15 +499,15 @@ HttpHdrRange::firstOffset() const * ranges are combined into one, for example FTP REST. * Use 0 for size if unknown */ -ssize_t -HttpHdrRange::lowestOffset(ssize_t size) const +int64_t +HttpHdrRange::lowestOffset(int64_t size) const { - ssize_t offset = HttpHdrRangeSpec::UnknownPosition; + int64_t offset = HttpHdrRangeSpec::UnknownPosition; const_iterator pos = begin(); assert(this); while (pos != end()) { - ssize_t current = (*pos)->offset; + int64_t current = (*pos)->offset; if (!known_spec(current)) { if ((*pos)->length > size || !known_spec((*pos)->length)) @@ -538,7 +538,7 @@ HttpHdrRange::offsetLimitExceeded() const /* not a range request */ return false; - if (-1 == (ssize_t)Config.rangeOffsetLimit) + if (-1 == Config.rangeOffsetLimit) /* disabled */ return false; @@ -546,7 +546,7 @@ HttpHdrRange::offsetLimitExceeded() const /* tail request */ return true; - if ((ssize_t)Config.rangeOffsetLimit >= firstOffset()) + if (Config.rangeOffsetLimit >= firstOffset()) /* below the limit */ return false; @@ -590,14 +590,14 @@ HttpHdrRangeIter::updateSpec() } } -ssize_t +int64_t HttpHdrRangeIter::debt() const { debugs(64, 3, "HttpHdrRangeIter::debt: debt is " << debt_size); return debt_size; } -void HttpHdrRangeIter::debt(ssize_t newDebt) +void HttpHdrRangeIter::debt(int64_t newDebt) { debugs(64, 3, "HttpHdrRangeIter::debt: was " << debt_size << " now " << newDebt); debt_size = newDebt; diff --git a/src/HttpHeader.cc b/src/HttpHeader.cc index c4a34ac740..3b6ad24125 100644 --- a/src/HttpHeader.cc +++ b/src/HttpHeader.cc @@ -1,6 +1,6 @@ /* - * $Id: HttpHeader.cc,v 1.133 2007/05/29 13:31:37 amosjeffries Exp $ + * $Id: HttpHeader.cc,v 1.134 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 55 HTTP Header * AUTHOR: Alex Rousskov @@ -86,7 +86,7 @@ static const HttpHeaderFieldAttrs HeadersAttrs[] = {"Content-Base", HDR_CONTENT_BASE, ftStr}, {"Content-Encoding", HDR_CONTENT_ENCODING, ftStr}, {"Content-Language", HDR_CONTENT_LANGUAGE, ftStr}, - {"Content-Length", HDR_CONTENT_LENGTH, ftInt}, + {"Content-Length", HDR_CONTENT_LENGTH, ftInt64}, {"Content-Location", HDR_CONTENT_LOCATION, ftStr}, {"Content-MD5", HDR_CONTENT_MD5, ftStr}, /* for now */ {"Content-Range", HDR_CONTENT_RANGE, ftPContRange}, @@ -1024,6 +1024,15 @@ HttpHeader::putInt(http_hdr_type id, int number) addEntry(new HttpHeaderEntry(id, NULL, xitoa(number))); } +void +HttpHeader::putInt64(http_hdr_type id, int64_t number) +{ + assert_eid(id); + assert(Headers[id].type == ftInt64); /* must be of an appropriate type */ + assert(number >= 0); + addEntry(new HttpHeaderEntry(id, NULL, xint64toa(number))); +} + void HttpHeader::putTime(http_hdr_type id, time_t htime) { @@ -1156,6 +1165,19 @@ HttpHeader::getInt(http_hdr_type id) const return -1; } +int64_t +HttpHeader::getInt64(http_hdr_type id) const +{ + assert_eid(id); + assert(Headers[id].type == ftInt64); /* must be of an appropriate type */ + HttpHeaderEntry *e; + + if ((e = findEntry(id))) + return e->getInt64(); + + return -1; +} + time_t HttpHeader::getTime(http_hdr_type id) const { @@ -1510,6 +1532,20 @@ HttpHeaderEntry::getInt() const return val; } +int64_t +HttpHeaderEntry::getInt64() const +{ + assert_eid (id); + assert (Headers[id].type == ftInt64); + int64_t val = -1; + int ok = httpHeaderParseOffset(value.buf(), &val); + httpHeaderNoteParsedEntry(id, value, !ok); + /* XXX: Should we check ok - ie + * return ok ? -1 : value; + */ + return val; +} + static void httpHeaderNoteParsedEntry(http_hdr_type id, String const &context, int error) { diff --git a/src/HttpHeader.h b/src/HttpHeader.h index 667518f4ba..ddd3ac2915 100644 --- a/src/HttpHeader.h +++ b/src/HttpHeader.h @@ -1,6 +1,6 @@ /* - * $Id: HttpHeader.h,v 1.22 2007/08/01 22:52:45 amosjeffries Exp $ + * $Id: HttpHeader.h,v 1.23 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -124,6 +124,7 @@ typedef enum { typedef enum { ftInvalid = HDR_ENUM_END, /* to catch nasty errors with hdr_id<->fld_type clashes */ ftInt, + ftInt64, ftStr, ftDate_1123, ftETag, @@ -181,6 +182,7 @@ public: HttpHeaderEntry *clone() const; void packInto(Packer *p) const; int getInt() const; + int64_t getInt64() const; MEMPROXY_CLASS(HttpHeaderEntry); http_hdr_type id; String name; @@ -219,6 +221,7 @@ public: String getListMember(http_hdr_type id, const char *member, const char separator) const; int has(http_hdr_type id) const; void putInt(http_hdr_type id, int number); + void putInt64(http_hdr_type id, int64_t number); void putTime(http_hdr_type id, time_t htime); void insertTime(http_hdr_type id, time_t htime); void putStr(http_hdr_type id, const char *str); @@ -229,6 +232,7 @@ public: void putSc(HttpHdrSc *sc); void putExt(const char *name, const char *value); int getInt(http_hdr_type id) const; + int64_t getInt64(http_hdr_type id) const; time_t getTime(http_hdr_type id) const; const char *getStr(http_hdr_type id) const; const char *getLastStr(http_hdr_type id) const; diff --git a/src/HttpHeaderRange.h b/src/HttpHeaderRange.h index d1576938a9..aeb96881ec 100644 --- a/src/HttpHeaderRange.h +++ b/src/HttpHeaderRange.h @@ -1,6 +1,6 @@ /* - * $Id: HttpHeaderRange.h,v 1.10 2007/05/29 13:31:37 amosjeffries Exp $ + * $Id: HttpHeaderRange.h,v 1.11 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -48,19 +48,19 @@ class HttpHdrRangeSpec public: MEMPROXY_CLASS(HttpHdrRangeSpec); - typedef Range HttpRange; - static ssize_t const UnknownPosition; + typedef Range HttpRange; + static int64_t const UnknownPosition; HttpHdrRangeSpec(); static HttpHdrRangeSpec *Create(const char *field, int fieldLen); bool parseInit(const char *field, int flen); - int canonize(size_t clen); + int canonize(int64_t clen); void outputInfo( char const *note) const; void packInto(Packer * p) const; bool mergeWith(const HttpHdrRangeSpec * donor); - ssize_t offset; - ssize_t length; + int64_t offset; + int64_t length; }; MEMPROXY_CLASS_INLINE(HttpHdrRangeSpec) @@ -93,7 +93,7 @@ public: const_iterator end() const; /* adjust specs after the length is known */ - int canonize(size_t); + int canonize(int64_t); int canonize(HttpReply *rep); /* returns true if ranges are valid; inits HttpHdrRange */ bool parseInit(const String * range_spec); @@ -101,8 +101,8 @@ public: /* other */ bool isComplex() const; bool willBeComplex() const; - ssize_t firstOffset() const; - ssize_t lowestOffset(ssize_t) const; + int64_t firstOffset() const; + int64_t lowestOffset(int64_t) const; bool offsetLimitExceeded() const; bool contains(HttpHdrRangeSpec& r) const; Vector specs; @@ -110,7 +110,7 @@ public: private: void getCanonizedSpecs (Vector ©); void merge (Vector &basis); - ssize_t clen; + int64_t clen; }; MEMPROXY_CLASS_INLINE(HttpHdrRange) @@ -124,9 +124,9 @@ public: HttpHdrRange::iterator pos; const HttpHdrRangeSpec *currentSpec() const; void updateSpec(); - ssize_t debt() const; - void debt(ssize_t); - ssize_t debt_size; /* bytes left to send from the current spec */ + int64_t debt() const; + void debt(int64_t); + int64_t debt_size; /* bytes left to send from the current spec */ String boundary; /* boundary for multipart responses */ bool valid; }; diff --git a/src/HttpHeaderTools.cc b/src/HttpHeaderTools.cc index 5f208b0cd3..1a158e4904 100644 --- a/src/HttpHeaderTools.cc +++ b/src/HttpHeaderTools.cc @@ -1,6 +1,6 @@ /* - * $Id: HttpHeaderTools.cc,v 1.61 2007/05/29 13:31:37 amosjeffries Exp $ + * $Id: HttpHeaderTools.cc,v 1.62 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 66 HTTP Header Tools * AUTHOR: Alex Rousskov @@ -143,7 +143,7 @@ httpHeaderPutStrvf(HttpHeader * hdr, http_hdr_type id, const char *fmt, va_list /* wrapper arrounf PutContRange */ void -httpHeaderAddContRange(HttpHeader * hdr, HttpHdrRangeSpec spec, ssize_t ent_len) +httpHeaderAddContRange(HttpHeader * hdr, HttpHdrRangeSpec spec, int64_t ent_len) { HttpHdrContRange *cr = httpHdrContRangeCreate(); assert(hdr && ent_len >= 0); @@ -342,6 +342,16 @@ httpHeaderParseSize(const char *start, ssize_t * value) return res; } +int +httpHeaderParseOffset(const char *start, int64_t * value) +{ + int64_t res = strtoll(start, NULL, 10); + if (!res && EINVAL == errno) /* maybe not portable? */ + return 0; + *value = res; + return 1; +} + /* Parses a quoted-string field (RFC 2616 section 2.2), complains if * something went wrong, returns non-zero on success. diff --git a/src/HttpMsg.cc b/src/HttpMsg.cc index 98bbd73bf8..c742daa55b 100644 --- a/src/HttpMsg.cc +++ b/src/HttpMsg.cc @@ -1,6 +1,6 @@ /* - * $Id: HttpMsg.cc,v 1.42 2007/04/28 22:26:37 hno Exp $ + * $Id: HttpMsg.cc,v 1.43 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 74 HTTP Message * AUTHOR: Alex Rousskov @@ -355,7 +355,7 @@ void HttpMsg::packInto(Packer *p, bool full_uri) const void HttpMsg::hdrCacheInit() { - content_length = header.getInt(HDR_CONTENT_LENGTH); + content_length = header.getInt64(HDR_CONTENT_LENGTH); assert(NULL == cache_control); cache_control = header.getCc(); } diff --git a/src/HttpMsg.h b/src/HttpMsg.h index aa1ba0c6be..719c00a6b7 100644 --- a/src/HttpMsg.h +++ b/src/HttpMsg.h @@ -1,6 +1,6 @@ /* - * $Id: HttpMsg.h,v 1.15 2007/04/06 04:50:04 rousskov Exp $ + * $Id: HttpMsg.h,v 1.16 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -67,7 +67,7 @@ public: * Also used to report parsed header size if parse() is successful */ int hdr_sz; - int content_length; + int64_t content_length; protocol_t protocol; @@ -86,7 +86,7 @@ public: virtual int httpMsgParseError(); - virtual bool expectingBody(method_t, ssize_t&) const = 0; + virtual bool expectingBody(method_t, int64_t&) const = 0; void firstLineBuf(MemBuf&); diff --git a/src/HttpReply.cc b/src/HttpReply.cc index 709d420a04..8bff5faf59 100644 --- a/src/HttpReply.cc +++ b/src/HttpReply.cc @@ -1,6 +1,6 @@ /* - * $Id: HttpReply.cc,v 1.95 2007/05/29 13:31:37 amosjeffries Exp $ + * $Id: HttpReply.cc,v 1.96 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 58 HTTP Reply (Response) * AUTHOR: Alex Rousskov @@ -155,7 +155,7 @@ HttpReply::pack() MemBuf * httpPackedReply(HttpVersion ver, http_status status, const char *ctype, - int clen, time_t lmt, time_t expires) + int64_t clen, time_t lmt, time_t expires) { HttpReply *rep = new HttpReply; rep->setHeaders(ver, status, ctype, NULL, clen, lmt, expires); @@ -207,7 +207,7 @@ HttpReply::packed304Reply() void HttpReply::setHeaders(HttpVersion ver, http_status status, const char *reason, - const char *ctype, int clen, time_t lmt, time_t expires) + const char *ctype, int64_t clen, time_t lmt, time_t expires) { HttpHeader *hdr; httpStatusLineSet(&sline, ver, status, reason); @@ -223,7 +223,7 @@ HttpReply::setHeaders(HttpVersion ver, http_status status, const char *reason, content_type = String(); if (clen >= 0) - hdr->putInt(HDR_CONTENT_LENGTH, clen); + hdr->putInt64(HDR_CONTENT_LENGTH, clen); if (expires >= 0) hdr->putTime(HDR_EXPIRES, expires); @@ -249,7 +249,7 @@ HttpReply::redirect(http_status status, const char *loc) hdr = &header; hdr->putStr(HDR_SERVER, full_appname_string); hdr->putTime(HDR_DATE, squid_curtime); - hdr->putInt(HDR_CONTENT_LENGTH, 0); + hdr->putInt64(HDR_CONTENT_LENGTH, 0); hdr->putStr(HDR_LOCATION, loc); date = squid_curtime; content_length = 0; @@ -372,7 +372,7 @@ HttpReply::hdrCacheInit() { HttpMsg::hdrCacheInit(); - content_length = header.getInt(HDR_CONTENT_LENGTH); + content_length = header.getInt64(HDR_CONTENT_LENGTH); date = header.getTime(HDR_DATE); last_modified = header.getTime(HDR_LAST_MODIFIED); surrogate_control = header.getSc(); @@ -414,7 +414,7 @@ HttpReply::hdrCacheClean() /* * Returns the body size of a HTTP response */ -int +int64_t HttpReply::bodySize(method_t method) const { if (sline.version.major < 1) @@ -469,7 +469,7 @@ HttpReply::httpMsgParseError() * along with this response */ bool -HttpReply::expectingBody(method_t req_method, ssize_t& theSize) const +HttpReply::expectingBody(method_t req_method, int64_t& theSize) const { bool expectBody = true; diff --git a/src/HttpReply.h b/src/HttpReply.h index 29962625c2..6dfb3d0705 100644 --- a/src/HttpReply.h +++ b/src/HttpReply.h @@ -1,6 +1,6 @@ /* - * $Id: HttpReply.h,v 1.20 2007/05/29 13:31:37 amosjeffries Exp $ + * $Id: HttpReply.h,v 1.21 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -39,7 +39,7 @@ extern void httpReplyInitModule(void); /* do everything in one call: init, set, pack, clean, return MemBuf */ -extern MemBuf *httpPackedReply(HttpVersion ver, http_status status, const char *ctype, int clen, time_t lmt, time_t expires); +extern MemBuf *httpPackedReply(HttpVersion ver, http_status status, const char *ctype, int64_t clen, time_t lmt, time_t expires); /* Sync changes here with HttpReply.cc */ @@ -97,7 +97,7 @@ public: public: virtual int httpMsgParseError(); - virtual bool expectingBody(method_t, ssize_t&) const; + virtual bool expectingBody(method_t, int64_t&) const; void updateOnNotModified(HttpReply const *other); @@ -106,7 +106,7 @@ public: /* set commonly used info with one call */ void setHeaders(HttpVersion ver, http_status status, - const char *reason, const char *ctype, int clen, time_t lmt, time_t expires); + const char *reason, const char *ctype, int64_t clen, time_t lmt, time_t expires); /* mem-pack: returns a ready to use mem buffer with a packed reply */ MemBuf *pack(); @@ -116,7 +116,7 @@ public: void redirect(http_status, const char *); - int bodySize(method_t) const; + int64_t bodySize(method_t) const; int validatorsMatch (HttpReply const *other) const; diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc index 5f9333f0b6..80a840dd8d 100644 --- a/src/HttpRequest.cc +++ b/src/HttpRequest.cc @@ -1,6 +1,6 @@ /* - * $Id: HttpRequest.cc,v 1.76 2007/05/29 13:31:37 amosjeffries Exp $ + * $Id: HttpRequest.cc,v 1.77 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 73 HTTP Request * AUTHOR: Duane Wessels @@ -370,7 +370,7 @@ void HttpRequest::packFirstLineInto(Packer * p, bool full_uri) const * along with this request */ bool -HttpRequest::expectingBody(method_t unused, ssize_t& theSize) const +HttpRequest::expectingBody(method_t unused, int64_t& theSize) const { bool expectBody = false; diff --git a/src/HttpRequest.h b/src/HttpRequest.h index cfa1c389ba..b926f3449f 100644 --- a/src/HttpRequest.h +++ b/src/HttpRequest.h @@ -1,6 +1,6 @@ /* - * $Id: HttpRequest.h,v 1.29 2007/05/29 13:31:38 amosjeffries Exp $ + * $Id: HttpRequest.h,v 1.30 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -135,7 +135,7 @@ public: int parseHeader(const char *parse_start, int len); - virtual bool expectingBody(method_t unused, ssize_t&) const; + virtual bool expectingBody(method_t unused, int64_t&) const; bool bodyNibbled() const; // the request has a [partially] consumed body diff --git a/src/ICAP/ChunkedCodingParser.cc b/src/ICAP/ChunkedCodingParser.cc index 06608d2227..c71175143c 100644 --- a/src/ICAP/ChunkedCodingParser.cc +++ b/src/ICAP/ChunkedCodingParser.cc @@ -66,10 +66,10 @@ void ChunkedCodingParser::parseChunkBeg() if (findCrlf(crlfBeg, crlfEnd)) { debugs(93,7, "found chunk-size end: " << crlfBeg << "-" << crlfEnd); - int size = -1; + int64_t size = -1; const char *p = 0; - if (StringToInt(theIn->content(), size, &p, 16)) { + if (StringToInt64(theIn->content(), size, &p, 16)) { if (size < 0) { throw TexcHere("negative chunk size"); return; @@ -104,7 +104,7 @@ void ChunkedCodingParser::parseChunkBody() { Must(theLeftBodySize > 0); // Should, really - const size_t availSize = XMIN(theLeftBodySize, (size_t)theIn->contentSize()); + const size_t availSize = XMIN(theLeftBodySize, (uint64_t)theIn->contentSize()); const size_t safeSize = XMIN(availSize, (size_t)theOut->potentialSpaceSize()); doNeedMoreData = availSize < theLeftBodySize; diff --git a/src/ICAP/ChunkedCodingParser.h b/src/ICAP/ChunkedCodingParser.h index a935aa676d..d6e1a6fba4 100644 --- a/src/ICAP/ChunkedCodingParser.h +++ b/src/ICAP/ChunkedCodingParser.h @@ -1,6 +1,6 @@ /* - * $Id: ChunkedCodingParser.h,v 1.2 2005/11/21 23:46:27 wessels Exp $ + * $Id: ChunkedCodingParser.h,v 1.3 2007/08/13 17:20:53 hno Exp $ * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- @@ -82,8 +82,8 @@ private: MemBuf *theOut; Step theStep; - size_t theChunkSize; - size_t theLeftBodySize; + uint64_t theChunkSize; + uint64_t theLeftBodySize; bool doNeedMoreData; }; diff --git a/src/ICAP/ICAPModXact.cc b/src/ICAP/ICAPModXact.cc index 8b74ab9080..0c92672cd6 100644 --- a/src/ICAP/ICAPModXact.cc +++ b/src/ICAP/ICAPModXact.cc @@ -333,20 +333,20 @@ size_t ICAPModXact::virginContentSize(const VirginBodyAct &act) const { Must(act.active()); // asbolute start of unprocessed data - const size_t start = act.offset(); + const uint64_t start = act.offset(); // absolute end of buffered data - const size_t end = virginConsumed + virgin.body_pipe->buf().contentSize(); + const uint64_t end = virginConsumed + virgin.body_pipe->buf().contentSize(); Must(virginConsumed <= start && start <= end); - return end - start; + return static_cast(end - start); } // pointer to buffered virgin body data available for the specified activity const char *ICAPModXact::virginContentData(const VirginBodyAct &act) const { Must(act.active()); - const size_t start = act.offset(); + const uint64_t start = act.offset(); Must(virginConsumed <= start); - return virgin.body_pipe->buf().content() + (start-virginConsumed); + return virgin.body_pipe->buf().content() + static_cast(start-virginConsumed); } void ICAPModXact::virginConsume() @@ -374,8 +374,8 @@ void ICAPModXact::virginConsume() } const size_t have = static_cast(bp.buf().contentSize()); - const size_t end = virginConsumed + have; - size_t offset = end; + const uint64_t end = virginConsumed + have; + uint64_t offset = end; debugs(93, 9, HERE << "max virgin consumption offset=" << offset << " acts " << virginBodyWriting.active() << virginBodySending.active() << @@ -390,7 +390,7 @@ void ICAPModXact::virginConsume() Must(virginConsumed <= offset && offset <= end); - if (const size_t size = offset - virginConsumed) { + if (const size_t size = static_cast(offset - virginConsumed)) { debugs(93, 8, HERE << "consuming " << size << " out of " << have << " virgin body bytes"); bp.consume(size); @@ -1215,7 +1215,7 @@ void ICAPModXact::decideOnPreview() ad = 0; else if (virginBody.knownSize()) - ad = XMIN(ad, virginBody.size()); // not more than we have + ad = XMIN(static_cast(ad), virginBody.size()); // not more than we have debugs(93, 5, "ICAPModXact should offer " << ad << "-byte preview " << "(service wanted " << wantedSize << ")"); @@ -1370,7 +1370,7 @@ void ICAPModXact::estimateVirginBody() else method = METHOD_NONE; - ssize_t size; + int64_t size; // expectingBody returns true for zero-sized bodies, but we will not // get a pipe for that body, so we treat the message as bodyless if (method != METHOD_NONE && msg->expectingBody(method, size) && size) { @@ -1410,9 +1410,9 @@ SizedEstimate::SizedEstimate() : theData(dtUnexpected) {} -void SizedEstimate::expect(ssize_t aSize) +void SizedEstimate::expect(int64_t aSize) { - theData = (aSize >= 0) ? aSize : (ssize_t)dtUnknown; + theData = (aSize >= 0) ? aSize : (int64_t)dtUnknown; } bool SizedEstimate::expected() const @@ -1426,10 +1426,10 @@ bool SizedEstimate::knownSize() const return theData != dtUnknown; } -size_t SizedEstimate::size() const +uint64_t SizedEstimate::size() const { Must(knownSize()); - return static_cast(theData); + return static_cast(theData); } @@ -1453,13 +1453,13 @@ void VirginBodyAct::progress(size_t size) { Must(active()); Must(size >= 0); - theStart += size; + theStart += static_cast(size); } -size_t VirginBodyAct::offset() const +uint64_t VirginBodyAct::offset() const { Must(active()); - return theStart; + return static_cast(theStart); } diff --git a/src/ICAP/ICAPModXact.h b/src/ICAP/ICAPModXact.h index e80729970e..4127fd2f39 100644 --- a/src/ICAP/ICAPModXact.h +++ b/src/ICAP/ICAPModXact.h @@ -1,6 +1,6 @@ /* - * $Id: ICAPModXact.h,v 1.9 2007/06/19 21:12:15 rousskov Exp $ + * $Id: ICAPModXact.h,v 1.10 2007/08/13 17:20:53 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -59,17 +59,17 @@ class SizedEstimate public: SizedEstimate(); // not expected by default - void expect(ssize_t aSize); // expect with any, even unknown size + void expect(int64_t aSize); // expect with any, even unknown size bool expected() const; /* other members can be accessed iff expected() */ bool knownSize() const; - size_t size() const; // can be accessed iff knownSize() + uint64_t size() const; // can be accessed iff knownSize() private: enum { dtUnexpected = -2, dtUnknown = -1 }; - ssize_t theData; // combines expectation and size info to save RAM + int64_t theData; // combines expectation and size info to save RAM }; // Virgin body may be used for two activities: (a) writing preview or prime @@ -91,11 +91,11 @@ public: // methods below require active() - size_t offset() const; // the absolute beginning of not-yet-acted-on data + uint64_t offset() const; // the absolute beginning of not-yet-acted-on data void progress(size_t size); // note processed body bytes private: - size_t theStart; // unprocessed virgin body data offset + int64_t theStart; // unprocessed virgin body data offset typedef enum { stUndecided, stActive, stDisabled } State; State theState; @@ -255,7 +255,7 @@ private: SizedEstimate virginBody; VirginBodyAct virginBodyWriting; // virgin body writing state VirginBodyAct virginBodySending; // virgin body sending state - size_t virginConsumed; // virgin data consumed so far + uint64_t virginConsumed; // virgin data consumed so far ICAPPreview preview; // use for creating (writing) the preview ChunkedCodingParser *bodyParser; // ICAP response body parser diff --git a/src/Makefile.am b/src/Makefile.am index 24afb8321f..d3ddd9aa0f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,7 +1,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.am,v 1.183 2007/06/19 22:01:11 hno Exp $ +# $Id: Makefile.am,v 1.184 2007/08/13 17:20:51 hno Exp $ # # Uncomment and customize the following to suit your needs: # @@ -584,6 +584,8 @@ squid_SOURCES = \ StoreMetaMD5.h \ StoreMetaSTD.cc \ StoreMetaSTD.h \ + StoreMetaSTDLFS.cc \ + StoreMetaSTDLFS.h \ StoreMetaUnpacker.cc \ StoreMetaUnpacker.h \ StoreMetaURL.cc \ @@ -732,6 +734,7 @@ ufsdump_SOURCES = \ StoreMeta.cc \ StoreMetaMD5.cc \ StoreMetaSTD.cc \ + StoreMetaSTDLFS.cc \ StoreMetaUnpacker.cc \ StoreMetaURL.cc \ StoreMetaVary.cc \ @@ -1358,6 +1361,7 @@ tests_testCacheManager_SOURCES = \ StoreMeta.cc \ StoreMetaMD5.cc \ StoreMetaSTD.cc \ + StoreMetaSTDLFS.cc \ StoreMetaUnpacker.cc \ StoreMetaURL.cc \ StoreMetaVary.cc \ @@ -1522,6 +1526,7 @@ tests_testEvent_SOURCES = \ StoreMeta.cc \ StoreMetaMD5.cc \ StoreMetaSTD.cc \ + StoreMetaSTDLFS.cc \ StoreMetaUnpacker.cc \ StoreMetaURL.cc \ StoreMetaVary.cc \ @@ -1673,6 +1678,7 @@ tests_testEventLoop_SOURCES = \ StoreMeta.cc \ StoreMetaMD5.cc \ StoreMetaSTD.cc \ + StoreMetaSTDLFS.cc StoreMetaUnpacker.cc \ StoreMetaURL.cc \ StoreMetaVary.cc \ @@ -1852,6 +1858,7 @@ tests_test_http_range_SOURCES = \ StoreMeta.cc \ StoreMetaMD5.cc \ StoreMetaSTD.cc \ + StoreMetaSTDLFS.cc \ StoreMetaUnpacker.cc \ StoreMetaURL.cc \ StoreMetaVary.cc \ @@ -2006,6 +2013,7 @@ tests_testHttpRequest_SOURCES = \ StoreMeta.cc \ StoreMetaMD5.cc \ StoreMetaSTD.cc \ + StoreMetaSTDLFS.cc \ StoreMetaUnpacker.cc \ StoreMetaURL.cc \ StoreMetaVary.cc \ @@ -2155,6 +2163,7 @@ SWAP_TEST_SOURCES = \ StoreMeta.cc \ StoreMetaMD5.cc \ StoreMetaSTD.cc \ + StoreMetaSTDLFS.cc \ StoreMetaUnpacker.cc \ StoreMetaURL.cc \ StoreMetaVary.cc \ @@ -2341,6 +2350,7 @@ tests_testURL_SOURCES = \ StoreMeta.cc \ StoreMetaMD5.cc \ StoreMetaSTD.cc \ + StoreMetaSTDLFS.cc \ StoreMetaUnpacker.cc \ StoreMetaURL.cc \ StoreMetaVary.cc \ diff --git a/src/MemObject.cc b/src/MemObject.cc index cc3be90ffd..31b2f128ac 100644 --- a/src/MemObject.cc +++ b/src/MemObject.cc @@ -1,6 +1,6 @@ /* - * $Id: MemObject.cc,v 1.28 2007/05/29 13:31:38 amosjeffries Exp $ + * $Id: MemObject.cc,v 1.29 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 19 Store Memory Primitives * AUTHOR: Robert Collins @@ -186,7 +186,7 @@ MemObject::replaceHttpReply(HttpReply *newrep) struct LowestMemReader : public unary_function { - LowestMemReader(off_t seed):current(seed){} + LowestMemReader(int64_t seed):current(seed){} void operator() (store_client const &x) { @@ -194,7 +194,7 @@ struct LowestMemReader : public unary_function current = x.copyInto.offset; } - off_t current; + int64_t current; }; struct StoreClientStats : public unary_function @@ -215,27 +215,27 @@ MemObject::stat (MemBuf * mb) const { mb->Printf("\t%s %s\n", RequestMethodStr[method], log_url); - mb->Printf("\tinmem_lo: %d\n", (int) inmem_lo); - mb->Printf("\tinmem_hi: %d\n", (int) data_hdr.endOffset()); - mb->Printf("\tswapout: %d bytes queued\n", - (int) swapout.queue_offset); + mb->Printf("\tinmem_lo: %"PRId64"\n", inmem_lo); + mb->Printf("\tinmem_hi: %"PRId64"\n", data_hdr.endOffset()); + mb->Printf("\tswapout: %"PRId64" bytes queued\n", + swapout.queue_offset); if (swapout.sio.getRaw()) - mb->Printf("\tswapout: %d bytes written\n", - (int) swapout.sio->offset()); + mb->Printf("\tswapout: %"PRId64" bytes written\n", + (int64_t) swapout.sio->offset()); StoreClientStats statsVisitor(mb); for_each(clients, statsVisitor); } -off_t +int64_t MemObject::endOffset () const { return data_hdr.endOffset(); } -size_t +int64_t MemObject::size() const { if (object_sz < 0) @@ -254,7 +254,7 @@ MemObject::reset() } -off_t +int64_t MemObject::lowestMemReaderOffset() const { LowestMemReader lowest (endOffset() + 1); @@ -268,7 +268,7 @@ MemObject::lowestMemReaderOffset() const bool MemObject::readAheadPolicyCanRead() const { - return (size_t)endOffset() - getReply()->hdr_sz < lowestMemReaderOffset() + (Config.readAheadGap << 10); + return endOffset() - getReply()->hdr_sz < lowestMemReaderOffset() + Config.readAheadGap; } void @@ -290,7 +290,7 @@ MemObject::checkUrlChecksum () const /* * How much of the object data is on the disk? */ -size_t +int64_t MemObject::objectBytesOnDisk() const { /* @@ -308,25 +308,25 @@ MemObject::objectBytesOnDisk() const if (swapout.sio.getRaw() == NULL) return 0; - off_t nwritten = swapout.sio->offset(); + int64_t nwritten = swapout.sio->offset(); - if (nwritten <= (off_t)swap_hdr_sz) + if (nwritten <= swap_hdr_sz) return 0; - return (size_t) (nwritten - swap_hdr_sz); + return (nwritten - swap_hdr_sz); } -off_t +int64_t MemObject::policyLowestOffsetToKeep() const { /* * Careful. lowest_offset can be greater than endOffset(), such * as in the case of a range request. */ - off_t lowest_offset = lowestMemReaderOffset(); + int64_t lowest_offset = lowestMemReaderOffset(); if (endOffset() < lowest_offset || - endOffset() - inmem_lo > (ssize_t)Config.Store.maxInMemObjSize) + endOffset() - inmem_lo > Config.Store.maxInMemObjSize) return lowest_offset; return inmem_lo; @@ -335,7 +335,7 @@ MemObject::policyLowestOffsetToKeep() const void MemObject::trimSwappable() { - off_t new_mem_lo = policyLowestOffsetToKeep(); + int64_t new_mem_lo = policyLowestOffsetToKeep(); /* * We should only free up to what we know has been written * to disk, not what has been queued for writing. Otherwise @@ -344,7 +344,7 @@ MemObject::trimSwappable() * The -1 makes sure the page isn't freed until storeSwapOut has * walked to the next page. (mem->swapout.memnode) */ - off_t on_disk; + int64_t on_disk; if ((on_disk = objectBytesOnDisk()) - 1 < new_mem_lo) new_mem_lo = on_disk - 1; @@ -360,7 +360,7 @@ MemObject::trimSwappable() void MemObject::trimUnSwappable() { - off_t new_mem_lo = policyLowestOffsetToKeep(); + int64_t new_mem_lo = policyLowestOffsetToKeep(); assert (new_mem_lo > 0); data_hdr.freeDataUpto(new_mem_lo); @@ -371,7 +371,7 @@ MemObject::trimUnSwappable() bool MemObject::isContiguous() const { - bool result = data_hdr.hasContigousContentRange (Range(inmem_lo, endOffset())); + bool result = data_hdr.hasContigousContentRange (Range(inmem_lo, endOffset())); /* XXX : make this higher level */ debugs (19, result ? 4 :3, "MemObject::isContiguous: Returning " << (result ? "true" : "false")); return result; diff --git a/src/MemObject.h b/src/MemObject.h index 94e44f063a..1b204610c9 100644 --- a/src/MemObject.h +++ b/src/MemObject.h @@ -1,6 +1,6 @@ /* - * $Id: MemObject.h,v 1.14 2007/04/20 07:29:47 wessels Exp $ + * $Id: MemObject.h,v 1.15 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -63,17 +63,17 @@ public: HttpReply const *getReply() const; void replaceHttpReply(HttpReply *newrep); void stat (MemBuf * mb) const; - off_t endOffset () const; - size_t size() const; + int64_t endOffset () const; + int64_t size() const; void reset(); - off_t lowestMemReaderOffset() const; + int64_t lowestMemReaderOffset() const; bool readAheadPolicyCanRead() const; void addClient(store_client *); /* XXX belongs in MemObject::swapout, once swaphdrsz is managed * better */ - size_t objectBytesOnDisk() const; - off_t policyLowestOffsetToKeep() const; + int64_t objectBytesOnDisk() const; + int64_t policyLowestOffsetToKeep() const; void trimSwappable(); void trimUnSwappable(); bool isContiguous() const; @@ -93,7 +93,7 @@ public: method_t method; char *url; mem_hdr data_hdr; - off_t inmem_lo; + int64_t inmem_lo; dlink_list clients; /* TODO: move into .cc or .cci */ size_t clientCount() const {return nclients;} @@ -106,7 +106,7 @@ public: { public: - off_t queue_offset; /* relative to in-mem data */ + int64_t queue_offset; /* relative to in-mem data */ mem_node *memnode; /* which node we're currently paging out */ StoreIOState::Pointer sio; }; @@ -130,7 +130,7 @@ public: char *log_url; RemovalPolicyNode repl; int id; - ssize_t object_sz; + int64_t object_sz; size_t swap_hdr_sz; #if URL_CHECKSUM_DEBUG diff --git a/src/Parsing.cc b/src/Parsing.cc index cb689ba92e..d2b08ba03c 100644 --- a/src/Parsing.cc +++ b/src/Parsing.cc @@ -1,6 +1,6 @@ /* - * $Id: Parsing.cc,v 1.3 2006/10/08 13:10:34 serassio Exp $ + * $Id: Parsing.cc,v 1.4 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 3 Configuration File Parsing * AUTHOR: Harvest Derived @@ -125,3 +125,23 @@ StringToInt(const char *s, int &result, const char **p, int base) return false; } + +bool +StringToInt64(const char *s, int64_t &result, const char **p, int base) +{ + if (s) { + char *ptr = 0; + const int64_t h = (int64_t) strtoll(s, &ptr, base); + + if (ptr != s && ptr) { + result = h; + + if (p) + *p = ptr; + + return true; + } + } + + return false; +} diff --git a/src/Parsing.h b/src/Parsing.h index 91613bad54..76a77054c0 100644 --- a/src/Parsing.h +++ b/src/Parsing.h @@ -1,6 +1,6 @@ /* - * $Id: Parsing.h,v 1.3 2006/10/08 13:10:34 serassio Exp $ + * $Id: Parsing.h,v 1.4 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 3 Configuration File Parsing * AUTHOR: Harvest Derived @@ -47,5 +47,6 @@ extern u_short GetShort(void); // on success, returns true and sets *p (if any) to the end of the integer extern bool StringToInt(const char *str, int &result, const char **p, int base); +extern bool StringToInt64(const char *str, int64_t &result, const char **p, int base); #endif /* SQUID_PARSING_H */ diff --git a/src/Server.cc b/src/Server.cc index e58b5207fd..84251cb3da 100644 --- a/src/Server.cc +++ b/src/Server.cc @@ -1,5 +1,5 @@ /* - * $Id: Server.cc,v 1.21 2007/08/09 23:30:52 rousskov Exp $ + * $Id: Server.cc,v 1.22 2007/08/13 17:20:51 hno Exp $ * * DEBUG: * AUTHOR: Duane Wessels @@ -386,9 +386,9 @@ ServerStateData::startIcap(ICAPServiceRep::Pointer service, HttpRequest *cause) // check whether we should be sending a body as well // start body pipe to feed ICAP transaction if needed assert(!virginBodyDestination); - HttpReply *vrep = virginReply(); + HttpReply *vrep = virginReply(); assert(!vrep->body_pipe); - ssize_t size = 0; + int64_t size = 0; if (vrep->expectingBody(cause->method, size) && size) { virginBodyDestination = new BodyPipe(this); vrep->body_pipe = virginBodyDestination; diff --git a/src/Store.h b/src/Store.h index a501719675..6a2d1a740b 100644 --- a/src/Store.h +++ b/src/Store.h @@ -1,6 +1,6 @@ /* - * $Id: Store.h,v 1.35 2007/05/29 13:31:38 amosjeffries Exp $ + * $Id: Store.h,v 1.36 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -127,7 +127,7 @@ public: time_t lastref; time_t expires; time_t lastmod; - size_t swap_file_sz; + uint64_t swap_file_sz; u_short refcount; u_short flags; /* END OF ON-DISK STORE_META_STD */ @@ -178,8 +178,8 @@ public: /* reduce the memory lock count on the entry */ virtual int unlock(); /* increate the memory lock count on the entry */ - virtual ssize_t objectLen() const; - virtual int contentLen() const; + virtual int64_t objectLen() const; + virtual int64_t contentLen() const; virtual void lock() @@ -292,7 +292,7 @@ public: virtual void maintain() = 0; /* perform regular maintenance should be private and self registered ... */ /* These should really be private */ - virtual void updateSize(size_t size, int sign) = 0; + virtual void updateSize(int64_t size, int sign) = 0; private: static RefCount CurrentRoot; @@ -301,6 +301,9 @@ private: typedef RefCount StorePointer; SQUIDCEXTERN size_t storeEntryInUse(); +#if UNUSED_CODE_20070420 +SQUIDCEXTERN off_t storeLowestMemReaderOffset(const StoreEntry * entry); +#endif SQUIDCEXTERN const char *storeEntryFlags(const StoreEntry *); extern void storeEntryReplaceObject(StoreEntry *, HttpReply *); diff --git a/src/StoreClient.h b/src/StoreClient.h index 8568ae698a..f202dbfa7e 100644 --- a/src/StoreClient.h +++ b/src/StoreClient.h @@ -1,6 +1,6 @@ /* - * $Id: StoreClient.h,v 1.14 2006/05/22 19:58:51 wessels Exp $ + * $Id: StoreClient.h,v 1.15 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -64,7 +64,7 @@ public: void operator delete (void *); store_client(StoreEntry *); ~store_client(); - bool memReaderHasLowerOffset(off_t) const; + bool memReaderHasLowerOffset(int64_t) const; int getType() const; void fail(); void callback(ssize_t len, bool error = false); @@ -73,7 +73,7 @@ public: void copy(StoreEntry *, StoreIOBuffer, STCB *, void *); void dumpStats(MemBuf * output, int clientNumber) const; - off_t cmp_offset; + int64_t cmp_offset; #if STORE_CLIENT_LIST_DEBUG void *owner; diff --git a/src/StoreHashIndex.h b/src/StoreHashIndex.h index 8c9c647eb0..a17778d544 100644 --- a/src/StoreHashIndex.h +++ b/src/StoreHashIndex.h @@ -1,6 +1,6 @@ /* - * $Id: StoreHashIndex.h,v 1.4 2007/05/29 13:31:38 amosjeffries Exp $ + * $Id: StoreHashIndex.h,v 1.5 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -75,7 +75,7 @@ public: virtual void maintain(); - virtual void updateSize(size_t, int); + virtual void updateSize(int64_t, int); virtual StoreSearch *search(String const url, HttpRequest *); diff --git a/src/StoreIOBuffer.h b/src/StoreIOBuffer.h index 180596663e..601909497b 100644 --- a/src/StoreIOBuffer.h +++ b/src/StoreIOBuffer.h @@ -1,6 +1,6 @@ /* - * $Id: StoreIOBuffer.h,v 1.6 2006/10/14 13:34:29 serassio Exp $ + * $Id: StoreIOBuffer.h,v 1.7 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -45,7 +45,7 @@ class StoreIOBuffer public: StoreIOBuffer():length(0), offset (0), data (NULL){flags.error = 0;} - StoreIOBuffer(size_t aLength, off_t anOffset, char *someData) : + StoreIOBuffer(size_t aLength, int64_t anOffset, char *someData) : length (aLength), offset (anOffset), data (someData) { flags.error = 0; @@ -53,7 +53,7 @@ public: /* Create a StoreIOBuffer from a MemBuf and offset */ /* NOTE that MemBuf still "owns" the pointers, StoreIOBuffer is just borrowing them */ - StoreIOBuffer(MemBuf *aMemBuf, off_t anOffset) : + StoreIOBuffer(MemBuf *aMemBuf, int64_t anOffset) : length(aMemBuf->contentSize()), offset (anOffset), data(aMemBuf->content()) @@ -61,9 +61,9 @@ public: flags.error = 0; } - Range range() const + Range range() const { - return Range(offset, offset + length); + return Range(offset, offset + length); } void dump() const @@ -81,7 +81,7 @@ unsigned error: flags; size_t length; - off_t offset; + int64_t offset; char *data; }; diff --git a/src/StoreMeta.cc b/src/StoreMeta.cc index 89286ac81e..538aa3b6c5 100644 --- a/src/StoreMeta.cc +++ b/src/StoreMeta.cc @@ -1,6 +1,6 @@ /* - * $Id: StoreMeta.cc,v 1.4 2007/04/28 22:26:37 hno Exp $ + * $Id: StoreMeta.cc,v 1.5 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 20 Storage Manager Swapfile Metadata * AUTHOR: Kostas Anagnostakis @@ -40,6 +40,7 @@ #include "StoreMetaMD5.h" #include "StoreMetaURL.h" #include "StoreMetaSTD.h" +#include "StoreMetaSTDLFS.h" #include "StoreMetaVary.h" bool @@ -127,6 +128,10 @@ StoreMeta::Factory (char type, size_t len, void const *value) result = new StoreMetaSTD; break; + case STORE_META_STD_LFS: + result = new StoreMetaSTDLFS; + break; + case STORE_META_VARY_HEADERS: result = new StoreMetaVary; break; @@ -183,6 +188,12 @@ StoreMeta::checkConsistency(StoreEntry *e) const case STORE_META_STD: break; + case STORE_META_STD_LFS: + break; + + case STORE_META_OBJSIZE: + break; + default: debugs(20, 1, "WARNING: got unused STORE_META type " << getType()); break; diff --git a/src/StoreMetaSTD.cc b/src/StoreMetaSTD.cc index 010c0963f9..fa541c705c 100644 --- a/src/StoreMetaSTD.cc +++ b/src/StoreMetaSTD.cc @@ -1,6 +1,6 @@ /* - * $Id: StoreMetaSTD.cc,v 1.4 2004/08/30 05:12:31 robertc Exp $ + * $Id: StoreMetaSTD.cc,v 1.5 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 20 Storage Manager Swapfile Metadata * AUTHOR: Kostas Anagnostakis @@ -41,5 +41,5 @@ bool StoreMetaSTD::validLength(int len) const { - return len == STORE_HDR_METASIZE; + return len == STORE_HDR_METASIZE_OLD; } diff --git a/src/StoreMetaSTDLFS.cc b/src/StoreMetaSTDLFS.cc new file mode 100644 index 0000000000..1e097870fd --- /dev/null +++ b/src/StoreMetaSTDLFS.cc @@ -0,0 +1,44 @@ + +/* + * $Id: StoreMetaSTDLFS.cc,v 1.1 2007/08/13 17:22:30 hno Exp $ + * + * DEBUG: section 20 Storage Manager Swapfile Metadata + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "StoreMetaSTDLFS.h" +#include "Store.h" +#include "MemObject.h" + +bool +StoreMetaSTDLFS::validLength(int len) const +{ + return len == STORE_HDR_METASIZE; +} diff --git a/src/StoreMetaSTDLFS.h b/src/StoreMetaSTDLFS.h new file mode 100644 index 0000000000..e4c9a19e2c --- /dev/null +++ b/src/StoreMetaSTDLFS.h @@ -0,0 +1,53 @@ + +/* + * $Id: StoreMetaSTDLFS.h,v 1.1 2007/08/13 17:22:30 hno Exp $ + * + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#ifndef SQUID_STOREMETASTDLFS_H +#define SQUID_STOREMETASTDLFS_H + +#include "StoreMeta.h" + +class StoreMetaSTDLFS : public StoreMeta +{ + +public: + MEMPROXY_CLASS(StoreMetaSTDLFS); + + char getType() const {return STORE_META_STD_LFS;} + + bool validLength(int) const; + // bool checkConsistency(StoreEntry *) const; +}; + +MEMPROXY_CLASS_INLINE(StoreMetaSTDLFS) + +#endif /* SQUID_STOREMETASTDLFS_H */ diff --git a/src/StoreSwapLogData.cc b/src/StoreSwapLogData.cc index 18f7917bbf..12c61d8795 100644 --- a/src/StoreSwapLogData.cc +++ b/src/StoreSwapLogData.cc @@ -1,5 +1,5 @@ /* - * $Id: StoreSwapLogData.cc,v 1.3 2004/08/30 05:12:31 robertc Exp $ + * $Id: StoreSwapLogData.cc,v 1.4 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Duane Wessels @@ -38,3 +38,8 @@ StoreSwapLogData::StoreSwapLogData(): op(0), swap_filen (0), timestamp (0), last { memset (key, '\0', sizeof(key)); } + +StoreSwapLogHeader::StoreSwapLogHeader():op(SWAP_LOG_VERSION), version(1) +{ + record_size = sizeof(StoreSwapLogData); +} diff --git a/src/StoreSwapLogData.h b/src/StoreSwapLogData.h index 507eb39018..5babcaeedb 100644 --- a/src/StoreSwapLogData.h +++ b/src/StoreSwapLogData.h @@ -1,6 +1,6 @@ /* - * $Id: StoreSwapLogData.h,v 1.3 2004/08/30 05:12:31 robertc Exp $ + * $Id: StoreSwapLogData.h,v 1.4 2007/08/13 17:20:51 hno Exp $ * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- @@ -57,7 +57,7 @@ public: time_t lastref; time_t expires; time_t lastmod; - size_t swap_file_sz; + uint64_t swap_file_sz; u_short refcount; u_short flags; unsigned char key[MD5_DIGEST_CHARS]; @@ -65,4 +65,14 @@ public: MEMPROXY_CLASS_INLINE(StoreSwapLogData) +class StoreSwapLogHeader +{ +public: + StoreSwapLogHeader(); + char op; + int version; + int record_size; +}; + + #endif /* SQUID_STORESWAPLOGDATA_H */ diff --git a/src/SwapDir.cc b/src/SwapDir.cc index d23999c054..7024bc21a5 100644 --- a/src/SwapDir.cc +++ b/src/SwapDir.cc @@ -1,6 +1,6 @@ /* - * $Id: SwapDir.cc,v 1.13 2007/05/29 13:31:38 amosjeffries Exp $ + * $Id: SwapDir.cc,v 1.14 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 20 Swap Dir base object * AUTHOR: Robert Collins @@ -245,7 +245,7 @@ SwapDir::optionMaxSizeParse(char const *option, const char *value, int reconfigu if (!value) self_destruct(); - ssize_t size = xatoi(value); + int64_t size = strtoll(value, NULL, 10); if (reconfiguring && max_objsize != size) debugs(3, 1, "Cache dir '" << path << "' max object size now " << size); @@ -259,7 +259,7 @@ void SwapDir::optionMaxSizeDump(StoreEntry * e) const { if (max_objsize != -1) - storeAppendPrintf(e, " max-size=%ld", (long int) max_objsize); + storeAppendPrintf(e, " max-size=%"PRId64, max_objsize); } /* Swapdirs do not have an index of their own - thus they ask their parent.. diff --git a/src/SwapDir.h b/src/SwapDir.h index a76f04d655..4cb7baab0a 100644 --- a/src/SwapDir.h +++ b/src/SwapDir.h @@ -1,6 +1,6 @@ /* - * $Id: SwapDir.h,v 1.13 2007/05/29 13:31:38 amosjeffries Exp $ + * $Id: SwapDir.h,v 1.14 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -80,7 +80,7 @@ public: virtual void dereference(StoreEntry &); /* Unreference this object */ - virtual void updateSize(size_t size, int sign); + virtual void updateSize(int64_t size, int sign); /* the number of store dirs being rebuilt. */ static int store_dirs_rebuilding; @@ -142,7 +142,7 @@ virtual size_t maxSize() const { return max_size;} virtual void stat (StoreEntry &anEntry) const; virtual StoreSearch *search(String const url, HttpRequest *) = 0; - virtual void updateSize(size_t size, int sign); + virtual void updateSize(int64_t size, int sign); /* migrated from store_dir.cc */ bool objectSizeIsAcceptable(ssize_t objsize) const; @@ -164,7 +164,7 @@ public: int max_size; char *path; int index; /* This entry's index into the swapDirs array */ - ssize_t max_objsize; + int64_t max_objsize; RemovalPolicy *repl; int removals; int scanned; diff --git a/src/access_log.cc b/src/access_log.cc index 48d4bf08bb..8efb98117d 100644 --- a/src/access_log.cc +++ b/src/access_log.cc @@ -1,6 +1,6 @@ /* - * $Id: access_log.cc,v 1.126 2007/05/29 13:31:38 amosjeffries Exp $ + * $Id: access_log.cc,v 1.127 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 46 Access Log * AUTHOR: Duane Wessels @@ -531,6 +531,8 @@ accessLogCustom(AccessLogEntry * al, customlog * log) long int outint = 0; int doint = 0; int dofree = 0; + int64_t outoff = 0; + int dooff = 0; switch (fmt->type) { @@ -786,9 +788,9 @@ accessLogCustom(AccessLogEntry * al, customlog * log) /*case LFT_REQUEST_SIZE_BODY_NO_TE: */ case LFT_REPLY_SIZE_TOTAL: - outint = al->cache.size; + outoff = al->cache.size; - doint = 1; + dooff = 1; break; @@ -833,7 +835,11 @@ accessLogCustom(AccessLogEntry * al, customlog * log) break; } - if (doint) { + if (dooff) { + snprintf(tmp, sizeof(tmp), "%0*lld", fmt->zero ? (int) fmt->width : 0, outoff); + out = tmp; + + } else if (doint) { snprintf(tmp, sizeof(tmp), "%0*ld", fmt->zero ? (int) fmt->width : 0, outint); out = tmp; } @@ -1289,14 +1295,14 @@ accessLogSquid(AccessLogEntry * al, Logfile * logfile) safe_free(user); if (!Config.onoff.log_mime_hdrs) { - logfilePrintf(logfile, "%9ld.%03d %6d %s %s/%03d %ld %s %s %s %s%s/%s %s", + logfilePrintf(logfile, "%9ld.%03d %6d %s %s/%03d %"PRId64" %s %s %s %s%s/%s %s", (long int) current_time.tv_sec, (int) current_time.tv_usec / 1000, al->cache.msec, client, log_tags[al->cache.code], al->http.code, - (long int) al->cache.size, + al->cache.size, al->_private.method_str, al->url, user ? user : dash_str, @@ -1307,14 +1313,14 @@ accessLogSquid(AccessLogEntry * al, Logfile * logfile) } else { char *ereq = log_quote(al->headers.request); char *erep = log_quote(al->headers.reply); - logfilePrintf(logfile, "%9ld.%03d %6d %s %s/%03d %ld %s %s %s %s%s/%s %s [%s] [%s]", + logfilePrintf(logfile, "%9ld.%03d %6d %s %s/%03d %"PRId64" %s %s %s %s%s/%s %s [%s] [%s]", (long int) current_time.tv_sec, (int) current_time.tv_usec / 1000, al->cache.msec, client, log_tags[al->cache.code], al->http.code, - (long int) al->cache.size, + al->cache.size, al->_private.method_str, al->url, user ? user : dash_str, @@ -1347,7 +1353,7 @@ accessLogCommon(AccessLogEntry * al, Logfile * logfile) user2 = accessLogFormatName(al->cache.rfc931); - logfilePrintf(logfile, "%s %s %s [%s] \"%s %s HTTP/%d.%d\" %d %ld %s:%s", + logfilePrintf(logfile, "%s %s %s [%s] \"%s %s HTTP/%d.%d\" %d %"PRId64" %s:%s", client, user2 ? user2 : dash_str, user1 ? user1 : dash_str, @@ -1356,7 +1362,7 @@ accessLogCommon(AccessLogEntry * al, Logfile * logfile) al->url, al->http.version.major, al->http.version.minor, al->http.code, - (long int) al->cache.size, + al->cache.size, log_tags[al->cache.code], hier_strings[al->hier.code]); diff --git a/src/cache_cf.cc b/src/cache_cf.cc index 271b150d42..ec4f58f913 100644 --- a/src/cache_cf.cc +++ b/src/cache_cf.cc @@ -1,6 +1,6 @@ /* - * $Id: cache_cf.cc,v 1.514 2007/08/02 01:20:47 amosjeffries Exp $ + * $Id: cache_cf.cc,v 1.515 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 3 Configuration File Parsing * AUTHOR: Harvest Derived @@ -143,6 +143,7 @@ static int check_null_https_port_list(const https_port_list *); #endif #endif /* USE_SSL */ static void parse_b_size_t(size_t * var); +static void parse_b_int64_t(int64_t * var); /* * LegacyParser is a parser for legacy code that uses the global @@ -162,7 +163,7 @@ static void update_maxobjsize(void) { int i; - ssize_t ms = -1; + int64_t ms = -1; for (i = 0; i < Config.cacheSwap.n_configured; i++) { assert (Config.cacheSwap.swapDirs[i].getRaw()); @@ -171,7 +172,6 @@ update_maxobjsize(void) max_objsize > ms) ms = dynamic_cast(Config.cacheSwap.swapDirs[i].getRaw())->max_objsize; } - store_maxobjsize = ms; } @@ -699,13 +699,52 @@ parseTimeUnits(const char *unit) return 0; } +static void +parseBytesLine64(int64_t * bptr, const char *units) +{ + char *token; + double d; + int64_t m; + int64_t u; + + if ((u = parseBytesUnits(units)) == 0) + self_destruct(); + + if ((token = strtok(NULL, w_space)) == NULL) + self_destruct(); + + if (strcmp(token, "none") == 0 || strcmp(token, "-1") == 0) { + *bptr = static_cast(-1); + return; + } + + d = xatof(token); + + m = u; /* default to 'units' if none specified */ + + if (0.0 == d) + (void) 0; + else if ((token = strtok(NULL, w_space)) == NULL) + debugs(3, 0, "WARNING: No units on '" << + config_input_line << "', assuming " << + d << " " << units ); + else if ((m = parseBytesUnits(token)) == 0) + self_destruct(); + + *bptr = static_cast(m * d / u); + + if (static_cast(*bptr) * 2 != m * d / u * 2) + self_destruct(); +} + + static void parseBytesLine(size_t * bptr, const char *units) { char *token; double d; - size_t m; - size_t u; + int m; + int u; if ((u = parseBytesUnits(units)) == 0) self_destruct(); @@ -1028,7 +1067,7 @@ parse_acl_b_size_t(acl_size_t ** head) l = cbdataAlloc(acl_size_t); - parse_b_size_t(&l->size); + parse_b_int64_t(&l->size); aclParseAclList(LegacyParser, &l->aclList); @@ -2286,6 +2325,18 @@ dump_kb_size_t(StoreEntry * entry, const char *name, size_t var) storeAppendPrintf(entry, "%s %d %s\n", name, (int) var, B_KBYTES_STR); } +static void +dump_b_int64_t(StoreEntry * entry, const char *name, int64_t var) +{ + storeAppendPrintf(entry, "%s %"PRId64" %s\n", name, var, B_BYTES_STR); +} + +static void +dump_kb_int64_t(StoreEntry * entry, const char *name, int64_t var) +{ + storeAppendPrintf(entry, "%s %"PRId64" %s\n", name, var, B_KBYTES_STR); +} + static void parse_size_t(size_t * var) { @@ -2306,16 +2357,35 @@ parse_kb_size_t(size_t * var) parseBytesLine(var, B_KBYTES_STR); } +static void +parse_b_int64_t(int64_t * var) +{ + parseBytesLine64(var, B_BYTES_STR); +} + +static void +parse_kb_int64_t(int64_t * var) +{ + parseBytesLine64(var, B_KBYTES_STR); +} + static void free_size_t(size_t * var) { *var = 0; } +static void +free_b_int64_t(int64_t * var) +{ + *var = 0; +} + #define free_b_size_t free_size_t #define free_kb_size_t free_size_t #define free_mb_size_t free_size_t #define free_gb_size_t free_size_t +#define free_kb_int64_t free_b_int64_t static void dump_ushort(StoreEntry * entry, const char *name, u_short var) diff --git a/src/cf.data.pre b/src/cf.data.pre index ea3789bc65..7c19263bcf 100644 --- a/src/cf.data.pre +++ b/src/cf.data.pre @@ -1,6 +1,6 @@ # -# $Id: cf.data.pre,v 1.445 2007/08/02 22:58:15 amosjeffries Exp $ +# $Id: cf.data.pre,v 1.446 2007/08/13 17:20:51 hno Exp $ # # SQUID Web Proxy Cache http://www.squid-cache.org/ # ---------------------------------------------------------- @@ -929,7 +929,7 @@ DOC_END NAME: maximum_object_size COMMENT: (bytes) -TYPE: b_size_t +TYPE: b_int64_t DEFAULT: 4096 KB LOC: Config.Store.maxObjectSize DOC_START @@ -947,7 +947,7 @@ DOC_END NAME: minimum_object_size COMMENT: (bytes) -TYPE: b_size_t +TYPE: b_int64_t DEFAULT: 0 KB LOC: Config.Store.minObjectSize DOC_START @@ -2096,8 +2096,8 @@ DOC_START DOC_END NAME: request_body_max_size -COMMENT: (KB) -TYPE: b_size_t +COMMENT: (bytes) +TYPE: b_int64_t DEFAULT: 0 KB LOC: Config.maxRequestBodySize DOC_START @@ -2213,14 +2213,14 @@ DOC_END NAME: quick_abort_min COMMENT: (KB) -TYPE: kb_size_t +TYPE: kb_int64_t DEFAULT: 16 KB LOC: Config.quickAbort.min DOC_NONE NAME: quick_abort_max COMMENT: (KB) -TYPE: kb_size_t +TYPE: kb_int64_t DEFAULT: 16 KB LOC: Config.quickAbort.max DOC_NONE @@ -2261,7 +2261,7 @@ DOC_END NAME: read_ahead_gap COMMENT: buffer-size -TYPE: kb_size_t +TYPE: b_int64_t LOC: Config.readAheadGap DEFAULT: 16 KB DOC_START @@ -2304,7 +2304,7 @@ DOC_END NAME: range_offset_limit COMMENT: (bytes) -TYPE: b_size_t +TYPE: b_int64_t LOC: Config.rangeOffsetLimit DEFAULT: 0 KB DOC_START diff --git a/src/client_side.cc b/src/client_side.cc index 1bcc1f22ea..2c6d4f9374 100644 --- a/src/client_side.cc +++ b/src/client_side.cc @@ -1,6 +1,6 @@ /* - * $Id: client_side.cc,v 1.757 2007/07/26 21:17:01 rousskov Exp $ + * $Id: client_side.cc,v 1.758 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -139,8 +139,8 @@ static CSD clientSocketDetach; static void clientSetKeepaliveFlag(ClientHttpRequest *); static int clientIsContentLengthValid(HttpRequest * r); static bool okToAccept(); -static int clientIsRequestBodyValid(int bodyLength); -static int clientIsRequestBodyTooLargeForPolicy(size_t bodyLength); +static int clientIsRequestBodyValid(int64_t bodyLength); +static int clientIsRequestBodyTooLargeForPolicy(int64_t bodyLength); static void clientUpdateStatHistCounters(log_type logType, int svc_time); static void clientUpdateStatCounters(log_type logType); @@ -683,7 +683,7 @@ clientIsContentLengthValid(HttpRequest * r) } int -clientIsRequestBodyValid(int bodyLength) +clientIsRequestBodyValid(int64_t bodyLength) { if (bodyLength >= 0) return 1; @@ -692,7 +692,7 @@ clientIsRequestBodyValid(int bodyLength) } int -clientIsRequestBodyTooLargeForPolicy(size_t bodyLength) +clientIsRequestBodyTooLargeForPolicy(int64_t bodyLength) { if (Config.maxRequestBodySize && bodyLength > Config.maxRequestBodySize) @@ -748,9 +748,10 @@ ClientSocketContext::startOfOutput() const } size_t -ClientSocketContext::lengthToSend(Range const &available) +ClientSocketContext::lengthToSend(Range const &available) { - size_t maximum = available.size(); + /*the size of available range can always fit in a size_t type*/ + size_t maximum = (size_t)available.size(); if (!http->request->range) return maximum; @@ -763,10 +764,10 @@ ClientSocketContext::lengthToSend(Range const &available) assert (http->range_iter.debt() > 0); /* TODO this + the last line could be a range intersection calculation */ - if ((ssize_t)available.start < http->range_iter.currentSpec()->offset) + if (available.start < http->range_iter.currentSpec()->offset) return 0; - return XMIN(http->range_iter.debt(), (ssize_t)maximum); + return XMIN(http->range_iter.debt(), (int64_t)maximum); } void @@ -871,7 +872,7 @@ void ClientSocketContext::packRange(StoreIOBuffer const &source, MemBuf * mb) { HttpHdrRangeIter * i = &http->range_iter; - Range available (source.range()); + Range available (source.range()); char const *buf = source.data; while (i->currentSpec() && available.size()) { @@ -931,7 +932,7 @@ ClientSocketContext::packRange(StoreIOBuffer const &source, MemBuf * mb) return; } - off_t next = getNextRangeOffset(); + int64_t next = getNextRangeOffset(); assert (next >= http->out.offset); @@ -960,7 +961,7 @@ ClientSocketContext::packRange(StoreIOBuffer const &source, MemBuf * mb) int ClientHttpRequest::mRangeCLen() { - int clen = 0; + int64_t clen = 0; MemBuf mb; assert(memObject()); @@ -1110,9 +1111,10 @@ ClientSocketContext::buildRangeHeader(HttpReply * rep) request->range->contains(rep->content_range->spec) : true; const int spec_count = http->request->range->specs.count; - int actual_clen = -1; + int64_t actual_clen = -1; - debugs(33, 3, "clientBuildRangeHeader: range spec count: " << spec_count << " virgin clen: " << rep->content_length); + debugs(33, 3, "clientBuildRangeHeader: range spec count: " << + spec_count << " virgin clen: " << rep->content_length); assert(spec_count > 0); /* ETags should not be returned with Partial Content replies? */ hdr->delById(HDR_ETAG); @@ -1164,7 +1166,7 @@ ClientSocketContext::buildRangeHeader(HttpReply * rep) hdr->delById(HDR_CONTENT_LENGTH); - hdr->putInt(HDR_CONTENT_LENGTH, actual_clen); + hdr->putInt64(HDR_CONTENT_LENGTH, actual_clen); debugs(33, 3, "clientBuildRangeHeader: actual content length: " << actual_clen); @@ -1423,32 +1425,28 @@ ClientSocketContext::canPackMoreRanges() const return http->range_iter.currentSpec() ? true : false; } -off_t +int64_t ClientSocketContext::getNextRangeOffset() const { if (http->request->range) { /* offset in range specs does not count the prefix of an http msg */ - debugs(33, 5, "ClientSocketContext::getNextRangeOffset: http offset " << (long unsigned)http->out.offset); + debugs (33, 5, "ClientSocketContext::getNextRangeOffset: http offset " << http->out.offset); /* check: reply was parsed and range iterator was initialized */ assert(http->range_iter.valid); /* filter out data according to range specs */ assert (canPackMoreRanges()); { - off_t start; /* offset of still missing data */ + int64_t start; /* offset of still missing data */ assert(http->range_iter.currentSpec()); start = http->range_iter.currentSpec()->offset + http->range_iter.currentSpec()->length - http->range_iter.debt(); - debugs(33, 3, "clientPackMoreRanges: in: offset: " << - (long int) http->out.offset); - debugs(33, 3, "clientPackMoreRanges: out: start: " << - (long int) start << " spec[" << - (long int) (http->range_iter.pos - http->request->range->begin()) << - "]: [" << (long int) http->range_iter.currentSpec()->offset << - ", " << - (long int) (http->range_iter.currentSpec()->offset + http->range_iter.currentSpec()->length) << - "), len: " << - (long int) http->range_iter.currentSpec()->length << - " debt: " << (long int) http->range_iter.debt()); - + debugs(33, 3, "clientPackMoreRanges: in: offset: " << http->out.offset); + debugs(33, 3, "clientPackMoreRanges: out:" + " start: " << start << + " spec[" << http->range_iter.pos - http->request->range->begin() << "]:" << + " [" << http->range_iter.currentSpec()->offset << + ", " << http->range_iter.currentSpec()->offset + http->range_iter.currentSpec()->length << ")," + " len: " << http->range_iter.currentSpec()->length << + " debt: " << http->range_iter.debt()); if (http->range_iter.currentSpec()->length != -1) assert(http->out.offset <= start); /* we did not miss it */ diff --git a/src/client_side.h b/src/client_side.h index 705f00deca..5fd23acb53 100644 --- a/src/client_side.h +++ b/src/client_side.h @@ -1,6 +1,6 @@ /* - * $Id: client_side.h,v 1.22 2007/05/09 07:45:58 wessels Exp $ + * $Id: client_side.h,v 1.23 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -98,12 +98,12 @@ unsigned parsed_ok: 1; /* Was this parsed correctly? */ DeferredParams deferredparams; off_t writtenToSocket; void pullData(); - off_t getNextRangeOffset() const; + int64_t getNextRangeOffset() const; bool canPackMoreRanges() const; clientStream_status_t socketState(); void sendBody(HttpReply * rep, StoreIOBuffer bodyData); void sendStartOfMessage(HttpReply * rep, StoreIOBuffer bodyData); - size_t lengthToSend(Range const &available); + size_t lengthToSend(Range const &available); void noteSentBodyBytes(size_t); void buildRangeHeader(HttpReply * rep); int fd() const; diff --git a/src/client_side_reply.cc b/src/client_side_reply.cc index aeda2da45e..d2f5c0f25c 100644 --- a/src/client_side_reply.cc +++ b/src/client_side_reply.cc @@ -1,6 +1,6 @@ /* - * $Id: client_side_reply.cc,v 1.129 2007/05/29 13:31:39 amosjeffries Exp $ + * $Id: client_side_reply.cc,v 1.130 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 88 Client-side Reply Routines * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c) @@ -977,8 +977,13 @@ clientReplyContext::checkTransferDone() int clientReplyContext::storeOKTransferDone() const { - if (http->out.offset >= http->storeEntry()->objectLen() - headers_sz) + if (http->out.offset >= http->storeEntry()->objectLen() - headers_sz) { + debugs(88,3,HERE << "storeOKTransferDone " << + " out.offset=" << http->out.offset << + " objectLen()=" << http->storeEntry()->objectLen() << + " headers_sz=" << headers_sz); return 1; + } return 0; } @@ -1015,12 +1020,16 @@ clientReplyContext::storeNotOKTransferDone() const if (reply->content_length < 0) return 0; - size_t expectedLength = http->out.headers_sz + reply->content_length; + int64_t expectedLength = reply->content_length + http->out.headers_sz; if (http->out.size < expectedLength) return 0; - else + else { + debugs(88,3,HERE << "storeNotOKTransferDone " << + " out.size=" << http->out.size << + " expectedLength=" << expectedLength); return 1; + } } @@ -1033,23 +1042,22 @@ clientReplyContext::storeNotOKTransferDone() const int clientHttpRequestStatus(int fd, ClientHttpRequest const *http) { -#if SIZEOF_SIZE_T == 4 - +#if SIZEOF_INT64_T == 4 if (http->out.size > 0x7FFF0000) { - debugs(88, 1, "WARNING: closing FD " << fd << " to prevent counter overflow" ); - debugs(88, 1, "\tclient " << (inet_ntoa(http->getConn() != NULL ? http->getConn()->peer.sin_addr : no_addr)) ); - debugs(88, 1, "\treceived " << http->out.size << " bytes" ); - debugs(88, 1, "\tURI " << http->log_uri ); + debugs(88, 1, "WARNING: closing FD " << fd << " to prevent out.size counter overflow"); + debugs(88, 1, "\tclient " << (inet_ntoa(http->getConn() != NULL ? http->getConn()->peer.sin_addr : no_addr))); + debugs(88, 1, "\treceived " << http->out.size << " bytes"); + debugs(88, 1, "\tURI " << http->log_uri); return 1; } #endif -#if SIZEOF_OFF_T == 4 +#if SIZEOF_INT64_T == 4 if (http->out.offset > 0x7FFF0000) { - debugs(88, 1, "WARNING: closing FD " << fd << " to prevent counter overflow" ); - debugs(88, 1, "\tclient " << (inet_ntoa(http->getConn() != NULL ? http->getConn()->peer.sin_addr : no_addr)) ); - debugs(88, 1, "\treceived " << http->out.size << " bytes (offset " << http->out.offset << ")" ); - debugs(88, 1, "\tURI " << http->log_uri ); + debugs(88, 1, ("WARNING: closing FD " << fd < " to prevent out.offset counter overflow"); + debugs(88, 1, ("\tclient " << (inet_ntoa(http->getConn() != NULL ? http->getConn()->peer.sin_addr : no_addr))); + debugs(88, 1, ("\treceived " << http->out.size " << " bytes, offset " << http->out.offset); + debugs(88, 1, ("\tURI " << http->log_uri); return 1; } @@ -1915,9 +1923,12 @@ clientReplyContext::sendMoreData (StoreIOBuffer result) makeThisHead(); debugs(88, 5, "clientReplyContext::sendMoreData: " << http->uri << ", " << - (int) reqofs << " bytes (" << (unsigned int)result.length << + reqofs << " bytes (" << result.length << " new bytes)"); - debugs(88, 5, "clientReplyContext::sendMoreData: FD " << fd << " '" << entry->url() << "', out.offset=" << http->out.offset << " " ); + debugs(88, 5, "clientReplyContext::sendMoreData:" + " FD " << fd << + " '" << entry->url() << "'" << + " out.offset=" << http->out.offset); /* update size of the request */ reqsize = reqofs; diff --git a/src/client_side_request.cc b/src/client_side_request.cc index e647a22b85..04389a0a4c 100644 --- a/src/client_side_request.cc +++ b/src/client_side_request.cc @@ -1,6 +1,6 @@ /* - * $Id: client_side_request.cc,v 1.91 2007/07/19 00:35:27 hno Exp $ + * $Id: client_side_request.cc,v 1.92 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 85 Client-side Request Routines * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c) @@ -918,19 +918,19 @@ ClientHttpRequest::gotEnough() const } void -ClientHttpRequest::maxReplyBodySize(ssize_t clen) +ClientHttpRequest::maxReplyBodySize(int64_t clen) { maxReplyBodySize_ = clen; } -ssize_t +int64_t ClientHttpRequest::maxReplyBodySize() const { return maxReplyBodySize_; } bool -ClientHttpRequest::isReplyBodyTooLarge(ssize_t clen) const +ClientHttpRequest::isReplyBodyTooLarge(int64_t clen) const { if (0 == maxReplyBodySize()) return 0; /* disabled */ diff --git a/src/client_side_request.h b/src/client_side_request.h index 7358c0b798..92e1b7848d 100644 --- a/src/client_side_request.h +++ b/src/client_side_request.h @@ -1,6 +1,6 @@ /* - * $Id: client_side_request.h,v 1.31 2007/06/02 11:50:32 hno Exp $ + * $Id: client_side_request.h,v 1.32 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -99,8 +99,8 @@ public: struct { - off_t offset; - size_t size; + int64_t offset; + int64_t size; size_t headers_sz; } @@ -144,16 +144,16 @@ unsigned int purging: dlink_list client_stream; int mRangeCLen(); - ssize_t maxReplyBodySize() const; - void maxReplyBodySize(ssize_t size); - bool isReplyBodyTooLarge(ssize_t len) const; + int64_t maxReplyBodySize() const; + void maxReplyBodySize(int64_t size); + bool isReplyBodyTooLarge(int64_t len) const; ClientRequestContext *calloutContext; void doCallouts(); private: CBDATA_CLASS(ClientHttpRequest); - ssize_t maxReplyBodySize_; + int64_t maxReplyBodySize_; StoreEntry *entry_; StoreEntry *loggingEntry_; ConnStateData::Pointer conn_; @@ -196,7 +196,7 @@ SQUIDCEXTERN void clientAccessCheck(ClientHttpRequest *); /* ones that should be elsewhere */ SQUIDCEXTERN void redirectStart(ClientHttpRequest *, RH *, void *); -SQUIDCEXTERN void tunnelStart(ClientHttpRequest *, size_t *, int *); +SQUIDCEXTERN void tunnelStart(ClientHttpRequest *, int64_t *, int *); #ifdef _USE_INLINE_ #include "Store.h" diff --git a/src/defines.h b/src/defines.h index 1631f39d1f..50865647a2 100644 --- a/src/defines.h +++ b/src/defines.h @@ -1,6 +1,6 @@ /* - * $Id: defines.h,v 1.121 2006/08/21 00:50:41 robertc Exp $ + * $Id: defines.h,v 1.122 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -202,7 +202,8 @@ #define SwapMetaType(x) (char)x[0] #define SwapMetaSize(x) &x[sizeof(char)] #define SwapMetaData(x) &x[STORE_META_TLD_START] -#define STORE_HDR_METASIZE (4*sizeof(time_t)+2*sizeof(u_short)+sizeof(size_t)) +#define STORE_HDR_METASIZE (4*sizeof(time_t)+2*sizeof(u_short)+sizeof(uint64_t)) +#define STORE_HDR_METASIZE_OLD (4*sizeof(time_t)+2*sizeof(u_short)+sizeof(size_t)) #define PINGER_PAYLOAD_SZ 8192 diff --git a/src/enums.h b/src/enums.h index af0c1e36ec..cfa5078e73 100644 --- a/src/enums.h +++ b/src/enums.h @@ -1,6 +1,6 @@ /* - * $Id: enums.h,v 1.257 2007/05/04 22:12:55 wessels Exp $ + * $Id: enums.h,v 1.258 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -440,6 +440,8 @@ enum { STORE_META_HITMETERING, /* reserved for hit metering */ STORE_META_VALID, STORE_META_VARY_HEADERS, /* Stores Vary request headers */ + STORE_META_STD_LFS, /* standard metadata in lfs format */ + STORE_META_OBJSIZE, /* object size, not impleemented, squid26 compatibility */ STORE_META_END }; @@ -455,6 +457,7 @@ typedef enum { SWAP_LOG_NOP, SWAP_LOG_ADD, SWAP_LOG_DEL, + SWAP_LOG_VERSION, SWAP_LOG_MAX } swap_log_op; diff --git a/src/fde.cc b/src/fde.cc index 64f5fcb8bd..4966ff3bda 100644 --- a/src/fde.cc +++ b/src/fde.cc @@ -1,6 +1,6 @@ /* - * $Id: fde.cc,v 1.6 2007/04/25 11:30:18 adrian Exp $ + * $Id: fde.cc,v 1.7 2007/08/13 17:20:51 hno Exp $ * * DEBUG: none FDE * AUTHOR: Robert Collins @@ -56,11 +56,11 @@ fde::dumpStats (StoreEntry &dumpEntry, int fdNumber) #ifdef _SQUID_MSWIN_ - storeAppendPrintf(&dumpEntry, "%4d 0x%-8lX %-6.6s %4d %7d%c %7d%c %-21s %s\n", + storeAppendPrintf(&dumpEntry, "%4d 0x%-8lX %-6.6s %4d %7"PRId64"%c %7"PRId64"%c %-21s %s\n", fdNumber, win32.handle, #else - storeAppendPrintf(&dumpEntry, "%4d %-6.6s %4d %7d%c %7d%c %-21s %s\n", + storeAppendPrintf(&dumpEntry, "%4d %-6.6s %4d %7"PRId64"%c %7"PRId64"%c %-21s %s\n", fdNumber, #endif fdTypeStr[type], diff --git a/src/fde.h b/src/fde.h index a3e0353d09..d53a17c73b 100644 --- a/src/fde.h +++ b/src/fde.h @@ -1,6 +1,6 @@ /* - * $Id: fde.h,v 1.12 2006/09/15 20:40:29 serassio Exp $ + * $Id: fde.h,v 1.13 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -75,8 +75,8 @@ public: unsigned int write_pending:1; } flags; - int bytes_read; - int bytes_written; + int64_t bytes_read; + int64_t bytes_written; struct { int uses; /* ie # req's over persistent conn */ diff --git a/src/fs/coss/CossSwapDir.h b/src/fs/coss/CossSwapDir.h index fdbda43fc5..29faf19cc2 100644 --- a/src/fs/coss/CossSwapDir.h +++ b/src/fs/coss/CossSwapDir.h @@ -66,16 +66,16 @@ public: dlink_list membufs; CossMemBuf *current_membuf; - size_t current_offset; /* in Blocks */ + off_t current_offset; /* in Blocks */ int numcollisions; dlink_list cossindex; unsigned int blksz_bits; unsigned int blksz_mask; /* just 1< theFile; - char *storeCossMemPointerFromDiskOffset(size_t offset, CossMemBuf ** mb); + char *storeCossMemPointerFromDiskOffset(off_t offset, CossMemBuf ** mb); void storeCossMemBufUnlock(StoreIOState::Pointer); - CossMemBuf *createMemBuf(size_t start, sfileno curfn, int *collision); + CossMemBuf *createMemBuf(off_t start, sfileno curfn, int *collision); sfileno allocate(const StoreEntry * e, int which); void startMembuf(); diff --git a/src/fs/coss/store_coss.h b/src/fs/coss/store_coss.h index 48354a6f86..f3766beffe 100644 --- a/src/fs/coss/store_coss.h +++ b/src/fs/coss/store_coss.h @@ -24,8 +24,8 @@ public: void maybeWrite(CossSwapDir * SD); void write(CossSwapDir * SD); dlink_node node; - size_t diskstart; /* in blocks */ - size_t diskend; /* in blocks */ + off_t diskstart; /* in blocks */ + off_t diskend; /* in blocks */ CossSwapDir *SD; int lockcount; char buffer[COSS_MEMBUF_SZ]; @@ -61,7 +61,7 @@ public: char *requestbuf; size_t requestlen; size_t requestoffset; /* in blocks */ - sfileno reqdiskoffset; /* in blocks */ + int64_t reqdiskoffset; /* in blocks */ struct { @@ -76,7 +76,7 @@ unsigned int writing: flags; CossMemBuf *locked_membuf; - size_t st_size; + off_t st_size; void read_(char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data); void write(char const *buf, size_t size, off_t offset, FREE * free_func); void close(); diff --git a/src/fs/coss/store_dir_coss.cc b/src/fs/coss/store_dir_coss.cc index d52ef64b55..dc3794aa1a 100644 --- a/src/fs/coss/store_dir_coss.cc +++ b/src/fs/coss/store_dir_coss.cc @@ -1,6 +1,6 @@ /* - * $Id: store_dir_coss.cc,v 1.75 2007/05/29 13:31:47 amosjeffries Exp $ + * $Id: store_dir_coss.cc,v 1.76 2007/08/13 17:20:56 hno Exp $ * vim: set et : * * DEBUG: section 47 Store COSS Directory Routines @@ -81,7 +81,7 @@ static char *storeCossDirSwapLogFile(SwapDir *, const char *); static EVH storeCossRebuildFromSwapLog; static StoreEntry *storeCossAddDiskRestore(CossSwapDir * SD, const cache_key * key, int file_number, - size_t swap_file_sz, + uint64_t swap_file_sz, time_t expires, time_t timestamp, time_t lastref, @@ -514,7 +514,7 @@ storeCossRebuildFromSwapLog(void *data) static StoreEntry * storeCossAddDiskRestore(CossSwapDir * SD, const cache_key * key, int file_number, - size_t swap_file_sz, + uint64_t swap_file_sz, time_t expires, time_t timestamp, time_t lastref, @@ -1034,7 +1034,7 @@ CossSwapDir::parse(int anIndex, char *aPath) { unsigned int i; unsigned int size; - unsigned long max_offset; + off_t max_offset; i = GetInteger(); size = i << 10; /* Mbytes to Kbytes */ @@ -1066,9 +1066,9 @@ CossSwapDir::parse(int anIndex, char *aPath) * largest possible sfileno, assuming sfileno is a 25-bit * signed integer, as defined in structs.h. */ - max_offset = (unsigned long) 0xFFFFFF << blksz_bits; + max_offset = (off_t) 0xFFFFFF << blksz_bits; - if ((unsigned long)max_size > (unsigned long)(max_offset>>10)) { + if ((off_t)max_size > (max_offset>>10)) { debugs(47, 0, "COSS block-size = " << (1< ((size_t)max_size << 10)) { + if ((current_offset + allocsize) > ((off_t)max_size << 10)) { /* * tried to allocate past the end of the disk, so wrap * back to the beginning @@ -163,9 +163,9 @@ CossSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, sio->swap_dirn = index; sio->swap_filen = allocate(&e, COSS_ALLOC_ALLOCATE); debugs(79, 3, "storeCossCreate: offset " << - (long int) storeCossFilenoToDiskOffset(sio->swap_filen) << + storeCossFilenoToDiskOffset(sio->swap_filen) << ", size " << (long int) cstate->st_size << ", end " << - (long int) (sio->swap_filen + cstate->st_size)); + (sio->swap_filen + cstate->st_size)); /* assume allocate() always succeeds */ assert(-1 != sio->swap_filen); @@ -301,7 +301,7 @@ CossState::read_(char *buf, size_t size, off_t offset, STRCB * callback, void *c offset_ = offset; flags.reading = 1; - if ((offset + size) > st_size) + if ((offset + (off_t)size) > st_size) size = st_size - offset; requestlen = size; @@ -431,7 +431,7 @@ CossState::doCallback(int errflag) } char * -CossSwapDir::storeCossMemPointerFromDiskOffset(size_t offset, CossMemBuf ** mb) +CossSwapDir::storeCossMemPointerFromDiskOffset(off_t offset, CossMemBuf ** mb) { CossMemBuf *t; dlink_node *m; @@ -485,7 +485,7 @@ CossSwapDir::sync() { CossMemBuf *t; dlink_node *m; - int end; + off_t end; /* First, flush pending IO ops */ io->sync(); @@ -505,7 +505,7 @@ CossSwapDir::sync() end = (t == current_membuf) ? current_offset : t->diskend; - if ((size_t)end > t->diskstart) + if (end > t->diskstart) theFile->write(new CossWrite(WriteRequest((char const *)&t->buffer, t->diskstart, end - t->diskstart, NULL), t)); /* and flush */ @@ -539,7 +539,7 @@ CossMemBuf::write(CossSwapDir * SD) } CossMemBuf * -CossSwapDir::createMemBuf(size_t start, sfileno curfn, int *collision) +CossSwapDir::createMemBuf(off_t start, sfileno curfn, int *collision) { CossMemBuf *newmb; CossMemBuf *t; diff --git a/src/fs/ufs/store_dir_ufs.cc b/src/fs/ufs/store_dir_ufs.cc index f70ac81714..ea7d3c8701 100644 --- a/src/fs/ufs/store_dir_ufs.cc +++ b/src/fs/ufs/store_dir_ufs.cc @@ -1,6 +1,6 @@ /* - * $Id: store_dir_ufs.cc,v 1.84 2007/05/29 13:31:47 amosjeffries Exp $ + * $Id: store_dir_ufs.cc,v 1.85 2007/08/13 17:20:57 hno Exp $ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Duane Wessels @@ -682,7 +682,7 @@ UFSSwapDir::validL2(int anInt) const StoreEntry * UFSSwapDir::addDiskRestore(const cache_key * key, sfileno file_number, - size_t swap_file_sz, + uint64_t swap_file_sz, time_t expires, time_t timestamp, time_t lastref, @@ -753,6 +753,13 @@ UFSSwapDir::closeTmpSwapLog() debugs(47, 3, "Cache Dir #" << index << " log opened on FD " << fd); } +static void +FreeHeader(void *address) +{ + StoreSwapLogHeader *anObject = static_cast (address); + delete anObject; +} + FILE * UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) { @@ -765,6 +772,7 @@ UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) struct stat clean_sb; FILE *fp; int fd; + StoreSwapLogHeader *head; if (::stat(swaplog_path, &log_sb) < 0) { debugs(47, 1, "Cache Dir #" << index << ": No log file"); @@ -782,13 +790,19 @@ UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) /* open a write-only FD for the new log */ fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); - + if (fd < 0) { debugs(50, 1, "" << new_path << ": " << xstrerror()); fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); } - + swaplog_fd = fd; + + head = new StoreSwapLogHeader; + + file_write(swaplog_fd, -1, head, head->record_size, + NULL, NULL, FreeHeader); + /* open a read-only stream of the old log */ fp = fopen(swaplog_path, "rb"); @@ -850,6 +864,7 @@ int UFSSwapDir::writeCleanStart() { UFSCleanLog *state = new UFSCleanLog(this); + StoreSwapLogHeader header; #if HAVE_FCHMOD struct stat sb; @@ -869,6 +884,10 @@ UFSSwapDir::writeCleanStart() state->cln = xstrdup(logFile(".last-clean")); state->outbuf = (char *)xcalloc(CLEAN_BUF_SZ, 1); state->outbuf_offset = 0; + /*copy the header */ + xmemcpy(state->outbuf, &header, sizeof(StoreSwapLogHeader)); + state->outbuf_offset += header.record_size; + state->walker = repl->WalkInit(repl); ::unlink(state->cln); debugs(47, 3, "storeDirWriteCleanLogs: opened " << state->newLog << ", FD " << state->fd); @@ -878,9 +897,9 @@ UFSSwapDir::writeCleanStart() fchmod(state->fd, sb.st_mode); #endif + cleanLog = state; - return 0; } diff --git a/src/fs/ufs/ufscommon.cc b/src/fs/ufs/ufscommon.cc index f2dfd0d402..c68c810acd 100644 --- a/src/fs/ufs/ufscommon.cc +++ b/src/fs/ufs/ufscommon.cc @@ -1,5 +1,5 @@ /* - * $Id: ufscommon.cc,v 1.12 2007/08/01 23:21:07 amosjeffries Exp $ + * $Id: ufscommon.cc,v 1.13 2007/08/13 17:20:57 hno Exp $ * vim: set et : * * DEBUG: section 47 Store Directory Routines @@ -45,7 +45,129 @@ CBDATA_CLASS_INIT(RebuildState); -RebuildState::RebuildState (RefCount aSwapDir) : sd (aSwapDir), e(NULL), fromLog(true), _done (false) + +class UFSSwapLogParser_old:public UFSSwapLogParser{ +public: + struct StoreSwapLogDataOld{ + char op; + sfileno swap_filen; + time_t timestamp; + time_t lastref; + time_t expires; + time_t lastmod; + size_t swap_file_sz; + u_short refcount; + u_short flags; + unsigned char key[MD5_DIGEST_CHARS]; + }; + UFSSwapLogParser_old(FILE *fp):UFSSwapLogParser(fp) + { + record_size = sizeof(UFSSwapLogParser_old::StoreSwapLogDataOld); + } + bool ReadRecord(StoreSwapLogData &swapData); +}; + + +bool UFSSwapLogParser_old::ReadRecord(StoreSwapLogData &swapData){ + UFSSwapLogParser_old::StoreSwapLogDataOld readData; + int bytes = sizeof(UFSSwapLogParser_old::StoreSwapLogDataOld); + + assert(log); + + if (fread(&readData, bytes, 1, log) != 1){ + return false; + } + swapData.op = readData.op; + swapData.swap_filen = readData.swap_filen; + swapData.timestamp = readData.timestamp; + swapData.lastref = readData.lastref; + swapData.expires = readData.expires; + swapData.lastmod = readData.lastmod; + swapData.swap_file_sz = readData.swap_file_sz; + swapData.refcount = readData.refcount; + swapData.flags = readData.flags; + xmemcpy(swapData.key, readData.key, MD5_DIGEST_CHARS); + return true; +} + + +class UFSSwapLogParser_v1:public UFSSwapLogParser{ +public: + UFSSwapLogParser_v1(FILE *fp):UFSSwapLogParser(fp) + { + record_size = sizeof(StoreSwapLogData); + } + bool ReadRecord(StoreSwapLogData &swapData); +}; + + +bool UFSSwapLogParser_v1::ReadRecord(StoreSwapLogData &swapData) +{ + int bytes = sizeof(StoreSwapLogData); + + assert(log); + + if (fread(&swapData, bytes, 1, log) != 1){ + return false; + } + return true; +} + + +UFSSwapLogParser *UFSSwapLogParser::GetUFSSwapLogParser(FILE *fp) +{ + StoreSwapLogHeader header; + + assert(fp); + + if (fread(&header, sizeof(StoreSwapLogHeader), 1, fp) != 1) + return NULL; + + if (header.op != SWAP_LOG_VERSION){ + debugs(47, 1, "Old swap file detected... "); + fseek(fp, 0, SEEK_SET); + return new UFSSwapLogParser_old(fp); + } + + if (header.version == 1){ + if (fseek(fp, header.record_size, SEEK_SET) != 0) + return NULL; + + if (header.record_size == sizeof(struct UFSSwapLogParser_old::StoreSwapLogDataOld)){ + debugs(47, 1, "Version 1 of swap file without LFS support detected... "); + return new UFSSwapLogParser_old(fp); + } + + if (header.record_size == sizeof(StoreSwapLogData)){ + debugs(47, 1, "Version 1 of swap file with LFS support detected... "); + return new UFSSwapLogParser_v1(fp); + } + + debugs(47, 1, "The swap file has wrong format!... "); + return NULL; + } + + return NULL; +} + +int UFSSwapLogParser::SwapLogEntries(){ + struct stat sb; + + if (log_entries >= 0) + return log_entries; + + if (log && record_size && 0 == fstat(fileno(log), &sb)){ + log_entries = sb.st_size/record_size; + return log_entries; + } + + return 0; +} + + + + +RebuildState::RebuildState (RefCount aSwapDir) : sd (aSwapDir),LogParser(NULL), e(NULL), fromLog(true), _done (false) { speed = opt_foreground_rebuild ? 1 << 30 : 50; /* @@ -58,16 +180,18 @@ RebuildState::RebuildState (RefCount aSwapDir) : sd (aSwapDir), e(NU int zeroLengthLog = 0; FILE *fp = sd->openTmpSwapLog(&clean, &zeroLengthLog); - if (fp == NULL || zeroLengthLog) { + if (fp && !zeroLengthLog) + LogParser = UFSSwapLogParser::GetUFSSwapLogParser(fp); + + if (LogParser == NULL ) { fromLog = false; if (fp != NULL) fclose(fp); } else { - fromLog = true; - log = fp; - flags.clean = (unsigned int) clean; + fromLog = true; + flags.clean = (unsigned int) clean; } if (!clean) @@ -79,8 +203,9 @@ RebuildState::RebuildState (RefCount aSwapDir) : sd (aSwapDir), e(NU RebuildState::~RebuildState() { sd->closeTmpSwapLog(); - /* now thats closed we DONT want to keep our secondary pointer to it */ - log = NULL; + + if (LogParser) + delete LogParser; } void @@ -121,7 +246,28 @@ struct InitStoreEntry : public unary_function break; case STORE_META_STD: - assert(x.length == STORE_HDR_METASIZE); + struct old_metahdr{ + time_t timestamp; + time_t lastref; + time_t expires; + time_t lastmod; + size_t swap_file_sz; + u_short refcount; + u_short flags; + } *tmp; + tmp = (struct old_metahdr *)x.value; + assert(x.length == STORE_HDR_METASIZE_OLD); + what->timestamp = tmp->timestamp; + what->lastref = tmp->lastref; + what->expires = tmp->expires; + what->lastmod = tmp->lastmod; + what->swap_file_sz = tmp->swap_file_sz; + what->refcount = tmp->refcount; + what->flags = tmp->flags; + break; + + case STORE_META_STD_LFS: + assert(x.length == STORE_HDR_METASIZE); xmemcpy(&what->timestamp, x.value, STORE_HDR_METASIZE); break; @@ -232,13 +378,13 @@ RebuildState::rebuildFromDirectory() /* check sizes */ if (tmpe.swap_file_sz == 0) { - tmpe.swap_file_sz = (size_t) sb.st_size; - } else if (tmpe.swap_file_sz == (size_t)(sb.st_size - swap_hdr_len)) { - tmpe.swap_file_sz = (size_t) sb.st_size; - } else if (tmpe.swap_file_sz != (size_t)sb.st_size) { + tmpe.swap_file_sz = (uint64_t) sb.st_size; + } else if (tmpe.swap_file_sz == (uint64_t)(sb.st_size - swap_hdr_len)) { + tmpe.swap_file_sz = (uint64_t) sb.st_size; + } else if (tmpe.swap_file_sz != (uint64_t)sb.st_size) { debugs(47, 1, "commonUfsDirRebuildFromDirectory: SIZE MISMATCH " << - (long int) tmpe.swap_file_sz << "!=" << - (long int) sb.st_size); + tmpe.swap_file_sz << "!=" << + sb.st_size); sd->unlinkFile(filn); continue; @@ -283,7 +429,7 @@ RebuildState::rebuildFromDirectory() } counts.objcount++; - tmpe.dump(5); + // tmpe.dump(5); currentEntry(sd->addDiskRestore(key, filn, tmpe.swap_file_sz, @@ -320,12 +466,12 @@ RebuildState::rebuildFromSwapLog() for (int count = 0; count < speed; count++) { StoreSwapLogData swapData; - size_t ss = sizeof(StoreSwapLogData); - if (fread(&swapData, ss, 1, log) != 1) { + if (LogParser->ReadRecord(swapData) != 1) { debugs(47, 1, "Done reading " << sd->path << " swaplog (" << n_read << " entries)"); - fclose(log); - log = NULL; + LogParser->Close(); + delete LogParser; + LogParser = NULL; _done = true; return; } @@ -405,11 +551,11 @@ RebuildState::rebuildFromSwapLog() if ((++counts.scancount & 0xFFF) == 0) { - struct stat sb; + int swap_entries = LogParser->SwapLogEntries(); - if (0 == fstat(fileno(log), &sb)) + if (0 != swap_entries ) storeRebuildProgress(sd->index, - (int) sb.st_size / ss, n_read); + swap_entries, n_read); } if (!sd->validFileno(swapData.swap_filen, 0)) { diff --git a/src/fs/ufs/ufscommon.h b/src/fs/ufs/ufscommon.h index 6fb02639d8..53f7b5eccf 100644 --- a/src/fs/ufs/ufscommon.h +++ b/src/fs/ufs/ufscommon.h @@ -1,6 +1,6 @@ /* - * $Id: ufscommon.h,v 1.11 2007/05/29 13:31:47 amosjeffries Exp $ + * $Id: ufscommon.h,v 1.12 2007/08/13 17:20:57 hno Exp $ * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- @@ -97,7 +97,7 @@ public: void mapBitSet(sfileno filn); StoreEntry *addDiskRestore(const cache_key * key, sfileno file_number, - size_t swap_file_sz, + uint64_t swap_file_sz, time_t expires, time_t timestamp, time_t lastref, @@ -321,6 +321,31 @@ private: bool _done; }; +class StoreSwapLogData; +class UFSSwapLogParser{ + +public: + FILE *log; + int log_entries; + int record_size; + + UFSSwapLogParser(FILE *fp):log(fp),log_entries(-1), record_size(0){ + } + virtual ~UFSSwapLogParser(){}; + + static UFSSwapLogParser *GetUFSSwapLogParser(FILE *fp); + + virtual bool ReadRecord(StoreSwapLogData &swapData) = 0; + int SwapLogEntries(); + void Close() + { + if(log){ + fclose(log); + log = NULL; + } + } +}; + class RebuildState : public RefCountable { @@ -343,7 +368,8 @@ public: RefCount sd; int n_read; - FILE *log; +/* FILE *log;*/ + UFSSwapLogParser *LogParser; int speed; int curlvl1; int curlvl2; diff --git a/src/ftp.cc b/src/ftp.cc index 6d3194eda6..8313983a55 100644 --- a/src/ftp.cc +++ b/src/ftp.cc @@ -1,6 +1,6 @@ /* - * $Id: ftp.cc,v 1.437 2007/08/13 10:31:19 serassio Exp $ + * $Id: ftp.cc,v 1.438 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 9 File Transfer Protocol (FTP) * AUTHOR: Harvest Derived @@ -126,12 +126,12 @@ public: int login_att; ftp_state_t state; time_t mdtm; - int size; + int64_t theSize; wordlist *pathcomps; char *filepath; char *dirpath; - int restart_offset; - int restarted_offset; + int64_t restart_offset; + int64_t restarted_offset; char *proxy_host; size_t list_width; wordlist *cwd_message; @@ -145,7 +145,7 @@ public: int fd; char *buf; size_t size; - off_t offset; + size_t offset; wordlist *message; char *last_command; char *last_reply; @@ -192,7 +192,7 @@ public: int checkAuth(const HttpHeader * req_hdr); void checkUrlpath(); void buildTitleUrl(); - void writeReplyBody(const char *, int len); + void writeReplyBody(const char *, size_t len); void printfReplyBody(const char *fmt, ...); virtual int dataDescriptor() const; virtual void maybeReadVirginBody(); @@ -211,7 +211,7 @@ public: static IOCB ftpReadControlReply; static IOCB ftpWriteCommandCallback; static HttpReply *ftpAuthRequired(HttpRequest * request, const char *realm); - static wordlist *ftpParseControlReply(char *, size_t, int *, int *); + static wordlist *ftpParseControlReply(char *, size_t, int *, size_t *); // sending of the request body to the server virtual void sentRequestBody(int fd, size_t size, comm_err_t errflag); @@ -247,7 +247,7 @@ FtpStateData::operator delete (void *address) typedef struct { char type; - int size; + int64_t size; char *date; char *name; char *showname; @@ -370,7 +370,7 @@ FtpStateData::FtpStateData(FwdState *theFwdState) : ServerStateData(theFwdState) statCounter.server.ftp.requests++; ctrl.fd = theFwdState->server_fd; data.fd = -1; - size = -1; + theSize = -1; mdtm = -1; if (Config.Ftp.passive && !theFwdState->ftpPasvFailed()) @@ -715,7 +715,7 @@ ftpListParseParts(const char *buf, struct _ftp_flags flags) if ((copyFrom = strstr(buf, tbuf))) { p->type = *tokens[0]; - p->size = atoi(size); + p->size = strtoll(size, NULL, 10); p->date = xstrdup(tbuf); if (flags.skip_whitespace) { @@ -752,7 +752,7 @@ ftpListParseParts(const char *buf, struct _ftp_flags flags) p->type = 'd'; } else { p->type = '-'; - p->size = atoi(tokens[2]); + p->size = strtoll(tokens[2], NULL, 10); } snprintf(tbuf, 128, "%s %s", tokens[0], tokens[1]); @@ -1043,7 +1043,7 @@ FtpStateData::htmlifyListEntry(const char *line) snprintf(icon, 2048, "\"%-6s\"", mimeGetIconURL(parts->name), "[FILE]"); - snprintf(size, 2048, " %6dk", parts->size); + snprintf(size, 2048, " %6"PRId64"k", parts->size); break; } @@ -1313,7 +1313,7 @@ FtpStateData::processReplyBody() { debugs(9, 5, HERE << "FtpStateData::processReplyBody starting."); - if (request->method == METHOD_HEAD && (flags.isdir || size != -1)) { + if (request->method == METHOD_HEAD && (flags.isdir || theSize != -1)) { serverComplete(); return; } @@ -1598,7 +1598,7 @@ FtpStateData::ftpWriteCommandCallback(int fd, char *buf, size_t size, comm_err_t } wordlist * -FtpStateData::ftpParseControlReply(char *buf, size_t len, int *codep, int *used) +FtpStateData::ftpParseControlReply(char *buf, size_t len, int *codep, size_t *used) { char *s; char *sbuf; @@ -1608,7 +1608,7 @@ FtpStateData::ftpParseControlReply(char *buf, size_t len, int *codep, int *used) wordlist *head = NULL; wordlist *list; wordlist **tail = &head; - off_t offset; + size_t offset; size_t linelen; int code = -1; debugs(9, 5, "ftpParseControlReply"); @@ -1673,7 +1673,7 @@ FtpStateData::ftpParseControlReply(char *buf, size_t len, int *codep, int *used) tail = &list->next; } - *used = (int) (s - sbuf); + *used = (size_t) (s - sbuf); safe_free(sbuf); if (!complete) @@ -1734,7 +1734,7 @@ FtpStateData::ftpReadControlReply(int fd, char *buf, size_t len, comm_err_t errf return; } - assert(ftpState->ctrl.offset < (off_t)ftpState->ctrl.size); + assert(ftpState->ctrl.offset < ftpState->ctrl.size); if (errflag == COMM_OK && len > 0) { fd_bytes(fd, len, FD_READ); @@ -1777,7 +1777,7 @@ void FtpStateData::handleControlReply() { wordlist **W; - int bytes_used = 0; + size_t bytes_used = 0; wordlistDestroy(&ctrl.message); ctrl.message = ftpParseControlReply(ctrl.buf, ctrl.offset, &ctrl.replycode, &bytes_used); @@ -1785,7 +1785,7 @@ FtpStateData::handleControlReply() if (ctrl.message == NULL) { /* didn't get complete reply yet */ - if (ctrl.offset == (off_t)ctrl.size) { + if (ctrl.offset == ctrl.size) { ctrl.buf = (char *)memReallocBuf(ctrl.buf, ctrl.size << 1, &ctrl.size); } @@ -2208,14 +2208,13 @@ ftpReadSize(FtpStateData * ftpState) if (code == 213) { ftpState->unhack(); - ftpState->size = atoi(ftpState->ctrl.last_reply); + ftpState->theSize = strtoll(ftpState->ctrl.last_reply, NULL, 10); - if (ftpState->size == 0) { + if (ftpState->theSize == 0) { debugs(9, 2, "ftpReadSize: SIZE reported " << - ftpState->ctrl.last_reply << " on " << - ftpState->title_url.buf()); - - ftpState->size = -1; + ftpState->ctrl.last_reply << " on " << + ftpState->title_url.buf()); + ftpState->theSize = -1; } } else if (code < 0) { ftpFail(ftpState); @@ -2237,7 +2236,7 @@ ftpSendPasv(FtpStateData * ftpState) debugs(9, 3, HERE << "ftpSendPasv started"); - if (ftpState->request->method == METHOD_HEAD && (ftpState->flags.isdir || ftpState->size != -1)) { + if (ftpState->request->method == METHOD_HEAD && (ftpState->flags.isdir || ftpState->theSize != -1)) { ftpState->processHeadResponse(); // may call serverComplete return; } @@ -2645,7 +2644,7 @@ ftpSendStor(FtpStateData * ftpState) snprintf(cbuf, 1024, "STOR %s\r\n", ftpState->filepath); ftpState->writeCommand(cbuf); ftpState->state = SENT_STOR; - } else if (ftpState->request->header.getInt(HDR_CONTENT_LENGTH) > 0) { + } else if (ftpState->request->header.getInt64(HDR_CONTENT_LENGTH) > 0) { /* File upload without a filename. use STOU to generate one */ snprintf(cbuf, 1024, "STOU\r\n"); ftpState->writeCommand(cbuf); @@ -2705,7 +2704,7 @@ ftpSendRest(FtpStateData * ftpState) if(!ftpState || !ftpState->haveControlChannel("ftpSendRest")) return; - snprintf(cbuf, 1024, "REST %d\r\n", ftpState->restart_offset); + snprintf(cbuf, 1024, "REST %"PRId64"\r\n", ftpState->restart_offset); ftpState->writeCommand(cbuf); ftpState->state = SENT_REST; } @@ -2722,15 +2721,15 @@ FtpStateData::restartable() if (!flags.binary) return 0; - if (size <= 0) + if (theSize <= 0) return 0; - int desired_offset = request->range->lowestOffset((size_t) size); + int64_t desired_offset = request->range->lowestOffset(theSize); if (desired_offset <= 0) return 0; - if (desired_offset >= size) + if (desired_offset >= theSize) return 0; restart_offset = desired_offset; @@ -3016,7 +3015,7 @@ ftpFail(FtpStateData *ftpState) if (!ftpState->flags.isdir && /* Not a directory */ !ftpState->flags.try_slash_hack && /* Not in slash hack */ - ftpState->mdtm <= 0 && ftpState->size < 0 && /* Not known as a file */ + ftpState->mdtm <= 0 && ftpState->theSize < 0 && /* Not known as a file */ ftpState->request->urlpath.caseCmp("/%2f", 4) != 0) { /* No slash encoded */ switch (ftpState->state) { @@ -3226,28 +3225,28 @@ FtpStateData::appendSuccessHeader() if (0 == restarted_offset) { /* Full reply */ reply->setHeaders(version, HTTP_OK, "Gatewaying", - mime_type, size, mdtm, -2); - } else if (size < restarted_offset) { + mime_type, theSize, mdtm, -2); + } else if (theSize < restarted_offset) { /* * DPW 2007-05-04 - * offset should not be larger than size. We should + * offset should not be larger than theSize. We should * not be seeing this condition any more because we'll only - * send REST if we know the size and if it is less than size. + * send REST if we know the theSize and if it is less than theSize. */ debugs(0,0,HERE << "Whoops! " << " restarted_offset=" << restarted_offset << - ", but size=" << size << + ", but theSize=" << theSize << ". assuming full content response"); reply->setHeaders(version, HTTP_OK, "Gatewaying", - mime_type, size, mdtm, -2); + mime_type, theSize, mdtm, -2); } else { /* Partial reply */ HttpHdrRangeSpec range_spec; range_spec.offset = restarted_offset; - range_spec.length = size - restarted_offset; + range_spec.length = theSize - restarted_offset; reply->setHeaders(version, HTTP_PARTIAL_CONTENT, "Gatewaying", - mime_type, size - restarted_offset, mdtm, -2); - httpHeaderAddContRange(&reply->header, range_spec, size); + mime_type, theSize - restarted_offset, mdtm, -2); + httpHeaderAddContRange(&reply->header, range_spec, theSize); } /* additional info */ @@ -3344,7 +3343,7 @@ FtpStateData::printfReplyBody(const char *fmt, ...) * which should be sent to either StoreEntry, or to ICAP... */ void -FtpStateData::writeReplyBody(const char *data, int len) +FtpStateData::writeReplyBody(const char *data, size_t len) { debugs(9,5,HERE << "writing " << len << " bytes to the reply"); addVirginReplyBody(data, len); diff --git a/src/globals.h b/src/globals.h index 74100c2705..d34a28d165 100644 --- a/src/globals.h +++ b/src/globals.h @@ -1,6 +1,6 @@ /* - * $Id: globals.h,v 1.140 2007/04/15 14:46:16 serassio Exp $ + * $Id: globals.h,v 1.141 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -156,7 +156,7 @@ extern "C" extern int store_swap_low; /* 0 */ extern int store_swap_high; /* 0 */ extern size_t store_pages_max; /* 0 */ - extern ssize_t store_maxobjsize; /* -1 */ + extern int64_t store_maxobjsize; /* -1 */ extern hash_table *proxy_auth_username_cache; /* NULL */ extern int incoming_sockets_accepted; #ifdef _SQUID_MSWIN_ diff --git a/src/http.cc b/src/http.cc index 189b625391..509df70b93 100644 --- a/src/http.cc +++ b/src/http.cc @@ -1,6 +1,6 @@ /* - * $Id: http.cc,v 1.536 2007/08/09 23:30:52 rousskov Exp $ + * $Id: http.cc,v 1.537 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -899,7 +899,7 @@ HttpStateData::persistentConnStatus() const if (eof) // already reached EOF return COMPLETE_NONPERSISTENT_MSG; - const int clen = vrep->bodySize(request->method); + const int64_t clen = vrep->bodySize(request->method); debugs(11, 5, "persistentConnStatus: clen=" << clen); @@ -911,7 +911,7 @@ HttpStateData::persistentConnStatus() const if (clen > 0) { // old technique: // if (entry->mem_obj->endOffset() < vrep->content_length + vrep->hdr_sz) - const int body_bytes_read = reply_bytes_read - header_bytes_read; + const int64_t body_bytes_read = reply_bytes_read - header_bytes_read; debugs(11,5, "persistentConnStatus: body_bytes_read=" << body_bytes_read << " content_length=" << vrep->content_length); diff --git a/src/mem_node.cc b/src/mem_node.cc index 834aa5b5c1..dc44b48978 100644 --- a/src/mem_node.cc +++ b/src/mem_node.cc @@ -1,6 +1,6 @@ /* - * $Id: mem_node.cc,v 1.9 2006/09/20 00:59:27 adrian Exp $ + * $Id: mem_node.cc,v 1.10 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 19 Store Memory Primitives * AUTHOR: Robert Collins @@ -66,7 +66,7 @@ memNodeWriteComplete(void* d) n->write_pending = 0; } -mem_node::mem_node(off_t offset):nodeBuffer(0,offset,data) +mem_node::mem_node(int64_t offset):nodeBuffer(0,offset,data) {} mem_node::~mem_node() @@ -80,23 +80,23 @@ mem_node::InUseCount() return Pool().inUseCount(); } -size_t +int64_t mem_node::start() const { assert (nodeBuffer.offset >= 0); return nodeBuffer.offset; } -size_t +int64_t mem_node::end() const { return nodeBuffer.offset + nodeBuffer.length; } -Range +Range mem_node::dataRange() const { - return Range (start(), end()); + return Range (start(), end()); } size_t @@ -106,7 +106,7 @@ mem_node::space() const } bool -mem_node::contains (size_t const &location) const +mem_node::contains (int64_t const &location) const { if (start() <= location && end() > location) return true; @@ -116,7 +116,7 @@ mem_node::contains (size_t const &location) const /* nodes can not be sparse */ bool -mem_node::canAccept (size_t const &location) const +mem_node::canAccept (int64_t const &location) const { if (location == end() && space() > 0) return true; diff --git a/src/mem_node.h b/src/mem_node.h index 3ef21ab63f..15a20c98e9 100644 --- a/src/mem_node.h +++ b/src/mem_node.h @@ -1,6 +1,6 @@ /* - * $Id: mem_node.h,v 1.9 2005/09/14 18:23:21 wessels Exp $ + * $Id: mem_node.h,v 1.10 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -45,14 +45,14 @@ public: static unsigned long store_mem_size; /* 0 */ MEMPROXY_CLASS(mem_node); - mem_node(off_t); + mem_node(int64_t); ~mem_node(); size_t space() const; - size_t start() const; - size_t end() const; - Range dataRange() const; - bool contains (size_t const &location) const; - bool canAccept (size_t const &location) const; + int64_t start() const; + int64_t end() const; + Range dataRange() const; + bool contains (int64_t const &location) const; + bool canAccept (int64_t const &location) const; bool operator < (mem_node const & rhs) const; /* public */ StoreIOBuffer nodeBuffer; diff --git a/src/mime.cc b/src/mime.cc index 97bf0a3fbf..d528b670d1 100644 --- a/src/mime.cc +++ b/src/mime.cc @@ -1,6 +1,6 @@ /* - * $Id: mime.cc,v 1.131 2007/04/28 22:26:37 hno Exp $ + * $Id: mime.cc,v 1.132 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 25 MIME Parsing * AUTHOR: Harvest Derived @@ -586,7 +586,7 @@ MimeIcon::created (StoreEntry *newEntry) HttpVersion version(1, 0); reply->setHeaders(version, HTTP_OK, NULL, - mimeGetContentType(icon), (int) sb.st_size, sb.st_mtime, -1); + mimeGetContentType(icon), sb.st_size, sb.st_mtime, -1); reply->cache_control = httpHdrCcCreate(); diff --git a/src/peer_digest.cc b/src/peer_digest.cc index 6b4caadda3..1e69696984 100644 --- a/src/peer_digest.cc +++ b/src/peer_digest.cc @@ -1,6 +1,6 @@ /* - * $Id: peer_digest.cc,v 1.125 2007/05/29 13:31:40 amosjeffries Exp $ + * $Id: peer_digest.cc,v 1.126 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 72 Peer Digest Routines * AUTHOR: Alex Rousskov @@ -787,7 +787,7 @@ peerDigestFetchedEnough(DigestFetchState * fetch, char *buf, ssize_t size, const if (!reason && !size) { if (!pd->cd) reason = "null digest?!"; - else if (fetch->mask_offset != (off_t)pd->cd->mask_size) + else if (fetch->mask_offset != (int)pd->cd->mask_size) reason = "premature end of digest?!"; else if (!peerDigestUseful(pd)) reason = "useless digest"; diff --git a/src/protos.h b/src/protos.h index 791c594ead..f36bd7ca28 100644 --- a/src/protos.h +++ b/src/protos.h @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.547 2007/05/29 13:31:40 amosjeffries Exp $ + * $Id: protos.h,v 1.548 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -245,6 +245,7 @@ SQUIDCEXTERN int strListGetItem(const String * str, char del, const char **item, SQUIDCEXTERN const char *getStringPrefix(const char *str, const char *end); SQUIDCEXTERN int httpHeaderParseInt(const char *start, int *val); SQUIDCEXTERN int httpHeaderParseSize(const char *start, ssize_t * sz); +SQUIDCEXTERN int httpHeaderParseOffset(const char *start, int64_t * off); #if STDC_HEADERS SQUIDCEXTERN void httpHeaderPutStrf(HttpHeader * hdr, http_hdr_type id, const char *fmt,...) PRINTF_FORMAT_ARG3; diff --git a/src/squid.h b/src/squid.h index 45cba9bf6c..b564e12f05 100644 --- a/src/squid.h +++ b/src/squid.h @@ -1,6 +1,6 @@ /* - * $Id: squid.h,v 1.261 2007/04/24 15:04:22 hno Exp $ + * $Id: squid.h,v 1.262 2007/08/13 17:20:51 hno Exp $ * * AUTHOR: Duane Wessels * @@ -35,6 +35,11 @@ #ifndef SQUID_H #define SQUID_H +/* + * On linux this must be defined to get PRId64 and friends + */ +#define __STDC_FORMAT_MACROS + #include "config.h" #ifdef _SQUID_MSWIN_ diff --git a/src/stat.cc b/src/stat.cc index 73317aa545..02bfc12bf4 100644 --- a/src/stat.cc +++ b/src/stat.cc @@ -1,5 +1,5 @@ /* - * $Id: stat.cc,v 1.407 2007/05/29 13:31:40 amosjeffries Exp $ + * $Id: stat.cc,v 1.408 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 18 Cache Manager Statistics * AUTHOR: Harvest Derived @@ -1657,7 +1657,7 @@ statClientRequests(StoreEntry * s) if (conn != NULL) { fd = conn->fd; - storeAppendPrintf(s, "\tFD %d, read %d, wrote %d\n", fd, + storeAppendPrintf(s, "\tFD %d, read %"PRId64", wrote %"PRId64"\n", fd, fd_table[fd].bytes_read, fd_table[fd].bytes_written); storeAppendPrintf(s, "\tFD desc: %s\n", fd_table[fd].desc); storeAppendPrintf(s, "\tin: buf %p, offset %ld, size %ld\n", diff --git a/src/stmem.cc b/src/stmem.cc index c34eebe984..7d964771a2 100644 --- a/src/stmem.cc +++ b/src/stmem.cc @@ -1,6 +1,6 @@ /* - * $Id: stmem.cc,v 1.90 2007/04/28 22:26:37 hno Exp $ + * $Id: stmem.cc,v 1.91 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 19 Store Memory Primitives * AUTHOR: Harvest Derived @@ -55,7 +55,7 @@ mem_hdr::NodeGet(mem_node * aNode) return aNode->data; } -int +int64_t mem_hdr::lowestOffset () const { const SplayNode *theStart = nodes.start(); @@ -66,10 +66,10 @@ mem_hdr::lowestOffset () const return 0; } -off_t +int64_t mem_hdr::endOffset () const { - off_t result = 0; + int64_t result = 0; const SplayNode *theEnd = nodes.finish(); if (theEnd) @@ -100,8 +100,8 @@ mem_hdr::unlink(mem_node *aNode) return true; } -int -mem_hdr::freeDataUpto(int target_offset) +int64_t +mem_hdr::freeDataUpto(int64_t target_offset) { /* keep the last one to avoid change to other part of code */ @@ -111,7 +111,7 @@ mem_hdr::freeDataUpto(int target_offset) if (theStart == nodes.finish()) break; - if (theStart->data->end() > (size_t) target_offset ) + if (theStart->data->end() > target_offset ) break; if (!unlink(theStart->data)) @@ -131,7 +131,7 @@ mem_hdr::appendToNode(mem_node *aNode, const char *data, int maxLength) } size_t -mem_hdr::writeAvailable(mem_node *aNode, size_t location, size_t amount, char const *source) +mem_hdr::writeAvailable(mem_node *aNode, int64_t location, size_t amount, char const *source) { /* if we attempt to overwrite existing data or leave a gap within a node */ assert (location == aNode->nodeBuffer.offset + aNode->nodeBuffer.length); @@ -144,7 +144,7 @@ mem_hdr::writeAvailable(mem_node *aNode, size_t location, size_t amount, char co xmemcpy(aNode->nodeBuffer.data + aNode->nodeBuffer.length, source, copyLen); - if (inmem_hi <= (off_t) location) + if (inmem_hi <= location) inmem_hi = location + copyLen; /* Adjust the ptr and len according to what was deposited in the page */ @@ -194,7 +194,7 @@ mem_hdr::internalAppend(const char *data, int len) * If no node contains the start, it returns NULL. */ mem_node * -mem_hdr::getBlockContainingLocation (size_t location) const +mem_hdr::getBlockContainingLocation (int64_t location) const { mem_node target (location); target.nodeBuffer.length = 1; @@ -207,12 +207,12 @@ mem_hdr::getBlockContainingLocation (size_t location) const } size_t -mem_hdr::copyAvailable(mem_node *aNode, size_t location, size_t amount, char *target) const +mem_hdr::copyAvailable(mem_node *aNode, int64_t location, size_t amount, char *target) const { - if (aNode->nodeBuffer.offset > (off_t) location) + if (aNode->nodeBuffer.offset > location) return 0; - assert (aNode->nodeBuffer.offset <= (off_t) location); + assert (aNode->nodeBuffer.offset <= location); assert (aNode->end() > location); @@ -246,6 +246,7 @@ ssize_t mem_hdr::copy(StoreIOBuffer const &target) const { + assert(target.range().end > target.range().start); debugs(19, 6, "memCopy: " << target.range()); /* we shouldn't ever ask for absent offsets */ @@ -261,7 +262,7 @@ mem_hdr::copy(StoreIOBuffer const &target) const assert(target.length > 0); /* Seek our way into store */ - mem_node *p = getBlockContainingLocation((size_t)target.offset); + mem_node *p = getBlockContainingLocation(target.offset); if (!p) { debugs(19, 1, "memCopy: could not find start of " << target.range() << @@ -273,7 +274,7 @@ mem_hdr::copy(StoreIOBuffer const &target) const size_t bytes_to_go = target.length; char *ptr_to_buf = target.data; - off_t location = target.offset; + int64_t location = target.offset; /* Start copying begining with this block until * we're satiated */ @@ -300,9 +301,9 @@ mem_hdr::copy(StoreIOBuffer const &target) const } bool -mem_hdr::hasContigousContentRange(Range const & range) const +mem_hdr::hasContigousContentRange(Range const & range) const { - size_t currentStart = range.start; + int64_t currentStart = range.start; while (mem_node *curr = getBlockContainingLocation(currentStart)) { currentStart = curr->end(); @@ -324,7 +325,7 @@ mem_hdr::unionNotEmpty(StoreIOBuffer const &candidate) } mem_node * -mem_hdr::nodeToRecieve(off_t offset) +mem_hdr::nodeToRecieve(int64_t offset) { /* case 1: Nothing in memory */ @@ -375,7 +376,7 @@ mem_hdr::write (StoreIOBuffer const &writeBuffer) assert (writeBuffer.offset >= 0); mem_node *target; - off_t currentOffset = writeBuffer.offset; + int64_t currentOffset = writeBuffer.offset; char *currentSource = writeBuffer.data; size_t len = writeBuffer.length; diff --git a/src/stmem.h b/src/stmem.h index 3624ae283e..c4e30ce757 100644 --- a/src/stmem.h +++ b/src/stmem.h @@ -1,6 +1,6 @@ /* - * $Id: stmem.h,v 1.9 2005/09/14 18:23:21 wessels Exp $ + * $Id: stmem.h,v 1.10 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -49,18 +49,18 @@ public: mem_hdr(); ~mem_hdr(); void freeContent(); - int lowestOffset () const; - off_t endOffset () const; - int freeDataUpto (int); + int64_t lowestOffset () const; + int64_t endOffset () const; + int64_t freeDataUpto (int64_t); ssize_t copy (StoreIOBuffer const &) const; - bool hasContigousContentRange(Range const &range) const; + bool hasContigousContentRange(Range const &range) const; /* success or fail */ bool write (StoreIOBuffer const &); void dump() const; size_t size() const; /* Not an iterator - thus the start, not begin() */ mem_node const *start() const; - mem_node *getBlockContainingLocation (size_t location) const; + mem_node *getBlockContainingLocation (int64_t location) const; /* access the contained nodes - easier than punning * as a contianer ourselves */ @@ -78,11 +78,11 @@ private: void makeAppendSpace(); int appendToNode(mem_node *aNode, const char *data, int maxLength); void appendNode (mem_node *aNode); - size_t copyAvailable(mem_node *aNode, size_t location, size_t amount, char *target) const; + size_t copyAvailable(mem_node *aNode, int64_t location, size_t amount, char *target) const; bool unionNotEmpty (StoreIOBuffer const &); - mem_node *nodeToRecieve(off_t offset); - size_t writeAvailable(mem_node *aNode, size_t location, size_t amount, char const *source); - off_t inmem_hi; + mem_node *nodeToRecieve(int64_t offset); + size_t writeAvailable(mem_node *aNode, int64_t location, size_t amount, char const *source); + int64_t inmem_hi; Splay nodes; }; diff --git a/src/store.cc b/src/store.cc index dad68fb3db..61336d0671 100644 --- a/src/store.cc +++ b/src/store.cc @@ -1,6 +1,6 @@ /* - * $Id: store.cc,v 1.615 2007/05/29 13:31:40 amosjeffries Exp $ + * $Id: store.cc,v 1.616 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 20 Storage Manager * AUTHOR: Harvest Derived @@ -940,11 +940,10 @@ StoreEntry::checkTooSmall() if (STORE_OK == store_status) if (mem_obj->object_sz < 0 || - static_cast(mem_obj->object_sz) - < Config.Store.minObjectSize) + mem_obj->object_sz < Config.Store.minObjectSize) return 1; if (getReply()->content_length > -1) - if (getReply()->content_length < (int) Config.Store.minObjectSize) + if (getReply()->content_length < Config.Store.minObjectSize) return 1; return 0; } @@ -970,12 +969,12 @@ StoreEntry::checkCachable() store_check_cachable_hist.no.negative_cached++; return 0; /* avoid release call below */ } else if ((getReply()->content_length > 0 && - static_cast(getReply()->content_length) + getReply()->content_length > Config.Store.maxObjectSize) || - static_cast(mem_obj->endOffset()) > Config.Store.maxObjectSize) { + mem_obj->endOffset() > Config.Store.maxObjectSize) { debugs(20, 2, "StoreEntry::checkCachable: NO: too big"); store_check_cachable_hist.no.too_big++; - } else if (getReply()->content_length > (int) Config.Store.maxObjectSize) { + } else if (getReply()->content_length > Config.Store.maxObjectSize) { debugs(20, 2, "StoreEntry::checkCachable: NO: too big"); store_check_cachable_hist.no.too_big++; } else if (checkTooSmall()) { @@ -1356,7 +1355,7 @@ StoreEntry::locked() const bool StoreEntry::validLength() const { - int diff; + int64_t diff; const HttpReply *reply; assert(mem_obj != NULL); reply = getReply(); @@ -1659,14 +1658,14 @@ StoreEntry::flush() } } -ssize_t +int64_t StoreEntry::objectLen() const { assert(mem_obj != NULL); return mem_obj->object_sz; } -int +int64_t StoreEntry::contentLen() const { assert(mem_obj != NULL); diff --git a/src/store_client.cc b/src/store_client.cc index 559848f5ba..8ba6833972 100644 --- a/src/store_client.cc +++ b/src/store_client.cc @@ -1,6 +1,6 @@ /* - * $Id: store_client.cc,v 1.157 2007/04/30 16:56:09 wessels Exp $ + * $Id: store_client.cc,v 1.158 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 90 Storage Manager Client-Side Interface * AUTHOR: Duane Wessels @@ -81,7 +81,7 @@ store_client::operator delete (void *address) } bool -store_client::memReaderHasLowerOffset(off_t anOffset) const +store_client::memReaderHasLowerOffset(int64_t anOffset) const { return getType() == STORE_MEM_CLIENT && copyInto.offset < anOffset; } @@ -229,7 +229,7 @@ store_client::copy(StoreEntry * anEntry, assert (data); assert(!EBIT_TEST(entry->flags, ENTRY_ABORTED)); debugs(90, 3, "store_client::copy: " << entry->getMD5Text() << ", from " << - (unsigned long) copyRequest.offset << ", for length " << + copyRequest.offset << ", for length " << (int) copyRequest.length << ", cb " << callback_fn << ", cbdata " << data); @@ -276,7 +276,7 @@ store_client::copy(StoreEntry * anEntry, static int storeClientNoMoreToSend(StoreEntry * e, store_client * sc) { - ssize_t len; + int64_t len; if (e->store_status == STORE_PENDING) return 0; @@ -341,11 +341,12 @@ store_client::doCopy(StoreEntry *anEntry) MemObject *mem = entry->mem_obj; debugs(33, 5, "store_client::doCopy: co: " << - (unsigned long) copyInto.offset << ", hi: " << - (long int) mem->endOffset()); + copyInto.offset << ", hi: " << + mem->endOffset()); if (storeClientNoMoreToSend(entry, this)) { /* There is no more to send! */ + debugs(33, 3, HERE << "There is no more to send!"); callback(0); flags.store_copying = 0; return; @@ -459,7 +460,7 @@ store_client::fileRead() if (mem->swap_hdr_sz != 0) if (entry->swap_status == SWAPOUT_WRITING) - assert(mem->swapout.sio->offset() > copyInto.offset + (off_t)mem->swap_hdr_sz); + assert(mem->swapout.sio->offset() > copyInto.offset + (int64_t)mem->swap_hdr_sz); storeRead(swapin_sio, copyInto.data, @@ -580,7 +581,7 @@ store_client::readHeader(char const *buf, ssize_t len) */ size_t body_sz = len - mem->swap_hdr_sz; - if (static_cast(copyInto.offset) < body_sz) { + if (copyInto.offset < static_cast(body_sz)) { /* * we have (part of) what they want */ @@ -697,6 +698,14 @@ storeUnregister(store_client * sc, StoreEntry * e, void *data) return 1; } +#if UNUSED_CODE_20070420 +off_t +storeLowestMemReaderOffset(const StoreEntry * entry) +{ + return entry->mem_obj->lowestMemReaderOffset(); +} +#endif + /* Call handlers waiting for data to be appended to E. */ void StoreEntry::invokeHandlers() @@ -756,17 +765,15 @@ CheckQuickAbort2(StoreEntry * entry) return 1; } - size_t expectlen = entry->getReply()->content_length + entry->getReply()->hdr_sz; + int64_t expectlen = entry->getReply()->content_length + entry->getReply()->hdr_sz; if (expectlen < 0) /* expectlen is < 0 if *no* information about the object has been recieved */ return 1; - size_t curlen = (size_t) mem->endOffset (); - - size_t minlen = (size_t) Config.quickAbort.min << 10; + int64_t curlen = mem->endOffset (); - if (minlen < 0) { + if (Config.quickAbort.min < 0) { debugs(90, 3, "CheckQuickAbort2: NO disabled"); return 0; } @@ -776,7 +783,7 @@ CheckQuickAbort2(StoreEntry * entry) return 1; } - if ((expectlen - curlen) < minlen) { + if ((expectlen - curlen) < (Config.quickAbort.min << 10)) { debugs(90, 3, "CheckQuickAbort2: NO only little more left"); return 0; } @@ -791,7 +798,7 @@ CheckQuickAbort2(StoreEntry * entry) return 0; } - if ((curlen / (expectlen / 100)) > (size_t)Config.quickAbort.pct) { + if ((curlen / (expectlen / 100)) > (Config.quickAbort.pct)) { debugs(90, 3, "CheckQuickAbort2: NO past point of no return"); return 0; } @@ -828,8 +835,8 @@ store_client::dumpStats(MemBuf * output, int clientNumber) const output->Printf("\tClient #%d, %p\n", clientNumber, _callback.callback_data); - output->Printf("\t\tcopy_offset: %lu\n", - (unsigned long) copyInto.offset); + output->Printf("\t\tcopy_offset: %"PRId64"\n", + copyInto.offset); output->Printf("\t\tcopy_size: %d\n", (int) copyInto.length); diff --git a/src/store_digest.cc b/src/store_digest.cc index 73054ca36c..21dba733b5 100644 --- a/src/store_digest.cc +++ b/src/store_digest.cc @@ -1,6 +1,6 @@ /* - * $Id: store_digest.cc,v 1.75 2007/04/30 16:56:09 wessels Exp $ + * $Id: store_digest.cc,v 1.76 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 71 Store Digest Manager * AUTHOR: Alex Rousskov @@ -247,7 +247,7 @@ storeDigestAddable(const StoreEntry * e) } /* do not digest huge objects */ - if (e->swap_file_sz > Config.Store.maxObjectSize) { + if (e->swap_file_sz > (uint64_t )Config.Store.maxObjectSize) { debugs(71, 6, "storeDigestAddable: NO: too big"); return 0; } diff --git a/src/store_dir.cc b/src/store_dir.cc index 4ced1ed63c..4003ccdf49 100644 --- a/src/store_dir.cc +++ b/src/store_dir.cc @@ -1,6 +1,6 @@ /* - * $Id: store_dir.cc,v 1.161 2007/05/29 13:31:41 amosjeffries Exp $ + * $Id: store_dir.cc,v 1.162 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Duane Wessels @@ -330,13 +330,13 @@ storeDirSwapLog(const StoreEntry * e, int op) } void -StoreController::updateSize(size_t size, int sign) +StoreController::updateSize(int64_t size, int sign) { fatal("StoreController has no independent size\n"); } void -SwapDir::updateSize(size_t size, int sign) +SwapDir::updateSize(int64_t size, int sign) { int blks = (size + fs.blksize - 1) / fs.blksize; int k = (blks * fs.blksize >> 10) * sign; @@ -707,6 +707,8 @@ StoreController::get StoreHashIndex::StoreHashIndex() { + if (store_table) + abort(); assert (store_table == NULL); } @@ -880,7 +882,7 @@ StoreHashIndex::maintain() } void -StoreHashIndex::updateSize(size_t, int) +StoreHashIndex::updateSize(int64_t, int) {} void diff --git a/src/store_log.cc b/src/store_log.cc index 075820c3fd..2d1be05e20 100644 --- a/src/store_log.cc +++ b/src/store_log.cc @@ -1,6 +1,6 @@ /* - * $Id: store_log.cc,v 1.34 2007/05/29 13:31:41 amosjeffries Exp $ + * $Id: store_log.cc,v 1.35 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 20 Storage Manager Logging Functions * AUTHOR: Duane Wessels @@ -76,7 +76,7 @@ storeLog(int tag, const StoreEntry * e) * Because if we print it before the swap file number, it'll break * the existing log format. */ - logfilePrintf(storelog, "%9d.%03d %-7s %02d %08X %s %4d %9d %9d %9d %s %d/%d %s %s\n", + logfilePrintf(storelog, "%9d.%03d %-7s %02d %08X %s %4d %9d %9d %9d %s %"PRId64"/%"PRId64" %s %s\n", (int) current_time.tv_sec, (int) current_time.tv_usec / 1000, storeLogTags[tag], diff --git a/src/store_swapmeta.cc b/src/store_swapmeta.cc index 2b3472a8e5..c8a95e9f5d 100644 --- a/src/store_swapmeta.cc +++ b/src/store_swapmeta.cc @@ -1,6 +1,6 @@ /* - * $Id: store_swapmeta.cc,v 1.25 2007/04/28 22:26:38 hno Exp $ + * $Id: store_swapmeta.cc,v 1.26 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 20 Storage Manager Swapfile Metadata * AUTHOR: Kostas Anagnostakis @@ -73,7 +73,7 @@ storeSwapMetaBuild(StoreEntry * e) } T = StoreMeta::Add(T, t); - t = StoreMeta::Factory(STORE_META_STD,STORE_HDR_METASIZE,&e->timestamp); + t = StoreMeta::Factory(STORE_META_STD_LFS,STORE_HDR_METASIZE,&e->timestamp); if (!t) { storeSwapTLVFree(TLV); diff --git a/src/store_swapout.cc b/src/store_swapout.cc index 5d77441c4a..25fc6a685d 100644 --- a/src/store_swapout.cc +++ b/src/store_swapout.cc @@ -1,6 +1,6 @@ /* - * $Id: store_swapout.cc,v 1.117 2007/04/30 16:56:09 wessels Exp $ + * $Id: store_swapout.cc,v 1.118 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 20 Storage Manager Swapout Functions * AUTHOR: Duane Wessels @@ -165,7 +165,7 @@ doPages(StoreEntry *anEntry) if (anEntry->swap_status != SWAPOUT_WRITING) break; - ssize_t swapout_size = (ssize_t) (mem->endOffset() - mem->swapout.queue_offset); + int64_t swapout_size = mem->endOffset() - mem->swapout.queue_offset; if (anEntry->store_status == STORE_PENDING) if (swapout_size < SM_PAGE_SIZE) @@ -189,20 +189,20 @@ StoreEntry::swapOut() if (!swapoutPossible()) return; - debugs(20, 7, "storeSwapOut: mem_obj->inmem_lo = " << mem_obj->inmem_lo); - debugs(20, 7, "storeSwapOut: mem_obj->endOffset() = " << mem_obj->endOffset()); - debugs(20, 7, "storeSwapOut: swapout.queue_offset = " << mem_obj->swapout.queue_offset); + debugs(20, 7, HERE << "storeSwapOut: mem->inmem_lo = " << mem_obj->inmem_lo); + debugs(20, 7, HERE << "storeSwapOut: mem->endOffset() = " << mem_obj->endOffset()); + debugs(20, 7, HERE << "storeSwapOut: swapout.queue_offset = " << mem_obj->swapout.queue_offset); if (mem_obj->swapout.sio != NULL) debugs(20, 7, "storeSwapOut: storeOffset() = " << mem_obj->swapout.sio->offset() ); - ssize_t swapout_maxsize = (ssize_t) (mem_obj->endOffset() - mem_obj->swapout.queue_offset); + int64_t swapout_maxsize = mem_obj->endOffset() - mem_obj->swapout.queue_offset; assert(swapout_maxsize >= 0); - off_t const lowest_offset = mem_obj->lowestMemReaderOffset(); + int64_t const lowest_offset = mem_obj->lowestMemReaderOffset(); - debugs(20, 7, "storeSwapOut: lowest_offset = " << lowest_offset); + debugs(20, 7, HERE << "storeSwapOut: lowest_offset = " << lowest_offset); /* * Grab the swapout_size and check to see whether we're going to defer @@ -224,17 +224,17 @@ StoreEntry::swapOut() } trimMemory(); -#if SIZEOF_OFF_T == 4 +#if SIZEOF_INT64_T == 4 if (mem_obj->endOffset() > 0x7FFF0000) { - debugs(20, 0, "WARNING: preventing off_t overflow for " << url() ); + debugs(20, 0, "WARNING: preventing int64_t overflow for %s\n", url()); abort(); return; } #endif if (swap_status == SWAPOUT_WRITING) - assert(mem_obj->inmem_lo <= (off_t)mem_obj->objectBytesOnDisk() ); + assert(mem_obj->inmem_lo <= mem_obj->objectBytesOnDisk() ); if (!swapOutAble()) return; diff --git a/src/structs.h b/src/structs.h index 9078f13ca1..6ec0f64ed3 100644 --- a/src/structs.h +++ b/src/structs.h @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.560 2007/08/02 01:16:51 amosjeffries Exp $ + * $Id: structs.h,v 1.561 2007/08/13 17:20:51 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -107,7 +107,7 @@ struct _acl_size_t { acl_size_t *next; acl_list *aclList; - size_t size; + int64_t size; }; struct _ushortlist @@ -209,13 +209,13 @@ struct _SquidConfig struct { - size_t min; + int64_t min; int pct; - size_t max; + int64_t max; } quickAbort; - size_t readAheadGap; + int64_t readAheadGap; RemovalPolicySettings *replPolicy; RemovalPolicySettings *memPolicy; time_t negativeTtl; @@ -254,7 +254,7 @@ struct _SquidConfig Timeout; size_t maxRequestHeaderSize; - size_t maxRequestBodySize; + int64_t maxRequestBodySize; size_t maxReplyHeaderSize; acl_size_t *ReplyBodySize; @@ -476,8 +476,8 @@ struct _SquidConfig { int objectsPerBucket; size_t avgObjectSize; - size_t maxObjectSize; - size_t minObjectSize; + int64_t maxObjectSize; + int64_t minObjectSize; size_t maxInMemObjSize; } @@ -660,7 +660,7 @@ struct _SquidConfig comm_incoming; int max_open_disk_fds; int uri_whitespace; - size_t rangeOffsetLimit; + int64_t rangeOffsetLimit; #if MULTICAST_MISS_STREAM struct diff --git a/src/tests/stub_HttpReply.cc b/src/tests/stub_HttpReply.cc index e153d37453..6afc0d2d9f 100644 --- a/src/tests/stub_HttpReply.cc +++ b/src/tests/stub_HttpReply.cc @@ -1,5 +1,5 @@ /* - * $Id: stub_HttpReply.cc,v 1.3 2006/05/27 00:35:05 robertc Exp $ + * $Id: stub_HttpReply.cc,v 1.4 2007/08/13 17:20:58 hno Exp $ * * DEBUG: section 84 Helper process maintenance * AUTHOR: Robert Collins @@ -53,7 +53,7 @@ HttpReply::absorb(HttpReply * new_rep) void HttpReply::setHeaders(HttpVersion ver, http_status status, const char *reason, - const char *ctype, int clen, time_t lmt, time_t expires) + const char *ctype, int64_t clen, time_t lmt, time_t expires) { fatal ("Not implemented"); } @@ -90,7 +90,7 @@ HttpReply::httpMsgParseError() } bool -HttpReply::expectingBody(method_t, ssize_t&) const +HttpReply::expectingBody(method_t, int64_t&) const { fatal ("Not implemented"); return false; diff --git a/src/tests/stub_HttpRequest.cc b/src/tests/stub_HttpRequest.cc index 7748cb2253..5e290d4c4f 100644 --- a/src/tests/stub_HttpRequest.cc +++ b/src/tests/stub_HttpRequest.cc @@ -1,5 +1,5 @@ /* - * $Id: stub_HttpRequest.cc,v 1.3 2006/04/18 12:46:13 robertc Exp $ + * $Id: stub_HttpRequest.cc,v 1.4 2007/08/13 17:20:58 hno Exp $ * * DEBUG: section 28 Access Control * AUTHOR: Robert Collins @@ -75,7 +75,7 @@ HttpRequest::reset() } bool -HttpRequest::expectingBody(method_t unused, ssize_t&) const +HttpRequest::expectingBody(method_t unused, int64_t&) const { fatal("Not implemented"); return false; diff --git a/src/tunnel.cc b/src/tunnel.cc index 8d62b69471..02e35a8787 100644 --- a/src/tunnel.cc +++ b/src/tunnel.cc @@ -1,6 +1,6 @@ /* - * $Id: tunnel.cc,v 1.172 2007/06/02 12:21:57 hno Exp $ + * $Id: tunnel.cc,v 1.173 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 26 Secure Sockets Layer Proxy * AUTHOR: Duane Wessels @@ -90,7 +90,7 @@ public: void dataSent (size_t amount); int len; char *buf; - size_t *size_ptr; /* pointer to size in an ConnStateData for logging */ + int64_t *size_ptr; /* pointer to size in an ConnStateData for logging */ private: int fd_; @@ -579,7 +579,7 @@ tunnelConnectDone(int fdnotused, comm_err_t status, int xerrno, void *data) } void -tunnelStart(ClientHttpRequest * http, size_t * size_ptr, int *status_ptr) +tunnelStart(ClientHttpRequest * http, int64_t * size_ptr, int *status_ptr) { /* Create state structure. */ TunnelStateData *tunnelState = NULL; diff --git a/src/ufsdump.cc b/src/ufsdump.cc index 479f057df1..4f8ab804b2 100644 --- a/src/ufsdump.cc +++ b/src/ufsdump.cc @@ -1,6 +1,6 @@ /* - * $Id: ufsdump.cc,v 1.9 2006/09/13 19:05:11 serassio Exp $ + * $Id: ufsdump.cc,v 1.10 2007/08/13 17:20:51 hno Exp $ * * DEBUG: section 0 UFS Store Dump * AUTHOR: Robert Collins @@ -56,6 +56,26 @@ eventAdd(const char *name, EVH * func, void *arg, double when, int, bool cbdata) #endif /* end stub functions */ +struct MetaStd{ + time_t timestamp; + time_t lastref; + time_t expires; + time_t lastmod; + size_t swap_file_sz; + u_short refcount; + u_short flags; +}; + +struct MetaStdLfs{ + time_t timestamp; + time_t lastref; + time_t expires; + time_t lastmod; + uint64_t swap_file_sz; + u_short refcount; + u_short flags; +}; + struct DumpStoreMeta : public unary_function { DumpStoreMeta(){} @@ -69,13 +89,27 @@ struct DumpStoreMeta : public unary_function break; case STORE_META_STD: + std::cout << "STD, Size:" << ((struct MetaStd*)x.value)->swap_file_sz << + " Flags: 0x" << std::hex << ((struct MetaStd*)x.value)->flags << std::dec << + " Refcount: " << ((struct MetaStd*)x.value)->refcount << + std::endl; + break; + + case STORE_META_STD_LFS: + std::cout << "STD_LFS, Size: " << ((struct MetaStdLfs*)x.value)->swap_file_sz << + " Flags: 0x" << std::hex << ((struct MetaStdLfs*)x.value)->flags << std::dec << + " Refcount: " << ((struct MetaStdLfs*)x.value)->refcount << + std::endl; break; case STORE_META_URL: assert (((char *)x.value)[x.length - 1] == 0); std::cout << "URL: " << (char *)x.value << std::endl; + break; default: + std::cout << "Unknown store meta type: " << (int)x.getType() << + " of length " << x.length << std::endl; break; } }