diff -u -r -N squid-3.2.7/ChangeLog squid-3.2.8/ChangeLog --- squid-3.2.7/ChangeLog 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/ChangeLog 2013-03-02 14:46:03.000000000 +1300 @@ -1,3 +1,19 @@ +Changes to squid-3.2.8 (02 Mar 2013): + + - Bug 3767: tcp_outgoing_tos/mark ACLs do not obey acl_uses_indirect_client + - Bug 3763: diskd Error: no filename in shm buffer + - Bug 3752: objects that cannot be cached in memory are not cached on disk + - Bug 3753: Removes the domain from the cache_peer server pconn key + - Bug 3749: IDENT lookup using wrong ports to identify the user + - Bug 3723: tcp_outgoing_tos/mark broken for CONNECT requests + - Bug 3686: cache_dir max-size default fails + - Bug 3515: crash in FtpStateData::ftpTimeout + - Bug 3329: Quieten orphan Comm::Connection messages + - Make squid -z for cache_dir rock preserve the rock DB + - Fixed several server connect problems + - ... and some build issues on Solaris, OpenIndiana, MacOS X + - ... and some documentation and debugs polishing + Changes to squid-3.2.7 (01 Feb 2013): - Bug 3736: Floating point exception due to divide by zero diff -u -r -N squid-3.2.7/compat/xstrto.h squid-3.2.8/compat/xstrto.h --- squid-3.2.7/compat/xstrto.h 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/compat/xstrto.h 2013-03-02 14:46:03.000000000 +1300 @@ -1,6 +1,9 @@ #ifndef _SQUID_XSTRTO_H #define _SQUID_XSTRTO_H +// these functions are not used by the remaining Squid C code. +#if defined(__cplusplus) + #if HAVE_STDBOOL_H #include #endif @@ -27,4 +30,5 @@ bool xstrtoui(const char *s, char **end, unsigned int *value, unsigned int min, unsigned int max); +#endif /* __cplusplus */ #endif /* _SQUID_XSTRTO_H */ diff -u -r -N squid-3.2.7/configure squid-3.2.8/configure --- squid-3.2.7/configure 2013-02-01 23:57:00.000000000 +1300 +++ squid-3.2.8/configure 2013-03-02 14:47:21.000000000 +1300 @@ -1,7 +1,7 @@ #! /bin/sh # From configure.ac Revision. # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.68 for Squid Web Proxy 3.2.7. +# Generated by GNU Autoconf 2.68 for Squid Web Proxy 3.2.8. # # Report bugs to . # @@ -575,8 +575,8 @@ # Identity of this package. PACKAGE_NAME='Squid Web Proxy' PACKAGE_TARNAME='squid' -PACKAGE_VERSION='3.2.7' -PACKAGE_STRING='Squid Web Proxy 3.2.7' +PACKAGE_VERSION='3.2.8' +PACKAGE_STRING='Squid Web Proxy 3.2.8' PACKAGE_BUGREPORT='http://bugs.squid-cache.org/' PACKAGE_URL='' @@ -1571,7 +1571,7 @@ # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Squid Web Proxy 3.2.7 to adapt to many kinds of systems. +\`configure' configures Squid Web Proxy 3.2.8 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1641,7 +1641,7 @@ if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Squid Web Proxy 3.2.7:";; + short | recursive ) echo "Configuration of Squid Web Proxy 3.2.8:";; esac cat <<\_ACEOF @@ -2019,7 +2019,7 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Squid Web Proxy configure 3.2.7 +Squid Web Proxy configure 3.2.8 generated by GNU Autoconf 2.68 Copyright (C) 2010 Free Software Foundation, Inc. @@ -3115,7 +3115,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Squid Web Proxy $as_me 3.2.7, which was +It was created by Squid Web Proxy $as_me 3.2.8, which was generated by GNU Autoconf 2.68. Invocation command line was $ $0 $@ @@ -3934,7 +3934,7 @@ # Define the identity of the package. PACKAGE='squid' - VERSION='3.2.7' + VERSION='3.2.8' cat >>confdefs.h <<_ACEOF @@ -30894,7 +30894,7 @@ # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Squid Web Proxy $as_me 3.2.7, which was +This file was extended by Squid Web Proxy $as_me 3.2.8, which was generated by GNU Autoconf 2.68. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -30960,7 +30960,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Squid Web Proxy config.status 3.2.7 +Squid Web Proxy config.status 3.2.8 configured by $0, generated by GNU Autoconf 2.68, with options \\"\$ac_cs_config\\" diff -u -r -N squid-3.2.7/configure.ac squid-3.2.8/configure.ac --- squid-3.2.7/configure.ac 2013-02-01 23:56:59.000000000 +1300 +++ squid-3.2.8/configure.ac 2013-03-02 14:47:21.000000000 +1300 @@ -1,4 +1,4 @@ -AC_INIT([Squid Web Proxy],[3.2.7],[http://bugs.squid-cache.org/],[squid]) +AC_INIT([Squid Web Proxy],[3.2.8],[http://bugs.squid-cache.org/],[squid]) AC_PREREQ(2.61) AC_CONFIG_HEADERS([include/autoconf.h]) AC_CONFIG_AUX_DIR(cfgaux) diff -u -r -N squid-3.2.7/helpers/basic_auth/DB/basic_db_auth.8 squid-3.2.8/helpers/basic_auth/DB/basic_db_auth.8 --- squid-3.2.7/helpers/basic_auth/DB/basic_db_auth.8 2013-02-02 00:21:45.000000000 +1300 +++ squid-3.2.8/helpers/basic_auth/DB/basic_db_auth.8 2013-03-02 15:07:58.000000000 +1300 @@ -124,7 +124,7 @@ .\" ======================================================================== .\" .IX Title "BASIC_DB_AUTH 1" -.TH BASIC_DB_AUTH 1 "2013-02-01" "perl v5.10.1" "User Contributed Perl Documentation" +.TH BASIC_DB_AUTH 1 "2013-03-01" "perl v5.10.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-3.2.7/helpers/external_acl/wbinfo_group/ext_wbinfo_group_acl.8 squid-3.2.8/helpers/external_acl/wbinfo_group/ext_wbinfo_group_acl.8 --- squid-3.2.7/helpers/external_acl/wbinfo_group/ext_wbinfo_group_acl.8 2013-02-02 00:22:05.000000000 +1300 +++ squid-3.2.8/helpers/external_acl/wbinfo_group/ext_wbinfo_group_acl.8 2013-03-02 15:08:02.000000000 +1300 @@ -124,7 +124,7 @@ .\" ======================================================================== .\" .IX Title "EXT_WBINFO_GROUP_ACL.PL.IN 1" -.TH EXT_WBINFO_GROUP_ACL.PL.IN 1 "2013-02-01" "perl v5.10.1" "User Contributed Perl Documentation" +.TH EXT_WBINFO_GROUP_ACL.PL.IN 1 "2013-03-01" "perl v5.10.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-3.2.7/include/version.h squid-3.2.8/include/version.h --- squid-3.2.7/include/version.h 2013-02-01 23:57:00.000000000 +1300 +++ squid-3.2.8/include/version.h 2013-03-02 14:47:21.000000000 +1300 @@ -9,7 +9,7 @@ */ #ifndef SQUID_RELEASE_TIME -#define SQUID_RELEASE_TIME 1359716114 +#define SQUID_RELEASE_TIME 1362188760 #endif #ifndef APP_SHORTNAME diff -u -r -N squid-3.2.7/src/cache_cf.cc squid-3.2.8/src/cache_cf.cc --- squid-3.2.7/src/cache_cf.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/cache_cf.cc 2013-03-02 14:46:03.000000000 +1300 @@ -223,16 +223,23 @@ static void update_maxobjsize(void) { - int i; int64_t ms = -1; - for (i = 0; i < Config.cacheSwap.n_configured; ++i) { + // determine the maximum size object that can be stored to disk + for (int i = 0; i < Config.cacheSwap.n_configured; ++i) { assert (Config.cacheSwap.swapDirs[i].getRaw()); - if (dynamic_cast(Config.cacheSwap.swapDirs[i].getRaw())-> - max_objsize > ms) - ms = dynamic_cast(Config.cacheSwap.swapDirs[i].getRaw())->max_objsize; + const int64_t storeMax = dynamic_cast(Config.cacheSwap.swapDirs[i].getRaw())->maxObjectSize(); + if (ms < storeMax) + ms = storeMax; } + + // Ensure that we do not discard objects which could be stored only in memory. + // It is governed by maximum_object_size_in_memory (for now) + // TODO: update this to check each in-memory location (SMP and local memory limits differ) + if (ms < static_cast(Config.Store.maxInMemObjSize)) + ms = Config.Store.maxInMemObjSize; + store_maxobjsize = ms; } diff -u -r -N squid-3.2.7/src/cf.data.pre squid-3.2.8/src/cf.data.pre --- squid-3.2.7/src/cf.data.pre 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/cf.data.pre 2013-03-02 14:46:03.000000000 +1300 @@ -104,6 +104,61 @@ across all Squid processes. COMMENT_END +# options still not yet ported from 2.7 to 3.x +NAME: broken_vary_encoding +TYPE: obsolete +DOC_START + This option is not yet supported by Squid-3. +DOC_END + +NAME: cache_vary +TYPE: obsolete +DOC_START + This option is not yet supported by Squid-3. +DOC_END + +NAME: collapsed_forwarding +TYPE: obsolete +DOC_START + This option is not yet supported by Squid-3. see http://bugs.squid-cache.org/show_bug.cgi?id=3495 +DOC_END + +NAME: error_map +TYPE: obsolete +DOC_START + This option is not yet supported by Squid-3. +DOC_END + +NAME: external_refresh_check +TYPE: obsolete +DOC_START + This option is not yet supported by Squid-3. +DOC_END + +NAME: ignore_ims_on_miss +TYPE: obsolete +DOC_START + This option is not yet supported by Squid-3. +DOC_END + +NAME: location_rewrite_program location_rewrite_access location_rewrite_children location_rewrite_concurrency +TYPE: obsolete +DOC_START + This option is not yet supported by Squid-3. +DOC_END + +NAME: refresh_stale_hit +TYPE: obsolete +DOC_START + This option is not yet supported by Squid-3. +DOC_END + +NAME: storeurl_access storeurl_rewrite_program storeurl_rewrite_concurrency storeurl_rewrite_children +TYPE: obsolete +DOC_START + This option is not yet supported by this version of Squid-3. Please try a later release. +DOC_END + # Options Removed in 3.2 NAME: ignore_expect_100 TYPE: obsolete @@ -114,7 +169,7 @@ NAME: dns_v4_fallback TYPE: obsolete DOC_START - Remove this line. + Remove this line. Squid performs a 'Happy Eyeballs' algorithm, the 'fallback' algorithm is no longer relevant. DOC_END NAME: ftp_list_width @@ -129,6 +184,12 @@ Replaced by connect_retries. The behaviour has changed, please read the documentation before altering. DOC_END +NAME: update_headers +TYPE: obsolete +DOC_START + Remove this line. The feature is supported by default in storage types where update is implemented. +DOC_END + NAME: url_rewrite_concurrency TYPE: obsolete DOC_START @@ -190,6 +251,18 @@ Since squid-3.0 use the 'disable-pmtu-discovery' flag on http_port instead. DOC_END +NAME: wais_relay_host +TYPE: obsolete +DOC_START + Replace this line with 'cache_peer' configuration. +DOC_END + +NAME: wais_relay_port +TYPE: obsolete +DOC_START + Replace this line with 'cache_peer' configuration. +DOC_END + COMMENT_START OPTIONS FOR AUTHENTICATION ----------------------------------------------------------------------------- @@ -257,7 +330,7 @@ auth_param basic program @DEFAULT_PREFIX@/libexec/ncsa_auth @DEFAULT_PREFIX@/etc/passwd "utf8" on|off - HTTP uses iso-latin-1 as characterset, while some authentication + HTTP uses iso-latin-1 as character set, while some authentication backends such as LDAP expects UTF-8. If this is set to on Squid will translate the HTTP iso-latin-1 charset to UTF-8 before sending the username & password to the helper. @@ -280,7 +353,7 @@ supports one request at a time. Setting this to a number greater than 0 changes the protocol used to include a channel number first on the request/response line, allowing multiple requests to be sent to the - same helper in parallell without wating for the response. + same helper in parallel without waiting for the response. Must not be set unless it's known the helper supports this. auth_param basic children 20 startup=0 idle=1 @@ -330,7 +403,7 @@ auth_param digest program @DEFAULT_PREFIX@/bin/digest_pw_auth @DEFAULT_PREFIX@/etc/digpass "utf8" on|off - HTTP uses iso-latin-1 as characterset, while some authentication + HTTP uses iso-latin-1 as character set, while some authentication backends such as LDAP expects UTF-8. If this is set to on Squid will translate the HTTP iso-latin-1 charset to UTF-8 before sending the username & password to the helper. @@ -353,7 +426,7 @@ supports one request at a time. Setting this to a number greater than 0 changes the protocol used to include a channel number first on the request/response line, allowing multiple requests to be sent to the - same helper in parallell without wating for the response. + same helper in parallel without waiting for the response. Must not be set unless it's known the helper supports this. auth_param digest children 20 startup=0 idle=1 @@ -380,7 +453,7 @@ "nonce_strictness" on|off Determines if squid requires strict increment-by-1 behavior for nonce counts, or just incrementing (off - for use when - useragents generate nonce counts that occasionally miss 1 + user agents generate nonce counts that occasionally miss 1 (ie, 1,2,4,6)). Default off. "check_nonce_count" on|off @@ -451,7 +524,7 @@ The maximum number of authenticator processes to spawn (default 5). If you start too few Squid will have to wait for them to process a backlog of credential verifications, slowing it - down. When crendential verifications are done via a (slow) + down. When credential verifications are done via a (slow) network you are likely to need lots of authenticator processes. @@ -503,7 +576,7 @@ LOC: Config.authenticateGCInterval DOC_START The time period between garbage collection across the username cache. - This is a tradeoff between memory utilization (long intervals - say + This is a trade-off between memory utilization (long intervals - say 2 days) and CPU (short intervals - say 1 minute). Only change if you have good reason to. DOC_END @@ -528,7 +601,7 @@ this directive controls how long Squid remembers the IP addresses associated with each user. Use a small value (e.g., 60 seconds) if your users might change addresses - quickly, as is the case with dialups. You might be safe + quickly, as is the case with dialup. You might be safe using a larger value (e.g., 2 hours) in a corporate LAN environment with relatively static address assignments. DOC_END @@ -2735,7 +2808,7 @@ replacement policies. NOTE: if using the LFUDA replacement policy you should increase - the value of maximum_object_size above its default of 4096 KB to + the value of maximum_object_size above its default of 4 MB to to maximize the potential byte hit rate improvement of LFUDA. For more information about the GDSF and LFUDA cache replacement @@ -2833,6 +2906,12 @@ slot size is specified in bytes using the max-size option. See below for more info on the max-size option. + If possible, Squid using Rock Store creates a dedicated kid + process called "disker" to avoid blocking Squid worker(s) on disk + I/O. One disker kid is created for each rock cache_dir. Diskers + are created only when Squid, running in daemon mode, has support + for the IpcIo disk I/O module. + swap-timeout=msec: Squid will not start writing a miss to or reading a hit from disk if it estimates that the swap operation will take more than the specified number of milliseconds. By @@ -2934,14 +3013,18 @@ NAME: maximum_object_size COMMENT: (bytes) TYPE: b_int64_t -DEFAULT: 4096 KB +DEFAULT: 4 MB LOC: Config.Store.maxObjectSize DOC_START - Objects larger than this size will NOT be saved on disk. The - value is specified in kilobytes, and the default is 4MB. If - you wish to get a high BYTES hit ratio, you should probably + The default limit on size of objects stored to disk. + This size is used for cache_dir where max-size is not set. + The value is specified in bytes, and the default is 4 MB. + + If you wish to get a high BYTES hit ratio, you should probably increase this (one 32 MB object hit counts for 3200 10KB - hits). If you wish to increase speed more than your want to + hits). + + If you wish to increase hit ratio more than you want to save bandwidth you should leave this low. NOTE: if using the LFUDA replacement policy you should increase diff -u -r -N squid-3.2.7/src/comm/Connection.cc squid-3.2.8/src/comm/Connection.cc --- squid-3.2.7/src/comm/Connection.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/comm/Connection.cc 2013-03-02 14:46:03.000000000 +1300 @@ -27,8 +27,8 @@ Comm::Connection::~Connection() { if (fd >= 0) { - debugs(5, DBG_CRITICAL, "BUG #3329: Orphan Comm::Connection: " << *this); - debugs(5, DBG_CRITICAL, "NOTE: " << ++lost_conn << " Orphans since last started."); + debugs(5, 4, "BUG #3329: Orphan Comm::Connection: " << *this); + debugs(5, 4, "NOTE: " << ++lost_conn << " Orphans since last started."); close(); } diff -u -r -N squid-3.2.7/src/comm/ConnOpener.cc squid-3.2.8/src/comm/ConnOpener.cc --- squid-3.2.7/src/comm/ConnOpener.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/comm/ConnOpener.cc 2013-03-02 14:46:03.000000000 +1300 @@ -23,8 +23,7 @@ callback_(handler), totalTries_(0), failRetries_(0), - connectTimeout_(ctimeout), - connectStart_(0) + deadline_(squid_curtime + static_cast(ctimeout)) {} Comm::ConnOpener::~ConnOpener() @@ -45,36 +44,26 @@ return AsyncJob::doneAll(); } + // otherwise, we must be waiting for something + Must(temporaryFd_ >= 0 || calls_.sleep_); return false; } void Comm::ConnOpener::swanSong() { - // cancel any event watchers - // done here to get the "swanSong" mention in cancel debugging. - if (calls_.earlyAbort_ != NULL) { - calls_.earlyAbort_->cancel("Comm::ConnOpener::swanSong"); - calls_.earlyAbort_ = NULL; - } - if (calls_.timeout_ != NULL) { - calls_.timeout_->cancel("Comm::ConnOpener::swanSong"); - calls_.timeout_ = NULL; - } - if (callback_ != NULL) { - if (callback_->canceled()) - callback_ = NULL; - else - // inform the still-waiting caller we are dying - doneConnecting(COMM_ERR_CONNECT, 0); + // inform the still-waiting caller we are dying + sendAnswer(COMM_ERR_CONNECT, 0, "Comm::ConnOpener::swanSong"); } - // rollback what we can from the job state - if (temporaryFd_ >= 0) { - // doneConnecting() handles partial FD connection cleanup - doneConnecting(COMM_ERR_CONNECT, 0); - } + // did we abort with a temporary FD assigned? + if (temporaryFd_ >= 0) + closeFd(); + + // did we abort while waiting between retries? + if (calls_.sleep_) + cancelSleep(); AsyncJob::swanSong(); } @@ -100,14 +89,13 @@ /** * Connection attempt are completed. One way or the other. * Pass the results back to the external handler. - * NP: on errors the earlyAbort call should be cancelled first with a reason. */ void -Comm::ConnOpener::doneConnecting(comm_err_t status, int xerrno) +Comm::ConnOpener::sendAnswer(comm_err_t errFlag, int xerrno, const char *why) { // only mark the address good/bad AFTER connect is finished. if (host_ != NULL) { - if (xerrno == 0) + if (xerrno == 0) // XXX: should not we use errFlag instead? ipcacheMarkGoodAddr(host_, conn_->remote); else { ipcacheMarkBadAddr(host_, conn_->remote); @@ -119,34 +107,106 @@ } if (callback_ != NULL) { - typedef CommConnectCbParams Params; - Params ¶ms = GetCommParams(callback_); - params.conn = conn_; - params.flag = status; - params.xerrno = xerrno; - ScheduleCallHere(callback_); + // avoid scheduling cancelled callbacks, assuming they are common + // enough to make this extra check an optimization + if (callback_->canceled()) { + debugs(5, 4, conn_ << " not calling canceled " << *callback_ << + " [" << callback_->id << ']' ); + // TODO save the pconn to the pconnPool ? + } else { + typedef CommConnectCbParams Params; + Params ¶ms = GetCommParams(callback_); + params.conn = conn_; + params.flag = errFlag; + params.xerrno = xerrno; + ScheduleCallHere(callback_); + } callback_ = NULL; } - if (temporaryFd_ >= 0) { - debugs(5, 4, HERE << conn_ << " closing temp FD " << temporaryFd_); - // it never reached fully open, so cleanup the FD handlers - // Note that comm_close() sequence does not happen for partially open FD - Comm::SetSelect(temporaryFd_, COMM_SELECT_WRITE, NULL, NULL, 0); + // The job will stop without this call because nil callback_ makes + // doneAll() true, but this explicit call creates nicer debugging. + mustStop(why); +} + +/// cleans up this job I/O state without closing temporaryFd +/// required before closing temporaryFd or keeping it in conn_ +/// leaves FD bare so must only be called via closeFd() or keepFd() +void +Comm::ConnOpener::cleanFd() +{ + debugs(5, 4, HERE << conn_ << " closing temp FD " << temporaryFd_); + + Must(temporaryFd_ >= 0); + fde &f = fd_table[temporaryFd_]; + + // Our write_handler was set without using Comm::Write API, so we cannot + // use a cancellable Pointer-free job callback and simply cancel it here. + if (f.write_handler) { + + /* XXX: We are about to remove write_handler, which was responsible + * for deleting write_data, so we have to delete write_data + * ourselves. Comm currently calls SetSelect handlers synchronously + * so if write_handler is set, we know it has not been called yet. + * ConnOpener converts that sync call into an async one, but only + * after deleting ptr, so that is not a problem. + */ + + delete static_cast(f.write_data); + f.write_data = NULL; + f.write_handler = NULL; + } + // Comm::DoSelect does not do this when calling and resetting write_handler + // (because it expects more writes to come?). We could mimic that + // optimization by resetting Comm "Select" state only when the FD is + // actually closed. + Comm::SetSelect(temporaryFd_, COMM_SELECT_WRITE, NULL, NULL, 0); + + if (calls_.timeout_ != NULL) { + calls_.timeout_->cancel("Comm::ConnOpener::cleanFd"); + calls_.timeout_ = NULL; + } + // Comm checkTimeouts() and commCloseAllSockets() do not clear .timeout + // when calling timeoutHandler (XXX fix them), so we clear unconditionally. + f.timeoutHandler = NULL; + f.timeout = 0; + + if (calls_.earlyAbort_ != NULL) { + comm_remove_close_handler(temporaryFd_, calls_.earlyAbort_); calls_.earlyAbort_ = NULL; - if (calls_.timeout_ != NULL) { - calls_.timeout_->cancel("Comm::ConnOpener::doneConnecting"); - calls_.timeout_ = NULL; - } - fd_table[temporaryFd_].timeoutHandler = NULL; - fd_table[temporaryFd_].timeout = 0; - close(temporaryFd_); - fd_close(temporaryFd_); - temporaryFd_ = -1; } +} + +/// cleans I/O state and ends I/O for temporaryFd_ +void +Comm::ConnOpener::closeFd() +{ + if (temporaryFd_ < 0) + return; + + cleanFd(); + + // comm_close() below uses COMMIO_FD_WRITECB(fd)->active() to clear Comm + // "Select" state. It will not clear ours. XXX: It should always clear + // because a callback may have been active but was called before comm_close + // Update: we now do this in cleanFd() + // Comm::SetSelect(temporaryFd_, COMM_SELECT_WRITE, NULL, NULL, 0); + + comm_close(temporaryFd_); + temporaryFd_ = -1; +} + +/// cleans I/O state and moves temporaryFd_ to the conn_ for long-term use +void +Comm::ConnOpener::keepFd() +{ + Must(conn_ != NULL); + Must(temporaryFd_ >= 0); - /* ensure cleared local state, we are done. */ - conn_ = NULL; + cleanFd(); + + conn_->fd = temporaryFd_; + temporaryFd_ = -1; } void @@ -154,17 +214,41 @@ { Must(conn_ != NULL); - /* get a socket open ready for connecting with */ + /* outbound sockets have no need to be protocol agnostic. */ + if (!(Ip::EnableIpv6&IPV6_SPECIAL_V4MAPPING) && conn_->remote.IsIPv4()) { + conn_->local.SetIPv4(); + } + + if (createFd()) + connect(); +} + +/// called at the end of Comm::ConnOpener::DelayedConnectRetry event +void +Comm::ConnOpener::restart() +{ + debugs(5, 5, conn_ << " restarting after sleep"); + calls_.sleep_ = false; + + if (createFd()) + connect(); +} + +/// Create a socket for the future connection or return false. +/// If false is returned, done() is guaranteed to return true and end the job. +bool +Comm::ConnOpener::createFd() +{ + Must(temporaryFd_ < 0); + + // our initators signal abort by cancelling their callbacks + if (callback_ == NULL || callback_->canceled()) + return false; + + temporaryFd_ = comm_openex(SOCK_STREAM, IPPROTO_TCP, conn_->local, conn_->flags, conn_->tos, conn_->nfmark, host_); if (temporaryFd_ < 0) { - /* outbound sockets have no need to be protocol agnostic. */ - if (!(Ip::EnableIpv6&IPV6_SPECIAL_V4MAPPING) && conn_->remote.IsIPv4()) { - conn_->local.SetIPv4(); - } - temporaryFd_ = comm_openex(SOCK_STREAM, IPPROTO_TCP, conn_->local, conn_->flags, conn_->tos, conn_->nfmark, host_); - if (temporaryFd_ < 0) { - doneConnecting(COMM_ERR_CONNECT, 0); - return; - } + sendAnswer(COMM_ERR_CONNECT, 0, "Comm::ConnOpener::createFd"); + return false; } typedef CommCbMemFunT abortDialer; @@ -173,26 +257,25 @@ typedef CommCbMemFunT timeoutDialer; calls_.timeout_ = JobCallback(5, 4, timeoutDialer, this, Comm::ConnOpener::timeout); - debugs(5, 3, HERE << conn_ << " timeout " << connectTimeout_); + debugs(5, 3, conn_ << " will timeout in " << (deadline_ - squid_curtime)); - // Update the fd_table directly because conn_ is not yet storing the FD + // Update the fd_table directly because commSetConnTimeout() needs open conn_ assert(temporaryFd_ < Squid_MaxFD); assert(fd_table[temporaryFd_].flags.open); typedef CommTimeoutCbParams Params; Params ¶ms = GetCommParams(calls_.timeout_); params.conn = conn_; fd_table[temporaryFd_].timeoutHandler = calls_.timeout_; - fd_table[temporaryFd_].timeout = squid_curtime + (time_t) connectTimeout_; + fd_table[temporaryFd_].timeout = deadline_; - connectStart_ = squid_curtime; - connect(); + return true; } void Comm::ConnOpener::connected() { - conn_->fd = temporaryFd_; - temporaryFd_ = -1; + Must(temporaryFd_ >= 0); + keepFd(); /* * stats.conn_open is used to account for the number of @@ -210,64 +293,81 @@ * Also, legacy code still depends on comm_local_port() with no access to Comm::Connection * when those are done comm_local_port can become one of our member functions to do the below. */ - fd_table[conn_->fd].flags.open = 1; + Must(fd_table[conn_->fd].flags.open); fd_table[conn_->fd].local_addr = conn_->local; + + sendAnswer(COMM_OK, 0, "Comm::ConnOpener::connected"); } -/** Make an FD connection attempt. - * Handles the case(s) when a partially setup connection gets closed early. - */ +/// Make an FD connection attempt. void Comm::ConnOpener::connect() { Must(conn_ != NULL); - - // our parent Jobs signal abort by cancelling their callbacks. - if (callback_ == NULL || callback_->canceled()) - return; + Must(temporaryFd_ >= 0); ++ totalTries_; switch (comm_connect_addr(temporaryFd_, conn_->remote) ) { case COMM_INPROGRESS: - // check for timeout FIRST. - if (squid_curtime - connectStart_ > connectTimeout_) { - debugs(5, 5, HERE << conn_ << ": * - ERR took too long already."); - calls_.earlyAbort_->cancel("Comm::ConnOpener::connect timed out"); - doneConnecting(COMM_TIMEOUT, errno); - return; - } else { - debugs(5, 5, HERE << conn_ << ": COMM_INPROGRESS"); - Comm::SetSelect(temporaryFd_, COMM_SELECT_WRITE, Comm::ConnOpener::InProgressConnectRetry, new Pointer(this), 0); - } + debugs(5, 5, HERE << conn_ << ": COMM_INPROGRESS"); + Comm::SetSelect(temporaryFd_, COMM_SELECT_WRITE, Comm::ConnOpener::InProgressConnectRetry, new Pointer(this), 0); break; case COMM_OK: debugs(5, 5, HERE << conn_ << ": COMM_OK - connected"); connected(); - doneConnecting(COMM_OK, 0); break; - default: + default: { + const int xerrno = errno; + ++failRetries_; + debugs(5, 7, conn_ << ": failure #" << failRetries_ << " <= " << + Config.connect_retries << ": " << xstrerr(xerrno)); - // check for timeout FIRST. - if (squid_curtime - connectStart_ > connectTimeout_) { - debugs(5, 5, HERE << conn_ << ": * - ERR took too long to receive response."); - calls_.earlyAbort_->cancel("Comm::ConnOpener::connect timed out"); - doneConnecting(COMM_TIMEOUT, errno); - } else if (failRetries_ < Config.connect_retries) { + if (failRetries_ < Config.connect_retries) { debugs(5, 5, HERE << conn_ << ": * - try again"); - eventAdd("Comm::ConnOpener::DelayedConnectRetry", Comm::ConnOpener::DelayedConnectRetry, new Pointer(this), 0.05, 0, false); + sleep(); return; } else { // send ERROR back to the upper layer. debugs(5, 5, HERE << conn_ << ": * - ERR tried too many times already."); - calls_.earlyAbort_->cancel("Comm::ConnOpener::connect failed"); - doneConnecting(COMM_ERR_CONNECT, errno); + sendAnswer(COMM_ERR_CONNECT, xerrno, "Comm::ConnOpener::connect"); } } + } +} + +/// Close and wait a little before trying to open and connect again. +void +Comm::ConnOpener::sleep() +{ + Must(!calls_.sleep_); + closeFd(); + calls_.sleep_ = true; + eventAdd("Comm::ConnOpener::DelayedConnectRetry", + Comm::ConnOpener::DelayedConnectRetry, + new Pointer(this), 0.05, 0, false); +} + +/// cleans up this job sleep state +void +Comm::ConnOpener::cancelSleep() +{ + if (calls_.sleep_) { + // It would be nice to delete the sleep event, but it might be out of + // the event queue and in the async queue already, so (a) we do not know + // whether we can safely delete the call ptr here and (b) eventDelete() + // will assert if the event went async. Thus, we let the event run so + // that it deletes the call ptr [after this job is gone]. Note that we + // are called only when the job ends so this "hanging event" will do + // nothing but deleting the call ptr. TODO: Revise eventDelete() API. + // eventDelete(Comm::ConnOpener::DelayedConnectRetry, calls_.sleep); + calls_.sleep_ = false; + debugs(5, 9, conn_ << " stops sleeping"); + } } /** @@ -298,7 +398,9 @@ Comm::ConnOpener::earlyAbort(const CommCloseCbParams &io) { debugs(5, 3, HERE << io.conn); - doneConnecting(COMM_ERR_CLOSING, io.xerrno); // NP: is closing or shutdown better? + calls_.earlyAbort_ = NULL; + // NP: is closing or shutdown better? + sendAnswer(COMM_ERR_CLOSING, io.xerrno, "Comm::ConnOpener::earlyAbort"); } /** @@ -308,7 +410,9 @@ void Comm::ConnOpener::timeout(const CommTimeoutCbParams &) { - connect(); + debugs(5, 5, HERE << conn_ << ": * - ERR took too long to receive response."); + calls_.timeout_ = NULL; + sendAnswer(COMM_TIMEOUT, ETIMEDOUT, "Comm::ConnOpener::timeout"); } /* Legacy Wrapper for the retry event after COMM_INPROGRESS @@ -330,7 +434,7 @@ } /* Legacy Wrapper for the retry event with small delay after errors. - * XXX: As soon as eventAdd() accepts Async calls we can use a ConnOpener::connect call + * XXX: As soon as eventAdd() accepts Async calls we can use a ConnOpener::restart call */ void Comm::ConnOpener::DelayedConnectRetry(void *data) @@ -341,7 +445,7 @@ // Ew. we are now outside the all AsyncJob protections. // get back inside by scheduling another call... typedef NullaryMemFunT Dialer; - AsyncCall::Pointer call = JobCallback(5, 4, Dialer, cs, Comm::ConnOpener::connect); + AsyncCall::Pointer call = JobCallback(5, 4, Dialer, cs, Comm::ConnOpener::restart); ScheduleCallHere(call); } delete ptr; diff -u -r -N squid-3.2.7/src/comm/ConnOpener.h squid-3.2.8/src/comm/ConnOpener.h --- squid-3.2.7/src/comm/ConnOpener.h 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/comm/ConnOpener.h 2013-03-02 14:46:03.000000000 +1300 @@ -40,13 +40,23 @@ void earlyAbort(const CommCloseCbParams &); void timeout(const CommTimeoutCbParams &); - void doneConnecting(comm_err_t status, int xerrno); + void sendAnswer(comm_err_t errFlag, int xerrno, const char *why); static void InProgressConnectRetry(int fd, void *data); static void DelayedConnectRetry(void *data); void connect(); void connected(); void lookupLocalAddress(); + void sleep(); + void restart(); + + bool createFd(); + void closeFd(); + void keepFd(); + void cleanFd(); + + void cancelSleep(); + private: char *host_; ///< domain name we are trying to connect to. int temporaryFd_; ///< the FD being opened. Do NOT set conn_->fd until it is fully open. @@ -56,19 +66,16 @@ int totalTries_; ///< total number of connection attempts over all destinations so far. int failRetries_; ///< number of retries current destination has been tried. - /** - * time at which to abandon the connection. - * the connection-done callback will be passed COMM_TIMEOUT - */ - time_t connectTimeout_; - - /// time at which this series of connection attempts was started. - time_t connectStart_; + /// if we are not done by then, we will call back with COMM_TIMEOUT + time_t deadline_; /// handles to calls which we may need to cancel. struct Calls { AsyncCall::Pointer earlyAbort_; AsyncCall::Pointer timeout_; + /// Whether we are idling before retrying to connect; not yet a call + /// [that we can cancel], but it will probably become one eventually. + bool sleep_; } calls_; CBDATA_CLASS2(ConnOpener); diff -u -r -N squid-3.2.7/src/DiskIO/DiskDaemon/diskd.cc squid-3.2.8/src/DiskIO/DiskDaemon/diskd.cc --- squid-3.2.7/src/DiskIO/DiskDaemon/diskd.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/DiskIO/DiskDaemon/diskd.cc 2013-03-02 14:46:03.000000000 +1300 @@ -266,7 +266,7 @@ if (s->shm_offset > -1) buf = shmbuf + s->shm_offset; - else { + else if (r->mtype != _MQD_CLOSE) { fprintf(stderr, "%d UNLNK id(%u) Error: no filename in shm buffer\n", (int) mypid, s->id); return; } diff -u -r -N squid-3.2.7/src/forward.cc squid-3.2.8/src/forward.cc --- squid-3.2.7/src/forward.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/forward.cc 2013-03-02 14:46:03.000000000 +1300 @@ -265,11 +265,12 @@ if ( Config.accessList.miss && !request->client_addr.IsNoAddr() && request->protocol != AnyP::PROTO_INTERNAL && request->protocol != AnyP::PROTO_CACHE_OBJECT) { /** - * Check if this host is allowed to fetch MISSES from us (miss_access) + * Check if this host is allowed to fetch MISSES from us (miss_access). + * Intentionally replace the src_addr automatically selected by the checklist code + * we do NOT want the indirect client address to be tested here. */ ACLFilledChecklist ch(Config.accessList.miss, request, NULL); ch.src_addr = request->client_addr; - ch.my_addr = request->my_addr; if (ch.fastCheck() == ACCESS_DENIED) { err_type page_id; page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, 1); @@ -918,12 +919,9 @@ } // Use pconn to avoid opening a new connection. - const char *host; - if (serverDestinations[0]->getPeer()) { - host = serverDestinations[0]->getPeer()->host; - } else { + const char *host = NULL; + if (!serverDestinations[0]->getPeer()) host = request->GetHost(); - } Comm::ConnectionPointer temp; // Avoid pconns after races so that the same client does not suffer twice. @@ -988,7 +986,8 @@ calls.connector = commCbCall(17,3, "fwdConnectDoneWrapper", CommConnectCbPtrFun(fwdConnectDoneWrapper, this)); Comm::ConnOpener *cs = new Comm::ConnOpener(serverDestinations[0], calls.connector, ctimeout); - cs->setHost(host); + if (host) + cs->setHost(host); AsyncJob::Start(cs); } @@ -1228,7 +1227,7 @@ FwdState::pconnPush(Comm::ConnectionPointer &conn, const char *domain) { if (conn->getPeer()) { - fwdPconnPool->push(conn, conn->getPeer()->name); + fwdPconnPool->push(conn, NULL); } else { fwdPconnPool->push(conn, domain); } @@ -1350,12 +1349,6 @@ GetTosToServer(HttpRequest * request) { ACLFilledChecklist ch(NULL, request, NULL); - - if (request) { - ch.src_addr = request->client_addr; - ch.my_addr = request->my_addr; - } - return aclMapTOS(Ip::Qos::TheConfig.tosToServer, &ch); } @@ -1363,11 +1356,5 @@ GetNfmarkToServer(HttpRequest * request) { ACLFilledChecklist ch(NULL, request, NULL); - - if (request) { - ch.src_addr = request->client_addr; - ch.my_addr = request->my_addr; - } - return aclMapNfmark(Ip::Qos::TheConfig.nfmarkToServer, &ch); } diff -u -r -N squid-3.2.7/src/fs/rock/RockIoState.cc squid-3.2.8/src/fs/rock/RockIoState.cc --- squid-3.2.7/src/fs/rock/RockIoState.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/fs/rock/RockIoState.cc 2013-03-02 14:46:03.000000000 +1300 @@ -25,7 +25,7 @@ { e = anEntry; // swap_filen, swap_dirn, diskOffset, and payloadEnd are set by the caller - slotSize = dir->max_objsize; + slotSize = dir->maxObjectSize(); file_callback = cbFile; callback = cbIo; callback_data = cbdataReference(data); diff -u -r -N squid-3.2.7/src/fs/rock/RockSwapDir.cc squid-3.2.8/src/fs/rock/RockSwapDir.cc --- squid-3.2.7/src/fs/rock/RockSwapDir.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/fs/rock/RockSwapDir.cc 2013-03-02 14:46:03.000000000 +1300 @@ -152,8 +152,19 @@ debugs (47,3, HERE << "creating in " << path); - struct stat swap_sb; - if (::stat(path, &swap_sb) < 0) { + struct stat dir_sb; + if (::stat(path, &dir_sb) == 0) { + struct stat file_sb; + if (::stat(filePath, &file_sb) == 0) { + debugs (47, DBG_IMPORTANT, "Skipping existing Rock db: " << filePath); + return; + } + // else the db file is not there or is not accessible, and we will try + // to create it later below, generating a detailed error on failures. + } else { // path does not exist or is inaccessible + // If path exists but is not accessible, mkdir() below will fail, and + // the admin should see the error and act accordingly, so there is + // no need to distinguish ENOENT from other possible stat() errors. debugs (47, DBG_IMPORTANT, "Creating Rock db directory: " << path); const int res = mkdir(path, 0700); if (res != 0) { @@ -163,6 +174,7 @@ } } + debugs (47, DBG_IMPORTANT, "Creating Rock db: " << filePath); #if SLOWLY_FILL_WITH_ZEROS char block[1024]; Must(maxSize() % sizeof(block) == 0); diff -u -r -N squid-3.2.7/src/ftp.cc squid-3.2.8/src/ftp.cc --- squid-3.2.7/src/ftp.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/ftp.cc 2013-03-02 14:46:03.000000000 +1300 @@ -673,10 +673,17 @@ if (abortOnBadEntry("entry went bad while waiting for a timeout")) return; - if (SENT_PASV == state && io.conn->fd == data.conn->fd) { - /* stupid ftp.netscape.com */ + if (SENT_PASV == state) { + /* stupid ftp.netscape.com, of FTP server behind stupid firewall rules */ flags.pasv_supported = false; debugs(9, DBG_IMPORTANT, "ftpTimeout: timeout in SENT_PASV state" ); + + // cancel the data connection setup. + if (data.opener != NULL) { + data.opener->cancel("timeout"); + data.opener = NULL; + } + data.close(); } failed(ERR_READ_TIMEOUT, 0); diff -u -r -N squid-3.2.7/src/http.cc squid-3.2.8/src/http.cc --- squid-3.2.7/src/http.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/http.cc 2013-03-02 14:46:03.000000000 +1300 @@ -418,7 +418,7 @@ // HTTPbis pt6 section 3.2: a response CC:s-maxage is present } else if (rep->cache_control->sMaxAge()) { - debugs(22, 3, HERE << " Authenticated but server reply Cache-Control:s-maxage"); + debugs(22, 3, HERE << "Authenticated but server reply Cache-Control:s-maxage"); mayStore = true; } @@ -1449,7 +1449,7 @@ request->clientConnectionManager->pinConnection(serverConnection, request, _peer, (request->flags.connection_auth != 0)); } else { - fwd->pconnPush(serverConnection, request->peer_host ? request->peer_host : request->GetHost()); + fwd->pconnPush(serverConnection, request->GetHost()); } serverConnection = NULL; diff -u -r -N squid-3.2.7/src/HttpHdrSc.cc squid-3.2.8/src/HttpHdrSc.cc --- squid-3.2.7/src/HttpHdrSc.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/HttpHdrSc.cc 2013-03-02 14:46:03.000000000 +1300 @@ -196,22 +196,23 @@ int ma; if (p && httpHeaderParseInt(p, &ma)) { sct->maxAge(ma); + + if ((p = strchr (p, '+'))) { + int ms; + ++p; //skip the + char + if (httpHeaderParseInt(p, &ms)) { + sct->maxStale(ms); + } else { + debugs(90, 2, "sc: invalid max-stale specs near '" << item << "'"); + sct->clearMaxStale(); + /* leave the max-age alone */ + } + } } else { debugs(90, 2, "sc: invalid max-age specs near '" << item << "'"); sct->clearMaxAge(); } - if ((p = strchr (p, '+'))) { - int ms; - ++p; //skip the + char - if (httpHeaderParseInt(p, &ms)) { - sct->maxStale(ms); - } else { - debugs(90, 2, "sc: invalid max-stale specs near '" << item << "'"); - sct->clearMaxStale(); - /* leave the max-age alone */ - } - } break; } diff -u -r -N squid-3.2.7/src/ident/Ident.cc squid-3.2.8/src/ident/Ident.cc --- squid-3.2.7/src/ident/Ident.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/ident/Ident.cc 2013-03-02 14:46:03.000000000 +1300 @@ -61,6 +61,7 @@ typedef struct _IdentStateData { hash_link hash; /* must be first */ Comm::ConnectionPointer conn; + MemBuf queryMsg; ///< the lookup message sent to IDENT server IdentClient *clients; char buf[IDENT_BUFSIZE]; } IdentStateData; @@ -149,14 +150,9 @@ comm_add_close_handler(conn->fd, Ident::Close, state); - MemBuf mb; - mb.init(); - mb.Printf("%d, %d\r\n", - conn->remote.GetPort(), - conn->local.GetPort()); AsyncCall::Pointer writeCall = commCbCall(5,4, "Ident::WriteFeedback", CommIoCbPtrFun(Ident::WriteFeedback, state)); - Comm::Write(conn, &mb, writeCall); + Comm::Write(conn, &state->queryMsg, writeCall); AsyncCall::Pointer readCall = commCbCall(5,4, "Ident::ReadReply", CommIoCbPtrFun(Ident::ReadReply, state)); comm_read(conn, state->buf, IDENT_BUFSIZE, readCall); @@ -266,6 +262,10 @@ state->conn->local.SetPort(0); state->conn->remote.SetPort(IDENT_PORT); + // build our query from the original connection details + state->queryMsg.init(); + state->queryMsg.Printf("%d, %d\r\n", conn->remote.GetPort(), conn->local.GetPort()); + ClientAdd(state, callback, data); hash_join(ident_hash, &state->hash); diff -u -r -N squid-3.2.7/src/main.cc squid-3.2.8/src/main.cc --- squid-3.2.7/src/main.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/main.cc 2013-03-02 14:46:03.000000000 +1300 @@ -283,7 +283,7 @@ " Enable logging to syslog.\n" " -u port Specify ICP port number (default: %d), disable with 0.\n" " -v Print version.\n" - " -z Create swap directories\n" + " -z Create missing swap directories and then exit.\n" " -C Do not catch fatal signals.\n" " -D OBSOLETE. Scheduled for removal.\n" " -F Don't serve any requests until store is rebuilt.\n" @@ -1429,7 +1429,7 @@ } setEffectiveUser(); - debugs(0, 0, "Creating Swap Directories"); + debugs(0, DBG_CRITICAL, "Creating missing swap directories"); Store::Root().create(); return 0; diff -u -r -N squid-3.2.7/src/squid.8.in squid-3.2.8/src/squid.8.in --- squid-3.2.7/src/squid.8.in 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/squid.8.in 2013-03-02 14:46:03.000000000 +1300 @@ -166,7 +166,21 @@ . .if !'po4a'hide' .TP .if !'po4a'hide' .B \-z -Create swap directories +Create missing swap directories and other missing cache_dir structures, +then exit. All cache_dir types create the configured top-level directory if +it is missing. Other actions are type-specific. For example, ufs-based +storage systems create missing L1 and L2 directories while Rock creates +the missing database file. +.IP +This option does not enable validation of any present swap structures. Its +focus is on creation of missing pieces. If nothing is missing, squid -z +just exits. If you suspect cache_dir corruption, you must delete the top-level +cache_dir directory before running squid -z. +.IP +By default, squid -z runs in daemon mode (so that configuration macros and +other SMP features work as expected). Use +.B \-N +option to overwrite this. . .SH FILES Squid configuration files located in @SYSCONFDIR@/: diff -u -r -N squid-3.2.7/src/store.cc squid-3.2.8/src/store.cc --- squid-3.2.7/src/store.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/store.cc 2013-03-02 14:46:03.000000000 +1300 @@ -988,9 +988,8 @@ ++store_check_cachable_hist.no.negative_cached; return 0; /* avoid release call below */ } else if ((getReply()->content_length > 0 && - getReply()->content_length - > Config.Store.maxObjectSize) || - mem_obj->endOffset() > Config.Store.maxObjectSize) { + getReply()->content_length > store_maxobjsize) || + mem_obj->endOffset() > store_maxobjsize) { debugs(20, 2, "StoreEntry::checkCachable: NO: too big"); ++store_check_cachable_hist.no.too_big; } else if (checkTooSmall()) { diff -u -r -N squid-3.2.7/src/store_dir.cc squid-3.2.8/src/store_dir.cc --- squid-3.2.7/src/store_dir.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/store_dir.cc 2013-03-02 14:46:03.000000000 +1300 @@ -270,10 +270,10 @@ /* If the load is equal, then look in more details */ if (load == least_load) { - /* closest max_objsize fit */ + /* closest max-size fit */ if (least_objsize != -1) - if (SD->max_objsize > least_objsize || SD->max_objsize == -1) + if (SD->maxObjectSize() > least_objsize) continue; /* most free */ @@ -282,7 +282,7 @@ } least_load = load; - least_objsize = SD->max_objsize; + least_objsize = SD->maxObjectSize(); most_free = cur_free; dirn = i; } diff -u -r -N squid-3.2.7/src/store_swapout.cc squid-3.2.8/src/store_swapout.cc --- squid-3.2.7/src/store_swapout.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/store_swapout.cc 2013-03-02 14:46:03.000000000 +1300 @@ -200,7 +200,7 @@ Store::Root().maybeTrimMemory(*this, weAreOrMayBeSwappingOut); - if (!weAreOrMayBeSwappingOut) + if (mem_obj->swapout.decision != MemObject::SwapOut::swPossible) return; // nothing else to do // Aborted entries have STORE_OK, but swapoutPossible rejects them. Thus, @@ -360,8 +360,6 @@ bool StoreEntry::mayStartSwapOut() { - dlink_node *node; - // must be checked in the caller assert(!EBIT_TEST(flags, ENTRY_ABORTED)); assert(!swappingOut()); @@ -403,6 +401,18 @@ return false; } + if (mem_obj->inmem_lo > 0) { + debugs(20, 3, "storeSwapOut: (inmem_lo > 0) imem_lo:" << mem_obj->inmem_lo); + decision = MemObject::SwapOut::swImpossible; + return false; + } + + if (!mem_obj->isContiguous()) { + debugs(20, 3, "storeSwapOut: not Contiguous"); + decision = MemObject::SwapOut::swImpossible; + return false; + } + // check cache_dir max-size limit if all cache_dirs have it if (store_maxobjsize >= 0) { // TODO: add estimated store metadata size to be conservative @@ -426,69 +436,25 @@ return false; // already does not fit and may only get bigger } - // prevent default swPossible answer for yet unknown length - if (expectedEnd < 0) { - debugs(20, 3, HERE << "wait for more info: " << - store_maxobjsize); - return false; // may fit later, but will be rejected now - } - - if (store_status != STORE_OK) { - const int64_t maxKnownSize = expectedEnd < 0 ? - mem_obj->availableForSwapOut() : expectedEnd; + // prevent final default swPossible answer for yet unknown length + if (expectedEnd < 0 && store_status != STORE_OK) { + const int64_t maxKnownSize = mem_obj->availableForSwapOut(); debugs(20, 7, HERE << "maxKnownSize= " << maxKnownSize); - if (maxKnownSize < store_maxobjsize) { - /* - * NOTE: the store_maxobjsize here is the max of optional - * max-size values from 'cache_dir' lines. It is not the - * same as 'maximum_object_size'. By default, store_maxobjsize - * will be set to -1. However, I am worried that this - * deferance may consume a lot of memory in some cases. - * Should we add an option to limit this memory consumption? - */ - debugs(20, 5, HERE << "Deferring swapout start for " << - (store_maxobjsize - maxKnownSize) << " bytes"); - return false; - } - } - } - - if (mem_obj->inmem_lo > 0) { - debugs(20, 3, "storeSwapOut: (inmem_lo > 0) imem_lo:" << mem_obj->inmem_lo); - decision = MemObject::SwapOut::swImpossible; - return false; - } - - /* - * If there are DISK clients, we must write to disk - * even if its not cachable - * RBC: Surely we should not create disk client on non cacheable objects? - * therefore this should be an assert? - * RBC 20030708: We can use disk to avoid mem races, so this shouldn't be - * an assert. - * - * XXX: Not clear what "mem races" the above refers to, especially when - * dealing with non-cachable objects that cannot have multiple clients. - * - * XXX: If STORE_DISK_CLIENT needs SwapOut::swPossible, we have to check - * for that flag earlier, but forcing swapping may contradict max-size or - * other swapability restrictions. Change storeClientType() and/or its - * callers to take swap-in availability into account. - */ - for (node = mem_obj->clients.head; node; node = node->next) { - if (((store_client *) node->data)->getType() == STORE_DISK_CLIENT) { - debugs(20, 3, HERE << "DISK client found"); - decision = MemObject::SwapOut::swPossible; - return true; + /* + * NOTE: the store_maxobjsize here is the global maximum + * size of object cacheable in any of Squid cache stores + * both disk and memory stores. + * + * However, I am worried that this + * deferance may consume a lot of memory in some cases. + * Should we add an option to limit this memory consumption? + */ + debugs(20, 5, HERE << "Deferring swapout start for " << + (store_maxobjsize - maxKnownSize) << " bytes"); + return true; // may still fit, but no final decision yet } } - if (!mem_obj->isContiguous()) { - debugs(20, 3, "storeSwapOut: not Contiguous"); - decision = MemObject::SwapOut::swImpossible; - return false; - } - decision = MemObject::SwapOut::swPossible; return true; } diff -u -r -N squid-3.2.7/src/SwapDir.cc squid-3.2.8/src/SwapDir.cc --- squid-3.2.7/src/SwapDir.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/SwapDir.cc 2013-03-02 14:46:03.000000000 +1300 @@ -39,8 +39,8 @@ #include "ConfigOption.h" SwapDir::SwapDir(char const *aType): theType(aType), - max_size(0), - path(NULL), index(-1), disker(-1), min_objsize(0), max_objsize (-1), + max_size(0), min_objsize(0), max_objsize (-1), + path(NULL), index(-1), disker(-1), repl(NULL), removals(0), scanned(0), cleanLog(NULL) { @@ -111,6 +111,39 @@ return ((maxSize() * Config.Swap.lowWaterMark) / 100); } +int64_t +SwapDir::maxObjectSize() const +{ + // per-store max-size=N value is authoritative + if (max_objsize > -1) + return max_objsize; + + // store with no individual max limit is limited by configured maximum_object_size + // or the total store size, whichever is smaller + return min(static_cast(maxSize()), Config.Store.maxObjectSize); +} + +void +SwapDir::maxObjectSize(int64_t newMax) +{ + // negative values mean no limit (-1) + if (newMax < 0) { + max_objsize = -1; // set explicitly in case it had a non-default value previously + return; + } + + // prohibit values greater than total storage area size + // but set max_objsize to the maximum allowed to override maximum_object_size global config + if (static_cast(newMax) > maxSize()) { + debugs(47, DBG_PARSE_NOTE(2), "WARNING: Ignoring 'max-size' option for " << path << + " which is larger than total cache_dir size of " << maxSize() << " bytes."); + max_objsize = maxSize(); + return; + } + + max_objsize = newMax; +} + void SwapDir::reference(StoreEntry &) {} @@ -288,6 +321,10 @@ if (strcmp(option, "no-store") != 0 && strcmp(option, "read-only") != 0) return false; + if (strcmp(option, "read-only") == 0) { + debugs(3, DBG_PARSE_NOTE(3), "UPGRADE WARNING: Replace cache_dir option 'read-only' with 'no-store'."); + } + int read_only = 0; if (value) diff -u -r -N squid-3.2.7/src/SwapDir.h squid-3.2.8/src/SwapDir.h --- squid-3.2.7/src/SwapDir.h 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/SwapDir.h 2013-03-02 14:46:03.000000000 +1300 @@ -149,7 +149,13 @@ virtual uint64_t minSize() const; - virtual int64_t maxObjectSize() const { return max_objsize; } + /// The maximum size of object which may be stored here. + /// Larger objects will not be added and may be purged. + virtual int64_t maxObjectSize() const; + + /// configure the maximum object size for this storage area. + /// May be any size up to the total storage area. + void maxObjectSize(int64_t newMax); virtual void getStats(StoreInfoStats &stats) const; virtual void stat (StoreEntry &anEntry) const; @@ -181,13 +187,13 @@ protected: uint64_t max_size; ///< maximum allocatable size of the storage area + int64_t min_objsize; ///< minimum size of any object stored here (-1 for no limit) + int64_t max_objsize; ///< maximum size of any object stored here (-1 for no limit) public: char *path; int index; /* This entry's index into the swapDirs array */ int disker; ///< disker kid id dedicated to this SwapDir or -1 - int64_t min_objsize; - int64_t max_objsize; RemovalPolicy *repl; int removals; int scanned; diff -u -r -N squid-3.2.7/src/tests/testCoss.cc squid-3.2.8/src/tests/testCoss.cc --- squid-3.2.7/src/tests/testCoss.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/tests/testCoss.cc 2013-03-02 14:46:03.000000000 +1300 @@ -17,7 +17,7 @@ #include #endif -#define TESTDIR "testCoss__testCossSearch" +#define TESTDIR "testCoss_Store" CPPUNIT_TEST_SUITE_REGISTRATION( testCoss ); diff -u -r -N squid-3.2.7/src/tests/testRock.cc squid-3.2.8/src/tests/testRock.cc --- squid-3.2.7/src/tests/testRock.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/tests/testRock.cc 2013-03-02 14:46:03.000000000 +1300 @@ -21,7 +21,7 @@ #include #endif -#define TESTDIR "testRock__testRockSearch" +#define TESTDIR "testRock_Store" CPPUNIT_TEST_SUITE_REGISTRATION( testRock ); @@ -65,6 +65,7 @@ strtok(config_line, w_space); store->parse(0, path); + store_maxobjsize = 1024*1024*2; safe_free(path); @@ -173,8 +174,7 @@ StoreEntry *const pe = storeCreateEntry(url, "dummy log url", flags, METHOD_GET); HttpReply *const rep = const_cast(pe->getReply()); - rep->setHeaders(HTTP_OK, "dummy test object", "x-squid-internal/test", - -1, -1, squid_curtime + 100000); + rep->setHeaders(HTTP_OK, "dummy test object", "x-squid-internal/test", 0, -1, squid_curtime + 100000); pe->setPublicKey(); diff -u -r -N squid-3.2.7/src/tests/testUfs.cc squid-3.2.8/src/tests/testUfs.cc --- squid-3.2.7/src/tests/testUfs.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/tests/testUfs.cc 2013-03-02 14:46:03.000000000 +1300 @@ -15,7 +15,7 @@ #include #endif -#define TESTDIR "testUfs__testUfsSearch" +#define TESTDIR "testUfs_Store" CPPUNIT_TEST_SUITE_REGISTRATION( testUfs ); @@ -108,6 +108,7 @@ strtok(config_line, w_space); aStore->parse(0, path); + store_maxobjsize = 1024*1024*2; safe_free(path); @@ -142,7 +143,7 @@ flags.cachable = 1; StoreEntry *pe = storeCreateEntry("dummy url", "dummy log url", flags, METHOD_GET); HttpReply *rep = (HttpReply *) pe->getReply(); // bypass const - rep->setHeaders(HTTP_OK, "dummy test object", "x-squid-internal/test", -1, -1, squid_curtime + 100000); + rep->setHeaders(HTTP_OK, "dummy test object", "x-squid-internal/test", 0, -1, squid_curtime + 100000); pe->setPublicKey(); diff -u -r -N squid-3.2.7/src/tools.cc squid-3.2.8/src/tools.cc --- squid-3.2.7/src/tools.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/tools.cc 2013-03-02 14:46:03.000000000 +1300 @@ -114,28 +114,27 @@ { FILE *fp = NULL; static char command[256]; -#if HAVE_MKSTEMP + const mode_t prev_umask=umask(S_IRWXU); + +#if HAVE_MKSTEMP char filename[] = "/tmp/squid-XXXXXX"; int tfd = mkstemp(filename); - - if (tfd < 0) + if (tfd < 0 || (fp = fdopen(tfd, "w")) == NULL) { + umask(prev_umask); return; - - if ((fp = fdopen(tfd, "w")) == NULL) - return; - + } #else - char *filename; - - if ((filename = tempnam(NULL, APP_SHORTNAME)) == NULL) + // XXX tempnam is obsolete since POSIX.2008-1 + // tmpfile is not an option, we want the created files to stick around + if ((filename = tempnam(NULL, APP_SHORTNAME)) == NULL || + (fp = fopen(filename, "w")) == NULL) { + umask(prev_umask); return; - - if ((fp = fopen(filename, "w")) == NULL) - return; - + } #endif + umask(prev_umask); if (Config.EmailFrom) fprintf(fp, "From: %s\n", Config.EmailFrom); @@ -143,16 +142,15 @@ fprintf(fp, "From: %s@%s\n", APP_SHORTNAME, uniqueHostname()); fprintf(fp, "To: %s\n", Config.adminEmail); - fprintf(fp, "Subject: %s\n", dead_msg()); - fclose(fp); snprintf(command, 256, "%s %s < %s", Config.EmailProgram, Config.adminEmail, filename); - if (system(command)) {} /* XXX should avoid system(3) */ - unlink(filename); +#if !HAVE_MKSTEMP + xfree(filename); // tempnam() requires us to free its allocation +#endif } void diff -u -r -N squid-3.2.7/src/tunnel.cc squid-3.2.8/src/tunnel.cc --- squid-3.2.7/src/tunnel.cc 2013-02-01 23:55:31.000000000 +1300 +++ squid-3.2.8/src/tunnel.cc 2013-03-02 14:46:03.000000000 +1300 @@ -50,6 +50,7 @@ #include "client_side.h" #include "MemBuf.h" #include "http.h" +#include "ip/QosConfig.h" #include "PeerSelectState.h" #include "StatCounters.h" @@ -558,6 +559,15 @@ tunnelState->serverDestinations.shift(); if (status != COMM_TIMEOUT && tunnelState->serverDestinations.size() > 0) { /* Try another IP of this destination host */ + + if (Ip::Qos::TheConfig.isAclTosActive()) { + tunnelState->serverDestinations[0]->tos = GetTosToServer(tunnelState->request); + } + +#if SO_MARK && USE_LIBCAP + tunnelState->serverDestinations[0]->nfmark = GetNfmarkToServer(tunnelState->request); +#endif + debugs(26, 4, HERE << "retry with : " << tunnelState->serverDestinations[0]); AsyncCall::Pointer call = commCbCall(26,3, "tunnelConnectDone", CommConnectCbPtrFun(tunnelConnectDone, tunnelState)); Comm::ConnOpener *cs = new Comm::ConnOpener(tunnelState->serverDestinations[0], call, Config.Timeout.connect); @@ -724,6 +734,14 @@ } delete err; + if (Ip::Qos::TheConfig.isAclTosActive()) { + tunnelState->serverDestinations[0]->tos = GetTosToServer(tunnelState->request); + } + +#if SO_MARK && USE_LIBCAP + tunnelState->serverDestinations[0]->nfmark = GetNfmarkToServer(tunnelState->request); +#endif + debugs(26, 3, HERE << "paths=" << peer_paths->size() << ", p[0]={" << (*peer_paths)[0] << "}, serverDest[0]={" << tunnelState->serverDestinations[0] << "}");