Patch file generated Tue Apr 1 20:24:25 UTC 2008 from CVS branch s2_delaywork CVS base branch HEAD CVS repository: adri@cvs.devel.squid-cache.org:/cvsroot/squid CVS module: squid cvs -q rdiff -u -kk -r Z-s2_delaywork_merge_HEAD -r s2_delaywork squid Index: squid/src/cache_cf.c diff -u squid/src/cache_cf.c:1.114 squid/src/cache_cf.c:1.114.2.2 --- squid/src/cache_cf.c:1.114 Mon Feb 25 20:51:48 2008 +++ squid/src/cache_cf.c Sun Mar 30 06:41:02 2008 @@ -2510,6 +2510,30 @@ dlinkAddTail(bs, &bs->node, bodylist); } +#if DELAY_POOLS +CBDATA_TYPE(delay_body_size); + +static void +parse_delay_body_size_t(dlink_list * bodylist) +{ + delay_body_size *dbs; + ushort pool; + CBDATA_INIT_TYPE(delay_body_size); + dbs = cbdataAlloc(delay_body_size); + dbs->maxsize = GetOffT(); + + parse_ushort(&pool); + if (pool < 1 || pool > Config.Delay.pools) { + debug(3, 0) ("parse_delay_body_size_t: Ignoring pool %d not in 1 .. %d\n", pool, Config.Delay.pools); + return; + } + dbs->pool = pool -1; + aclParseAccessLine(&dbs->access_list); + + dlinkAddTail(dbs, &dbs->node, bodylist); +} +#endif + static void dump_body_size_t(StoreEntry * entry, const char *name, dlink_list bodylist) { @@ -2533,6 +2557,32 @@ } } +#if DELAY_POOLS +static void +dump_delay_body_size_t(StoreEntry * entry, const char *name, dlink_list bodylist) +{ + delay_body_size *dbs; + dbs = (delay_body_size *) bodylist.head; + while (dbs) { + acl_list *l; + acl_access *head = dbs->access_list; + while (head != NULL) { + storeAppendPrintf(entry, "%s %" PRINTF_OFF_T " %u %s", + name, dbs->maxsize, dbs->pool, + head->allow ? "Allow" : "Deny"); + for (l = head->acl_list; l != NULL; l = l->next) { + storeAppendPrintf(entry, " %s%s", + l->op ? null_string : "!", + l->acl->name); + } + storeAppendPrintf(entry, "\n"); + head = head->next; + } + dbs = (delay_body_size *) dbs->node.next; + } +} +#endif + static void free_body_size_t(dlink_list * bodylist) { @@ -2548,12 +2598,39 @@ } } +#if DELAY_POOLS +static void +free_delay_body_size_t(dlink_list * bodylist) +{ + delay_body_size *dbs, *tempnode; + dbs = (delay_body_size *) bodylist->head; + while (dbs) { + dbs->maxsize = 0; + dbs->pool = 0; + aclDestroyAccessList(&dbs->access_list); + tempnode = (delay_body_size *) dbs->node.next; + dlinkDelete(&dbs->node, bodylist); + cbdataFree(dbs); + dbs = tempnode; + } +} +#endif + + static int check_null_body_size_t(dlink_list bodylist) { return bodylist.head == NULL; } +#if DELAY_POOLS +static int +check_null_delay_body_size_t(dlink_list bodylist) +{ + return bodylist.head == NULL; +} +#endif + static void parse_kb_size_t(squid_off_t * var) Index: squid/src/cf.data.depend diff -u squid/src/cf.data.depend:1.3 squid/src/cf.data.depend:1.3.2.1 --- squid/src/cf.data.depend:1.3 Mon Feb 25 20:51:48 2008 +++ squid/src/cf.data.depend Tue Mar 25 04:33:29 2008 @@ -49,6 +49,7 @@ wccp2_service_info wordlist body_size_t acl +delay_body_size_t acl programline extension_method errormap Index: squid/src/cf.data.pre diff -u squid/src/cf.data.pre:1.240 squid/src/cf.data.pre:1.240.2.5 --- squid/src/cf.data.pre:1.240 Mon Mar 17 20:52:14 2008 +++ squid/src/cf.data.pre Tue Apr 1 05:22:56 2008 @@ -3959,6 +3959,7 @@ LOC: Config.Delay DOC_START This is used to determine which delay pool a request falls into. + It operates on the HTTP request and not the HTTP reply. delay_access is sorted per pool and the matching starts with pool 1, then pool 2, ..., and finally pool N. The first delay pool where the @@ -4050,6 +4051,29 @@ "seen" by squid). DOC_END +NAME: delay_body_max_size +COMMENT: bytes delay_pool allow|deny acl acl... +TYPE: delay_body_size_t +IFDEF: DELAY_POOLS +DEFAULT: none +LOC: Config.DelayBodySize +DOC_START + This option allows for a limit to be placed on replies before the + request is placed in a delay pool. It operates on the HTTP request, + not the HTTP reply. + + It can be used to prevent users from downloading very large files at + high speed. When the delay headers are received, the reply_body_max_size + lines are processed, and the first line with a result of "allow" is used + as the maximum body size for this reply to be unassigned in delay_pool. + The client connection is inserted into the delay pool once the amount + of data sent to the client exceeds the value specified. + + There will be no limit imposed by a rule if the "bytes" value is set to 0. +DOC_END + + + COMMENT_START WCCPv1 AND WCCPv2 CONFIGURATION OPTIONS ----------------------------------------------------------------------------- Index: squid/src/client_side.c diff -u squid/src/client_side.c:1.223 squid/src/client_side.c:1.223.2.2 --- squid/src/client_side.c:1.223 Wed Mar 19 03:54:12 2008 +++ squid/src/client_side.c Sun Mar 30 06:41:06 2008 @@ -2681,6 +2681,38 @@ } } +#if DELAY_POOLS +/* + * Calculates the delay maximum size allowed for an HTTP response + */ +static void +clientDelayMaxBodySize(request_t * request, clientHttpRequest * http, HttpReply * reply) +{ + delay_body_size *dbs; + aclCheck_t *checklist; + if (http->log_type == LOG_TCP_DENIED) + return; + dbs = (delay_body_size *) Config.DelayBodySize.head; + while (dbs) { + checklist = clientAclChecklistCreate(dbs->access_list, http); + + checklist->reply = reply; + if (1 != aclCheckFast(dbs->access_list, checklist)) { + /* deny - skip this entry */ + dbs = (delay_body_size *) dbs->node.next; + } else { + /* Allow - use this entry */ + http->delayMaxBodySize = dbs->maxsize; + http->delayAssignedPool = dbs->pool; + dbs = NULL; + debug(58, 3) ("httpDelayBodyBuildSize: Setting delayMaxBodySize to %ld\n", + (long int) http->delayMaxBodySize); + } + aclChecklistFree(checklist); + } +} +#endif + static int clientReplyBodyTooLarge(clientHttpRequest * http, squid_off_t clen) { @@ -2693,6 +2725,20 @@ return 0; } +#if DELAY_POOLS +static int +clientDelayBodyTooLarge(clientHttpRequest * http, squid_off_t clen) +{ + if (0 == http->delayMaxBodySize) + return 0; /* disabled */ + if (clen < 0) + return 0; /* unknown */ + if (clen > http->delayMaxBodySize) + return 1; /* too large */ + return 0; +} +#endif + static int clientRequestBodyTooLarge(squid_off_t clen) { @@ -2783,6 +2829,9 @@ return; } clientMaxBodySize(http->request, http, rep); +#if DELAY_POOLS + clientDelayMaxBodySize(http->request, http, rep); +#endif if (http->log_type != LOG_TCP_DENIED && clientReplyBodyTooLarge(http, rep->content_length)) { ErrorState *err = errorCon(ERR_TOO_BIG, HTTP_FORBIDDEN, http->orig_request); storeClientUnregister(http->sc, http->entry, http); @@ -3251,6 +3300,16 @@ /* 4096 is a margin for the HTTP headers included in out.offset */ comm_close(fd); } else { +#if DELAY_POOLS + debug(33, 5) ("clientWriteComplete : Normal\n"); + if (clientDelayBodyTooLarge(http, http->out.offset - 4096)) { + debug(33, 5) ("clientWriteComplete: we should put this into the pool: DelayId=%i\n", + http->sc->delay_id); + delayUnregisterDelayIdPtr(&http->sc->delay_id); + delaySetStoreClient(http->sc, delayPoolClient(http->delayAssignedPool, + (in_addr_t) http->conn->peer.sin_addr.s_addr)); + } +#endif /* More data will be coming from primary server; register with * storage manager. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) Index: squid/src/delay_pools.c diff -u squid/src/delay_pools.c:1.18 squid/src/delay_pools.c:1.18.12.5 --- squid/src/delay_pools.c:1.18 Mon Aug 27 06:53:40 2007 +++ squid/src/delay_pools.c Tue Apr 1 05:24:00 2008 @@ -41,6 +41,7 @@ struct _class1DelayPool { int class; int aggregate; + uint64_t aggregate_bytes; }; #define IND_MAP_SZ 256 @@ -48,12 +49,14 @@ struct _class2DelayPool { int class; int aggregate; + uint64_t aggregate_bytes; /* OK: -1 is terminator. individual[255] is always host 255. */ /* 255 entries + 1 terminator byte */ unsigned char individual_map[IND_MAP_SZ]; unsigned char individual_255_used; /* 256 entries */ int individual[IND_MAP_SZ]; + uint64_t individual_bytes[IND_MAP_SZ]; }; #define NET_MAP_SZ 256 @@ -62,18 +65,21 @@ struct _class3DelayPool { int class; int aggregate; + uint64_t aggregate_bytes; /* OK: -1 is terminator. network[255] is always host 255. */ /* 255 entries + 1 terminator byte */ unsigned char network_map[NET_MAP_SZ]; unsigned char network_255_used; /* 256 entries */ int network[256]; + uint64_t network_bytes[256]; /* 256 sets of (255 entries + 1 terminator byte) */ unsigned char individual_map[NET_MAP_SZ][IND_MAP_SZ]; /* Pack this into one bit per net */ unsigned char individual_255_used[32]; /* largest entry = (255<<8)+255 = 65535 */ int individual[C3_IND_SZ]; + uint64_t individual_bytes[C3_IND_SZ]; }; typedef struct _class1DelayPool class1DelayPool; @@ -308,11 +314,7 @@ { request_t *r; aclCheck_t ch; - int i; - int j; - unsigned int host; - unsigned short pool, position; - unsigned char class, net; + ushort pool; assert(http); r = http->request; @@ -329,13 +331,25 @@ } if (pool == Config.Delay.pools) return delayId(0, 0); + return delayPoolClient(pool, ch.src_addr.s_addr); +} + +delay_id +delayPoolClient(unsigned short pool, in_addr_t addr) +{ + int i; + int j; + unsigned int host; + unsigned short position; + unsigned char class, net; class = Config.Delay.class[pool]; + debug(77,2) ("delayPoolClient: pool %u , class %u\n", pool, class); if (class == 0) return delayId(0, 0); if (class == 1) return delayId(pool + 1, 0); if (class == 2) { - host = ntohl(ch.src_addr.s_addr) & 0xff; + host = ntohl(addr) & 0xff; if (host == 255) { if (!delay_data[pool].class2->individual_255_used) { delay_data[pool].class2->individual_255_used = 1; @@ -361,7 +375,7 @@ return delayId(pool + 1, i); } /* class == 3 */ - host = ntohl(ch.src_addr.s_addr) & 0xffff; + host = ntohl(addr) & 0xffff; net = host >> 8; host &= 0xff; if (net == 255) { @@ -622,15 +636,25 @@ switch (class) { case 1: delay_data[pool].class1->aggregate -= qty; + + delay_data[pool].class1->aggregate_bytes += qty; return; case 2: delay_data[pool].class2->aggregate -= qty; delay_data[pool].class2->individual[position] -= qty; + + delay_data[pool].class2->aggregate_bytes += qty; + delay_data[pool].class2->individual_bytes[position] += qty; return; case 3: delay_data[pool].class3->aggregate -= qty; delay_data[pool].class3->network[position >> 8] -= qty; delay_data[pool].class3->individual[position] -= qty; + + delay_data[pool].class3->aggregate_bytes += qty; + delay_data[pool].class3->network_bytes[position >> 8] += qty; + delay_data[pool].class3->individual_bytes[position] += qty; + return; } fatalf("delayBytesWanted: Invalid class %d\n", class); @@ -687,7 +711,7 @@ } static void -delayPoolStatsAg(StoreEntry * sentry, delaySpecSet * rate, int ag) +delayPoolStatsAg(StoreEntry * sentry, delaySpecSet * rate, int ag, uint64_t bytes) { /* note - always pass delaySpecSet's by reference as may be incomplete */ if (rate->aggregate.restore_bps == -1) { @@ -697,7 +721,8 @@ storeAppendPrintf(sentry, "\tAggregate:\n"); storeAppendPrintf(sentry, "\t\tMax: %d\n", rate->aggregate.max_bytes); storeAppendPrintf(sentry, "\t\tRestore: %d\n", rate->aggregate.restore_bps); - storeAppendPrintf(sentry, "\t\tCurrent: %d\n\n", ag); + storeAppendPrintf(sentry, "\t\tCurrent: %d\n", ag); + storeAppendPrintf(sentry, "\t\tBytes Transferred: %" PRIu64 "\n\n", bytes); } static void @@ -707,7 +732,7 @@ delaySpecSet *rate = Config.Delay.rates[pool]; storeAppendPrintf(sentry, "Pool: %d\n\tClass: 1\n\n", pool + 1); - delayPoolStatsAg(sentry, rate, delay_data[pool].class1->aggregate); + delayPoolStatsAg(sentry, rate, delay_data[pool].class1->aggregate, delay_data[pool].class1->aggregate_bytes); } static void @@ -720,7 +745,7 @@ unsigned int i; storeAppendPrintf(sentry, "Pool: %d\n\tClass: 2\n\n", pool + 1); - delayPoolStatsAg(sentry, rate, class2->aggregate); + delayPoolStatsAg(sentry, rate, class2->aggregate, class2->aggregate_bytes); if (rate->individual.restore_bps == -1) { storeAppendPrintf(sentry, "\tIndividual:\n\t\tDisabled.\n\n"); return; @@ -732,12 +757,12 @@ for (i = 0; i < IND_MAP_SZ; i++) { if (class2->individual_map[i] == 255) break; - storeAppendPrintf(sentry, "%d:%d ", class2->individual_map[i], - class2->individual[i]); + storeAppendPrintf(sentry, "%d:%d (%" PRIu64 ") ", class2->individual_map[i], + class2->individual[i], class2->individual_bytes[i]); shown = 1; } if (class2->individual_255_used) { - storeAppendPrintf(sentry, "%d:%d ", 255, class2->individual[255]); + storeAppendPrintf(sentry, "%d:%d (%" PRIu64 ")", 255, class2->individual[255], class2->individual_bytes[255]); shown = 1; } if (!shown) @@ -756,7 +781,7 @@ unsigned int j; storeAppendPrintf(sentry, "Pool: %d\n\tClass: 3\n\n", pool + 1); - delayPoolStatsAg(sentry, rate, class3->aggregate); + delayPoolStatsAg(sentry, rate, class3->aggregate, class3->aggregate_bytes); if (rate->network.restore_bps == -1) { storeAppendPrintf(sentry, "\tNetwork:\n\t\tDisabled."); } else { @@ -767,12 +792,12 @@ for (i = 0; i < NET_MAP_SZ; i++) { if (class3->network_map[i] == 255) break; - storeAppendPrintf(sentry, "%d:%d ", class3->network_map[i], - class3->network[i]); + storeAppendPrintf(sentry, "%d:%d (%" PRIu64 ") ", class3->network_map[i], + class3->network[i], class3->network_bytes[i]); shown = 1; } if (class3->network_255_used) { - storeAppendPrintf(sentry, "%d:%d ", 255, class3->network[255]); + storeAppendPrintf(sentry, "%d:%d (%" PRIu64 ") ", 255, class3->network[255], class3->network_bytes[255]); shown = 1; } if (!shown) @@ -795,11 +820,11 @@ for (j = 0; j < IND_MAP_SZ; j++) { if (class3->individual_map[i][j] == 255) break; - storeAppendPrintf(sentry, "%d:%d ", class3->individual_map[i][j], - class3->individual[(i << 8) | j]); + storeAppendPrintf(sentry, "%d:%d (%" PRIu64 ") ", class3->individual_map[i][j], + class3->individual[(i << 8) | j], class3->individual_bytes[(i << 8) | j]); } if (class3->individual_255_used[i / 8] & (1 << (i % 8))) { - storeAppendPrintf(sentry, "%d:%d ", 255, class3->individual[(i << 8) | 255]); + storeAppendPrintf(sentry, "%d:%d (%" PRIu64 ") ", 255, class3->individual[(i << 8) | 255], class3->individual_bytes[(i << 8) | 255]); } storeAppendPrintf(sentry, "\n"); } Index: squid/src/protos.h diff -u squid/src/protos.h:1.162 squid/src/protos.h:1.162.2.1 --- squid/src/protos.h:1.162 Mon Mar 3 17:22:35 2008 +++ squid/src/protos.h Tue Mar 25 04:33:36 2008 @@ -522,6 +522,7 @@ extern int httpReplyHasCc(const HttpReply * rep, http_hdr_cc_type type); extern void httpRedirectReply(HttpReply *, http_status, const char *); extern squid_off_t httpReplyBodySize(method_t, const HttpReply *); +extern squid_off_t httpDelayBodySize(method_t, const HttpReply *); extern HttpReply *httpReplyClone(HttpReply * src); /* Http Request */ @@ -1324,6 +1325,7 @@ extern void delayClearNoDelay(int fd); extern int delayIsNoDelay(int fd); extern delay_id delayClient(clientHttpRequest *); +extern delay_id delayPoolClient(unsigned short pool, in_addr_t client); extern EVH delayPoolsUpdate; extern int delayBytesWanted(delay_id d, int min, int max); extern void delayBytesIn(delay_id, int qty); Index: squid/src/structs.h diff -u squid/src/structs.h:1.177 squid/src/structs.h:1.177.2.1 --- squid/src/structs.h:1.177 Mon Mar 17 20:52:14 2008 +++ squid/src/structs.h Tue Mar 25 04:33:37 2008 @@ -242,6 +242,14 @@ squid_off_t maxsize; }; +struct _delay_body_size { + dlink_node node; + acl_access *access_list; + squid_off_t maxsize; + ushort pool; +}; + + struct _http_version_t { unsigned int major; unsigned int minor; @@ -480,6 +488,7 @@ squid_off_t maxRequestBodySize; squid_off_t maxReplyHeaderSize; dlink_list ReplyBodySize; + dlink_list DelayBodySize; struct { u_short icp; #if USE_HTCP @@ -1261,6 +1270,8 @@ } redirect; dlink_node active; squid_off_t maxBodySize; + squid_off_t delayMaxBodySize; + ushort delayAssignedPool; mem_node_ref nr; STHCB *header_callback; /* Temporarily here for storeClientCopyHeaders */ StoreEntry *header_entry; /* Temporarily here for storeClientCopyHeaders */ Index: squid/src/typedefs.h diff -u squid/src/typedefs.h:1.50 squid/src/typedefs.h:1.50.2.1 --- squid/src/typedefs.h:1.50 Mon Feb 25 20:51:52 2008 +++ squid/src/typedefs.h Tue Mar 25 04:33:37 2008 @@ -184,6 +184,7 @@ typedef struct _http_state_flags http_state_flags; typedef struct _header_mangler header_mangler; typedef struct _body_size body_size; +typedef struct _delay_body_size delay_body_size; typedef struct _request_t request_t; typedef struct _AccessLogEntry AccessLogEntry; typedef struct _cachemgr_passwd cachemgr_passwd;