New configure parameter "spread" for ns_pools:

  Background: Requests are distributed to idle threads via CondSignal,
  which delivers subsequent signals to idle threads in a circular
  manner (i.e. t1, t2, .. tn, then t1 again) determined by the
  scheduling policy of the operating system. This round robin behavior
  has the disadvantage that multiple threads terminate about at the
  same time, based on the maxconnections (or maxconns) value. The idle
  timeout will never be used on a sufficiently busy server.  The mass
  termination of threads is performance-wise bad, especially when many
  connections threads are configured, since it is likely that about
  the same amount of threads will be recreated.

  The "spread" parameter introduces a random factor into threadtimeout
  and maxconnections to avoid these termination cycles.  It adds +/-
  the specified percentage to maxconnections and threadtimeout (e.g. a
  spread of 20 means a random value between 0.80 to 1.20 of the
  specified value of maxconnections and threadtimeout for every
  connection thread).

nsd/nsd.h

@@ -568,6 +568,7 @@ typedef struct Pool {
 	int 	    	    starting;
 	int 	    	    timeout;
 	int		    maxconns;
+	int		    spread;
     	unsigned int	    queued;
     } threads;
 

nsd/pools.c

@@ -33,7 +33,7 @@
  *  Routines for the managing the connection thread pools.
  */
 
-static const char *RCSID = "@(#) $Header: /Users/dossy/Desktop/cvs/aolserver/nsd/pools.c,v 1.13 2008/12/05 08:51:43 gneumann Exp $, compiled: " __DATE__ " " __TIME__;
+static const char *RCSID = "@(#) $Header: /Users/dossy/Desktop/cvs/aolserver/nsd/pools.c,v 1.14 2008/12/27 00:36:38 gneumann Exp $, compiled: " __DATE__ " " __TIME__;
 
 #include "nsd.h"
 
@@ -119,10 +119,10 @@ NsTclPoolsObjCmd(ClientData data, Tcl_Interp *interp, int objc, Tcl_Obj **objv)
         PGetIdx, PSetIdx, PListIdx, PRegisterIdx
     } opt;
     static CONST char *cfgs[] = {
-        "-maxthreads", "-minthreads", "-maxconns", "-timeout", NULL
+        "-maxthreads", "-minthreads", "-maxconns", "-timeout", "-spread", NULL
     };
     enum {
-        PCMaxThreadsIdx, PCMinThreadsIdx, PCMaxConnsIdx, PCTimeoutIdx
+        PCMaxThreadsIdx, PCMinThreadsIdx, PCMaxConnsIdx, PCTimeoutIdx, PCSpreadIdx
     } cfg;
 
     if (objc < 2) {
@@ -182,6 +182,10 @@ NsTclPoolsObjCmd(ClientData data, Tcl_Interp *interp, int objc, Tcl_Obj **objv)
             case PCMaxConnsIdx:
                 poolPtr->threads.maxconns = val;
                 break;
+
+            case PCSpreadIdx:
+                poolPtr->threads.spread = val;
+                break;
             }
         }
         /* catch unsane values */
@@ -197,6 +201,14 @@ NsTclPoolsObjCmd(ClientData data, Tcl_Interp *interp, int objc, Tcl_Obj **objv)
             Tcl_SetResult(interp, "timeout cannot be less than 1", TCL_STATIC);
             return TCL_ERROR;
         }
+        if (poolPtr->threads.maxconns < 1) {
+            Tcl_SetResult(interp, "maxconns cannot be less than 1", TCL_STATIC);
+            return TCL_ERROR;
+        }
+        if (poolPtr->threads.spread < 0 || poolPtr->threads.spread > 100 ) {
+            Tcl_SetResult(interp, "spread must be between 0 and 100", TCL_STATIC);
+            return TCL_ERROR;
+        }
         if (PoolResult(interp, poolPtr) != TCL_OK) {
             return TCL_ERROR;
         }
@@ -390,6 +402,7 @@ CreatePool(char *name)
     	poolPtr->threads.max = 10;      
     	poolPtr->threads.timeout = 120; /* NB: Exit after 2 minutes idle. */
     	poolPtr->threads.maxconns = 0;  /* NB: Never exit thread. */
+    	poolPtr->threads.spread = 20;   /* NB: +-20% random variance on timeout and maxconns. */
    }
     return poolPtr;
 }
@@ -420,7 +433,9 @@ PoolResult(Tcl_Interp *interp, Pool *poolPtr)
         !AppendPool(interp, "current", poolPtr->threads.current) ||
         !AppendPool(interp, "maxconns", poolPtr->threads.maxconns) ||
         !AppendPool(interp, "queued", poolPtr->threads.queued) ||
-        !AppendPool(interp, "timeout", poolPtr->threads.timeout)) {
+        !AppendPool(interp, "timeout", poolPtr->threads.timeout) ||
+        !AppendPool(interp, "spread", poolPtr->threads.spread)
+      ) {
     	return TCL_ERROR;
     }
     return TCL_OK;

nsd/queue.c

@@ -34,7 +34,7 @@
  *	and service threads.
  */
 
-static const char *RCSID = "@(#) $Header: /Users/dossy/Desktop/cvs/aolserver/nsd/queue.c,v 1.45 2008/09/24 11:25:33 gneumann Exp $, compiled: " __DATE__ " " __TIME__;
+static const char *RCSID = "@(#) $Header: /Users/dossy/Desktop/cvs/aolserver/nsd/queue.c,v 1.46 2008/12/27 00:36:38 gneumann Exp $, compiled: " __DATE__ " " __TIME__;
 
 #include "nsd.h"
 
@@ -370,6 +370,7 @@ NsConnThread(void *arg)
     char             name[100];
     int              status, ncons;
     char            *msg;
+    double           spread;
 
     /*
      * Set the conn thread name.
@@ -380,7 +381,12 @@ NsConnThread(void *arg)
     sprintf(name, "-%s:%d-", poolPtr->name, poolPtr->threads.nextid++);
     Ns_MutexUnlock(&poolPtr->lock);
     Ns_ThreadSetName(name);
-    ncons = poolPtr->threads.maxconns;
+
+    /* spread is a value of 1.0 +- specified percentage, 
+       i.e. between 0.0 and 2.0 when the configured percentage is 100 */
+    spread = 1.0 + (2 * poolPtr->threads.spread * Ns_DRand() - poolPtr->threads.spread) / 100.0;
+
+    ncons = round(poolPtr->threads.maxconns * spread);
     msg = "exceeded max connections per thread";
 
     /*
@@ -403,7 +409,7 @@ NsConnThread(void *arg)
 	    timePtr = NULL;
 	} else {
 	    Ns_GetTime(&wait);
-	    Ns_IncrTime(&wait, poolPtr->threads.timeout, 0);
+	    Ns_IncrTime(&wait, round(poolPtr->threads.timeout * spread), 0);
 	    timePtr = &wait;
 	}
 

tcl/pools.tcl

@@ -26,7 +26,7 @@
 # If you do not delete the provisions above, a recipient may use your
 # version of this file under either the License or the GPL.
 #
-# $Header: /Users/dossy/Desktop/cvs/aolserver/tcl/pools.tcl,v 1.4 2007/08/01 22:30:51 michael_andrews Exp $
+# $Header: /Users/dossy/Desktop/cvs/aolserver/tcl/pools.tcl,v 1.5 2008/12/27 00:36:39 gneumann Exp $
 #
 
 set cfgsection "ns/server/[ns_info server]"
@@ -35,7 +35,8 @@ set minthreads [ns_config $cfgsection minthreads 0]
 set maxthreads [ns_config $cfgsection maxthreads 10]
 set maxconns [ns_config $cfgsection maxconnections 0]
 set timeout [ns_config $cfgsection threadtimeout 0]
+set spread [ns_config $cfgsection spread 20]
 
-ns_pools set default -minthreads $minthreads -maxthreads $maxthreads -maxconns $maxconns -timeout $timeout
+ns_pools set default -minthreads $minthreads -maxthreads $maxthreads -maxconns $maxconns -timeout $timeout -spread $spread
 
 ns_log notice "default thread pool: [ns_pools get default]"