WIP

buffer_test.go

@@ -20,6 +20,7 @@ func TestBatchIsRefreshedWhenTheTimeoutExpires(t *testing.T) {
 	evictionPercentage := 10
 	minRefreshDelay := time.Minute * 5
 	maxRefreshDelay := time.Minute * 10
+	synchronousRefreshDelay := time.Minute * 30
 	refreshRetryInterval := time.Millisecond * 10
 	batchSize := 10
 	batchBufferTimeout := time.Minute
@@ -34,7 +35,7 @@ func TestBatchIsRefreshedWhenTheTimeoutExpires(t *testing.T) {
 	//    2. The 'batchBufferTimeout' threshold is exceeded.
 	client := sturdyc.New[string](capacity, numShards, ttl, evictionPercentage,
 		sturdyc.WithNoContinuousEvictions(),
-		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, refreshRetryInterval),
+		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, synchronousRefreshDelay, refreshRetryInterval),
 		sturdyc.WithMissingRecordStorage(),
 		sturdyc.WithRefreshCoalescing(batchSize, batchBufferTimeout),
 		sturdyc.WithClock(clock),
@@ -86,6 +87,7 @@ func TestBatchIsRefreshedWhenTheBufferSizeIsReached(t *testing.T) {
 	ttl := time.Hour
 	minRefreshDelay := time.Minute * 5
 	maxRefreshDelay := time.Minute * 10
+	synchronousRefreshDelay := time.Minute * 30
 	refreshRetryInterval := time.Millisecond * 10
 	batchSize := 10
 	batchBufferTimeout := time.Minute
@@ -100,7 +102,7 @@ func TestBatchIsRefreshedWhenTheBufferSizeIsReached(t *testing.T) {
 	//    2. The 'batchBufferTimeout' threshold is exceeded.
 	client := sturdyc.New[string](capacity, numShards, ttl, evictionPercentage,
 		sturdyc.WithNoContinuousEvictions(),
-		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, refreshRetryInterval),
+		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, synchronousRefreshDelay, refreshRetryInterval),
 		sturdyc.WithMissingRecordStorage(),
 		sturdyc.WithRefreshCoalescing(batchSize, batchBufferTimeout),
 		sturdyc.WithClock(clock),
@@ -180,6 +182,7 @@ func TestBatchIsNotRefreshedByDuplicates(t *testing.T) {
 	evictionPercentage := 10
 	minRefreshDelay := time.Minute * 5
 	maxRefreshDelay := time.Minute * 10
+	synchronousRefreshDelay := time.Minute * 30
 	refreshRetryInterval := time.Millisecond * 10
 	batchSize := 10
 	batchBufferTimeout := time.Minute
@@ -194,7 +197,7 @@ func TestBatchIsNotRefreshedByDuplicates(t *testing.T) {
 	//    2. The 'batchBufferTimeout' threshold is exceeded.
 	client := sturdyc.New[string](capacity, numShards, ttl, evictionPercentage,
 		sturdyc.WithNoContinuousEvictions(),
-		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, refreshRetryInterval),
+		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, synchronousRefreshDelay, refreshRetryInterval),
 		sturdyc.WithMissingRecordStorage(),
 		sturdyc.WithRefreshCoalescing(batchSize, batchBufferTimeout),
 		sturdyc.WithClock(clock),
@@ -250,6 +253,7 @@ func TestBatchesAreGroupedByPermutations(t *testing.T) {
 	evictionPercentage := 15
 	minRefreshDelay := time.Minute * 5
 	maxRefreshDelay := time.Minute * 10
+	synchronousRefreshDelay := time.Minute * 30
 	refreshRetryInterval := time.Millisecond * 10
 	batchSize := 5
 	batchBufferTimeout := time.Minute
@@ -264,7 +268,7 @@ func TestBatchesAreGroupedByPermutations(t *testing.T) {
 	//    2. The 'batchBufferTimeout' threshold is exceeded.
 	c := sturdyc.New[any](capacity, numShards, ttl, evictionPercentage,
 		sturdyc.WithNoContinuousEvictions(),
-		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, refreshRetryInterval),
+		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, synchronousRefreshDelay, refreshRetryInterval),
 		sturdyc.WithMissingRecordStorage(),
 		sturdyc.WithRefreshCoalescing(batchSize, batchBufferTimeout),
 		sturdyc.WithClock(clock),
@@ -339,6 +343,7 @@ func TestLargeBatchesAreChunkedCorrectly(t *testing.T) {
 	evictionPercentage := 23
 	minRefreshDelay := time.Minute * 5
 	maxRefreshDelay := time.Minute * 10
+	synchronousRefreshDelay := time.Minute * 30
 	refreshRetryInterval := time.Millisecond * 10
 	batchSize := 5
 	batchBufferTimeout := time.Minute
@@ -353,7 +358,7 @@ func TestLargeBatchesAreChunkedCorrectly(t *testing.T) {
 	//    2. The 'batchBufferTimeout' threshold is exceeded.
 	client := sturdyc.New[string](capacity, numShards, ttl, evictionPercentage,
 		sturdyc.WithNoContinuousEvictions(),
-		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, refreshRetryInterval),
+		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, synchronousRefreshDelay, refreshRetryInterval),
 		sturdyc.WithMissingRecordStorage(),
 		sturdyc.WithRefreshCoalescing(batchSize, batchBufferTimeout),
 		sturdyc.WithClock(clock),
@@ -401,6 +406,7 @@ func TestValuesAreUpdatedCorrectly(t *testing.T) {
 	evictionPercentage := 10
 	minRefreshDelay := time.Minute * 5
 	maxRefreshDelay := time.Minute * 10
+	synchronousRefreshDelay := time.Minute * 50
 	refreshRetryInterval := time.Millisecond * 10
 	batchSize := 10
 	batchBufferTimeout := time.Minute
@@ -415,7 +421,7 @@ func TestValuesAreUpdatedCorrectly(t *testing.T) {
 	//    2. The 'batchBufferTimeout' threshold is exceeded.
 	client := sturdyc.New[any](capacity, numShards, ttl, evictionPercentage,
 		sturdyc.WithNoContinuousEvictions(),
-		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, refreshRetryInterval),
+		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, synchronousRefreshDelay, refreshRetryInterval),
 		sturdyc.WithMissingRecordStorage(),
 		sturdyc.WithRefreshCoalescing(batchSize, batchBufferTimeout),
 		sturdyc.WithClock(clock),

cache.go

@@ -29,11 +29,12 @@ type Config struct {
 	metricsRecorder            DistributedMetricsRecorder
 	log                        Logger
 
-	refreshInBackground bool
-	minRefreshTime      time.Duration
-	maxRefreshTime      time.Duration
-	retryBaseDelay      time.Duration
-	storeMissingRecords bool
+	earlyRefreshes         bool
+	minRefreshTime         time.Duration
+	maxRefreshTime         time.Duration
+	synchronousRefreshTime time.Duration
+	retryBaseDelay         time.Duration
+	storeMissingRecords    bool
 
 	bufferRefreshes      bool
 	batchMutex           sync.Mutex
@@ -127,11 +128,11 @@ func (c *Client[T]) getShard(key string) *shard[T] {
 // getWithState retrieves a single value from the cache and returns additional
 // information about the state of the record. The state includes whether the record
 // exists, if it has been marked as missing, and if it is due for a refresh.
-func (c *Client[T]) getWithState(key string) (value T, exists, markedAsMissing, refresh bool) {
+func (c *Client[T]) getWithState(key string) (value T, exists, markedAsMissing, backgroundRefresh, synchronousRefresh bool) {
 	shard := c.getShard(key)
-	val, exists, markedAsMissing, refresh := shard.get(key)
-	c.reportCacheHits(exists, markedAsMissing, refresh)
-	return val, exists, markedAsMissing, refresh
+	val, exists, markedAsMissing, backgroundRefresh, synchronousRefresh := shard.get(key)
+	c.reportCacheHits(exists, markedAsMissing, backgroundRefresh, synchronousRefresh)
+	return val, exists, markedAsMissing, backgroundRefresh, synchronousRefresh
 }
 
 // Get retrieves a single value from the cache.
@@ -145,8 +146,8 @@ func (c *Client[T]) getWithState(key string) (value T, exists, markedAsMissing,
 //	The value corresponding to the key and a boolean indicating if the value was found.
 func (c *Client[T]) Get(key string) (T, bool) {
 	shard := c.getShard(key)
-	val, ok, markedAsMissing, refresh := shard.get(key)
-	c.reportCacheHits(ok, markedAsMissing, refresh)
+	val, ok, markedAsMissing, backgroundRefresh, synchronousRefresh := shard.get(key)
+	c.reportCacheHits(ok, markedAsMissing, backgroundRefresh, synchronousRefresh)
 	return val, ok && !markedAsMissing
 }
 

distribution_test.go

@@ -779,6 +779,7 @@ func TestPartialResponseForRefreshesDoesNotResultInMissingRecords(t *testing.T)
 	ttl := time.Hour
 	minRefreshDelay := time.Minute * 5
 	maxRefreshDelay := time.Minute * 10
+	synchronousRefreshDelay := time.Minute * 30
 	refreshRetryInterval := time.Millisecond * 10
 	batchSize := 10
 	batchBufferTimeout := time.Minute
@@ -788,7 +789,7 @@ func TestPartialResponseForRefreshesDoesNotResultInMissingRecords(t *testing.T)
 
 	c := sturdyc.New[string](capacity, numShards, ttl, evictionPercentage,
 		sturdyc.WithNoContinuousEvictions(),
-		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, refreshRetryInterval),
+		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, synchronousRefreshDelay, refreshRetryInterval),
 		sturdyc.WithMissingRecordStorage(),
 		sturdyc.WithRefreshCoalescing(batchSize, batchBufferTimeout),
 		sturdyc.WithDistributedStorageEarlyRefreshes(distributedStorage, refreshAfter),

examples/batch/main.go

@@ -54,12 +54,14 @@ func main() {
 	// used to spread out the refreshes for entries evenly over time.
 	minRefreshDelay := time.Second
 	maxRefreshDelay := time.Second * 2
+	// Set a synchronous refresh delay for when we want a refresh to happen synchronously.
+	synchronousRefreshDelay := time.Second * 30
 	// The base for exponential backoff when retrying a refresh.
 	retryBaseDelay := time.Millisecond * 10
 
 	// Create a cache client with the specified configuration.
 	cacheClient := sturdyc.New[string](capacity, numShards, ttl, evictionPercentage,
-		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, retryBaseDelay),
+		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, synchronousRefreshDelay, retryBaseDelay),
 	)
 
 	// Create a new API instance with the cache client.

examples/buffering/main.go

@@ -56,6 +56,8 @@ func main() {
 	// used to spread out the refreshes for entries evenly over time.
 	minRefreshDelay := time.Second
 	maxRefreshDelay := time.Second * 2
+	// Set a synchronous refresh delay for when we want a refresh to happen synchronously.
+	synchronousRefreshDelay := time.Second * 30
 	// The base for exponential backoff when retrying a refresh.
 	retryBaseDelay := time.Millisecond * 10
 	// Whether to store misses in the sturdyc. This can be useful to
@@ -68,7 +70,7 @@ func main() {
 
 	// Create a new cache client with the specified configuration.
 	cacheClient := sturdyc.New[string](capacity, numShards, ttl, evictionPercentage,
-		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, retryBaseDelay),
+		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, synchronousRefreshDelay, retryBaseDelay),
 		sturdyc.WithRefreshCoalescing(batchSize, batchBufferTimeout),
 	)
 

examples/distributed-early-refreshes/client.go

@@ -19,9 +19,10 @@ const (
 
 // Configuration for the early in-memory refreshes.
 const (
-	minRefreshTime = 2 * time.Second
-	maxRefreshTime = 4 * time.Second
-	retryBaseDelay = 5 * time.Second
+	minRefreshTime         = 2 * time.Second
+	maxRefreshTime         = 4 * time.Second
+	synchronousRefreshTime = 30 * time.Second
+	retryBaseDelay         = 5 * time.Second
 )
 
 // Configuration for the refresh coalescing.
@@ -36,7 +37,7 @@ const refreshAfter = time.Second
 func newAPIClient(distributedStorage sturdyc.DistributedStorageWithDeletions) *apiClient {
 	return &apiClient{
 		cache: sturdyc.New[any](capacity, numberOfShards, ttl, percentageOfRecordsToEvictWhenFull,
-			sturdyc.WithEarlyRefreshes(minRefreshTime, maxRefreshTime, retryBaseDelay),
+			sturdyc.WithEarlyRefreshes(minRefreshTime, maxRefreshTime, synchronousRefreshTime, retryBaseDelay),
 			sturdyc.WithRefreshCoalescing(idealBufferSize, bufferTimeout),
 			sturdyc.WithDistributedStorageEarlyRefreshes(distributedStorage, refreshAfter),
 			// NOTE: Uncommenting this line will make the cache mark the records as

examples/distribution/client.go

@@ -22,9 +22,10 @@ const (
 
 // Configuration for the early in-memory refreshes.
 const (
-	minRefreshTime = 100 * time.Millisecond
-	maxRefreshTime = 500 * time.Millisecond
-	retryBaseDelay = time.Second
+	minRefreshTime         = 100 * time.Millisecond
+	maxRefreshTime         = 500 * time.Millisecond
+	synchronousRefreshTime = 30 * time.Second
+	retryBaseDelay         = time.Second
 )
 
 // Configuration for the refresh coalescing.
@@ -37,7 +38,7 @@ func newAPIClient(distributedStorage sturdyc.DistributedStorage) *apiClient {
 	return &apiClient{
 		cache: sturdyc.New[any](capacity, numberOfShards, ttl, percentageOfRecordsToEvictWhenFull,
 			sturdyc.WithMissingRecordStorage(),
-			sturdyc.WithEarlyRefreshes(minRefreshTime, maxRefreshTime, retryBaseDelay),
+			sturdyc.WithEarlyRefreshes(minRefreshTime, maxRefreshTime, synchronousRefreshTime, retryBaseDelay),
 			sturdyc.WithRefreshCoalescing(idealBufferSize, bufferTimeout),
 			sturdyc.WithDistributedStorage(distributedStorage),
 		),

examples/generics/main.go

@@ -54,12 +54,14 @@ func main() {
 	// used to spread out the refreshes for entries evenly over time.
 	minRefreshDelay := time.Second
 	maxRefreshDelay := time.Second * 2
+	// Set a synchronous refresh delay for when we want a refresh to happen synchronously.
+	synchronousRefreshDelay := time.Second * 30
 	// The base for exponential backoff when retrying a refresh.
 	retryBaseDelay := time.Millisecond * 10
 
 	// Create a new cache client with the specified configuration.
 	cacheClient := sturdyc.New[any](capacity, numShards, ttl, evictionPercentage,
-		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, retryBaseDelay),
+		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, synchronousRefreshDelay, retryBaseDelay),
 		sturdyc.WithRefreshCoalescing(10, time.Second*15),
 	)
 

examples/missing/main.go

@@ -51,12 +51,14 @@ func main() {
 	// used to spread out the refreshes for entries evenly over time.
 	minRefreshDelay := time.Millisecond * 10
 	maxRefreshDelay := time.Millisecond * 30
+	// Set a synchronous refresh delay for when we want a refresh to happen synchronously.
+	synchronousRefreshDelay := time.Second * 30
 	// The base for exponential backoff when retrying a refresh.
 	retryBaseDelay := time.Millisecond * 10
 
 	// Create a cache client with the specified configuration.
 	cacheClient := sturdyc.New[string](capacity, numShards, ttl, evictionPercentage,
-		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, retryBaseDelay),
+		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, synchronousRefreshDelay, retryBaseDelay),
 		sturdyc.WithMissingRecordStorage(),
 	)
 

examples/permutations/main.go

@@ -56,12 +56,14 @@ func main() {
 	// used to spread out the refreshes for entries evenly over time.
 	minRefreshDelay := time.Second
 	maxRefreshDelay := time.Second * 2
+	// Set a synchronous refresh delay for when we want a refresh to happen synchronously.
+	synchronousRefreshDelay := time.Second * 30
 	// The base for exponential backoff when retrying a refresh.
 	retryBaseDelay := time.Millisecond * 10
 
 	// Create a new cache client with the specified configuration.
 	cacheClient := sturdyc.New[string](capacity, numShards, ttl, evictionPercentage,
-		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, retryBaseDelay),
+		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, synchronousRefreshDelay, retryBaseDelay),
 	)
 
 	// We will fetch these IDs using various option sets, meaning that the ID alone

examples/refreshes/main.go

@@ -49,6 +49,8 @@ func main() {
 	// We don't want our outgoing requests graph to look like a comb.
 	minRefreshDelay := time.Millisecond * 10
 	maxRefreshDelay := time.Millisecond * 30
+	// Set a synchronous refresh delay for when we want a refresh to happen synchronously.
+	synchronousRefreshDelay := time.Second * 30
 	// The base used for exponential backoff when retrying a refresh. Most of the
 	// time, we perform refreshes well in advance of the records expiry time.
 	// Hence, we can use this to make it easier for a system that is having
@@ -60,7 +62,7 @@ func main() {
 
 	// Create a cache client with the specified configuration.
 	cacheClient := sturdyc.New[string](capacity, numShards, ttl, evictionPercentage,
-		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, retryBaseDelay),
+		sturdyc.WithEarlyRefreshes(minRefreshDelay, maxRefreshDelay, synchronousRefreshDelay, retryBaseDelay),
 	)
 
 	// Create a new API instance with the cache client.