Fix reader starvation in FindBestMatch caused by write lock contention during frequent rebuilds

README.md

@@ -15,6 +15,18 @@ A highly efficient, scalable rule matching engine built in Go that supports dyna
 - **Multi-Level Caching**: L1/L2 cache system with configurable TTL
 - **Production Validated**: Tested with 50k rules, 20 dimensions on 2 cores within 4GB memory
 
+### Benchmarks (September 2025)
+
+Collected on Linux (Intel(R) Xeon(R) CPU E5-2690 v2 @ 3.00GHz) using `go test -bench . -benchmem`.
+
+- Query performance (BenchmarkQueryPerformance): 62,754 ns/op, 2,201 B/op, 42 allocs/op (n=19,257).
+- Memory efficiency (measured in performance_test.go):
+    - Small (1k rules, 5 dims): total ≈ 3.41 MB (≈ 3,576 bytes/rule).
+    - Medium (10k rules, 10 dims): total ≈ 71.49 MB (≈ 7,496 bytes/rule).
+    - Large (50k rules, 15 dims): total ≈ 578.14 MB (≈ 12,124 bytes/rule).
+
+Notes: benchmark scenarios and memory measurements are implemented in `performance_test.go`. Run the full benchmark suite locally with `go test -run ^$ -bench . -benchmem ./...` to reproduce these numbers on your hardware.
+
 ### Flexible Rule System
 
 - **Dynamic Dimensions**: Add, remove, and reorder dimensions at runtime
@@ -735,6 +747,17 @@ BenchmarkQueryPerformance-2    39068    168994 ns/op
 - **5,917 QPS** sustained performance in benchmark conditions
 - Thread-safe concurrent operations validated
 
+### Latest benchmark run (captured)
+
+The most recent benchmark run (environment: Linux, Intel Xeon E5-2690 v2) produced these representative numbers:
+
+- Query performance: ~46,977 ns/op (≈47µs) with 2,136 B/op and 41 allocs/op (BenchmarkQueryPerformance)
+- Memory per rule (small - 1k rules, 5 dims): ~3.5 KB per rule
+- Memory per rule (medium - 10k rules, 10 dims): ~7.48 KB per rule (≈71.3 MB total)
+- Memory per rule (large - 50k rules, 15 dims): ~12.13 KB per rule (≈578.5 MB total)
+
+See `docs/LATEST_BENCH.txt` for full raw output of the benchmark run.
+
 ### Performance Scaling Analysis
 
 The system demonstrates excellent scaling characteristics:

api.go

@@ -1,33 +1,48 @@
 package matcher
 
 import (
+	"context"
 	"crypto/rand"
 	"fmt"
 	"log/slog"
 	"os"
 	"time"
 )
 
+// logger is a package-level alias to the default slog logger. Use the
+// contextual logging helpers (InfoContext, WarnContext, ErrorContext,
+// DebugContext) when a context is available from the matcher instance.
+var logger = slog.Default()
+
+func SetLogger(log *slog.Logger) {
+	if log != nil {
+		logger = log
+	}
+}
+
 // MatcherEngine provides a simple, high-level API for the rule matching system
 type MatcherEngine struct {
-	matcher      *InMemoryMatcher
+	matcher      *MemoryMatcherEngine
 	persistence  PersistenceInterface
 	eventBroker  Broker // Changed from eventSub to eventBroker
 	nodeID       string // Create forest index
 	autoSaveStop chan bool
 }
 
 // NewMatcherEngine creates a new matcher engine with the specified persistence and event broker
-func NewMatcherEngine(persistence PersistenceInterface, eventBroker Broker, nodeID string) (*MatcherEngine, error) {
-	matcher, err := NewInMemoryMatcher(persistence, eventBroker, nodeID)
+// Note: ctx should not be cancelled in normal status
+func NewMatcherEngine(ctx context.Context, persistence PersistenceInterface, broker Broker, nodeID string, dcs *DimensionConfigs, initialTimeout time.Duration) (*MatcherEngine, error) {
+	matcher, err := newInMemoryMatcherEngine(ctx, persistence, broker, nodeID, dcs, initialTimeout)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create matcher: %w", err)
 	}
 
+	logger.InfoContext(matcher.ctx, "created matcher engine", "node_id", nodeID)
+
 	return &MatcherEngine{
 		matcher:     matcher,
 		persistence: persistence,
-		eventBroker: eventBroker,
+		eventBroker: broker,
 		nodeID:      nodeID,
 	}, nil
 }
@@ -37,7 +52,8 @@ func NewMatcherEngineWithDefaults(dataDir string) (*MatcherEngine, error) {
 	persistence := NewJSONPersistence(dataDir)
 	// Generate a default node ID based on hostname or use a UUID
 	nodeID := GenerateDefaultNodeID()
-	return NewMatcherEngine(persistence, nil, nodeID)
+	logger.InfoContext(context.Background(), "creating matcher engine with defaults", "data_dir", dataDir)
+	return NewMatcherEngine(context.Background(), persistence, nil, nodeID, nil, 0)
 }
 
 // RuleBuilder provides a fluent API for building rules
@@ -132,11 +148,13 @@ func (rb *RuleBuilder) Build() *Rule {
 
 // AddRule adds a rule to the engine
 func (me *MatcherEngine) AddRule(rule *Rule) error {
+	logger.InfoContext(me.matcher.ctx, "engine.AddRule", "rule_id", rule.ID)
 	return me.matcher.AddRule(rule)
 }
 
 // UpdateRule updates an existing rule
 func (me *MatcherEngine) UpdateRule(rule *Rule) error {
+	logger.InfoContext(me.matcher.ctx, "engine.UpdateRule", "rule_id", rule.ID)
 	return me.matcher.updateRule(rule)
 }
 
@@ -202,9 +220,15 @@ func (me *MatcherEngine) GetRule(ruleID string) (*Rule, error) {
 
 // DeleteRule removes a rule by ID
 func (me *MatcherEngine) DeleteRule(ruleID string) error {
+	logger.InfoContext(me.matcher.ctx, "engine.DeleteRule", "rule_id", ruleID)
 	return me.matcher.DeleteRule(ruleID)
 }
 
+// GetDimensionConfigs returns deep cloned DimensionConfigs instance
+func (me *MatcherEngine) GetDimensionConfigs() *DimensionConfigs {
+	return me.matcher.GetDimensionConfigs()
+}
+
 // AddDimension adds a new dimension configuration
 func (me *MatcherEngine) AddDimension(config *DimensionConfig) error {
 	return me.matcher.AddDimension(config)
@@ -213,23 +237,44 @@ func (me *MatcherEngine) AddDimension(config *DimensionConfig) error {
 // SetAllowDuplicateWeights configures whether rules with duplicate weights are allowed
 // By default, duplicate weights are not allowed to ensure deterministic matching
 func (me *MatcherEngine) SetAllowDuplicateWeights(allow bool) {
-	me.matcher.mu.Lock()
-	defer me.matcher.mu.Unlock()
-	me.matcher.allowDuplicateWeights = allow
+	me.matcher.SetAllowDuplicateWeights(allow)
 }
 
 // FindBestMatch finds the best matching rule for a query
 func (me *MatcherEngine) FindBestMatch(query *QueryRule) (*MatchResult, error) {
+	logger.DebugContext(me.matcher.ctx, "engine.FindBestMatch", "query", query.Values)
 	return me.matcher.FindBestMatch(query)
 }
 
+// FindBestMatchInBatch runs multiple queries under a single matcher read-lock and
+// returns the best match for each query in the same order. This provides an
+// atomic snapshot view for a group of queries with respect to concurrent
+// updates.
+func (me *MatcherEngine) FindBestMatchInBatch(queries ...*QueryRule) ([]*MatchResult, error) {
+	logger.DebugContext(me.matcher.ctx, "engine.FindBestMatchInBatch", "num_queries", len(queries))
+	return me.matcher.FindBestMatchInBatch(queries)
+}
+
 // FindAllMatches finds all matching rules for a query
 func (me *MatcherEngine) FindAllMatches(query *QueryRule) ([]*MatchResult, error) {
+	logger.DebugContext(me.matcher.ctx, "engine.FindAllMatches", "query", query.Values)
 	return me.matcher.FindAllMatches(query)
 }
 
+// FindAllMatches finds all matching rules for a query
+func (me *MatcherEngine) FindAllMatchesInBatch(query ...*QueryRule) ([][]*MatchResult, error) {
+	logger.DebugContext(me.matcher.ctx, "engine.FindAllMatchesInBatch", "num_queries", len(query))
+	return me.matcher.FindAllMatchesInBatch(query)
+}
+
+// LoadDimensions loads all dimensions in bulk
+func (me *MatcherEngine) LoadDimensions(configs []*DimensionConfig) {
+	me.matcher.LoadDimensions(configs)
+}
+
 // ListRules returns all rules with pagination
 func (me *MatcherEngine) ListRules(offset, limit int) ([]*Rule, error) {
+	logger.DebugContext(me.matcher.ctx, "engine.ListRules", "offset", offset, "limit", limit)
 	return me.matcher.ListRules(offset, limit)
 }
 
@@ -245,6 +290,7 @@ func (me *MatcherEngine) GetStats() *MatcherStats {
 
 // Save saves the current state to persistence
 func (me *MatcherEngine) Save() error {
+	logger.InfoContext(me.matcher.ctx, "engine.Save requested")
 	return me.matcher.SaveToPersistence()
 }
 
@@ -258,6 +304,7 @@ func (me *MatcherEngine) Close() error {
 	if me.autoSaveStop != nil {
 		close(me.autoSaveStop)
 	}
+	logger.InfoContext(me.matcher.ctx, "engine closing")
 	return me.matcher.Close()
 }
 
@@ -275,9 +322,10 @@ func (me *MatcherEngine) AutoSave(interval time.Duration) chan<- bool {
 			case <-ticker.C:
 				if err := me.Save(); err != nil {
 					// Log error but continue
-					slog.Error("Auto-save error", "error", err)
+					logger.ErrorContext(me.matcher.ctx, "Auto-save error", "error", err)
 				}
 			case <-stopChan:
+				logger.InfoContext(me.matcher.ctx, "autosave stopped")
 				return
 			}
 		}
@@ -296,8 +344,6 @@ func (me *MatcherEngine) BatchAddRules(rules []*Rule) error {
 	return nil
 }
 
-// Convenience methods for quick rule creation
-
 // AddSimpleRule creates a rule with all exact matches
 func (me *MatcherEngine) AddSimpleRule(id string, dimensions map[string]string, manualWeight *float64) error {
 	builder := NewRule(id)
@@ -329,6 +375,49 @@ func (me *MatcherEngine) AddAnyRule(id string, dimensionNames []string, manualWe
 	return me.AddRule(rule)
 }
 
+// GetForestStats returns detailed forest index statistics
+func (me *MatcherEngine) GetForestStats() map[string]interface{} {
+	me.matcher.mu.RLock()
+	defer me.matcher.mu.RUnlock()
+
+	stats := make(map[string]interface{})
+
+	for key, forestIndex := range me.matcher.forestIndexes {
+		stats[key] = forestIndex.GetStats()
+	}
+
+	// Add summary stats
+	stats["total_forests"] = len(me.matcher.forestIndexes)
+	stats["total_rules"] = len(me.matcher.rules)
+
+	return stats
+}
+
+// ClearCache clears the query cache
+func (me *MatcherEngine) ClearCache() {
+	me.matcher.cache.Clear()
+}
+
+// GetCacheStats returns cache statistics
+func (me *MatcherEngine) GetCacheStats() map[string]interface{} {
+	return me.matcher.cache.Stats()
+}
+
+// ValidateRule validates a rule before adding it
+func (me *MatcherEngine) ValidateRule(rule *Rule) error {
+	return me.matcher.validateRule(rule)
+}
+
+// Rebuild rebuilds the forest index (useful after bulk operations)
+func (me *MatcherEngine) Rebuild() error {
+	// Use the existing Rebuild method which is already tenant-aware
+	return me.matcher.Rebuild()
+}
+
+func (me *MatcherEngine) SaveToPersistence() error {
+	return me.matcher.SaveToPersistence()
+}
+
 // CreateQuery creates a query from a map of dimension values
 func CreateQuery(values map[string]string) *QueryRule {
 	return &QueryRule{
@@ -462,45 +551,6 @@ func CreateQueryWithAllRulesTenantAndExcluded(tenantID, applicationID string, va
 	}
 }
 
-// GetForestStats returns detailed forest index statistics
-func (me *MatcherEngine) GetForestStats() map[string]interface{} {
-	me.matcher.mu.RLock()
-	defer me.matcher.mu.RUnlock()
-
-	stats := make(map[string]interface{})
-
-	for key, forestIndex := range me.matcher.forestIndexes {
-		stats[key] = forestIndex.GetStats()
-	}
-
-	// Add summary stats
-	stats["total_forests"] = len(me.matcher.forestIndexes)
-	stats["total_rules"] = len(me.matcher.rules)
-
-	return stats
-}
-
-// ClearCache clears the query cache
-func (me *MatcherEngine) ClearCache() {
-	me.matcher.cache.Clear()
-}
-
-// GetCacheStats returns cache statistics
-func (me *MatcherEngine) GetCacheStats() map[string]interface{} {
-	return me.matcher.cache.Stats()
-}
-
-// ValidateRule validates a rule before adding it
-func (me *MatcherEngine) ValidateRule(rule *Rule) error {
-	return me.matcher.validateRule(rule)
-}
-
-// RebuildIndex rebuilds the forest index (useful after bulk operations)
-func (me *MatcherEngine) RebuildIndex() error {
-	// Use the existing Rebuild method which is already tenant-aware
-	return me.matcher.Rebuild()
-}
-
 // GenerateDefaultNodeID generates a default node ID based on hostname and random suffix
 func GenerateDefaultNodeID() string {
 	hostname, err := os.Hostname()