From 0ee7b80b489383cb7a3496214c3638b3e1fe8970 Mon Sep 17 00:00:00 2001 From: Matthew Rothenberg Date: Mon, 7 Aug 2023 17:41:15 +0100 Subject: [PATCH 1/4] bench: separate benchmarks for Pick and PickSource As discussed in #26, the changes in go1.21 will remove rand contention when the (now deprecated) rand.Seed() is not called. In order to check if this brings performance parity, start to benchmark both Pick and PickSource functions in parallel, and no longer rand.Seed in test init. --- weightedrand_test.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/weightedrand_test.go b/weightedrand_test.go index 0eb4206..749cce7 100644 --- a/weightedrand_test.go +++ b/weightedrand_test.go @@ -33,10 +33,6 @@ func Example() { * Tests *******************************************************************************/ -func init() { - rand.Seed(time.Now().UTC().UnixNano()) // only necessary prior to go1.20 -} - const ( testChoices = 10 testIterations = 1000000 @@ -246,6 +242,24 @@ func BenchmarkPick(b *testing.B) { } func BenchmarkPickParallel(b *testing.B) { + for n := BMMinChoices; n <= BMMaxChoices; n *= 10 { + b.Run(strconv.Itoa(n), func(b *testing.B) { + choices := mockChoices(n) + chooser, err := NewChooser(choices...) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _ = chooser.Pick() + } + }) + }) + } +} + +func BenchmarkPickSourceParallel(b *testing.B) { for n := BMMinChoices; n <= BMMaxChoices; n *= 10 { b.Run(strconv.Itoa(n), func(b *testing.B) { choices := mockChoices(n) From c2e673119ae8d7af6ac50bbb0c8feb0114c10df7 Mon Sep 17 00:00:00 2001 From: Matthew Rothenberg Date: Sat, 12 Aug 2023 15:54:08 +0100 Subject: [PATCH 2/4] docs: mark PickSource as deprecated since go1.21 --- weightedrand.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/weightedrand.go b/weightedrand.go index 52fd89b..ec9a73b 100644 --- a/weightedrand.go +++ b/weightedrand.go @@ -3,11 +3,9 @@ // each element to be selected not being equal, but defined by relative // "weights" (or probabilities). This is called weighted random selection. // -// Compare this package with (github.com/jmcvetta/randutil).WeightedChoice, -// which is optimized for the single operation case. In contrast, this package -// creates a presorted cache optimized for binary search, allowing for repeated -// selections from the same set to be significantly faster, especially for large -// data sets. +// This package creates a presorted cache optimized for binary search, allowing +// for repeated selections from the same set to be significantly faster, +// especially for large data sets. package weightedrand import ( @@ -93,7 +91,7 @@ var ( // Pick returns a single weighted random Choice.Item from the Chooser. // -// Utilizes global rand as the source of randomness. +// Utilizes global rand as the source of randomness. Safe for concurrent usage. func (c Chooser[T, W]) Pick() T { r := rand.Intn(c.max) + 1 i := searchInts(c.totals, r) @@ -109,6 +107,10 @@ func (c Chooser[T, W]) Pick() T { // // It is the responsibility of the caller to ensure the provided rand.Source is // free from thread safety issues. +// +// Deprecated: Since go1.21 global rand no longer suffers from lock contention +// when used in multiple high throughput goroutines, as long as you don't +// manually seed it. Use [Chooser.Pick] instead. func (c Chooser[T, W]) PickSource(rs *rand.Rand) T { r := rs.Intn(c.max) + 1 i := searchInts(c.totals, r) @@ -122,7 +124,9 @@ func (c Chooser[T, W]) PickSource(rs *rand.Rand) T { // overhead. // // Thus, this is essentially manually inlined version. In our use case here, it -// results in a up to ~33% overall throughput increase for Pick(). +// results in a significant throughput increase for Pick. +// +// See also github.com/mroth/xsort. func searchInts(a []int, x int) int { // Possible further future optimization for searchInts via SIMD if we want // to write some Go assembly code: http://0x80.pl/articles/simd-search.html From 60f6145735cf98a720c07e2289f123ac5c10e34e Mon Sep 17 00:00:00 2001 From: Matthew Rothenberg Date: Sun, 13 Aug 2023 11:49:31 -0400 Subject: [PATCH 3/4] docs: update examples/compbench for go1.21 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit results on my Apple M2 Pro laptop: $ make analysis benchstat -filter=".name:Multiple" -col="/lib" -row="/size" -table="/concurrency" results.txt /concurrency: single │ randutil │ weightedrand │ │ sec/op │ sec/op vs base │ 1e1 455.70n ± 0% 19.16n ± 2% -95.80% (p=0.000 n=10) 1e2 438.20n ± 0% 32.81n ± 0% -92.51% (p=0.000 n=10) 1e3 1179.00n ± 0% 46.00n ± 0% -96.10% (p=0.000 n=10) 1e4 6591.00n ± 1% 60.34n ± 0% -99.08% (p=0.000 n=10) 1e5 61188.00n ± 0% 84.83n ± 0% -99.86% (p=0.000 n=10) 1e6 631928.5n ± 1% 112.6n ± 2% -99.98% (p=0.000 n=10) 1e7 6402254.0n ± 1% 240.7n ± 0% -100.00% (p=0.000 n=10) geomean 16.84µ 63.16n -99.62% /concurrency: parallel │ randutil │ weightedrand │ │ sec/op │ sec/op vs base │ 1e1 589.300n ± 0% 2.026n ± 1% -99.66% (p=0.000 n=10) 1e2 490.000n ± 0% 3.337n ± 1% -99.32% (p=0.000 n=10) 1e3 766.500n ± 0% 4.518n ± 1% -99.41% (p=0.000 n=10) 1e4 808.700n ± 0% 5.951n ± 2% -99.26% (p=0.000 n=10) 1e5 6438.500n ± 1% 8.184n ± 1% -99.87% (p=0.000 n=10) 1e6 108285.50n ± 1% 12.00n ± 1% -99.99% (p=0.000 n=10) 1e7 2011263.00n ± 2% 28.95n ± 1% -100.00% (p=0.000 n=10) geomean 5.907µ 6.548n -99.89% --- examples/compbench/.gitignore | 1 + examples/compbench/Makefile | 12 +++ examples/compbench/bench_test.go | 133 ++++++++++++++++--------------- examples/compbench/go.mod | 2 +- 4 files changed, 83 insertions(+), 65 deletions(-) create mode 100644 examples/compbench/.gitignore create mode 100644 examples/compbench/Makefile diff --git a/examples/compbench/.gitignore b/examples/compbench/.gitignore new file mode 100644 index 0000000..e2b3ee7 --- /dev/null +++ b/examples/compbench/.gitignore @@ -0,0 +1 @@ +results.txt diff --git a/examples/compbench/Makefile b/examples/compbench/Makefile new file mode 100644 index 0000000..aa3ce12 --- /dev/null +++ b/examples/compbench/Makefile @@ -0,0 +1,12 @@ +.PHONY: analysis clean +FILE=results.txt +COUNT=10 + +analysis: $(FILE) + benchstat -filter=".name:Multiple" -col="/lib" -row="/size" -table="/concurrency" $< + +clean: + rm -f $(FILE) + +$(FILE): + go test -bench=. -count=${COUNT} | tee $@ diff --git a/examples/compbench/bench_test.go b/examples/compbench/bench_test.go index c8e417b..6ff1126 100644 --- a/examples/compbench/bench_test.go +++ b/examples/compbench/bench_test.go @@ -3,10 +3,10 @@ package compbench import ( + "fmt" + "math" "math/rand" - "strconv" "testing" - "time" "github.com/jmcvetta/randutil" "github.com/mroth/weightedrand/v2" @@ -16,86 +16,85 @@ const BMMinChoices = 10 const BMMaxChoices = 10_000_000 func BenchmarkMultiple(b *testing.B) { - b.Run("jmc_randutil", func(b *testing.B) { - for n := BMMinChoices; n <= BMMaxChoices; n *= 10 { - b.Run(strconv.Itoa(n), func(b *testing.B) { - choices := convertChoices(b, mockChoices(b, n)) - b.ResetTimer() - for i := 0; i < b.N; i++ { - randutil.WeightedChoice(choices) - } - }) - } - }) - - b.Run("weightedrand", func(b *testing.B) { - for n := BMMinChoices; n <= BMMaxChoices; n *= 10 { - b.Run(strconv.Itoa(n), func(b *testing.B) { - choices := mockChoices(b, n) - chs, err := weightedrand.NewChooser(choices...) - if err != nil { - b.Fatal(err) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - chs.Pick() - } + for n := BMMinChoices; n <= BMMaxChoices; n *= 10 { + b.Run(fmt.Sprintf("size=%s", fmt1eN(n)), func(b *testing.B) { + wr_choices := mockChoices(b, n) + ru_choices := convertChoices(b, wr_choices) + + b.Run("concurrency=single", func(b *testing.B) { + b.Run("lib=randutil", func(b *testing.B) { + for i := 0; i < b.N; i++ { + randutil.WeightedChoice(ru_choices) + } + }) + + b.Run("lib=weightedrand", func(b *testing.B) { + chs, err := weightedrand.NewChooser(wr_choices...) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + chs.Pick() + } + }) }) - } - }) - - b.Run("wr-parallel", func(b *testing.B) { - for n := BMMinChoices; n <= BMMaxChoices; n *= 10 { - b.Run(strconv.Itoa(n), func(b *testing.B) { - choices := mockChoices(b, n) - chs, err := weightedrand.NewChooser(choices...) - if err != nil { - b.Fatal(err) - } - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - rs := rand.New(rand.NewSource(time.Now().UTC().UnixNano())) - for pb.Next() { - chs.PickSource(rs) + + b.Run("concurrency=parallel", func(b *testing.B) { + b.Run("lib=weightedrand", func(b *testing.B) { + chs, err := weightedrand.NewChooser(wr_choices...) + if err != nil { + b.Fatal(err) } + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + chs.Pick() + } + }) + }) + + b.Run("lib=randutil", func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + randutil.WeightedChoice(ru_choices) + } + }) }) }) - } - }) + }) + } } -// The single usage case is an anti-pattern for the intended usage of this -// library. Might as well keep some optional benchmarks for that to illustrate -// the point. +// THE SINGLE USAGE CASE IS AN ANTI-PATTERN FOR THE INTENDED USAGE OF THIS +// LIBRARY. Provide some optional benchmarks for that to illustrate the point. func BenchmarkSingle(b *testing.B) { if testing.Short() { b.Skip() } - b.Run("jmc_randutil", func(b *testing.B) { - for n := BMMinChoices; n <= BMMaxChoices; n *= 10 { - b.Run(strconv.Itoa(n), func(b *testing.B) { - choices := convertChoices(b, mockChoices(b, n)) - b.ResetTimer() + for n := BMMinChoices; n <= BMMaxChoices; n *= 10 { + b.Run(fmt.Sprintf("size=%s", fmt1eN(n)), func(b *testing.B) { + wr_choices := mockChoices(b, n) + ru_choices := convertChoices(b, wr_choices) + + b.Run("lib=randutil", func(b *testing.B) { for i := 0; i < b.N; i++ { - randutil.WeightedChoice(choices) + randutil.WeightedChoice(ru_choices) } }) - } - }) - - b.Run("weightedrand", func(b *testing.B) { - for n := BMMinChoices; n <= BMMaxChoices; n *= 10 { - b.Run(strconv.Itoa(n), func(b *testing.B) { - choices := mockChoices(b, n) - b.ResetTimer() + + b.Run("lib=weightedrand", func(b *testing.B) { for i := 0; i < b.N; i++ { - chs, _ := weightedrand.NewChooser(choices...) + // never actually do this, this is not how the library is used + chs, _ := weightedrand.NewChooser(wr_choices...) chs.Pick() } }) - } - }) + }) + } } func mockChoices(tb testing.TB, n int) []weightedrand.Choice[rune, uint] { @@ -118,3 +117,9 @@ func convertChoices(tb testing.TB, cs []weightedrand.Choice[rune, uint]) []randu } return res } + +// fmt1eN returns simplified order of magnitude scientific notation for n, +// e.g. "1e2" for 100, "1e7" for 10 million. +func fmt1eN(n int) string { + return fmt.Sprintf("1e%d", int(math.Log10(float64(n)))) +} diff --git a/examples/compbench/go.mod b/examples/compbench/go.mod index bc67fc8..d39d54f 100644 --- a/examples/compbench/go.mod +++ b/examples/compbench/go.mod @@ -1,6 +1,6 @@ module github.com/mroth/weightedrand/examples/compbench -go 1.18 +go 1.21 require ( github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff From b306b1813297446d360509e4a3e42da8a9d51e9d Mon Sep 17 00:00:00 2001 From: Matthew Rothenberg Date: Sun, 13 Aug 2023 12:06:01 -0400 Subject: [PATCH 4/4] port benchstat format from examples/compbench key-value format is nicer for later filtering in benchstat --- weightedrand_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/weightedrand_test.go b/weightedrand_test.go index 749cce7..6dc642b 100644 --- a/weightedrand_test.go +++ b/weightedrand_test.go @@ -2,8 +2,8 @@ package weightedrand import ( "fmt" + "math" "math/rand" - "strconv" "sync" "testing" "time" @@ -209,11 +209,11 @@ func verifyFrequencyCounts(t *testing.T, counts map[int]int, choices []Choice[in *******************************************************************************/ const BMMinChoices = 10 -const BMMaxChoices = 1000000 +const BMMaxChoices = 10_000_000 func BenchmarkNewChooser(b *testing.B) { for n := BMMinChoices; n <= BMMaxChoices; n *= 10 { - b.Run(strconv.Itoa(n), func(b *testing.B) { + b.Run(fmt.Sprintf("size=%s", fmt1eN(n)), func(b *testing.B) { choices := mockChoices(n) b.ResetTimer() @@ -226,7 +226,7 @@ func BenchmarkNewChooser(b *testing.B) { func BenchmarkPick(b *testing.B) { for n := BMMinChoices; n <= BMMaxChoices; n *= 10 { - b.Run(strconv.Itoa(n), func(b *testing.B) { + b.Run(fmt.Sprintf("size=%s", fmt1eN(n)), func(b *testing.B) { choices := mockChoices(n) chooser, err := NewChooser(choices...) if err != nil { @@ -243,7 +243,7 @@ func BenchmarkPick(b *testing.B) { func BenchmarkPickParallel(b *testing.B) { for n := BMMinChoices; n <= BMMaxChoices; n *= 10 { - b.Run(strconv.Itoa(n), func(b *testing.B) { + b.Run(fmt.Sprintf("size=%s", fmt1eN(n)), func(b *testing.B) { choices := mockChoices(n) chooser, err := NewChooser(choices...) if err != nil { @@ -261,7 +261,7 @@ func BenchmarkPickParallel(b *testing.B) { func BenchmarkPickSourceParallel(b *testing.B) { for n := BMMinChoices; n <= BMMaxChoices; n *= 10 { - b.Run(strconv.Itoa(n), func(b *testing.B) { + b.Run(fmt.Sprintf("size=%s", fmt1eN(n)), func(b *testing.B) { choices := mockChoices(n) chooser, err := NewChooser(choices...) if err != nil { @@ -288,3 +288,9 @@ func mockChoices(n int) []Choice[rune, int] { } return choices } + +// fmt1eN returns simplified order of magnitude scientific notation for n, +// e.g. "1e2" for 100, "1e7" for 10 million. +func fmt1eN(n int) string { + return fmt.Sprintf("1e%d", int(math.Log10(float64(n)))) +}