fix(go-runner): count all timed segments of a b.Loop() iteration

example/setup_test.go

@@ -74,6 +74,18 @@ func BenchmarkWithOutlierMeasurementModern(b *testing.B) {
 	}
 }
 
+// The timer is restarted after the teardown, so the timed work is not the last timed segment of
+// the iteration.
+func BenchmarkWithTeardownInLoopModern(b *testing.B) {
+	for b.Loop() {
+		time.Sleep(1 * time.Millisecond)
+
+		b.StopTimer()
+		time.Sleep(10 * time.Millisecond)
+		b.StartTimer()
+	}
+}
+
 func BenchmarkWithoutStartupModern(b *testing.B) {
 	time.Sleep(10 * time.Millisecond)
 	b.ResetTimer()

go-runner/overlay/benchmark1.24.0.go

@@ -174,6 +174,7 @@ func (b *B) ResetTimer() {
 		b.startTimestamp = CurrentTimestamp()
 	}
 	b.duration = 0
+	b.savedDuration = 0
 	b.netAllocs = 0
 	b.netBytes = 0
 

go-runner/overlay/benchmark1.24.0.patch

@@ -72,14 +72,15 @@
  	}
  }
 
-@@ -161,10 +170,18 @@
+@@ -161,10 +170,19 @@
  		b.startAllocs = memStats.Mallocs
  		b.startBytes = memStats.TotalAlloc
  		b.start = highPrecisionTimeNow()
 +
 +		b.startTimestamp = CurrentTimestamp()
  	}
  	b.duration = 0
++	b.savedDuration = 0
  	b.netAllocs = 0
  	b.netBytes = 0
 +
@@ -91,7 +92,7 @@
  }
 
  // SetBytes records the number of bytes processed in a single operation.
-@@ -180,6 +197,11 @@
+@@ -180,6 +198,11 @@
 
  // runN runs a single benchmark for the specified number of iterations.
  func (b *B) runN(n int) {
@@ -103,7 +104,7 @@
  	benchmarkLock.Lock()
  	defer benchmarkLock.Unlock()
  	ctx, cancelCtx := context.WithCancel(context.Background())
-@@ -192,7 +214,9 @@
+@@ -192,7 +215,9 @@
  	runtime.GC()
  	b.resetRaces()
  	b.N = n
@@ -114,7 +115,7 @@
  	b.ctx = ctx
  	b.cancelCtx = cancelCtx
 
-@@ -201,8 +225,13 @@
+@@ -201,8 +226,13 @@
  	b.StartTimer()
  	b.benchFunc(b)
  	b.StopTimer()
@@ -128,7 +129,7 @@
  }
 
  // run1 runs the first iteration of benchFunc. It reports whether more
-@@ -225,6 +254,8 @@
+@@ -225,6 +255,8 @@
  	}()
  	<-b.signal
  	if b.failed {
@@ -137,7 +138,7 @@
  		fmt.Fprintf(b.w, "%s--- FAIL: %s\n%s", b.chatty.prefix(), b.name, b.output)
  		return false
  	}
-@@ -253,6 +284,8 @@
+@@ -253,6 +285,8 @@
  // subbenchmarks. b must not have subbenchmarks.
  func (b *B) run() {
  	labelsOnce.Do(func() {
@@ -146,7 +147,7 @@
  		fmt.Fprintf(b.w, "goos: %s\n", runtime.GOOS)
  		fmt.Fprintf(b.w, "goarch: %s\n", runtime.GOARCH)
  		if b.importPath != "" {
-@@ -312,8 +345,8 @@
+@@ -312,8 +346,8 @@
  	}()
 
  	// b.Loop does its own ramp-up logic so we just need to run it once.
@@ -157,7 +158,7 @@
  		// Run the benchmark for at least the specified amount of time.
  		if b.benchTime.n > 0 {
  			// We already ran a single iteration in run1.
-@@ -323,18 +356,10 @@
+@@ -323,18 +357,10 @@
  				b.runN(b.benchTime.n)
  			}
  		} else {
@@ -178,7 +179,7 @@
  }
 
  // Elapsed returns the measured elapsed time of the benchmark.
-@@ -368,42 +393,93 @@
+@@ -368,42 +394,93 @@
  }
 
  func (b *B) stopOrScaleBLoop() bool {
@@ -293,7 +294,7 @@
  }
 
  // Loop returns true as long as the benchmark should continue running.
-@@ -440,13 +516,41 @@
+@@ -440,13 +517,41 @@
  // whereas b.N-based benchmarks must run the benchmark function (and any
  // associated setup and cleanup) several times.
  func (b *B) Loop() bool {
@@ -337,7 +338,7 @@
  // BenchmarkResult contains the results of a benchmark run.
  type BenchmarkResult struct {
  	N         int           // The number of iterations.
-@@ -455,6 +559,9 @@
+@@ -455,6 +560,9 @@
  	MemAllocs uint64        // The total number of memory allocations.
  	MemBytes  uint64        // The total number of bytes allocated.
 
@@ -347,7 +348,7 @@
  	// Extra records additional metrics reported by ReportMetric.
  	Extra map[string]float64
  }
-@@ -635,6 +742,9 @@
+@@ -635,6 +743,9 @@
  			w:     os.Stdout,
  			bench: true,
  		},
@@ -357,7 +358,7 @@
  		importPath: importPath,
  		benchFunc: func(b *B) {
  			for _, Benchmark := range bs {
-@@ -644,6 +754,9 @@
+@@ -644,6 +755,9 @@
  		benchTime: benchTime,
  		bstate:    bstate,
  	}
@@ -367,15 +368,15 @@
  	if Verbose() {
  		main.chatty = newChattyPrinter(main.w)
  	}
-@@ -672,6 +785,7 @@
+@@ -672,6 +786,7 @@
  						chatty: b.chatty,
  						bench:  true,
  					},
 +					codspeed:  b.codspeed,
  					benchFunc: b.benchFunc,
  					benchTime: b.benchTime,
  				}
-@@ -679,6 +793,8 @@
+@@ -679,6 +794,8 @@
  			}
  			r := b.doBench()
  			if b.failed {
@@ -384,7 +385,7 @@
  				// The output could be very long here, but probably isn't.
  				// We print it all, regardless, because we don't want to trim the reason
  				// the benchmark failed.
-@@ -686,6 +802,8 @@
+@@ -686,6 +803,8 @@
  				continue
  			}
  			results := r.String()
@@ -393,7 +394,7 @@
  			if b.chatty != nil {
  				fmt.Fprintf(b.w, "%-*s\t", s.maxLen, benchName)
  			}
-@@ -746,6 +864,7 @@
+@@ -746,6 +865,7 @@
  			chatty:  b.chatty,
  			bench:   true,
  		},

go-runner/overlay/benchmark1.25.0.go

@@ -174,6 +174,7 @@ func (b *B) ResetTimer() {
 		b.startTimestamp = CurrentTimestamp()
 	}
 	b.duration = 0
+	b.savedDuration = 0
 	b.netAllocs = 0
 	b.netBytes = 0
 

go-runner/overlay/benchmark1.25.0.patch

@@ -53,14 +53,15 @@
  	}
  }
 
-@@ -176,10 +170,18 @@
+@@ -176,10 +170,19 @@
  		b.startAllocs = memStats.Mallocs
  		b.startBytes = memStats.TotalAlloc
  		b.start = highPrecisionTimeNow()
 +
 +		b.startTimestamp = CurrentTimestamp()
  	}
  	b.duration = 0
++	b.savedDuration = 0
  	b.netAllocs = 0
  	b.netBytes = 0
 +
@@ -72,7 +73,7 @@
  }
 
  // SetBytes records the number of bytes processed in a single operation.
-@@ -195,6 +197,11 @@
+@@ -195,6 +198,11 @@
 
  // runN runs a single benchmark for the specified number of iterations.
  func (b *B) runN(n int) {
@@ -84,15 +85,15 @@
  	benchmarkLock.Lock()
  	defer benchmarkLock.Unlock()
  	ctx, cancelCtx := context.WithCancel(context.Background())
-@@ -218,6 +225,7 @@
+@@ -218,6 +226,7 @@
  	b.StartTimer()
  	b.benchFunc(b)
  	b.StopTimer()
 +	b.SaveMeasurement()
  	b.previousN = n
  	b.previousDuration = b.duration
 
-@@ -246,6 +254,8 @@
+@@ -246,6 +255,8 @@
  	}()
  	<-b.signal
  	if b.failed {
@@ -101,7 +102,7 @@
  		fmt.Fprintf(b.w, "%s--- FAIL: %s\n%s", b.chatty.prefix(), b.name, b.output)
  		return false
  	}
-@@ -274,6 +284,8 @@
+@@ -274,6 +285,8 @@
  // subbenchmarks. b must not have subbenchmarks.
  func (b *B) run() {
  	labelsOnce.Do(func() {
@@ -110,7 +111,7 @@
  		fmt.Fprintf(b.w, "goos: %s\n", runtime.GOOS)
  		fmt.Fprintf(b.w, "goarch: %s\n", runtime.GOARCH)
  		if b.importPath != "" {
-@@ -344,18 +356,10 @@
+@@ -344,18 +357,10 @@
  				b.runN(b.benchTime.n)
  			}
  		} else {
@@ -131,7 +132,7 @@
  }
 
  // Elapsed returns the measured elapsed time of the benchmark.
-@@ -408,9 +412,9 @@
+@@ -408,9 +413,9 @@
 
  func (b *B) loopSlowPath() bool {
  	// Consistency checks
@@ -144,7 +145,7 @@
  	if b.loop.i&loopPoisonMask != 0 {
  		panic(fmt.Sprintf("unknown loop stop condition: %#x", b.loop.i))
  	}
-@@ -426,7 +430,10 @@
+@@ -426,7 +431,10 @@
  		}
  		// Within a b.Loop loop, we don't use b.N (to avoid confusion).
  		b.N = 0
@@ -155,7 +156,7 @@
 
  		// Start the next iteration.
  		b.loop.i++
-@@ -445,16 +452,31 @@
+@@ -445,16 +453,31 @@
  		more = false
  	} else {
  		// Handle fixed time case
@@ -189,7 +190,7 @@
  	// Start the next iteration.
  	b.loop.i++
  	return true
-@@ -495,6 +517,8 @@
+@@ -495,6 +518,8 @@
  // whereas b.N-based benchmarks must run the benchmark function (and any
  // associated setup and cleanup) several times.
  func (b *B) Loop() bool {
@@ -198,15 +199,15 @@
  	// This is written such that the fast path is as fast as possible and can be
  	// inlined.
  	//
-@@ -509,6 +533,7 @@
+@@ -509,6 +534,7 @@
  	//   path can do consistency checks and fail.
  	if b.loop.i < b.loop.n {
  		b.loop.i++
 +		b.StartTimerWithoutMarker()
  		return true
  	}
  	return b.loopSlowPath()
-@@ -535,6 +560,9 @@
+@@ -535,6 +561,9 @@
  	MemAllocs uint64        // The total number of memory allocations.
  	MemBytes  uint64        // The total number of bytes allocated.
 
@@ -216,7 +217,7 @@
  	// Extra records additional metrics reported by ReportMetric.
  	Extra map[string]float64
  }
-@@ -715,6 +743,9 @@
+@@ -715,6 +744,9 @@
  			w:     os.Stdout,
  			bench: true,
  		},
@@ -226,7 +227,7 @@
  		importPath: importPath,
  		benchFunc: func(b *B) {
  			for _, Benchmark := range bs {
-@@ -724,6 +755,9 @@
+@@ -724,6 +756,9 @@
  		benchTime: benchTime,
  		bstate:    bstate,
  	}
@@ -236,15 +237,15 @@
  	if Verbose() {
  		main.chatty = newChattyPrinter(main.w)
  	}
-@@ -752,6 +786,7 @@
+@@ -752,6 +787,7 @@
  						chatty: b.chatty,
  						bench:  true,
  					},
 +					codspeed:  b.codspeed,
  					benchFunc: b.benchFunc,
  					benchTime: b.benchTime,
  				}
-@@ -760,6 +795,8 @@
+@@ -760,6 +796,8 @@
  			}
  			r := b.doBench()
  			if b.failed {
@@ -253,7 +254,7 @@
  				// The output could be very long here, but probably isn't.
  				// We print it all, regardless, because we don't want to trim the reason
  				// the benchmark failed.
-@@ -767,6 +804,8 @@
+@@ -767,6 +805,8 @@
  				continue
  			}
  			results := r.String()
@@ -262,7 +263,7 @@
  			if b.chatty != nil {
  				fmt.Fprintf(b.w, "%-*s\t", s.maxLen, benchName)
  			}
-@@ -827,6 +866,7 @@
+@@ -827,6 +867,7 @@
  			chatty:  b.chatty,
  			bench:   true,
  		},

go-runner/overlay/benchmark1.26.0.go

@@ -174,6 +174,7 @@ func (b *B) ResetTimer() {
 		b.startTimestamp = CurrentTimestamp()
 	}
 	b.duration = 0
+	b.savedDuration = 0
 	b.netAllocs = 0
 	b.netBytes = 0