Bisect a performance test case with known commits

.ci/atime/bisect-7687.R

@@ -0,0 +1,25 @@
+pkg.path <- "~/R/data.table"
+Test <- "DT[by] fixed in #4558"
+library(data.table)
+if(FALSE){
+  remotes::install_github("tdhock/atime@bisect")
+  atime::atime_versions_remove("data.table")
+  atime::bisect(pkg.path, Test, "PR7401", "Fast") #should error
+  atime::bisect(pkg.path, Test, "PR4491", "Parent7401") #should find PR5463
+}
+
+(bisect.dt <- rowwiseDT(
+  old=, new=, label=,
+  "PR4558", "Parent5463", "PR4491",
+  "PR4164", "Parent7401", "PR5463",
+  "Before", "Parent4558", "PR4164",#returns PR4164, consistent with https://github.com/Rdatatable/data.table/issues/4200#issuecomment-646111420
+  "PR4558", "Fast", "PR7401",#returns PR7401, consistent with https://github.com/Rdatatable/data.table/issues/7687#issuecomment-4162931822
+  "PR4164", "Fast", "PR7401",#two speed increases between these two commits, bisect finds the larger one: PR7401.
+  "PR4491", "Parent7401", "PR5463",#should find one small speed increase in PR4558? Finds PR5463??
+  ##"PR4164", "Parent5463", "PR4491",#expected speed increase, found PR4491.
+  "PR4164", "Parent4491", "PR4558"))#expected speed increase, found PR4558. DONE.
+for(bisect.i in 1:nrow(bisect.dt)){
+  print(bisect.row <- bisect.dt[bisect.i])
+  bisect.row[, atime::bisect(pkg.path, Test, old, new)]
+}
+

.ci/atime/bisect-viz.R

@@ -0,0 +1,168 @@
+library(animint2)
+library(data.table)
+Test.dir.vec <- Sys.glob("bisect/*")
+Test.dir.vec <- "~/R/data.table/.ci/atime/bisect/DT_by__fixed_in__4558_PR4164_Parent4491"
+for(Test.dir in Test.dir.vec){
+  print(Test.dir)
+  (results.wide <- data.table(
+    csv=Sys.glob(file.path(Test.dir, "csv", "*"))
+  )[
+  , fread(csv), by=csv
+  ][
+  , rank := rank(time)
+  ][order(rank)][, let(
+    Rank = factor(rank, labels=format(time)),
+    Status = factor(status)
+  )][])
+  (results.long <- melt(
+    results.wide,
+    measure.vars=measure(variable, version, sep=".")))
+  show.versions <- unique(results.long$version)
+  details.dt.list <- list()
+  dot.dt.list <- list()
+  common.dt.list <- list()
+  for(result.RData in Sys.glob(file.path(Test.dir, "RData", "*"))){
+    (objs <- load(result.RData))
+    commit <- sub(".RData", "", basename(result.RData))
+    details <- tres.or.status$measurements[
+      show.versions,
+      on="version",
+      data.table(commit, expr.name, version, N, median, min, max)
+    ]
+    common.dt.list[[commit]] <- details[, .(
+      versions=.N
+    ), by=N][
+      versions==max(versions)
+    ][.N, data.table(commit, N)]
+    evers <- unique(details[, .(expr.name, version)])
+    details.dt.list[[commit]] <- details
+    dot.dt.list[[commit]] <- data.table(
+      commit,
+      tpred$prediction[evers, on="expr.name"])
+  }
+  (details.dt <- rbindlist(details.dt.list))
+  (dot.dt <- rbindlist(dot.dt.list))
+  (common.dt <- rbindlist(common.dt.list))
+  limit.dt <- data.table(seconds=tpred$seconds.limit)
+  version.colors <- c(
+    HEAD= "blue")
+  vers.dt <- nc::capture_first_vec(Test.dir, ".*_", old="[^_]+", "_", new="[^_]+")
+  version.colors[vers.dt$old] <- "violet"
+  version.colors[vers.dt$new] <- "red"
+  viz <- animint(
+    title=paste("Performance testing with git bisect in data.table", basename(Test.dir)),
+    source="https://github.com/Rdatatable/data.table/pull/7912/files#diff-e7f10716f0ff9f29e3186a43cfed95d970b167dc4d6a92c00b9e0736bf4b05ae",
+    out.dir=file.path("bisect-viz", basename(Test.dir)),
+    overview=ggplot()+
+      ggtitle("Overview, select commit")+
+      theme_bw()+
+      geom_tallrect(aes(
+        xmin=rank-0.5, xmax=rank+0.5,
+        key=rank,
+        fill=Status),
+        color=NA,
+        data=results.wide)+
+      geom_line(aes(
+        rank, value, color=version, group=version),
+        data=results.long)+
+      facet_grid(variable ~ ., scales="free")+
+      geom_label_aligned(aes(
+        rank, value, label=version, color=version),
+        data=results.long[rank==max(rank)],
+        hjust=0,
+        alignment="vertical")+
+      scale_fill_manual(values=c(
+        "0"="grey",
+        "1"="white"))+
+      scale_color_manual(values=version.colors)+
+      scale_y_log10("")+
+      scale_x_continuous(
+        "commit rank over time",
+        breaks=results.wide$rank)+
+      geom_tallrect(aes(
+        xmin=rank-0.5, xmax=rank+0.5),
+        clickSelects="commit",
+        alpha=1,
+        alpha_off=0,
+        color="black",
+        color_off=NA,
+        fill=NA,
+        data=results.wide),
+    details=ggplot()+
+      ggtitle("Details for selected commit")+
+      theme_bw()+
+      theme(legend.position="none")+
+      theme_animint(width=500, last_in_row=TRUE)+
+      geom_hline(aes(
+        yintercept=seconds),
+        color="grey",
+        data=limit.dt)+
+      geom_text(aes(
+        0, seconds*1.1,
+        label=sprintf(
+          "time limit = %s seconds",
+          paste(seconds)
+        )),
+        hjust=0,
+        color="grey50",
+        data=limit.dt)+
+      geom_vline(aes(
+        xintercept=N,
+        key=1),
+        data=common.dt,
+        color="grey",
+        showSelected="commit")+
+      geom_text(aes(
+        N, 0, label=sprintf("largest common N=%s", paste(N)),
+        key=1),
+        data=common.dt,
+        hjust=1,
+        color="grey50",
+        showSelected="commit")+
+      geom_text(aes(
+        0, 0, label=substring(commit, 1, 10),
+        href=paste0("https://github.com/Rdatatable/data.table/commit/", commit),
+        key=1),
+        hjust=0,
+        vjust=0,
+        data=common.dt,
+        showSelected="commit")+
+      geom_ribbon(aes(
+        N, ymin=min, ymax=max,
+        key=version, fill=version, group=version),
+        showSelected=c("version","commit"),
+        alpha=0.2,
+        color=NA,
+        data=details.dt)+
+      geom_line(aes(
+        N, median,
+        key=version, color=version, group=version),
+        showSelected=c("version","commit"),
+        data=details.dt)+
+      geom_point(aes(
+        N, unit.value,
+        key=version, color=version),
+        showSelected=c("version","commit"),
+        fill="white",
+        data=dot.dt)+
+      geom_label_aligned(aes(
+        N, unit.value, label=version,
+        key=version, color=version),
+        showSelected=c("version","commit"),
+        alignment="horizontal",
+        vjust=-0.1,
+        alpha=0.5,
+        data=dot.dt)+
+      scale_color_manual(values=version.colors)+
+      scale_fill_manual(values=version.colors)+
+      scale_x_log10("N = number of data")+
+      scale_y_log10("seconds"),
+    duration=list(
+      commit=1000)
+  )
+  print(viz)
+}
+
+if(FALSE){
+  animint2::animint2pages(viz, paste0("2026-10-09-performance-bisect-", basename(Test.dir)), chromote_sleep_seconds=3)
+}

.ci/atime/tests.R

@@ -320,6 +320,8 @@ test.list <- atime::atime_test_list(
   # Test case adapted from https://github.com/Rdatatable/data.table/issues/4200#issuecomment-645980224 which is where the issue was reported.
   # Fixed in https://github.com/Rdatatable/data.table/pull/4558
   "DT[by] fixed in #4558" = atime::atime_test(
+    N=2^seq(1, 30),
+    seconds.limit=0.1,
     setup = {
       N9 <- as.integer(N * 0.9)
       d <- data.table(
@@ -328,10 +330,17 @@ test.list <- atime::atime_test_list(
         v2 = sample(5L, N, TRUE)
       )
     },
-    PR7401="0216983c51e03e3f61d5e6f08f4ba0c42cceb22c", # Merge commit (https://github.com/Rdatatable/data.table/commit/0216983c51e03e3f61d5e6f08f4ba0c42cceb22c) of a PR (https://github.com/Rdatatable/data.table/pull/7401) which increased speed and memory usage of this test (https://github.com/Rdatatable/data.table/issues/7687)
-    Before = "7a9eaf62ede487625200981018d8692be8c6f134", # Parent of the first commit (https://github.com/Rdatatable/data.table/commit/515de90a6068911a148e54343a3503043b8bb87c) in the PR (https://github.com/Rdatatable/data.table/pull/4164/commits) that introduced the regression
-    Regression = "c152ced0e5799acee1589910c69c1a2c6586b95d", # Parent of the first commit (https://github.com/Rdatatable/data.table/commit/15f0598b9828d3af2eb8ddc9b38e0356f42afe4f) in the PR (https://github.com/Rdatatable/data.table/pull/4558/commits) that fixes the regression
-    Fixed = "ba32f3cba38ec270587e395f6e6c26a80be36be6", # Merge commit of the PR (https://github.com/Rdatatable/data.table/pull/4558) that fixes the regression
+    Before = "166e0bab317e9d9f4f1844997a315e6482ab5fd9",#a few click of parent up from first commit in PR4164 gets to https://github.com/Rdatatable/data.table/commit/166e0bab317e9d9f4f1844997a315e6482ab5fd9
+    PR4164="4aadde8f5a51cd7c8f3889964e7280432ec65bbc", # Merge commit of the PR that causes the regression.
+    Parent4558="6f360be0b2a6cf425f6df751ca9a99ec5d35ed93", #parent of PR4558 (https://github.com/Rdatatable/data.table/commit/ba32f3cba38ec270587e395f6e6c26a80be36be6)
+    PR4558 = "ba32f3cba38ec270587e395f6e6c26a80be36be6", # Merge commit of the PR (https://github.com/Rdatatable/data.table/pull/4558) that fixes the regression
+    Parent4491="3fa8b20435d33b3d4b5c26fd9b0ac14c10b98800", # PR4491 is bisect result of PR4164 and Parent5463.
+    PR4491="85adf09e3463838d547977ae9bc75e3b37f9cbaf",
+    Parent5463="052f8da6", # PR5463 is bisect result of PR4164 and Parent7401.
+    PR5463="19b7866112614db53eb3e909c097407d91cd6738",
+    Parent7401="20b463c9592c07d05ad2e89a25f84f0b998d6154", # master parent of the PR7401 which caused the recent speedup.
+    PR7401="0216983c51e03e3f61d5e6f08f4ba0c42cceb22c",
+    Fast="a3eeb4c0caec51afd66bdf9bbc16817e12a47a40", #master 10 Sept 2026
     expr = data.table:::`[.data.table`(d, , max(v1) - min(v2), by = id)),
 
   # Issue with sorting again when already sorted, as reported in https://github.com/Rdatatable/data.table/issues/4498