diff --git a/.ci/atime/bisect-7687.R b/.ci/atime/bisect-7687.R new file mode 100644 index 0000000000..125a985353 --- /dev/null +++ b/.ci/atime/bisect-7687.R @@ -0,0 +1,25 @@ +pkg.path <- "~/R/data.table" +Test <- "DT[by] fixed in #4558" +library(data.table) +if(FALSE){ + remotes::install_github("tdhock/atime@bisect") + atime::atime_versions_remove("data.table") + atime::bisect(pkg.path, Test, "PR7401", "Fast") #should error + atime::bisect(pkg.path, Test, "PR4491", "Parent7401") #should find PR5463 +} + +(bisect.dt <- rowwiseDT( + old=, new=, label=, + "PR4558", "Parent5463", "PR4491", + "PR4164", "Parent7401", "PR5463", + "Before", "Parent4558", "PR4164",#returns PR4164, consistent with https://github.com/Rdatatable/data.table/issues/4200#issuecomment-646111420 + "PR4558", "Fast", "PR7401",#returns PR7401, consistent with https://github.com/Rdatatable/data.table/issues/7687#issuecomment-4162931822 + "PR4164", "Fast", "PR7401",#two speed increases between these two commits, bisect finds the larger one: PR7401. + "PR4491", "Parent7401", "PR5463",#should find one small speed increase in PR4558? Finds PR5463?? + ##"PR4164", "Parent5463", "PR4491",#expected speed increase, found PR4491. + "PR4164", "Parent4491", "PR4558"))#expected speed increase, found PR4558. DONE. +for(bisect.i in 1:nrow(bisect.dt)){ + print(bisect.row <- bisect.dt[bisect.i]) + bisect.row[, atime::bisect(pkg.path, Test, old, new)] +} + diff --git a/.ci/atime/bisect-viz.R b/.ci/atime/bisect-viz.R new file mode 100644 index 0000000000..f52a169ef1 --- /dev/null +++ b/.ci/atime/bisect-viz.R @@ -0,0 +1,168 @@ +library(animint2) +library(data.table) +Test.dir.vec <- Sys.glob("bisect/*") +Test.dir.vec <- "~/R/data.table/.ci/atime/bisect/DT_by__fixed_in__4558_PR4164_Parent4491" +for(Test.dir in Test.dir.vec){ + print(Test.dir) + (results.wide <- data.table( + csv=Sys.glob(file.path(Test.dir, "csv", "*")) + )[ + , fread(csv), by=csv + ][ + , rank := rank(time) + ][order(rank)][, let( + Rank = factor(rank, labels=format(time)), + Status = factor(status) + )][]) + (results.long <- melt( + results.wide, + measure.vars=measure(variable, version, sep="."))) + show.versions <- unique(results.long$version) + details.dt.list <- list() + dot.dt.list <- list() + common.dt.list <- list() + for(result.RData in Sys.glob(file.path(Test.dir, "RData", "*"))){ + (objs <- load(result.RData)) + commit <- sub(".RData", "", basename(result.RData)) + details <- tres.or.status$measurements[ + show.versions, + on="version", + data.table(commit, expr.name, version, N, median, min, max) + ] + common.dt.list[[commit]] <- details[, .( + versions=.N + ), by=N][ + versions==max(versions) + ][.N, data.table(commit, N)] + evers <- unique(details[, .(expr.name, version)]) + details.dt.list[[commit]] <- details + dot.dt.list[[commit]] <- data.table( + commit, + tpred$prediction[evers, on="expr.name"]) + } + (details.dt <- rbindlist(details.dt.list)) + (dot.dt <- rbindlist(dot.dt.list)) + (common.dt <- rbindlist(common.dt.list)) + limit.dt <- data.table(seconds=tpred$seconds.limit) + version.colors <- c( + HEAD= "blue") + vers.dt <- nc::capture_first_vec(Test.dir, ".*_", old="[^_]+", "_", new="[^_]+") + version.colors[vers.dt$old] <- "violet" + version.colors[vers.dt$new] <- "red" + viz <- animint( + title=paste("Performance testing with git bisect in data.table", basename(Test.dir)), + source="https://github.com/Rdatatable/data.table/pull/7912/files#diff-e7f10716f0ff9f29e3186a43cfed95d970b167dc4d6a92c00b9e0736bf4b05ae", + out.dir=file.path("bisect-viz", basename(Test.dir)), + overview=ggplot()+ + ggtitle("Overview, select commit")+ + theme_bw()+ + geom_tallrect(aes( + xmin=rank-0.5, xmax=rank+0.5, + key=rank, + fill=Status), + color=NA, + data=results.wide)+ + geom_line(aes( + rank, value, color=version, group=version), + data=results.long)+ + facet_grid(variable ~ ., scales="free")+ + geom_label_aligned(aes( + rank, value, label=version, color=version), + data=results.long[rank==max(rank)], + hjust=0, + alignment="vertical")+ + scale_fill_manual(values=c( + "0"="grey", + "1"="white"))+ + scale_color_manual(values=version.colors)+ + scale_y_log10("")+ + scale_x_continuous( + "commit rank over time", + breaks=results.wide$rank)+ + geom_tallrect(aes( + xmin=rank-0.5, xmax=rank+0.5), + clickSelects="commit", + alpha=1, + alpha_off=0, + color="black", + color_off=NA, + fill=NA, + data=results.wide), + details=ggplot()+ + ggtitle("Details for selected commit")+ + theme_bw()+ + theme(legend.position="none")+ + theme_animint(width=500, last_in_row=TRUE)+ + geom_hline(aes( + yintercept=seconds), + color="grey", + data=limit.dt)+ + geom_text(aes( + 0, seconds*1.1, + label=sprintf( + "time limit = %s seconds", + paste(seconds) + )), + hjust=0, + color="grey50", + data=limit.dt)+ + geom_vline(aes( + xintercept=N, + key=1), + data=common.dt, + color="grey", + showSelected="commit")+ + geom_text(aes( + N, 0, label=sprintf("largest common N=%s", paste(N)), + key=1), + data=common.dt, + hjust=1, + color="grey50", + showSelected="commit")+ + geom_text(aes( + 0, 0, label=substring(commit, 1, 10), + href=paste0("https://github.com/Rdatatable/data.table/commit/", commit), + key=1), + hjust=0, + vjust=0, + data=common.dt, + showSelected="commit")+ + geom_ribbon(aes( + N, ymin=min, ymax=max, + key=version, fill=version, group=version), + showSelected=c("version","commit"), + alpha=0.2, + color=NA, + data=details.dt)+ + geom_line(aes( + N, median, + key=version, color=version, group=version), + showSelected=c("version","commit"), + data=details.dt)+ + geom_point(aes( + N, unit.value, + key=version, color=version), + showSelected=c("version","commit"), + fill="white", + data=dot.dt)+ + geom_label_aligned(aes( + N, unit.value, label=version, + key=version, color=version), + showSelected=c("version","commit"), + alignment="horizontal", + vjust=-0.1, + alpha=0.5, + data=dot.dt)+ + scale_color_manual(values=version.colors)+ + scale_fill_manual(values=version.colors)+ + scale_x_log10("N = number of data")+ + scale_y_log10("seconds"), + duration=list( + commit=1000) + ) + print(viz) +} + +if(FALSE){ + animint2::animint2pages(viz, paste0("2026-10-09-performance-bisect-", basename(Test.dir)), chromote_sleep_seconds=3) +} diff --git a/.ci/atime/tests.R b/.ci/atime/tests.R index 2d342ded67..f6b974f292 100644 --- a/.ci/atime/tests.R +++ b/.ci/atime/tests.R @@ -320,6 +320,8 @@ test.list <- atime::atime_test_list( # Test case adapted from https://github.com/Rdatatable/data.table/issues/4200#issuecomment-645980224 which is where the issue was reported. # Fixed in https://github.com/Rdatatable/data.table/pull/4558 "DT[by] fixed in #4558" = atime::atime_test( + N=2^seq(1, 30), + seconds.limit=0.1, setup = { N9 <- as.integer(N * 0.9) d <- data.table( @@ -328,10 +330,17 @@ test.list <- atime::atime_test_list( v2 = sample(5L, N, TRUE) ) }, - PR7401="0216983c51e03e3f61d5e6f08f4ba0c42cceb22c", # Merge commit (https://github.com/Rdatatable/data.table/commit/0216983c51e03e3f61d5e6f08f4ba0c42cceb22c) of a PR (https://github.com/Rdatatable/data.table/pull/7401) which increased speed and memory usage of this test (https://github.com/Rdatatable/data.table/issues/7687) - Before = "7a9eaf62ede487625200981018d8692be8c6f134", # Parent of the first commit (https://github.com/Rdatatable/data.table/commit/515de90a6068911a148e54343a3503043b8bb87c) in the PR (https://github.com/Rdatatable/data.table/pull/4164/commits) that introduced the regression - Regression = "c152ced0e5799acee1589910c69c1a2c6586b95d", # Parent of the first commit (https://github.com/Rdatatable/data.table/commit/15f0598b9828d3af2eb8ddc9b38e0356f42afe4f) in the PR (https://github.com/Rdatatable/data.table/pull/4558/commits) that fixes the regression - Fixed = "ba32f3cba38ec270587e395f6e6c26a80be36be6", # Merge commit of the PR (https://github.com/Rdatatable/data.table/pull/4558) that fixes the regression + Before = "166e0bab317e9d9f4f1844997a315e6482ab5fd9",#a few click of parent up from first commit in PR4164 gets to https://github.com/Rdatatable/data.table/commit/166e0bab317e9d9f4f1844997a315e6482ab5fd9 + PR4164="4aadde8f5a51cd7c8f3889964e7280432ec65bbc", # Merge commit of the PR that causes the regression. + Parent4558="6f360be0b2a6cf425f6df751ca9a99ec5d35ed93", #parent of PR4558 (https://github.com/Rdatatable/data.table/commit/ba32f3cba38ec270587e395f6e6c26a80be36be6) + PR4558 = "ba32f3cba38ec270587e395f6e6c26a80be36be6", # Merge commit of the PR (https://github.com/Rdatatable/data.table/pull/4558) that fixes the regression + Parent4491="3fa8b20435d33b3d4b5c26fd9b0ac14c10b98800", # PR4491 is bisect result of PR4164 and Parent5463. + PR4491="85adf09e3463838d547977ae9bc75e3b37f9cbaf", + Parent5463="052f8da6", # PR5463 is bisect result of PR4164 and Parent7401. + PR5463="19b7866112614db53eb3e909c097407d91cd6738", + Parent7401="20b463c9592c07d05ad2e89a25f84f0b998d6154", # master parent of the PR7401 which caused the recent speedup. + PR7401="0216983c51e03e3f61d5e6f08f4ba0c42cceb22c", + Fast="a3eeb4c0caec51afd66bdf9bbc16817e12a47a40", #master 10 Sept 2026 expr = data.table:::`[.data.table`(d, , max(v1) - min(v2), by = id)), # Issue with sorting again when already sorted, as reported in https://github.com/Rdatatable/data.table/issues/4498