From 955f660c5b5ff9a5eb9cbe2d4b4c77229ba38e5d Mon Sep 17 00:00:00 2001 From: doswalt Date: Fri, 23 Oct 2026 10:22:27 -0400 Subject: [PATCH 1/4] quickburst updates to more finely distribute the burst amount curve and some other params to control the shape and rate. also, adds some clunky but necessary means of making sure the mark calls are meaningfully enrolling, in case that matters for the test. --- .../client/QuickTestBurst.java | 487 ++++++++++++++++-- 1 file changed, 457 insertions(+), 30 deletions(-) diff --git a/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurst.java b/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurst.java index 0de526122..12a6a3f3e 100644 --- a/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurst.java +++ b/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurst.java @@ -2,13 +2,23 @@ import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.eclipse.jdt.annotation.NonNull; @@ -37,6 +47,12 @@ * aliases) is not counted in any timing. Bursts never overlap. The sizes and waits come from {@code --seed} * (printed on every run), so a run can be repeated exactly, e.g. to compare two backend builds. * + * With {@code --ratePerSecond}, bursts overlap instead: a new user (setup, then its burst) starts every 1/rate + * seconds on average, whether or not earlier users have finished, the way users arrive in production. {@code --loops} + * is then the number of users started (e.g. 360 at 6/s is about a minute). Gaps between starts are random + * (exponential, i.e. Poisson arrivals) and come from {@code --seed}, or exactly 1/rate with {@code --evenArrivals}. + * The summary adds the arrival rate actually achieved and the peak users and pairs in flight at once. + * * Pairs are streamed: a new pair starts only when one of the {@code --concurrency} slots frees up, and nothing is * kept per pair once it finishes except its latency samples, so memory stays flat however many loops run. * @@ -46,18 +62,41 @@ * -Dexec.args="185 --concurrency 50 --env qa --context assign-prog" * mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTestBurst" \ * -Dexec.args="--loops 50 --minBurst 10 --maxBurst 180 --minBurstWait 100 --maxBurstWait 1000 --seed 42" + * mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTestBurst" \ + * -Dexec.args="--loops 360 --minBurst 1 --maxBurst 50 --ratePerSecond 6 --seed 42 --env qa" + * mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTestBurst" \ + * -Dexec.args="--loops 360 --burstTiers 90:1,8:2-10,1:11-30,1:31-200 --ratePerSecond 6 --seed 42 --env qa" * * - * Options (give either {@code } or both {@code --minBurst} and {@code --maxBurst}): + * Options (give {@code }, both {@code --minBurst} and {@code --maxBurst}, or {@code --burstTiers}): *
    *
  • {@code } number of alias assign/mark pairs per burst (first argument; same as * {@code --minBurst --maxBurst })
    • *
  • {@code --loops N} number of bursts (default 1)
    • *
  • {@code --minBurst N --maxBurst N} alias pairs per burst, picked at random in this range (inclusive)
    • + *
  • {@code --burstMean N} pick burst sizes on an exponential curve instead of uniformly: {@code --minBurst} is the + * most common size and each larger size is a little less likely than the one before, with the curve's steepness set + * so sizes average N (above {@code --minBurst}, below the range's middle). The percentiles this gives are printed at + * the start of the run.
    • + *
  • {@code --burstTiers P:SIZE,P:LOW-HIGH,...} pick burst sizes by tier instead, e.g. + * {@code 90:1,8:2-10,1:11-30,1:31-200}: 90% of bursts are 1 pair, 8% are 2-10, and so on, with every size inside a + * tier equally likely. The percents must add up to 100 and the ranges must not overlap. Sets the size range itself, so + * it replaces {@code }, {@code --minBurst / --maxBurst} and {@code --burstMean}. The mean and percentiles this + * gives are printed at the start of the run.
    • *
  • {@code --minBurstWait MS --maxBurstWait MS} wait after each burst, picked at random in this range * (inclusive; default 0)
    • *
  • {@code --seed N} seed for the burst sizes and waits (default: random, printed so it can be reused)
    • - *
  • {@code --concurrency N} pairs in flight at once (default 50)
    • + *
  • {@code --ratePerSecond N} start users at this average rate, overlapping, instead of one burst after another + * (decimals allowed; can't be combined with {@code --minBurstWait / --maxBurstWait})
    • + *
  • {@code --evenArrivals} with {@code --ratePerSecond}, start users exactly 1/rate seconds apart instead of at + * random gaps
    • + *
  • {@code --percentEnroll N} mark against the mock data instead: each alias (a coin flip from {@code --seed}) + * enrolls N% of the time, marking one of the expected targets (ws_site/ws_target_1..45, site/target_group_1, + * test_site/test_target_1) that its /assign returned, with the condition /assign gave it. Every other alias marks + * ws_site/ws_target_none with no condition. Any other target /assign returns to an enrolling alias is reported as + * an error, and the run exits non-zero. Without this flag, each alias marks one of whatever decision points /assign + * returned.
    • + *
  • {@code --concurrency N} pairs in flight at once, per burst (default 50)
    • *
  • {@code --env local|qa|staging} (default local; qa and staging read UPGRADE_QA_URL / UPGRADE_STAGING_URL, see * QuickTestHosts), or {@code --url }
    • *
  • {@code --context } (default assign-prog)
    • @@ -85,6 +124,25 @@ public class QuickTestBurst { private static String groupId = "test_class_group"; private static String authToken = "BearerToken"; private static boolean verbose = false; + private static double ratePerSecond = 0; + private static boolean evenArrivals = false; + private static double burstMean = 0; + private static String burstTiers = null; + // Cumulative probability of each burst size from minBurst up; null means uniform. Set from --burstMean or + // --burstTiers. + private static double[] burstSizeCdf = null; + // Null means marks work as before --percentEnroll existed (see markFromAssignments). + private static Double percentEnroll = null; + // The mock data's decision points an enrolling alias may mark, as decisionPointKey(site, target). + private static final Set ENROLL_DECISION_POINTS = enrollDecisionPoints(); + // Every non-enrolling alias marks this, with no condition. It isn't one of the mock data's targets, and being a + // single fixed decision point it doesn't add a new monitored decision point for every alias. + private static final String NULL_MARK_SITE = "ws_site"; + private static final String NULL_MARK_TARGET = "ws_target_none"; + private static final Set reportedUnexpected = ConcurrentHashMap.newKeySet(); + // Across every user's burst, so overlapping bursts in --ratePerSecond mode show their combined load. + private static final AtomicInteger pairsInFlight = new AtomicInteger(); + private static final AtomicInteger peakPairsInFlight = new AtomicInteger(); private static final String sessionId = "quicktest_burst_session_" + System.currentTimeMillis(); /** Timings and failures, collected per burst and for the whole run. */ @@ -95,6 +153,12 @@ private static final class Stats { final List burstMs = Collections.synchronizedList(new ArrayList<>()); final List errors = Collections.synchronizedList(new ArrayList<>()); final AtomicInteger emptyAssignments = new AtomicInteger(); + // --percentEnroll only: marks by kind, enrolling aliases that fell back to a null mark, and enrolling aliases + // whose /assign returned a decision point not in ENROLL_DECISION_POINTS. + final AtomicInteger enrollMarks = new AtomicInteger(); + final AtomicInteger nullMarks = new AtomicInteger(); + final AtomicInteger enrollFallbacks = new AtomicInteger(); + final AtomicInteger unexpectedDecisionPoints = new AtomicInteger(); void addAll(Stats other) { assignMs.addAll(other.assignMs); @@ -103,6 +167,10 @@ void addAll(Stats other) { burstMs.addAll(other.burstMs); errors.addAll(other.errors); emptyAssignments.addAndGet(other.emptyAssignments.get()); + enrollMarks.addAndGet(other.enrollMarks.get()); + nullMarks.addAndGet(other.nullMarks.get()); + enrollFallbacks.addAndGet(other.enrollFallbacks.get()); + unexpectedDecisionPoints.addAndGet(other.unexpectedDecisionPoints.get()); } } @@ -111,14 +179,38 @@ public static void main(String[] args) throws Exception { Random random = new Random(seed); String runId = "quicktest_burst_" + System.currentTimeMillis(); - System.out.printf("%d burst(s) of %s assign/mark pairs, %s ms between, %d in flight, context %s, host %s%n", - loops, range(minBurst, maxBurst), range(minBurstWait, maxBurstWait), concurrency, context, hostUrl); + if (ratePerSecond > 0) { + System.out.printf( + "%d burst(s) of %s assign/mark pairs, starting %s users/s (%s arrivals), %d in flight per user," + + " context %s, host %s%n", + loops, range(minBurst, maxBurst), ratePerSecond, evenArrivals ? "even" : "random", concurrency, + context, hostUrl); + } else { + System.out.printf("%d burst(s) of %s assign/mark pairs, %s ms between, %d in flight, context %s, host %s%n", + loops, range(minBurst, maxBurst), range(minBurstWait, maxBurstWait), concurrency, context, hostUrl); + } System.out.println("client_session_id: " + sessionId); System.out.println("seed: " + seed); + if (minBurst != maxBurst) { + System.out.println(describeBurstSizes()); + } + if (percentEnroll != null) { + System.out.printf("marks: %s%% enroll on one of the %d expected targets /assign returns," + + " the rest mark %s/%s with no condition%n", percentEnroll, ENROLL_DECISION_POINTS.size(), + NULL_MARK_SITE, NULL_MARK_TARGET); + } + + Stats total = ratePerSecond > 0 ? runAtRate(random, runId) : runSequentially(random, runId); + printSummary(total); + System.exit(total.errors.isEmpty() && total.unexpectedDecisionPoints.get() == 0 ? 0 : 1); + } + + /** Bursts one after another: the next user's setup starts only after the previous burst and its wait. */ + private static Stats runSequentially(Random random, String runId) throws InterruptedException { Stats total = new Stats(); for (int loop = 1; loop <= loops; loop++) { - int count = between(random, minBurst, maxBurst); + int count = nextBurstSize(random); int waitMs = between(random, minBurstWait, maxBurstWait); String userId = existingUserId != null ? existingUserId : runId + "_" + loop; @@ -130,9 +222,72 @@ public static void main(String[] args) throws Exception { Thread.sleep(waitMs); } } + return total; + } - printSummary(total); - System.exit(total.errors.isEmpty() ? 0 : 1); + /** + * Bursts that overlap: a new user (setup, then its burst) starts every 1/ratePerSecond seconds on average, + * whether or not earlier users have finished, the way users arrive in production. A slow server therefore + * means more users in flight, not fewer users started. Each start is scheduled against the run's start time, so + * one late start doesn't push every later one back. + */ + private static Stats runAtRate(Random random, String runId) throws Exception { + // Daemon threads, so an unexpected failure in one user ends the run instead of idle pool threads holding the + // JVM open. + ExecutorService users = Executors.newCachedThreadPool(task -> { + Thread thread = new Thread(task); + thread.setDaemon(true); + return thread; + }); + List> results = new ArrayList<>(); + AtomicInteger usersInFlight = new AtomicInteger(); + AtomicInteger peakUsersInFlight = new AtomicInteger(); + + long runStart = System.nanoTime(); + double nextStartNanos = 0; + long lastStart = runStart; + for (int loop = 1; loop <= loops; loop++) { + int count = nextBurstSize(random); + if (loop > 1) { + nextStartNanos += arrivalGapNanos(random); + } + long sleepNanos = runStart + (long) nextStartNanos - System.nanoTime(); + if (sleepNanos > 0) { + TimeUnit.NANOSECONDS.sleep(sleepNanos); + } + lastStart = System.nanoTime(); + + int user = loop; + String userId = existingUserId != null ? existingUserId : runId + "_" + loop; + results.add(users.submit(() -> { + peakUsersInFlight.accumulateAndGet(usersInFlight.incrementAndGet(), Math::max); + try { + Stats burst = runBurst(user, userId, count); + printBurstLine(user, count, burst); + return burst; + } finally { + usersInFlight.decrementAndGet(); + } + })); + } + double startedOverSeconds = (lastStart - runStart) / 1e9; + + Stats total = new Stats(); + for (Future result : results) { + total.addAll(result.get()); + } + users.shutdown(); + + String achieved = loops > 1 ? String.format("%.2f users/s", (loops - 1) / startedOverSeconds) : "n/a"; + System.out.printf("%n%d users started over %.1f s: %s (target %s), peak %d users and %d pairs in flight%n", + loops, startedOverSeconds, achieved, ratePerSecond, peakUsersInFlight.get(), peakPairsInFlight.get()); + return total; + } + + /** Time until the next user starts: exponential (Poisson arrivals), or exactly 1/rate with --evenArrivals. */ + private static double arrivalGapNanos(Random random) { + double gapSeconds = evenArrivals ? 1 / ratePerSecond : -Math.log(1 - random.nextDouble()) / ratePerSecond; + return gapSeconds * 1e9; } /** One burst: set up the user and its aliases, then send every alias's assign/mark pair. */ @@ -156,11 +311,16 @@ private static Stats runBurst(int loop, String userId, int count) throws Interru Semaphore inFlight = new Semaphore(concurrency); AtomicInteger completed = new AtomicInteger(); List> closes = Collections.synchronizedList(new ArrayList<>()); + // Separate from the run's Random, so the burst sizes and waits a seed gives don't change, and drawn here in + // alias order rather than as responses arrive, so each alias gets the same enroll/pick on every run. + Random markRandom = new Random(seed ^ (loop * 0x9E3779B97F4A7C15L)); long burstStart = System.nanoTime(); for (int i = 0; i < count; i++) { + boolean enroll = percentEnroll != null && markRandom.nextDouble() * 100 < percentEnroll; + double pick = markRandom.nextDouble(); inFlight.acquire(); try { - runPair(i, aliases.get(i), count, stats, completed, closes) + runPair(i, aliases.get(i), count, enroll, pick, stats, completed, closes) .whenComplete((v, e) -> inFlight.release()); } catch (RuntimeException e) { // runPair failed before its future existed, so nothing else will give this slot back. @@ -206,13 +366,100 @@ private static void setUpUser(String userId, List aliases) { } } + /** + * Without --percentEnroll: mark one of the decision points /assign returned (spread across aliases by index), + * with its assigned condition, or a placeholder target with no condition if /assign returned none. + */ + private static MarkExperimentRequestData markFromAssignments(int index, List assignments, + Stats stats) { + if (assignments.isEmpty()) { + stats.emptyAssignments.incrementAndGet(); + return new MarkExperimentRequestData("SelectSection", "quicktest_burst_target_" + index, + new Condition(null, null)); + } + ExperimentsResponse dp = assignments.get(index % assignments.size()); + Condition[] conditions = dp.getAssignedCondition(); + Condition markedCondition = conditions != null && conditions.length > 0 + ? new Condition(conditions[0].getConditionCode(), conditions[0].getExperimentId()) + : new Condition(null, null); + return new MarkExperimentRequestData(dp.getSite(), dp.getTarget(), markedCondition); + } + + /** + * With --percentEnroll. A non-enrolling alias marks NULL_MARK_SITE/NULL_MARK_TARGET, which isn't in the mock + * data, with no condition. An enrolling alias marks one of the ENROLL_DECISION_POINTS that /assign returned with a + * condition, picked by {@code pick}, with that condition; if /assign returned none of them, it falls back to the + * null mark. Any decision point /assign returns to an enrolling alias that isn't on the list is reported as an + * error, since the mock data shouldn't produce one. + */ + private static MarkExperimentRequestData markEnrollOrNull(String alias, List assignments, + boolean enroll, double pick, Stats stats) { + if (enroll) { + List enrollable = new ArrayList<>(); + List unexpected = new ArrayList<>(); + for (ExperimentsResponse dp : assignments) { + if (!ENROLL_DECISION_POINTS.contains(decisionPointKey(dp.getSite(), dp.getTarget()))) { + unexpected.add(dp.getSite() + "/" + dp.getTarget()); + } else if (assignedCondition(dp) != null) { + enrollable.add(dp); + } + } + if (!unexpected.isEmpty()) { + stats.unexpectedDecisionPoints.incrementAndGet(); + for (String dp : unexpected) { + // Once per decision point for the whole run, so a misconfigured context doesn't flood the output. + if (reportedUnexpected.add(dp.toLowerCase(Locale.ROOT))) { + System.err.printf("ERROR /assign returned %s, which isn't an expected enroll target" + + " (first seen for %s)%n", dp, alias); + } + } + } + if (!enrollable.isEmpty()) { + // Sorted, so the same pick lands on the same decision point whatever order /assign listed them in. + enrollable.sort(Comparator.comparing(dp -> decisionPointKey(dp.getSite(), dp.getTarget()))); + ExperimentsResponse dp = enrollable.get((int) (pick * enrollable.size())); + Condition condition = assignedCondition(dp); + stats.enrollMarks.incrementAndGet(); + return new MarkExperimentRequestData(dp.getSite(), dp.getTarget(), + new Condition(condition.getConditionCode(), condition.getExperimentId())); + } + stats.enrollFallbacks.incrementAndGet(); + } + stats.nullMarks.incrementAndGet(); + return new MarkExperimentRequestData(NULL_MARK_SITE, NULL_MARK_TARGET, new Condition(null, null)); + } + + /** The decision point's assigned condition, or null if /assign gave it none. */ + private static Condition assignedCondition(ExperimentsResponse dp) { + Condition[] conditions = dp.getAssignedCondition(); + return conditions != null && conditions.length > 0 && conditions[0].getConditionCode() != null + ? conditions[0] + : null; + } + + private static String decisionPointKey(String site, String target) { + // Lower-cased: the client matches site and target case-insensitively. + return (site + "/" + target).toLowerCase(Locale.ROOT); + } + + private static Set enrollDecisionPoints() { + Set keys = new HashSet<>(); + for (int i = 1; i <= 45; i++) { + keys.add(decisionPointKey("ws_site", "ws_target_" + i)); + } + keys.add(decisionPointKey("site", "target_group_1")); + keys.add(decisionPointKey("test_site", "test_target_1")); + return Collections.unmodifiableSet(keys); + } + /** One alias: /assign, then /mark on one of the decision points it was assigned. */ - private static CompletableFuture runPair(int index, String alias, int count, Stats stats, - AtomicInteger completed, List> closes) { + private static CompletableFuture runPair(int index, String alias, int count, boolean enroll, double pick, + Stats stats, AtomicInteger completed, List> closes) { ExperimentClient client = new ExperimentClient(alias, context, authToken, sessionId, hostUrl, Collections.emptyMap()); long pairStart = System.nanoTime(); long assignStart = System.nanoTime(); + peakPairsInFlight.accumulateAndGet(pairsInFlight.incrementAndGet(), Math::max); return call("assign", (ResponseCallback> cb) -> client .getAllExperimentConditions(true, cb)) @@ -223,24 +470,11 @@ private static CompletableFuture runPair(int index, String alias, int coun System.out.printf("[%d] assign %d ms, %d decision points%n", index, ms, assignments.size()); } - String site; - String target; - Condition markedCondition; - if (assignments.isEmpty()) { - stats.emptyAssignments.incrementAndGet(); - site = "SelectSection"; - target = "quicktest_burst_target_" + index; - markedCondition = new Condition(null, null); - } else { - ExperimentsResponse dp = assignments.get(index % assignments.size()); - Condition[] conditions = dp.getAssignedCondition(); - site = dp.getSite(); - target = dp.getTarget(); - markedCondition = conditions != null && conditions.length > 0 - ? new Condition(conditions[0].getConditionCode(), conditions[0].getExperimentId()) - : new Condition(null, null); - } - MarkExperimentRequestData markData = new MarkExperimentRequestData(site, target, markedCondition); + MarkExperimentRequestData markData = percentEnroll == null + ? markFromAssignments(index, assignments, stats) + : markEnrollOrNull(alias, assignments, enroll, pick, stats); + String site = markData.getSite(); + String target = markData.getTarget(); long markStart = System.nanoTime(); return call("mark", (ResponseCallback cb) -> client.markDecisionPoint(STATUS, @@ -254,6 +488,7 @@ private static CompletableFuture runPair(int index, String alias, int coun }); }) .handle((v, e) -> { + pairsInFlight.decrementAndGet(); if (e != null) { stats.errors.add("[" + index + "] " + rootMessage(e)); } else { @@ -315,6 +550,19 @@ private static void printSummary(Stats total) { System.out.printf("%d aliases got no assignments; their /mark used a placeholder target%n", total.emptyAssignments.get()); } + if (percentEnroll != null) { + int marks = total.enrollMarks.get() + total.nullMarks.get(); + System.out.printf("marks: %d enroll, %d null (%.1f%% enrolled, target %s%%)%n", total.enrollMarks.get(), + total.nullMarks.get(), marks == 0 ? 0.0 : 100.0 * total.enrollMarks.get() / marks, percentEnroll); + if (total.enrollFallbacks.get() > 0) { + System.out.printf("%d enrolling aliases got none of the expected targets with a condition from" + + " /assign, so made the null mark instead%n", total.enrollFallbacks.get()); + } + if (total.unexpectedDecisionPoints.get() > 0) { + System.out.printf("ERROR %d enrolling aliases got targets from /assign that aren't expected: %s%n", + total.unexpectedDecisionPoints.get(), reportedUnexpected); + } + } printLatency("assign", total.assignMs); printLatency("mark", total.markMs); printLatency("pair", total.pairMs); @@ -385,6 +633,10 @@ private static void parseArgs(String[] args) { verbose = true; continue; } + if (flag.equals("--evenArrivals")) { + evenArrivals = true; + continue; + } if (i + 1 >= args.length) { usage("missing value for " + flag); } @@ -402,6 +654,10 @@ private static void parseArgs(String[] args) { usage("--seed must be a number, got " + value); } } + case "--ratePerSecond" -> ratePerSecond = parseDouble(flag, value); + case "--burstMean" -> burstMean = parseDouble(flag, value); + case "--burstTiers" -> burstTiers = value; + case "--percentEnroll" -> percentEnroll = parseDouble(flag, value); case "--concurrency" -> concurrency = parseInt(flag, value); case "--env" -> { try { @@ -419,8 +675,15 @@ private static void parseArgs(String[] args) { default -> usage("unknown option " + flag); } } + if (burstTiers != null) { + if (minBurst != 0 || maxBurst != 0 || burstMean != 0) { + usage("--burstTiers sets the burst sizes, so it can't be combined with , --minBurst /" + + " --maxBurst or --burstMean"); + } + burstSizeCdf = tieredCdf(burstTiers); + } if (minBurst < 1 || maxBurst < 1) { - usage("give , or both --minBurst and --maxBurst, of at least 1"); + usage("give , both --minBurst and --maxBurst (of at least 1), or --burstTiers"); } if (maxBurst < minBurst || maxBurstWait < minBurstWait) { usage("--maxBurst / --maxBurstWait must be at least --minBurst / --minBurstWait"); @@ -428,6 +691,169 @@ private static void parseArgs(String[] args) { if (loops < 1 || concurrency < 1 || minBurstWait < 0) { usage("--loops and --concurrency must be at least 1, and --minBurstWait at least 0"); } + if (ratePerSecond < 0) { + usage("--ratePerSecond must be more than 0"); + } + if (ratePerSecond > 0 && maxBurstWait > 0) { + usage("--ratePerSecond sets when each burst starts, so it can't be combined with" + + " --minBurstWait / --maxBurstWait"); + } + if (evenArrivals && ratePerSecond == 0) { + usage("--evenArrivals needs --ratePerSecond"); + } + if (percentEnroll != null && (percentEnroll < 0 || percentEnroll > 100)) { + usage("--percentEnroll must be from 0 to 100, got " + percentEnroll); + } + if (burstMean != 0) { + double midpoint = (minBurst + maxBurst) / 2.0; + if (!(burstMean > minBurst && burstMean < midpoint)) { + usage(String.format("--burstMean must be above --minBurst (%d) and below the middle of the range (%s)," + + " where sizes would be uniform; got %s", minBurst, midpoint, burstMean)); + } + burstSizeCdf = geometricCdfWithMean(maxBurst - minBurst + 1, burstMean - minBurst); + } + } + + /** Alias pairs in the next burst: uniform in [minBurst, maxBurst], or from --burstMean's or --burstTiers' table. */ + private static int nextBurstSize(Random random) { + if (burstSizeCdf == null) { + return between(random, minBurst, maxBurst); + } + return minBurst + sizeIndexAt(random.nextDouble()); + } + + /** The smallest size offset whose cumulative probability reaches {@code p}. */ + private static int sizeIndexAt(double p) { + // Searching just below p, so a p that falls exactly on a tier boundary (e.g. 0.98 after a 90% and an 8% tier) + // lands on that tier's last size, not the next tier's first, despite the cumulative sums' rounding. + int i = Arrays.binarySearch(burstSizeCdf, p - 1e-9); + return i >= 0 ? i : -i - 1; + } + + /** + * Cumulative probabilities for sizes minBurst..maxBurst from a --burstTiers spec such as + * "90:1,8:2-10,1:11-30,1:31-200": each tier is a percent of bursts and a size or inclusive size range, and its + * percent is spread evenly over its sizes. Sets minBurst and maxBurst to the smallest and largest size covered. + * The percents must add up to 100 and the ranges must not overlap; sizes no tier covers never come up. + */ + private static double[] tieredCdf(String spec) { + List ranges = new ArrayList<>(); + List percents = new ArrayList<>(); + double totalPercent = 0; + for (String tier : spec.split(",")) { + String[] parts = tier.trim().split(":"); + String[] bounds = parts.length == 2 ? parts[1].trim().split("-") : new String[0]; + if (bounds.length < 1 || bounds.length > 2) { + usage("--burstTiers: each tier is : or :-, got \"" + tier + "\""); + } + double percent = parseDouble("--burstTiers percent", parts[0].trim()); + int low = parseInt("--burstTiers size", bounds[0].trim()); + int high = bounds.length == 2 ? parseInt("--burstTiers size", bounds[1].trim()) : low; + if (percent <= 0 || low < 1 || high < low) { + usage("--burstTiers: each tier needs a percent above 0 and sizes of at least 1, low to high, got \"" + + tier + "\""); + } + ranges.add(new int[] { low, high }); + percents.add(percent); + totalPercent += percent; + } + if (Math.abs(totalPercent - 100) > 1e-6) { + usage("--burstTiers percents must add up to 100, got " + totalPercent); + } + + minBurst = ranges.stream().mapToInt(r -> r[0]).min().getAsInt(); + maxBurst = ranges.stream().mapToInt(r -> r[1]).max().getAsInt(); + double[] weights = new double[maxBurst - minBurst + 1]; + for (int t = 0; t < ranges.size(); t++) { + int[] r = ranges.get(t); + for (int size = r[0]; size <= r[1]; size++) { + if (weights[size - minBurst] > 0) { + usage("--burstTiers ranges overlap at size " + size); + } + weights[size - minBurst] = percents.get(t) / (r[1] - r[0] + 1); + } + } + + double[] cdf = new double[weights.length]; + double cumulative = 0; + for (int k = 0; k < weights.length; k++) { + cumulative += weights[k]; + cdf[k] = cumulative / totalPercent; + } + // Exactly 1, so every random.nextDouble() (always below 1) lands inside the table despite rounding. + cdf[cdf.length - 1] = 1.0; + return cdf; + } + + /** + * Cumulative probabilities for offsets 0..sizes-1 from minBurst, each offset {@code ratio} times as likely as the + * one before it (a geometric distribution, the whole-number form of an exponential curve, cut off at maxBurst). + * The ratio is found by bisection so the mean offset comes out at {@code meanOffset}: the mean rises steadily + * from 0 (ratio near 0, every burst minBurst) to the range's midpoint (ratio 1, uniform). + */ + private static double[] geometricCdfWithMean(int sizes, double meanOffset) { + double low = 0; + double high = 1; + double[] cdf = null; + for (int i = 0; i < 100; i++) { + double ratio = (low + high) / 2; + cdf = geometricCdf(sizes, ratio); + if (meanOffset(cdf) < meanOffset) { + low = ratio; + } else { + high = ratio; + } + } + return cdf; + } + + private static double[] geometricCdf(int sizes, double ratio) { + double[] cdf = new double[sizes]; + double weight = 1; + double total = 0; + for (int k = 0; k < sizes; k++) { + total += weight; + cdf[k] = total; + weight *= ratio; + } + for (int k = 0; k < sizes; k++) { + cdf[k] /= total; + } + // Exactly 1, so every random.nextDouble() (always below 1) lands inside the table despite rounding. + cdf[sizes - 1] = 1.0; + return cdf; + } + + private static double meanOffset(double[] cdf) { + double mean = 0; + for (int k = 1; k < cdf.length; k++) { + mean += k * (cdf[k] - cdf[k - 1]); + } + return mean; + } + + private static String describeBurstSizes() { + if (burstSizeCdf == null) { + return "burst sizes: uniform " + range(minBurst, maxBurst); + } + String shape = burstTiers != null ? "tiers " + burstTiers : "exponential curve"; + return String.format("burst sizes: %s, %s: mean %.2f, p50=%d p90=%d p95=%d p99=%d p99.9=%d", + range(minBurst, maxBurst), shape, minBurst + meanOffset(burstSizeCdf), minBurst + sizeIndexAt(0.50), + minBurst + sizeIndexAt(0.90), minBurst + sizeIndexAt(0.95), minBurst + sizeIndexAt(0.99), + minBurst + sizeIndexAt(0.999)); + } + + private static double parseDouble(String name, String value) { + try { + double parsed = Double.parseDouble(value); + if (Double.isNaN(parsed) || Double.isInfinite(parsed)) { + usage(name + " must be a number, got " + value); + } + return parsed; + } catch (NumberFormatException e) { + usage(name + " must be a number, got " + value); + return 0; + } } private static int parseInt(String name, String value) { @@ -442,7 +868,8 @@ private static int parseInt(String name, String value) { private static void usage(String problem) { System.err.println(problem); System.err.println("usage: QuickTestBurst [] [--loops 1] [--minBurst N --maxBurst N]" - + " [--minBurstWait 0 --maxBurstWait 0] [--seed N] [--concurrency 50]" + + " [--minBurstWait 0 --maxBurstWait 0 | --ratePerSecond N [--evenArrivals]] [--seed N]" + + " [--burstMean N | --burstTiers 90:1,8:2-10,...] [--percentEnroll N] [--concurrency 50]" + " [--env local|qa|staging | --url ] [--context assign-prog] [--user ]" + " [--group-type schoolId] [--group-id test_class_group] [--token BearerToken] [--verbose]"); System.exit(2); From 9e212a767fc4f947dfff58e7364fa7969ad3067b Mon Sep 17 00:00:00 2001 From: doswalt Date: Fri, 23 Oct 2026 10:31:49 -0400 Subject: [PATCH 2/4] clearer usage of host urls --- clientlibs/java/CLAUDE.md | 4 ++ .../org/upgradeplatform/client/QuickTest.java | 24 ++++++++-- .../client/QuickTestBurst.java | 20 ++++++-- .../client/QuickTestHosts.java | 46 +++++++++++++++++-- 4 files changed, 82 insertions(+), 12 deletions(-) diff --git a/clientlibs/java/CLAUDE.md b/clientlibs/java/CLAUDE.md index feea73a24..946376f93 100644 --- a/clientlibs/java/CLAUDE.md +++ b/clientlibs/java/CLAUDE.md @@ -73,9 +73,13 @@ TestNG is the test framework but there is no `src/test/java` directory — `Quic To run a manual integration test: ```bash mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTest" # localhost:3030 +mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTest" -Dexec.args="local:3031" # another local port mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTest" -Dexec.args="qa" # or staging +mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTest" -Dexec.args="https://..." # any base URL ``` +`QuickTestBurst` takes the same values via `--env` (`--env local:3031`), or a base URL via `--url`. + Only localhost is in the code. `qa` / `staging` (here and in `QuickTestBurst --env`) read the base URL from `UPGRADE_QA_URL` / `UPGRADE_STAGING_URL`, which you export yourself (e.g. in your shell profile) — hosted environments' URLs stay out of this public repo. See `QuickTestHosts`. There is no automated CI for this library (unlike the JS client which publishes via GitHub Actions). Publishing is currently a manual process. diff --git a/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTest.java b/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTest.java index 582cbbf3e..572b5cb3a 100644 --- a/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTest.java +++ b/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTest.java @@ -20,12 +20,24 @@ * * 
      * mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTest"
+ * mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTest" -Dexec.args="local:3031"
  * mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTest" -Dexec.args="qa"
  * mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTest" -Dexec.args="staging"
+ * mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTest" -Dexec.args="https://some-host.example.com"
  *
    * - * The optional argument is local (default, http://localhost:3030), qa or staging; qa and staging read - * UPGRADE_QA_URL / UPGRADE_STAGING_URL from the environment (see QuickTestHosts). + * The optional argument is local (default, http://localhost:3030), local:<port> for localhost on another port, + * qa, staging, or a base URL starting with http:// or https://. + *

    + * qa and staging are not built in. Their URLs come from environment variables you set on your own machine, + * which you need to add to your shell profile (e.g. {@code ~/.zshrc}) once: + * + * 

    + * export UPGRADE_QA_URL=https://<qa backend host>
+ * export UPGRADE_STAGING_URL=https://<staging backend host>
+ *
    + * + * See QuickTestHosts for details. Passing a base URL directly needs no variables. */ public class QuickTest { private static final String userId = "quicktest_user_" + System.currentTimeMillis(); @@ -56,9 +68,13 @@ public class QuickTest { logRequest.add(logEntry); } - /** Optional first argument: local (default), qa or staging; see QuickTestHosts for the env vars qa/staging need. */ + /** + * Optional first argument: local (default), local:<port>, qa, staging, or a base URL; see QuickTestHosts for + * the env vars qa/staging need. + */ public static void main(String[] args) throws InterruptedException, ExecutionException { - String hostUrl = QuickTestHosts.forEnv(args.length > 0 ? args[0] : "local"); + String host = args.length > 0 ? args[0] : "local"; + String hostUrl = host.startsWith("http://") || host.startsWith("https://") ? host : QuickTestHosts.forEnv(host); System.out.println("host: " + hostUrl); System.out.println("client_session_id: " + sessionId); ExperimentClient client = new ExperimentClient(userId, context, "BearerToken", sessionId, hostUrl, diff --git a/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurst.java b/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurst.java index 12a6a3f3e..da3205a83 100644 --- a/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurst.java +++ b/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurst.java @@ -66,8 +66,20 @@ * -Dexec.args="--loops 360 --minBurst 1 --maxBurst 50 --ratePerSecond 6 --seed 42 --env qa" * mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTestBurst" \ * -Dexec.args="--loops 360 --burstTiers 90:1,8:2-10,1:11-30,1:31-200 --ratePerSecond 6 --seed 42 --env qa" + * mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTestBurst" \ + * -Dexec.args="185 --url https://some-host.example.com" + * + * + * {@code --env qa} and {@code --env staging} are not built in. Their URLs come from environment variables you + * set on your own machine, which you need to add to your shell profile (e.g. {@code ~/.zshrc}) once: + * + * 
    + * export UPGRADE_QA_URL=https://<qa backend host>
+ * export UPGRADE_STAGING_URL=https://<staging backend host>
  *
    * + * See QuickTestHosts for details. {@code --url} takes a base URL directly and needs no variables. + * * Options (give {@code }, both {@code --minBurst} and {@code --maxBurst}, or {@code --burstTiers}): *
      *
    • {@code } number of alias assign/mark pairs per burst (first argument; same as @@ -97,8 +109,9 @@ * an error, and the run exits non-zero. Without this flag, each alias marks one of whatever decision points /assign * returned.
      • *
    • {@code --concurrency N} pairs in flight at once, per burst (default 50)
      • - *
    • {@code --env local|qa|staging} (default local; qa and staging read UPGRADE_QA_URL / UPGRADE_STAGING_URL, see - * QuickTestHosts), or {@code --url }
      • + *
    • {@code --env local|local:|qa|staging} (default local, which is localhost:3030; {@code local:3031} uses + * another port; qa and staging read UPGRADE_QA_URL / UPGRADE_STAGING_URL from your environment, see above), or + * {@code --url } to give the URL directly
      • *
    • {@code --context } (default assign-prog)
      • *
    • {@code --user } use an already-initialized user for every burst instead of creating one per burst
      • *
    • {@code --group-type --group-id } group and working group for a new user (default schoolId / @@ -870,7 +883,8 @@ private static void usage(String problem) { System.err.println("usage: QuickTestBurst [] [--loops 1] [--minBurst N --maxBurst N]" + " [--minBurstWait 0 --maxBurstWait 0 | --ratePerSecond N [--evenArrivals]] [--seed N]" + " [--burstMean N | --burstTiers 90:1,8:2-10,...] [--percentEnroll N] [--concurrency 50]" - + " [--env local|qa|staging | --url ] [--context assign-prog] [--user ]" + + " [--env local|local:|qa|staging | --url ] [--context assign-prog]" + + " [--user ]" + " [--group-type schoolId] [--group-id test_class_group] [--token BearerToken] [--verbose]"); System.exit(2); } diff --git a/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestHosts.java b/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestHosts.java index 3eeb79087..caeb1643a 100644 --- a/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestHosts.java +++ b/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestHosts.java @@ -2,29 +2,65 @@ /** * Base URLs for QuickTest and QuickTestBurst. Only localhost is in the code; the hosted environments' URLs come - * from environment variables so they stay out of this public repo: + * from environment variables on your own machine so they stay out of this public repo: *
        *
      • {@code UPGRADE_QA_URL} for {@code qa}
        • *
      • {@code UPGRADE_STAGING_URL} for {@code staging}
        • *
      - * e.g. {@code export UPGRADE_QA_URL=https://...} in your shell profile. + * Add these to your shell profile (e.g. {@code ~/.zshrc}), with each environment's backend base URL: + * + * 
      + * export UPGRADE_QA_URL=https://<qa backend host>
+ * export UPGRADE_STAGING_URL=https://<staging backend host>
+ *
      + * + * then open a new terminal (or {@code source ~/.zshrc}). An IDE reads your profile only when it starts, so restart it + * before running from there. To skip the variables entirely, pass the base URL itself instead of an env name. + *

      + * {@code local} is localhost on port 3030; {@code local:} (e.g. {@code local:3031}) uses another port, such as a + * worktree's backend. */ final class QuickTestHosts { - static final String LOCAL_URL = "http://localhost:3030"; + static final String LOCAL_URL = localUrl(3030); + + private static final String LOCAL_WITH_PORT = "local:"; private QuickTestHosts() { } - /** The base URL for {@code local}, {@code qa} or {@code staging}; throws with a usable message otherwise. */ + /** + * The base URL for {@code local}, {@code local:}, {@code qa} or {@code staging}; throws with a usable message + * otherwise. + */ static String forEnv(String env) { + if (env.startsWith(LOCAL_WITH_PORT)) { + return localUrl(parsePort(env.substring(LOCAL_WITH_PORT.length()))); + } return switch (env) { case "local" -> LOCAL_URL; case "qa" -> requireEnv("UPGRADE_QA_URL"); case "staging" -> requireEnv("UPGRADE_STAGING_URL"); - default -> throw new IllegalArgumentException("unknown env " + env + " (local, qa, staging)"); + default -> throw new IllegalArgumentException("unknown env " + env + " (local, local:, qa, staging)"); }; } + private static String localUrl(int port) { + return "http://localhost:" + port; + } + + private static int parsePort(String value) { + int port; + try { + port = Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("port must be a number, got " + value); + } + if (port < 1 || port > 65535) { + throw new IllegalArgumentException("port must be 1-65535, got " + port); + } + return port; + } + private static String requireEnv(String name) { String value = System.getenv(name); if (value == null || value.isBlank()) { From 4403007d7421c29f3bfb1db4ae166f6a610e2e47 Mon Sep 17 00:00:00 2001 From: doswalt Date: Fri, 23 Oct 2026 10:59:05 -0400 Subject: [PATCH 3/4] quickburst ui launcher --- clientlibs/java/CLAUDE.md | 7 + .../client/QuickTestBurst.java | 45 +- .../client/QuickTestBurstUi.java | 283 +++++++++ .../client/quicktest-burst-ui.html | 543 ++++++++++++++++++ package.json | 3 +- scripts/quickburst-ui.sh | 16 + 6 files changed, 863 insertions(+), 34 deletions(-) create mode 100644 clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurstUi.java create mode 100644 clientlibs/java/src/main/resources/org/upgradeplatform/client/quicktest-burst-ui.html create mode 100755 scripts/quickburst-ui.sh diff --git a/clientlibs/java/CLAUDE.md b/clientlibs/java/CLAUDE.md index 946376f93..883fbc7f4 100644 --- a/clientlibs/java/CLAUDE.md +++ b/clientlibs/java/CLAUDE.md @@ -80,6 +80,13 @@ mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTest" -Dexec.arg `QuickTestBurst` takes the same values via `--env` (`--env local:3031`), or a base URL via `--url`. +`QuickTestBurstUi` serves a local form for `QuickTestBurst` at http://localhost:8090: it starts each run as a separate `QuickTestBurst` process, streams the output to the page, and shows the equivalent command line. Its page is `src/main/resources/org/upgradeplatform/client/quicktest-burst-ui.html`. From the repo root, `yarn quickburst:ui` (`scripts/quickburst-ui.sh`) compiles and starts it: + +```bash +yarn quickburst:ui # http://localhost:8090 +yarn quickburst:ui --port 8091 # another port +``` + Only localhost is in the code. `qa` / `staging` (here and in `QuickTestBurst --env`) read the base URL from `UPGRADE_QA_URL` / `UPGRADE_STAGING_URL`, which you export yourself (e.g. in your shell profile) — hosted environments' URLs stay out of this public repo. See `QuickTestHosts`. There is no automated CI for this library (unlike the JS client which publishes via GitHub Actions). Publishing is currently a manual process. diff --git a/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurst.java b/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurst.java index da3205a83..ccaf20007 100644 --- a/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurst.java +++ b/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurst.java @@ -13,7 +13,6 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -80,6 +79,8 @@ * * See QuickTestHosts for details. {@code --url} takes a base URL directly and needs no variables. * + * QuickTestBurstUi is a local web page for the same options, if you'd rather fill in a form than write the command. + * * Options (give {@code }, both {@code --minBurst} and {@code --maxBurst}, or {@code --burstTiers}): *

        *
      • {@code } number of alias assign/mark pairs per burst (first argument; same as @@ -105,9 +106,8 @@ *
      • {@code --percentEnroll N} mark against the mock data instead: each alias (a coin flip from {@code --seed}) * enrolls N% of the time, marking one of the expected targets (ws_site/ws_target_1..45, site/target_group_1, * test_site/test_target_1) that its /assign returned, with the condition /assign gave it. Every other alias marks - * ws_site/ws_target_none with no condition. Any other target /assign returns to an enrolling alias is reported as - * an error, and the run exits non-zero. Without this flag, each alias marks one of whatever decision points /assign - * returned.
        • + * ws_site/ws_target_none with no condition. Other targets /assign returns are never marked. Without this flag, each + * alias marks one of whatever decision points /assign returned. *
      • {@code --concurrency N} pairs in flight at once, per burst (default 50)
        • *
      • {@code --env local|local:|qa|staging} (default local, which is localhost:3030; {@code local:3031} uses * another port; qa and staging read UPGRADE_QA_URL / UPGRADE_STAGING_URL from your environment, see above), or @@ -152,7 +152,6 @@ public class QuickTestBurst { // single fixed decision point it doesn't add a new monitored decision point for every alias. private static final String NULL_MARK_SITE = "ws_site"; private static final String NULL_MARK_TARGET = "ws_target_none"; - private static final Set reportedUnexpected = ConcurrentHashMap.newKeySet(); // Across every user's burst, so overlapping bursts in --ratePerSecond mode show their combined load. private static final AtomicInteger pairsInFlight = new AtomicInteger(); private static final AtomicInteger peakPairsInFlight = new AtomicInteger(); @@ -166,12 +165,10 @@ private static final class Stats { final List burstMs = Collections.synchronizedList(new ArrayList<>()); final List errors = Collections.synchronizedList(new ArrayList<>()); final AtomicInteger emptyAssignments = new AtomicInteger(); - // --percentEnroll only: marks by kind, enrolling aliases that fell back to a null mark, and enrolling aliases - // whose /assign returned a decision point not in ENROLL_DECISION_POINTS. + // --percentEnroll only: marks by kind, and enrolling aliases that fell back to a null mark. final AtomicInteger enrollMarks = new AtomicInteger(); final AtomicInteger nullMarks = new AtomicInteger(); final AtomicInteger enrollFallbacks = new AtomicInteger(); - final AtomicInteger unexpectedDecisionPoints = new AtomicInteger(); void addAll(Stats other) { assignMs.addAll(other.assignMs); @@ -183,7 +180,6 @@ void addAll(Stats other) { enrollMarks.addAndGet(other.enrollMarks.get()); nullMarks.addAndGet(other.nullMarks.get()); enrollFallbacks.addAndGet(other.enrollFallbacks.get()); - unexpectedDecisionPoints.addAndGet(other.unexpectedDecisionPoints.get()); } } @@ -216,7 +212,7 @@ loops, range(minBurst, maxBurst), ratePerSecond, evenArrivals ? "even" : "random Stats total = ratePerSecond > 0 ? runAtRate(random, runId) : runSequentially(random, runId); printSummary(total); - System.exit(total.errors.isEmpty() && total.unexpectedDecisionPoints.get() == 0 ? 0 : 1); + System.exit(total.errors.isEmpty() ? 0 : 1); } /** Bursts one after another: the next user's setup starts only after the previous burst and its wait. */ @@ -402,31 +398,18 @@ private static MarkExperimentRequestData markFromAssignments(int index, List assignments, - boolean enroll, double pick, Stats stats) { + private static MarkExperimentRequestData markEnrollOrNull(List assignments, boolean enroll, + double pick, Stats stats) { if (enroll) { List enrollable = new ArrayList<>(); - List unexpected = new ArrayList<>(); for (ExperimentsResponse dp : assignments) { - if (!ENROLL_DECISION_POINTS.contains(decisionPointKey(dp.getSite(), dp.getTarget()))) { - unexpected.add(dp.getSite() + "/" + dp.getTarget()); - } else if (assignedCondition(dp) != null) { + if (ENROLL_DECISION_POINTS.contains(decisionPointKey(dp.getSite(), dp.getTarget())) + && assignedCondition(dp) != null) { enrollable.add(dp); } } - if (!unexpected.isEmpty()) { - stats.unexpectedDecisionPoints.incrementAndGet(); - for (String dp : unexpected) { - // Once per decision point for the whole run, so a misconfigured context doesn't flood the output. - if (reportedUnexpected.add(dp.toLowerCase(Locale.ROOT))) { - System.err.printf("ERROR /assign returned %s, which isn't an expected enroll target" - + " (first seen for %s)%n", dp, alias); - } - } - } if (!enrollable.isEmpty()) { // Sorted, so the same pick lands on the same decision point whatever order /assign listed them in. enrollable.sort(Comparator.comparing(dp -> decisionPointKey(dp.getSite(), dp.getTarget()))); @@ -485,7 +468,7 @@ private static CompletableFuture runPair(int index, String alias, int coun MarkExperimentRequestData markData = percentEnroll == null ? markFromAssignments(index, assignments, stats) - : markEnrollOrNull(alias, assignments, enroll, pick, stats); + : markEnrollOrNull(assignments, enroll, pick, stats); String site = markData.getSite(); String target = markData.getTarget(); @@ -571,10 +554,6 @@ private static void printSummary(Stats total) { System.out.printf("%d enrolling aliases got none of the expected targets with a condition from" + " /assign, so made the null mark instead%n", total.enrollFallbacks.get()); } - if (total.unexpectedDecisionPoints.get() > 0) { - System.out.printf("ERROR %d enrolling aliases got targets from /assign that aren't expected: %s%n", - total.unexpectedDecisionPoints.get(), reportedUnexpected); - } } printLatency("assign", total.assignMs); printLatency("mark", total.markMs); diff --git a/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurstUi.java b/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurstUi.java new file mode 100644 index 000000000..1b15a05c3 --- /dev/null +++ b/clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestBurstUi.java @@ -0,0 +1,283 @@ +package org.upgradeplatform.client; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +/** + * A basic local web page for QuickTestBurst: fill in the options, start a run, watch its output, stop it. + * + * 
        + * yarn quickburst:ui                  (from the repo root; compiles first, see scripts/quickburst-ui.sh)
+ * yarn quickburst:ui --port 8091
+ * mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTestBurstUi"
+ * mvn exec:java -Dexec.mainClass="org.upgradeplatform.client.QuickTestBurstUi" -Dexec.args="--port 8091"
+ *
        + * + * then open http://localhost:8090 (or the port given). Ctrl-C stops it, along with any run in progress. + * + * Each run is its own QuickTestBurst process, started with this one's classpath and environment, so {@code qa} and + * {@code staging} need UPGRADE_QA_URL / UPGRADE_STAGING_URL set (see QuickTestHosts) before this starts; the page + * shows which are missing. One run at a time. The page shows the command line for the options it sends, so any run + * can be repeated from a terminal. It listens on localhost only. + */ +public class QuickTestBurstUi { + private static final int DEFAULT_PORT = 8090; + // Every POST must carry this. A page on another site can't add it without a CORS preflight, which this never + // answers, so another site can't start runs. + private static final String REQUEST_HEADER = "X-QuickTestBurst"; + // Older lines are dropped past this, so a long --verbose run doesn't grow without bound. The summary is printed + // last, so it's always kept. + private static final int MAX_LINES = 20_000; + private static final String JSON = "application/json; charset=utf-8"; + + private static int port = DEFAULT_PORT; + private static String page; + private static int runCount = 0; + private static Run current = null; + + /** One QuickTestBurst process and the output it has printed so far. */ + private static final class Run { + final int id; + final List args; + final Process process; + // These three are guarded by this Run. + final List lines = new ArrayList<>(); + int droppedLines = 0; + Integer exitCode = null; + + Run(int id, List args, Process process) { + this.id = id; + this.args = args; + this.process = process; + } + + synchronized void add(String line) { + lines.add(line); + if (lines.size() > MAX_LINES) { + int drop = MAX_LINES / 2; + lines.subList(0, drop).clear(); + droppedLines += drop; + } + } + + synchronized void finish(int code) { + exitCode = code; + } + + /** Status as JSON, with the lines from line number {@code from} on. */ + synchronized String status(int from) { + int start = Math.max(from, droppedLines); + int next = droppedLines + lines.size(); + List newLines = start >= next ? List.of() : lines.subList(start - droppedLines, lines.size()); + return "{\"id\":" + id + ",\"running\":" + (exitCode == null) + ",\"exitCode\":" + exitCode + ",\"args\":" + + json(args) + ",\"from\":" + start + ",\"next\":" + next + ",\"lines\":" + json(newLines) + "}"; + } + } + + public static void main(String[] args) throws IOException { + if (args.length == 2 && args[0].equals("--port")) { + port = Integer.parseInt(args[1]); + } else if (args.length != 0) { + System.err.println("usage: QuickTestBurstUi [--port " + DEFAULT_PORT + "]"); + System.exit(2); + } + try (InputStream in = QuickTestBurstUi.class.getResourceAsStream("quicktest-burst-ui.html")) { + page = new String(in.readAllBytes(), StandardCharsets.UTF_8); + } + + HttpServer server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), port), 0); + server.createContext("/", QuickTestBurstUi::handle); + server.start(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + Run run = currentRun(); + if (run != null) { + run.process.destroy(); + } + })); + System.out.println("QuickTestBurst UI at http://localhost:" + port); + } + + private static void handle(HttpExchange exchange) throws IOException { + try { + if (!allowed(exchange)) { + send(exchange, 403, "text/plain", "forbidden"); + return; + } + String route = exchange.getRequestMethod() + " " + exchange.getRequestURI().getPath(); + switch (route) { + case "GET /" -> send(exchange, 200, "text/html; charset=utf-8", page); + case "GET /config" -> send(exchange, 200, JSON, "{\"qa\":" + isSet("UPGRADE_QA_URL") + ",\"staging\":" + + isSet("UPGRADE_STAGING_URL") + "}"); + case "GET /status" -> status(exchange); + case "POST /run" -> run(exchange); + case "POST /stop" -> stop(exchange); + default -> send(exchange, 404, "text/plain", "not found"); + } + } catch (Exception e) { + send(exchange, 500, "text/plain", String.valueOf(e.getMessage())); + } finally { + exchange.close(); + } + } + + /** + * Only requests addressed to localhost, so another site can't reach this by pointing its own name at 127.0.0.1, + * and only POSTs with {@link #REQUEST_HEADER}. + */ + private static boolean allowed(HttpExchange exchange) { + String host = exchange.getRequestHeaders().getFirst("Host"); + boolean local = ("localhost:" + port).equals(host) || ("127.0.0.1:" + port).equals(host); + boolean post = exchange.getRequestMethod().equals("POST"); + return local && (!post || exchange.getRequestHeaders().containsKey(REQUEST_HEADER)); + } + + /** {@code ?id=N&from=M}: the current run, with its output from line M on, or all of it if N isn't this run. */ + private static void status(HttpExchange exchange) throws IOException { + Run run = currentRun(); + if (run == null) { + send(exchange, 200, JSON, "{\"id\":null,\"running\":false}"); + return; + } + int id = -1; + int from = 0; + String query = exchange.getRequestURI().getQuery(); + for (String pair : query == null ? new String[0] : query.split("&")) { + String[] parts = pair.split("=", 2); + try { + if (parts[0].equals("id")) { + id = Integer.parseInt(parts[1]); + } else if (parts[0].equals("from")) { + from = Integer.parseInt(parts[1]); + } + } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) { + // Ignored: the run's whole output is sent. + } + } + send(exchange, 200, JSON, run.status(id == run.id ? from : 0)); + } + + /** Starts QuickTestBurst with the request body's lines as its arguments, unless a run is already going. */ + private static void run(HttpExchange exchange) throws IOException { + List args = Arrays.stream(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8) + .split("\r?\n")).filter(arg -> !arg.isEmpty()).toList(); + Run run; + synchronized (QuickTestBurstUi.class) { + if (current != null && current.process.isAlive()) { + send(exchange, 409, "text/plain", "a run is already going; stop it first"); + return; + } + List command = new ArrayList<>(List.of( + Path.of(System.getProperty("java.home"), "bin", "java").toString(), "-cp", classpath(), + QuickTestBurst.class.getName())); + command.addAll(args); + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + run = new Run(++runCount, args, process); + current = run; + } + Thread reader = new Thread(() -> readOutput(run), "quicktest-burst-output-" + run.id); + reader.setDaemon(true); + reader.start(); + System.out.println("run " + run.id + ": QuickTestBurst " + String.join(" ", args)); + send(exchange, 200, JSON, "{\"id\":" + run.id + "}"); + } + + private static void readOutput(Run run) { + try (BufferedReader in = new BufferedReader( + new InputStreamReader(run.process.getInputStream(), StandardCharsets.UTF_8))) { + for (String line = in.readLine(); line != null; line = in.readLine()) { + run.add(line); + } + } catch (IOException e) { + run.add("(stopped reading output: " + e.getMessage() + ")"); + } + try { + run.finish(run.process.waitFor()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private static void stop(HttpExchange exchange) throws IOException { + Run run = currentRun(); + if (run != null && run.process.isAlive()) { + run.add("(stopped from the page)"); + run.process.destroy(); + } + send(exchange, 200, JSON, "{}"); + } + + private static synchronized Run currentRun() { + return current; + } + + /** + * The classpath this class came from, for the QuickTestBurst process. Under mvn exec:java that's the exec plugin's + * class loader; java.class.path is Maven's own there. + */ + private static String classpath() { + if (QuickTestBurstUi.class.getClassLoader() instanceof URLClassLoader loader) { + List entries = new ArrayList<>(); + for (URL url : loader.getURLs()) { + try { + entries.add(Path.of(url.toURI()).toString()); + } catch (URISyntaxException e) { + throw new IllegalStateException("can't use classpath entry " + url, e); + } + } + return String.join(File.pathSeparator, entries); + } + return System.getProperty("java.class.path"); + } + + private static boolean isSet(String name) { + String value = System.getenv(name); + return value != null && !value.isBlank(); + } + + private static void send(HttpExchange exchange, int code, String contentType, String body) throws IOException { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", contentType); + exchange.getResponseHeaders().set("Cache-Control", "no-store"); + exchange.sendResponseHeaders(code, bytes.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(bytes); + } + } + + private static String json(List values) { + return values.stream().map(QuickTestBurstUi::json).collect(Collectors.joining(",", "[", "]")); + } + + private static String json(String value) { + StringBuilder out = new StringBuilder("\""); + for (char c : value.toCharArray()) { + switch (c) { + case '"' -> out.append("\\\""); + case '\\' -> out.append("\\\\"); + case '\n' -> out.append("\\n"); + case '\r' -> out.append("\\r"); + case '\t' -> out.append("\\t"); + default -> out.append(c < 0x20 ? String.format("\\u%04x", (int) c) : String.valueOf(c)); + } + } + return out.append('"').toString(); + } +} diff --git a/clientlibs/java/src/main/resources/org/upgradeplatform/client/quicktest-burst-ui.html b/clientlibs/java/src/main/resources/org/upgradeplatform/client/quicktest-burst-ui.html new file mode 100644 index 000000000..a95601dd5 --- /dev/null +++ b/clientlibs/java/src/main/resources/org/upgradeplatform/client/quicktest-burst-ui.html @@ -0,0 +1,543 @@ + + + + + +QuickTestBurst + + + +
        +

        QuickTestBurst

        +

        Runs QuickTestBurst with these options. One run at a time.

        +
        +
        +
        +
        + Target +
        + + + + + + + + + +
        +
        + +
        + Burst sizes (alias pairs per burst) +
        + + + + + + +
        +
        + +
        + Pacing +
        + + + + + + + +
        +
        + +
        + Other +
        + + + +
        +
        + +
        + +
        +
        + + +
        +
        + + + Connecting… +
        +
        + 
        
+  
        
+
        
+
+
+
+
diff --git a/package.json b/package.json
index 2a26a6221..4a9824cee 100644
--- a/package.json
+++ b/package.json
@@ -37,7 +37,8 @@
     "build:all": "yarn types build && concurrently \"yarn backend build\" \"yarn js-clientlib build\" \"yarn frontend build\"",
     "launch": "concurrently -p \"[{name}]\" -n \"types,backend,frontend\" -c \"blue,magenta,cyan\" \"yarn types build\" \"yarn backend dev\" \"yarn frontend start\"",
     "clean-launch": "yarn clean && yarn install:all && yarn build:all && yarn launch",
-    "cache-demo": "./scripts/cache-demo.sh"
+    "cache-demo": "./scripts/cache-demo.sh",
+    "quickburst:ui": "./scripts/quickburst-ui.sh"
   },
   "lint-staged": {
     "./packages/frontend/projects/upgrade/src/**/*.{ts,js,html,json,scss}": [
diff --git a/scripts/quickburst-ui.sh b/scripts/quickburst-ui.sh
new file mode 100755
index 000000000..b1191037e
--- /dev/null
+++ b/scripts/quickburst-ui.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# Starts QuickTestBurstUi, the local web page for QuickTestBurst runs (clientlibs/java).
+#
+#   yarn quickburst:ui               serve it at http://localhost:8090
+#   yarn quickburst:ui --port 8091   on another port, e.g. when one is already running in another worktree
+#
+# Compiles the Java client first, so the page always runs the current code. Needs Maven and a JDK (17+).
+# --env qa / staging need UPGRADE_QA_URL / UPGRADE_STAGING_URL exported in the shell this runs from; see
+# clientlibs/java/src/main/java/org/upgradeplatform/client/QuickTestHosts.java. Ctrl-C stops it, along with any run in
+# progress.
+
+set -euo pipefail
+
+cd "$(dirname "$0")/../clientlibs/java"
+exec mvn -q compile exec:java -Dexec.mainClass=org.upgradeplatform.client.QuickTestBurstUi ${1:+"-Dexec.args=$*"}

From e74e94b2428dc79909b96abb716bc33cb20e64ef Mon Sep 17 00:00:00 2001
From: danoswaltCL <97542869+danoswaltCL@users.noreply.github.com>
Date: Fri, 23 Oct 2026 11:27:18 -0400
Subject: [PATCH 4/4] Fix URL validation in run button click handler

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
---
 .../org/upgradeplatform/client/quicktest-burst-ui.html     | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/clientlibs/java/src/main/resources/org/upgradeplatform/client/quicktest-burst-ui.html b/clientlibs/java/src/main/resources/org/upgradeplatform/client/quicktest-burst-ui.html
index a95601dd5..4493fa391 100644
--- a/clientlibs/java/src/main/resources/org/upgradeplatform/client/quicktest-burst-ui.html
+++ b/clientlibs/java/src/main/resources/org/upgradeplatform/client/quicktest-burst-ui.html
@@ -497,8 +497,13 @@ 
        QuickTestBurst
        
   }
 
   runButton.addEventListener('click', async () => {
+    const values = formValues();
+    if (values.host === 'url' && !values.url.trim()) {
+      setState('Enter a base URL before starting a run.', 'bad');
+      return;
+    }
     runButton.disabled = true;
-    const res = await post('run', buildArgs(formValues()).join('\n'));
+    const res = await post('run', buildArgs(values).join('\n'));
     if (!res.ok) setState(await res.text(), 'bad');
     pollSoon();
   });