[PoC] Support fault injection in cloud acceptance runs

acceptance/README.md

@@ -15,6 +15,19 @@ The cloud run is keyed off `CLOUD_ENV`; see `getSkipReason` in `acceptance/accep
 
 To run tests against a real workspace: `deco env run -i -n aws-prod-ucws -- <go test command>` (requires the `deco` tool and access to a test env).
 
+## Fault injection
+
+Set `Proxy = true` in `test.toml` to use `fault.py` against either the fake server or a real workspace. Add `Cloud = true` for cloud runs. Recording is optional.
+
+```bash
+fault.py "POST /api/2.0/workspace/mkdirs" 403 0 1
+fault.py --after-handler "POST /api/2.2/jobs/run-now" 503 0 1
+```
+
+A normal fault prevents the upstream call. `--after-handler` forwards the call first and replaces its response, preserving its effects. Rules match the incoming test token; the proxy uses separate upstream credentials. Offsets and counts include client retries. Use precise patterns or `--body-contains` to select a resource, and clean up real resources after testing a failed response.
+
+The proxy requires localhost access and cannot be used with `RunsOnDbr`.
+
 ## Authoring
 
 To author a test,

acceptance/acceptance_test.go

@@ -919,6 +919,8 @@ func runTest(t *testing.T,
 	args := []string{"bash", "-euo", "pipefail", EntryPointScript}
 	cmd := exec.CommandContext(ctx, args[0], args[1:]...)
 
+	require.False(t, isTruePtr(config.RunsOnDbr) && LogRequests,
+		"RunsOnDbr cannot be combined with -logrequests: the proxy requires localhost access")
 	cfg, user, workspaceID := internal.PrepareServerAndClient(t, config, LogRequests, outputDir, testEnv)
 	testdiff.PrepareReplacementsUser(t, &repls, user)
 	testdiff.PrepareReplacementsWorkspaceConfig(t, &repls, cfg)

acceptance/bin/fault.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Set up a fault rule on the testserver for the current test token.
+"""Set up a fault rule on the testserver or test proxy for the current test token.
 
 Usage: fault.py [--after-handler] [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]
 
@@ -16,6 +16,7 @@
 
 The rule is scoped to the current DATABRICKS_TOKEN so it only affects
 the test that registers it, even when tests share a server.
+Cloud runs require Proxy = true in test.toml.
 """
 
 import json

acceptance/internal/config.go

@@ -78,8 +78,8 @@ type TestConfig struct {
 	// out.requests.txt
 	RecordRequests *bool
 
-	// If true, local runs route the CLI through the recording proxy (libs/testproxy)
-	// instead of straight to the testserver, matching the cloud topology.
+	// If true, route the CLI through libs/testproxy for fault injection,
+	// independently of recording. Supported locally and on cloud.
 	Proxy *bool
 
 	// List of request headers to include when recording requests.
@@ -252,10 +252,10 @@ func LoadConfig(t *testing.T, dir string) (TestConfig, string) {
 
 // validateConfig checks for incompatible configuration combinations.
 func validateConfig(t *testing.T, config TestConfig, configPath string) {
-	// RunsOnDbr and RecordRequests are incompatible because serverless does not
+	// RunsOnDbr cannot use the proxy because serverless does not
 	// allow access to localhost ports, which the test proxy server requires.
-	if isTruePtr(config.RunsOnDbr) && isTruePtr(config.RecordRequests) {
-		t.Fatalf("Invalid config %s: RunsOnDbr and RecordRequests cannot both be true. "+
+	if isTruePtr(config.RunsOnDbr) && (isTruePtr(config.RecordRequests) || isTruePtr(config.Proxy)) {
+		t.Fatalf("Invalid config %s: RunsOnDbr cannot be combined with RecordRequests or Proxy. "+
 			"Serverless does not allow access to localhost ports, which the test proxy server requires.", configPath)
 	}
 

acceptance/internal/prepare_server.go

@@ -129,10 +129,10 @@ func PrepareServerAndClient(t *testing.T, config TestConfig, logRequests bool, o
 		cfg := w.Config
 
 		// If we are running in a cloud environment AND we need to intercept requests
-		// (for recording or logging), start a proxy server.
-		if recordRequests || logRequests {
-			// An empty config resolves the real workspace from the environment.
-			host := startProxyServer(t, &sdkconfig.Config{}, recordRequests, logRequests, config.IncludeRequestHeaders, outputDir)
+		// (for fault injection, recording or logging), start a proxy server.
+		if isTruePtr(config.Proxy) || recordRequests || logRequests {
+			// Reuse the resolved cloud config for upstream auth and workspace routing.
+			host := startProxyServer(t, cfg, token, recordRequests, logRequests, config.IncludeRequestHeaders, outputDir)
 			cfg = &sdkconfig.Config{
 				Host:  host,
 				Token: token,
@@ -155,7 +155,7 @@ func PrepareServerAndClient(t *testing.T, config TestConfig, logRequests bool, o
 		host := startProxyServer(t, &sdkconfig.Config{
 			Host:  upstream,
 			Token: token,
-		}, recordRequests, logRequests, config.IncludeRequestHeaders, outputDir)
+		}, token, recordRequests, logRequests, config.IncludeRequestHeaders, outputDir)
 
 		cfg := &sdkconfig.Config{
 			Host:  host,
@@ -284,12 +284,13 @@ func startLocalServer(t *testing.T,
 
 func startProxyServer(t *testing.T,
 	upstream *sdkconfig.Config,
+	token string,
 	recordRequests bool,
 	logRequests bool,
 	includeHeaders []string,
 	outputDir string,
 ) string {
-	s := testproxy.New(t, upstream)
+	s := testproxy.New(t, upstream, token)
 
 	// Record API requests in out.requests.txt if RecordRequests is true in test.toml.
 	if recordRequests {

acceptance/selftest/fault_proxy/out.test.toml

@@ -0,0 +1 @@
+Cloud = true

acceptance/selftest/fault_proxy/output.txt

@@ -0,0 +1,38 @@
+
+=== a fault before forwarding prevents the mutation
+>>> musterr [CLI] workspace mkdirs /Users/[USERNAME]/fault-proxy-[UNIQUE_NAME]/blocked
+Error: Fault injected by test.
+
+Host:      [DATABRICKS_URL]
+Auth type: Personal Access Token (pat)
+
+Next steps:
+  - Verify you have the required permissions for this operation
+  - Check your identity: databricks auth describe
+  - Consider setting up a profile: databricks auth login --profile <name>
+blocked directory does not exist
+
+=== an after-handler fault preserves the upstream mutation
+>>> musterr [CLI] workspace mkdirs /Users/[USERNAME]/fault-proxy-[UNIQUE_NAME]/committed
+Error: Fault injected by test.
+
+Host:      [DATABRICKS_URL]
+Auth type: Personal Access Token (pat)
+
+Next steps:
+  - Verify you have the required permissions for this operation
+  - Check your identity: databricks auth describe
+  - Consider setting up a profile: databricks auth login --profile <name>
+
+>>> [CLI] workspace get-status /Users/[USERNAME]/fault-proxy-[UNIQUE_NAME]/committed
+{
+  "object_type": "DIRECTORY"
+}
+
+=== the CLI recovers from a retryable error after the mutation
+>>> [CLI] workspace mkdirs /Users/[USERNAME]/fault-proxy-[UNIQUE_NAME]/retried
+
+>>> [CLI] workspace get-status /Users/[USERNAME]/fault-proxy-[UNIQUE_NAME]/retried
+{
+  "object_type": "DIRECTORY"
+}

acceptance/selftest/fault_proxy/script

@@ -0,0 +1,22 @@
+root="/Users/$CURRENT_USER_NAME/fault-proxy-$UNIQUE_NAME"
+cleanup() {
+    $CLI workspace delete "$root" --recursive
+}
+$CLI workspace mkdirs "$root"
+trap cleanup EXIT
+
+title "a fault before forwarding prevents the mutation"
+fault.py --body-contains "$root/blocked" "POST /api/2.0/workspace/mkdirs" 403 0 1
+trace musterr $CLI workspace mkdirs "$root/blocked"
+musterr $CLI workspace get-status "$root/blocked" >/dev/null 2>&1
+echo "blocked directory does not exist"
+
+title "an after-handler fault preserves the upstream mutation"
+fault.py --after-handler --body-contains "$root/committed" "POST /api/2.0/workspace/mkdirs" 403 0 1
+trace musterr $CLI workspace mkdirs "$root/committed"
+trace $CLI workspace get-status "$root/committed" | jq '{object_type}'
+
+title "the CLI recovers from a retryable error after the mutation"
+fault.py --after-handler --body-contains "$root/retried" "POST /api/2.0/workspace/mkdirs" 503 0 1
+trace $CLI workspace mkdirs "$root/retried"
+trace $CLI workspace get-status "$root/retried" | jq '{object_type}'

acceptance/selftest/fault_proxy/test.toml

@@ -0,0 +1,4 @@
+Cloud = true
+Proxy = true
+RecordRequests = false
+Env.MSYS_NO_PATHCONV = '1'