Add --force-refresh support for Databricks CLI token fetching

Try `--force-refresh` before the regular CLI command so the SDK can
bypass the CLI's own token cache when the SDK considers its token stale.
If the CLI is too old to recognise `--force-refresh` (or `--profile`),
gracefully fall back to the next command in the chain.

Chain order:
- with profile: forceCmd (--profile --force-refresh) -> profileCmd (--profile) -> fallbackCmd (--host)
- without profile: forceCmd (--host --force-refresh) -> profileCmd (--host)

Azure CLI callers are unchanged; they use constructors that leave
forceCmd null, preserving existing behavior.

Signed-off-by: Mihai Mitrea <mihai.mitrea@databricks.com>

Author: mihaimitrea-db

NEXT_CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### New Features and Improvements
 * Added automatic detection of AI coding agents (Antigravity, Claude Code, Cline, Codex, Copilot CLI, Cursor, Gemini CLI, OpenCode) in the user-agent string. The SDK now appends `agent/<name>` to HTTP request headers when running inside a known AI agent environment.
+* Pass `--force-refresh` to Databricks CLI `auth token` command so the SDK always receives a fresh token instead of a potentially stale one from the CLI's internal cache. Falls back gracefully on older CLIs that do not support this flag.
 
 ### Bug Fixes
 * Fixed Databricks CLI authentication to detect when the cached token's scopes don't match the SDK's configured scopes. Previously, a scope mismatch was silently ignored, causing requests to use wrong permissions. The SDK now raises an error with instructions to re-authenticate.

databricks-sdk-java/src/main/java/com/databricks/sdk/core/CliTokenSource.java

@@ -26,15 +26,12 @@ public class CliTokenSource implements TokenSource {
   private static final Logger LOG = LoggerFactory.getLogger(CliTokenSource.class);
 
   private List<String> cmd;
+  private List<String> fallbackCmd;
+  private List<String> secondFallbackCmd;
   private String tokenTypeField;
   private String accessTokenField;
   private String expiryField;
   private Environment env;
-  // fallbackCmd is tried when the primary command fails with "unknown flag: --profile",
-  // indicating the CLI is too old to support --profile. Can be removed once support
-  // for CLI versions predating --profile is dropped.
-  // See: https://github.com/databricks/databricks-sdk-go/pull/1497
-  private List<String> fallbackCmd;
 
   /**
    * Internal exception that carries the clean stderr message but exposes full output for checks.
@@ -58,7 +55,7 @@ public CliTokenSource(
       String accessTokenField,
       String expiryField,
       Environment env) {
-    this(cmd, tokenTypeField, accessTokenField, expiryField, env, null);
+    this(cmd, tokenTypeField, accessTokenField, expiryField, env, null, null);
   }
 
   public CliTokenSource(
@@ -67,15 +64,19 @@ public CliTokenSource(
       String accessTokenField,
       String expiryField,
       Environment env,
-      List<String> fallbackCmd) {
-    super();
+      List<String> fallbackCmd,
+      List<String> secondFallbackCmd) {
     this.cmd = OSUtils.get(env).getCliExecutableCommand(cmd);
     this.tokenTypeField = tokenTypeField;
     this.accessTokenField = accessTokenField;
     this.expiryField = expiryField;
     this.env = env;
     this.fallbackCmd =
         fallbackCmd != null ? OSUtils.get(env).getCliExecutableCommand(fallbackCmd) : null;
+    this.secondFallbackCmd =
+        secondFallbackCmd != null
+            ? OSUtils.get(env).getCliExecutableCommand(secondFallbackCmd)
+            : null;
   }
 
   /**
@@ -153,27 +154,47 @@ private Token execCliCommand(List<String> cmdToRun) throws IOException {
     }
   }
 
+  private String getErrorText(IOException e) {
+    return e instanceof CliCommandException
+        ? ((CliCommandException) e).getFullOutput()
+        : e.getMessage();
+  }
+
+  private boolean isUnknownFlagError(String errorText) {
+    return errorText != null && errorText.contains("unknown flag:");
+  }
+
   @Override
   public Token getToken() {
     try {
       return execCliCommand(this.cmd);
     } catch (IOException e) {
-      String textToCheck =
-          e instanceof CliCommandException
-              ? ((CliCommandException) e).getFullOutput()
-              : e.getMessage();
-      if (fallbackCmd != null
-          && textToCheck != null
-          && textToCheck.contains("unknown flag: --profile")) {
+      if (fallbackCmd != null && isUnknownFlagError(getErrorText(e))) {
         LOG.warn(
-            "Databricks CLI does not support --profile flag. Falling back to --host. "
+            "CLI does not support some flags used by this SDK. "
+                + "Falling back to a compatible command. "
                 + "Please upgrade your CLI to the latest version.");
-        try {
-          return execCliCommand(this.fallbackCmd);
-        } catch (IOException fallbackException) {
-          throw new DatabricksException(fallbackException.getMessage(), fallbackException);
-        }
+      } else {
+        throw new DatabricksException(e.getMessage(), e);
+      }
+    }
+
+    try {
+      return execCliCommand(this.fallbackCmd);
+    } catch (IOException e) {
+      if (secondFallbackCmd != null && isUnknownFlagError(getErrorText(e))) {
+        LOG.warn(
+            "CLI does not support some flags used by this SDK. "
+                + "Falling back to a compatible command. "
+                + "Please upgrade your CLI to the latest version.");
+      } else {
+        throw new DatabricksException(e.getMessage(), e);
       }
+    }
+
+    try {
+      return execCliCommand(this.secondFallbackCmd);
+    } catch (IOException e) {
       throw new DatabricksException(e.getMessage(), e);
     }
   }

databricks-sdk-java/src/main/java/com/databricks/sdk/core/DatabricksCliCredentialsProvider.java

@@ -69,6 +69,17 @@ List<String> buildHostArgs(String cliPath, DatabricksConfig config) {
     return cmd;
   }
 
+  List<String> buildProfileArgs(String cliPath, DatabricksConfig config) {
+    return new ArrayList<>(
+        Arrays.asList(cliPath, "auth", "token", "--profile", config.getProfile()));
+  }
+
+  private static List<String> withForceRefresh(List<String> cmd) {
+    List<String> forceCmd = new ArrayList<>(cmd);
+    forceCmd.add("--force-refresh");
+    return forceCmd;
+  }
+
   private CliTokenSource getDatabricksCliTokenSource(DatabricksConfig config) {
     String cliPath = config.getDatabricksCliPath();
     if (cliPath == null) {
@@ -81,23 +92,27 @@ private CliTokenSource getDatabricksCliTokenSource(DatabricksConfig config) {
 
     List<String> cmd;
     List<String> fallbackCmd = null;
+    List<String> secondFallbackCmd = null;
 
     if (config.getProfile() != null) {
-      // When profile is set, use --profile as the primary command.
-      // The profile contains the full config (host, account_id, etc.).
-      cmd =
-          new ArrayList<>(
-              Arrays.asList(cliPath, "auth", "token", "--profile", config.getProfile()));
-      // Build a --host fallback for older CLIs that don't support --profile.
+      List<String> profileArgs = buildProfileArgs(cliPath, config);
+      cmd = withForceRefresh(profileArgs);
+      fallbackCmd = profileArgs;
       if (config.getHost() != null) {
-        fallbackCmd = buildHostArgs(cliPath, config);
+        secondFallbackCmd = buildHostArgs(cliPath, config);
       }
     } else {
       cmd = buildHostArgs(cliPath, config);
     }
 
     return new CliTokenSource(
-        cmd, "token_type", "access_token", "expiry", config.getEnv(), fallbackCmd);
+        cmd,
+        "token_type",
+        "access_token",
+        "expiry",
+        config.getEnv(),
+        fallbackCmd,
+        secondFallbackCmd);
   }
 
   @Override