Skip to content

fix: refresh EKS authentication token per request #4741

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hwayoungjun wants to merge 3 commits into
base: master
Choose a base branch
from
+245 −76
Open

fix: refresh EKS authentication token per request #4741

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e03ee9f
fix: refresh EKS authentication token per request
hwayoungjun May 6, 2026
11c4de8
fix: reuse RefreshAuthentication for EKS token refresh
hwayoungjun May 8, 2026
9af4ea0
fix: correct EKS auth token encoding and extract token supplier
hwayoungjun May 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 80 additions & 60 deletions util/src/main/java/io/kubernetes/client/util/credentials/EKSAuthentication.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,31 +12,30 @@
*/
package io.kubernetes.client.util.credentials;

import io.kubernetes.client.openapi.ApiClient;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Base64;
import java.util.function.Supplier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.http.SdkHttpMethod;
import software.amazon.awssdk.http.SdkHttpRequest;
import software.amazon.awssdk.http.auth.aws.signer.AwsV4FamilyHttpSigner;
import software.amazon.awssdk.http.auth.aws.signer.AwsV4HttpSigner;
import software.amazon.awssdk.http.auth.spi.signer.SignedRequest;
import software.amazon.awssdk.utils.http.SdkHttpUtils;

import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Base64;

/**
* EKS cluster authentication which generates a bearer token from AWS AK/SK. It doesn't require an "aws"
* command line tool in the $PATH.
*/
public class EKSAuthentication implements Authentication {
public class EKSAuthentication extends RefreshAuthentication {

private static final Logger log = LoggerFactory.getLogger(EKSAuthentication.class);
private static final int MAX_EXPIRY_SECONDS = 60 * 15;

/**
* Instantiates a new Eks authentication.
Expand All @@ -50,61 +49,82 @@ public EKSAuthentication(AwsCredentialsProvider provider, String region, String
}

public EKSAuthentication(AwsCredentialsProvider provider, String region, String clusterName, int expirySeconds) {
this.provider = provider;
this.region = region;
this.clusterName = clusterName;
if (expirySeconds > MAX_EXPIRY_SECONDS) {
expirySeconds = MAX_EXPIRY_SECONDS;
}
this.expirySeconds = expirySeconds;
this.stsEndpoint = URI.create("https://sts." + this.region + ".amazonaws.com");
Copy link
Copy Markdown
Contributor

@brendandburns brendandburns May 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's keep the STS endpoint as a member variable vs. constructing it every time.

Copy link
Copy Markdown
Author

@hwayoungjun hwayoungjun May 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@brendandburns This was also a side effect of using static methods instead of a separate supplier class. With the small supplier implementation outlined below, stsEndpoint could be kept as a field and would not need to be reconstructed each time. I can update it that way if that direction sounds right.

this(provider, region, clusterName, expirySeconds, Clock.systemUTC());
}

private static final int MAX_EXPIRY_SECONDS = 60 * 15;
private final AwsCredentialsProvider provider;
private final String region;
private final String clusterName;
private final URI stsEndpoint;

private final int expirySeconds;

@Override
public void provide(ApiClient client) {
SdkHttpRequest httpRequest = generateStsRequest();
String presignedUrl = requestToPresignedUrl(httpRequest);
String encodedUrl = presignedUrlToEncodedUrl(presignedUrl);
String token = "k8s-aws-v1." + encodedUrl;
client.setApiKeyPrefix("Bearer");
client.setApiKey(token);
log.info("Generated BEARER token for ApiClient, expiring at {}", Instant.now().plus(expirySeconds, ChronoUnit.SECONDS));
EKSAuthentication(
AwsCredentialsProvider provider, String region, String clusterName, int expirySeconds, Clock clock) {
super(
new EksTokenSupplier(provider, region, clusterName, cappedExpirySeconds(expirySeconds)),
Duration.of(cappedExpirySeconds(expirySeconds), ChronoUnit.SECONDS),
clock);
setExpiry(Instant.now(clock).plus(cappedExpirySeconds(expirySeconds), ChronoUnit.SECONDS));
}

private static String presignedUrlToEncodedUrl(String presignedUrl) {
return Base64.getUrlEncoder()
.withoutPadding()
.encodeToString(SdkHttpUtils.urlEncodeIgnoreSlashes(presignedUrl).getBytes(StandardCharsets.UTF_8));
Copy link
Copy Markdown
Contributor

@brendandburns brendandburns May 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are we removing the urlEncodeIgnoreSlashes call?

Copy link
Copy Markdown
Author

@hwayoungjun hwayoungjun May 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@brendandburns Thanks for catching this.

I checked the AWS SDK path locally. The signer adds the X-Amz-* values as raw query parameters, and SdkHttpRequest.getUri() builds the final URI through encodedQueryParameters(), so the returned presigned URL already contains percent-encoded query values such as X-Amz-Credential=...%2F....

That was why I initially removed the extra urlEncodeIgnoreSlashes step. It looked like another encoding pass on top of an already encoded presigned URL.

However, I should have called this out explicitly in the PR, and this is outside the refresh-related scope of the change.

Since the existing implementation included this extra encoding step, I agree it is safer to preserve the previous token payload behavior. I also should have considered compatibility with other AWS SDK versions and the existing behavior more carefully.

I will restore urlEncodeIgnoreSlashes.

Copy link
Copy Markdown
Author

@hwayoungjun hwayoungjun May 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@brendandburns

I looked into this more deeply with an actual EKS API request, and I think the issue is a bit broader than the refresh change itself.

What I found

It looks like the existing EKSAuthentication path may not have been producing a token payload that EKS accepts.

In my own usage, I had been using a custom Authentication implementation instead of EKSAuthentication, so I did not catch this earlier. While verifying this PR against a real EKS cluster, I found that the current EKSAuthentication token payload is rejected with 401 Unauthorized.

Token payload difference

The expected decoded token payload is the presigned STS URL itself:

https://sts.ap-northeast-2.amazonaws.com/?Version=2011-06-15&Action=GetCallerIdentity&X-Amz-Credential=...%2F...

However, with the extra SdkHttpUtils.urlEncodeIgnoreSlashes(presignedUrl) step, the decoded token payload becomes a URL-encoded string:

https%3A//sts.ap-northeast-2.amazonaws.com%3FVersion%3D2011-06-15%26Action%3DGetCallerIdentity%26X-Amz-Credential%3D...%252F...

That means the URL structure itself is encoded, and already encoded query values such as %2F become double-encoded as %252F.

STS path

I also changed the STS request construction to keep the endpoint URI pathless and set the request path explicitly:

.uri(stsEndpoint)
.encodedPath("/")

This produces the working presigned URL form:

https://sts...amazonaws.com/?...

Reproduction branch

I created a separate branch to make this easier to verify outside this PR:

https://github.com/hwayoungjun/java/tree/repro/eks-authentication-401

That branch compares:

  • the current EKSAuthentication
  • FixedEKSAuthentication, copied from this PR's implementation

The smoke test result against a real EKS cluster was:

Current master decoded payload prefix:
https%3A//sts.ap-northeast-2.amazonaws.com%3F...
Kubernetes API request failed with HTTP 401

Fixed decoded payload prefix:
https://sts.ap-northeast-2.amazonaws.com/?...
Kubernetes API request succeeded, namespaces=13

Run command:

./mvnw -pl util -Dtest=EKSAuthenticationSmokeTest -Dsurefire.failIfNoSpecifiedTests=false test

Conclusion

From the real EKS verification, it looks like the existing EKSAuthentication token payload is not accepted by EKS today.

Because of that, I committed the token payload fix here by removing the extra whole-URL encoding and making the STS root path explicit with encodedPath("/").

That said, I realize this is separate from the refresh behavior itself. Would you prefer that I split the token payload fix into a separate issue/PR, and keep this PR focused only on refresh support?

private static int cappedExpirySeconds(int expirySeconds) {
return Math.min(expirySeconds, MAX_EXPIRY_SECONDS);
}

private SdkHttpRequest generateStsRequest() {
return SdkHttpRequest.builder()
.uri(stsEndpoint)
.putRawQueryParameter("Version", "2011-06-15")
.putRawQueryParameter("Action", "GetCallerIdentity")
.method(SdkHttpMethod.GET)
.putHeader("x-k8s-aws-id", clusterName)
.build();
}
private static class EksTokenSupplier implements Supplier<String> {
private final AwsCredentialsProvider provider;
private final String region;
private final String clusterName;
private final int expirySeconds;
private final URI stsEndpoint;

EksTokenSupplier(AwsCredentialsProvider provider, String region, String clusterName, int expirySeconds) {
this.provider = provider;
this.region = region;
this.clusterName = clusterName;
this.expirySeconds = expirySeconds;
this.stsEndpoint = URI.create("https://sts." + this.region + ".amazonaws.com");
}

@Override
public String get() {
return generateToken();
}

private String requestToPresignedUrl(SdkHttpRequest httpRequest) {
AwsV4HttpSigner signer = AwsV4HttpSigner.create();
SignedRequest signedRequest =
signer.sign(r -> r.identity(this.provider.resolveCredentials())
.request(httpRequest)
.putProperty(AwsV4HttpSigner.SERVICE_SIGNING_NAME, "sts")
.putProperty(AwsV4HttpSigner.REGION_NAME, region)
.putProperty(AwsV4HttpSigner.AUTH_LOCATION, AwsV4HttpSigner.AuthLocation.QUERY_STRING)
.putProperty(AwsV4HttpSigner.EXPIRATION_DURATION, Duration.of(60, ChronoUnit.SECONDS)));
SdkHttpRequest request = signedRequest.request();
return request.getUri().toString();
private String generateToken() {
SdkHttpRequest httpRequest = generateStsRequest();
String presignedUrl = requestToPresignedUrl(httpRequest);
String encodedUrl = presignedUrlToEncodedUrl(presignedUrl);
log.info(
"Generated BEARER token for ApiClient, expiring at {}",
Instant.now().plus(expirySeconds, ChronoUnit.SECONDS));
return "k8s-aws-v1." + encodedUrl;
}

private static String presignedUrlToEncodedUrl(String presignedUrl) {
return Base64.getUrlEncoder()
.withoutPadding()
.encodeToString(presignedUrl.getBytes(StandardCharsets.UTF_8));
}

private SdkHttpRequest generateStsRequest() {
return SdkHttpRequest.builder()
.uri(stsEndpoint)
.encodedPath("/")
.putRawQueryParameter("Version", "2011-06-15")
.putRawQueryParameter("Action", "GetCallerIdentity")
.method(SdkHttpMethod.GET)
.putHeader("x-k8s-aws-id", clusterName)
.build();
}

private String requestToPresignedUrl(SdkHttpRequest httpRequest) {
AwsV4HttpSigner signer = AwsV4HttpSigner.create();
SignedRequest signedRequest =
signer.sign(r -> r.identity(this.provider.resolveCredentials())
.request(httpRequest)
.putProperty(AwsV4HttpSigner.SERVICE_SIGNING_NAME, "sts")
.putProperty(AwsV4HttpSigner.REGION_NAME, region)
.putProperty(AwsV4HttpSigner.AUTH_LOCATION, AwsV4HttpSigner.AuthLocation.QUERY_STRING)
.putProperty(
AwsV4HttpSigner.EXPIRATION_DURATION,
Duration.of(expirySeconds, ChronoUnit.SECONDS)));
SdkHttpRequest request = signedRequest.request();
return request.getUri().toString();
}
}
}
181 changes: 165 additions & 16 deletions util/src/test/java/io/kubernetes/client/util/credentials/EKSAuthenticationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,37 +12,186 @@
*/
package io.kubernetes.client.util.credentials;

import static com.github.tomakehurst.wiremock.client.WireMock.get;
import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor;
import static com.github.tomakehurst.wiremock.client.WireMock.matching;
import static com.github.tomakehurst.wiremock.client.WireMock.okForContentType;
import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo;
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import com.github.tomakehurst.wiremock.junit5.WireMockExtension;
import io.kubernetes.client.openapi.ApiClient;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.Configuration;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import java.nio.charset.StandardCharsets;
import java.time.Clock;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.util.Base64;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;

import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

@ExtendWith(MockitoExtension.class)
class EKSAuthenticationTest {

@Mock
private AwsCredentialsProvider provider;
private static final String REGION = "us-west-2";
private static final String CLUSTER_NAME = "test-2";
private static final String LIST_PODS_PATH = "/api/v1/pods";
private static final String BEARER_TOKEN_PREFIX = "Bearer k8s-aws-v1.";

@RegisterExtension
static WireMockExtension apiServer =
WireMockExtension.newInstance().options(options().dynamicPort()).build();

@Mock private AwsCredentialsProvider provider;

private ApiClient apiClient;
private Instant instant;
private MockClock clock;

@BeforeEach
void setup() {
this.apiClient = new ApiClient();
this.apiClient.setBasePath("http://localhost:" + apiServer.getPort());
Configuration.setDefaultApiClient(this.apiClient);

this.instant = Instant.now();
this.clock = new MockClock(this.instant);
}

@Test
void addsBearerTokenToRequests() throws ApiException {
when(provider.resolveCredentials()).thenReturn(credentials("ak", "session"));
CoreV1Api api = authenticatedApi(new EKSAuthentication(provider, REGION, CLUSTER_NAME));

stubListPods();
listPods(api);

apiServer.verify(
1,
getRequestedFor(urlPathEqualTo(LIST_PODS_PATH))
.withHeader("Authorization", matching("Bearer k8s-aws-v1\\..+")));
verify(provider).resolveCredentials();
}

@Test
void reusesTokenBeforeExpiry() throws ApiException {
when(provider.resolveCredentials()).thenReturn(credentials("ak", "session"));
CoreV1Api api = authenticatedApi(new EKSAuthentication(provider, REGION, CLUSTER_NAME, 60, clock));

stubListPods();
listPods(api);
listPods(api);

List<String> authorizations = authorizationHeaders();
assertThat(authorizations).hasSize(2);
assertThat(authorizations.get(0)).isEqualTo(authorizations.get(1));
verify(provider).resolveCredentials();
}

@Mock
private ApiClient apiClient;
@Test
void refreshesTokenAfterExpiry() throws ApiException {
when(provider.resolveCredentials())
.thenReturn(credentials("ak-1", "session-1"))
.thenReturn(credentials("ak-2", "session-2"));
CoreV1Api api = authenticatedApi(new EKSAuthentication(provider, REGION, CLUSTER_NAME, 60, clock));

private String region = "us-west-2";
stubListPods();
listPods(api);
clock.setInstant(instant.plusSeconds(70));
listPods(api);

private String clusterName = "test-2";
List<String> authorizations = authorizationHeaders();
assertThat(authorizations).hasSize(2);
assertThat(authorizations.get(0)).isNotEqualTo(authorizations.get(1));
assertThat(decodedTokenUrl(authorizations.get(0))).contains("X-Amz-Credential=ak-1%2F");
assertThat(decodedTokenUrl(authorizations.get(1))).contains("X-Amz-Credential=ak-2%2F");
verify(provider, times(2)).resolveCredentials();
}

@Test
void provideApiClient() {
when(provider.resolveCredentials()).thenReturn(AwsSessionCredentials.create("ak", "sk", "session"));
EKSAuthentication authentication = new EKSAuthentication(provider, region, clusterName);
authentication.provide(apiClient);
verify(apiClient).setApiKey(anyString());
verify(apiClient).setApiKeyPrefix(anyString());
void expirySecondsAreCapped() throws ApiException {
when(provider.resolveCredentials()).thenReturn(credentials("ak", "session"));
CoreV1Api api = authenticatedApi(new EKSAuthentication(provider, REGION, CLUSTER_NAME, 1_000));

stubListPods();
listPods(api);

assertThat(decodedTokenUrl(authorizationHeaders().get(0)))
.startsWith("https://sts.us-west-2.amazonaws.com/?")
.contains("X-Amz-Expires=900");
}

private CoreV1Api authenticatedApi(EKSAuthentication authentication) {
authentication.provide(apiClient);
return new CoreV1Api();
}

private AwsSessionCredentials credentials(String accessKeyId, String sessionToken) {
return AwsSessionCredentials.create(accessKeyId, "sk", sessionToken);
}

private void stubListPods() {
apiServer.stubFor(
get(urlPathEqualTo(LIST_PODS_PATH))
.willReturn(okForContentType("application/json", "{\"items\":[]}")));
}

private void listPods(CoreV1Api api) throws ApiException {
api.listPodForAllNamespaces().execute();
}

private List<String> authorizationHeaders() {
return apiServer.getAllServeEvents().stream()
.sorted(Comparator.comparing(event -> event.getRequest().getLoggedDate()))
.map(event -> event.getRequest().getHeader("Authorization"))
.collect(Collectors.toList());
}

private String decodedTokenUrl(String authorization) {
String encodedToken = authorization.substring(BEARER_TOKEN_PREFIX.length());
return new String(Base64.getUrlDecoder().decode(encodedToken), StandardCharsets.UTF_8);
}

static class MockClock extends Clock {
Instant now;

MockClock(Instant start) {
this.now = start;
}

void setInstant(Instant instant) {
this.now = instant;
}

@Override
public Instant instant() {
return now;
}

@Override
public ZoneId getZone() {
return ZoneOffset.UTC;
}

@Override
public Clock withZone(ZoneId zone) {
throw new UnsupportedOperationException();
}
}
}