From 1fe48e1da447e8e25060e3ef87e997c96b0423fd Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Wed, 11 Mar 2026 17:40:23 +0530 Subject: [PATCH 1/5] add user-agent header to template downloader request --- .../java/com/cloud/storage/template/HttpTemplateDownloader.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index 6fe001de72c0..90294fce7a71 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -125,6 +125,7 @@ private GetMethod createRequest(String downloadUrl) { GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); request.setFollowRedirects(followRedirects); + request.setRequestHeader("User-Agent", "CloudStack-Agent"); return request; } From c1eb8a5c571d38fece060cd413d46f7fe55b42a3 Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Thu, 12 Mar 2026 16:12:59 +0530 Subject: [PATCH 2/5] handle review comment --- api/src/main/java/org/apache/cloudstack/api/ApiConstants.java | 2 ++ .../com/cloud/storage/template/HttpTemplateDownloader.java | 3 ++- .../cloud/storage/template/MetalinkTemplateDownloader.java | 3 +++ .../cloud/storage/template/SimpleHttpMultiFileDownloader.java | 2 ++ .../direct/download/HttpDirectTemplateDownloader.java | 4 ++++ 5 files changed, 13 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 944b111eb70c..c288e43a3354 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1364,6 +1364,8 @@ public class ApiConstants { public static final String RECURSIVE_DOMAINS = "recursivedomains"; + public static final String CLOUDSTACK_USER_AGENT = "CloudStack-Agent"; + /** * This enum specifies IO Drivers, each option controls specific policies on I/O. * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0). diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index 90294fce7a71..056a0d8245bf 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -20,6 +20,7 @@ package com.cloud.storage.template; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; +import static org.apache.cloudstack.api.ApiConstants.CLOUDSTACK_USER_AGENT; import java.io.File; import java.io.IOException; @@ -125,7 +126,7 @@ private GetMethod createRequest(String downloadUrl) { GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); request.setFollowRedirects(followRedirects); - request.setRequestHeader("User-Agent", "CloudStack-Agent"); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, CLOUDSTACK_USER_AGENT); return request; } diff --git a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java index 95ed0d1e76d0..fee2143e2f02 100644 --- a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java @@ -18,6 +18,8 @@ // package com.cloud.storage.template; +import static org.apache.cloudstack.api.ApiConstants.CLOUDSTACK_USER_AGENT; + import com.cloud.storage.StorageLayer; import com.cloud.utils.UriUtils; import org.apache.commons.httpclient.HttpClient; @@ -59,6 +61,7 @@ protected GetMethod createRequest(String downloadUrl) { GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); request.setFollowRedirects(followRedirects); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, CLOUDSTACK_USER_AGENT); if (!toFileSet) { String[] parts = downloadUrl.split("/"); String filename = parts[parts.length - 1]; diff --git a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java index 8719947cb4f0..c36f4b2e18f9 100644 --- a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java @@ -18,6 +18,7 @@ package com.cloud.storage.template; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; +import static org.apache.cloudstack.api.ApiConstants.CLOUDSTACK_USER_AGENT; import java.io.File; import java.io.IOException; @@ -95,6 +96,7 @@ private GetMethod createRequest(String downloadUrl) { GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler); request.setFollowRedirects(followRedirects); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, CLOUDSTACK_USER_AGENT); return request; } diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java index c4a802ecdbc6..95bff46f26b7 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java @@ -19,6 +19,8 @@ package org.apache.cloudstack.direct.download; +import static org.apache.cloudstack.api.ApiConstants.CLOUDSTACK_USER_AGENT; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -39,6 +41,7 @@ import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.HeadMethod; +import org.apache.commons.httpclient.params.HttpMethodParams; import org.apache.commons.io.IOUtils; public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { @@ -68,6 +71,7 @@ public HttpDirectTemplateDownloader(String url, Long templateId, String destPool protected GetMethod createRequest(String downloadUrl, Map headers) { GetMethod request = new GetMethod(downloadUrl); request.setFollowRedirects(this.isFollowRedirects()); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, CLOUDSTACK_USER_AGENT); if (MapUtils.isNotEmpty(headers)) { for (String key : headers.keySet()) { request.setRequestHeader(key, headers.get(key)); From 6fd80d68c7d84e42b107e68b93d18850a83a2f9b Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Tue, 24 Mar 2026 16:34:54 +0530 Subject: [PATCH 3/5] add user agent adhereing to RFC format --- .../apache/cloudstack/api/ApiConstants.java | 2 - .../template/HttpTemplateDownloader.java | 4 +- .../template/MetalinkTemplateDownloader.java | 5 ++- .../SimpleHttpMultiFileDownloader.java | 4 +- .../HttpDirectTemplateDownloader.java | 4 +- .../net/HttpClientCloudStackUserAgent.java | 39 +++++++++++++++++++ 6 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 utils/src/main/java/com/cloud/utils/net/HttpClientCloudStackUserAgent.java diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index c288e43a3354..944b111eb70c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1364,8 +1364,6 @@ public class ApiConstants { public static final String RECURSIVE_DOMAINS = "recursivedomains"; - public static final String CLOUDSTACK_USER_AGENT = "CloudStack-Agent"; - /** * This enum specifies IO Drivers, each option controls specific policies on I/O. * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0). diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index 056a0d8245bf..71c329796d1b 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -20,7 +20,6 @@ package com.cloud.storage.template; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; -import static org.apache.cloudstack.api.ApiConstants.CLOUDSTACK_USER_AGENT; import java.io.File; import java.io.IOException; @@ -53,6 +52,7 @@ import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; import com.cloud.utils.net.Proxy; /** @@ -126,7 +126,7 @@ private GetMethod createRequest(String downloadUrl) { GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); request.setFollowRedirects(followRedirects); - request.getParams().setParameter(HttpMethodParams.USER_AGENT, CLOUDSTACK_USER_AGENT); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); return request; } diff --git a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java index fee2143e2f02..0dad2564779c 100644 --- a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java @@ -18,10 +18,11 @@ // package com.cloud.storage.template; -import static org.apache.cloudstack.api.ApiConstants.CLOUDSTACK_USER_AGENT; import com.cloud.storage.StorageLayer; import com.cloud.utils.UriUtils; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; + import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpMethodRetryHandler; @@ -61,7 +62,7 @@ protected GetMethod createRequest(String downloadUrl) { GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); request.setFollowRedirects(followRedirects); - request.getParams().setParameter(HttpMethodParams.USER_AGENT, CLOUDSTACK_USER_AGENT); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); if (!toFileSet) { String[] parts = downloadUrl.split("/"); String filename = parts[parts.length - 1]; diff --git a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java index c36f4b2e18f9..fcf22512c605 100644 --- a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java @@ -18,7 +18,6 @@ package com.cloud.storage.template; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; -import static org.apache.cloudstack.api.ApiConstants.CLOUDSTACK_USER_AGENT; import java.io.File; import java.io.IOException; @@ -45,6 +44,7 @@ import org.apache.commons.lang3.StringUtils; import com.cloud.storage.StorageLayer; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implements TemplateDownloader { private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager(); @@ -96,7 +96,7 @@ private GetMethod createRequest(String downloadUrl) { GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler); request.setFollowRedirects(followRedirects); - request.getParams().setParameter(HttpMethodParams.USER_AGENT, CLOUDSTACK_USER_AGENT); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); return request; } diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java index 95bff46f26b7..24918144eb17 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java @@ -19,7 +19,6 @@ package org.apache.cloudstack.direct.download; -import static org.apache.cloudstack.api.ApiConstants.CLOUDSTACK_USER_AGENT; import java.io.File; import java.io.FileOutputStream; @@ -34,6 +33,7 @@ import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; import com.cloud.utils.storage.QCOW2Utils; import org.apache.commons.collections.MapUtils; import org.apache.commons.httpclient.HttpClient; @@ -71,7 +71,7 @@ public HttpDirectTemplateDownloader(String url, Long templateId, String destPool protected GetMethod createRequest(String downloadUrl, Map headers) { GetMethod request = new GetMethod(downloadUrl); request.setFollowRedirects(this.isFollowRedirects()); - request.getParams().setParameter(HttpMethodParams.USER_AGENT, CLOUDSTACK_USER_AGENT); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); if (MapUtils.isNotEmpty(headers)) { for (String key : headers.keySet()) { request.setRequestHeader(key, headers.get(key)); diff --git a/utils/src/main/java/com/cloud/utils/net/HttpClientCloudStackUserAgent.java b/utils/src/main/java/com/cloud/utils/net/HttpClientCloudStackUserAgent.java new file mode 100644 index 000000000000..991aa296380a --- /dev/null +++ b/utils/src/main/java/com/cloud/utils/net/HttpClientCloudStackUserAgent.java @@ -0,0 +1,39 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.utils.net; + +import org.apache.logging.log4j.util.Strings; + +public class HttpClientCloudStackUserAgent { + public static final String CLOUDSTACK_USER_AGENT = buildUserAgent(); + + private static String buildUserAgent() { + String version = HttpClientCloudStackUserAgent.class + .getPackage() + .getImplementationVersion(); + + if (Strings.isBlank(version)) { + version = "unknown"; + } + return "CloudStack-Agent/" + version + " (Apache CloudStack)"; + } + + private HttpClientCloudStackUserAgent() {} +} From e7cf205d85f27870b9582a7022856f1b34c0275e Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Thu, 26 Mar 2026 16:36:01 +0530 Subject: [PATCH 4/5] add user agent in HEAD request as well --- .../cloud/storage/template/SimpleHttpMultiFileDownloader.java | 1 + .../cloudstack/direct/download/HttpDirectTemplateDownloader.java | 1 + 2 files changed, 2 insertions(+) diff --git a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java index fcf22512c605..6608754073a1 100644 --- a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java @@ -143,6 +143,7 @@ private void tryAndGetTotalRemoteSize() { continue; } HeadMethod headMethod = new HeadMethod(downloadUrl); + headMethod.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); try { if (client.executeMethod(headMethod) != HttpStatus.SC_OK) { continue; diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java index 24918144eb17..99b84bb645c8 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java @@ -115,6 +115,7 @@ protected Pair performDownload() { public boolean checkUrl(String url) { HeadMethod httpHead = new HeadMethod(url); httpHead.setFollowRedirects(this.isFollowRedirects()); + httpHead.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); try { int responseCode = client.executeMethod(httpHead); if (responseCode != HttpStatus.SC_OK) { From 0f31e3eeea569d5df211c896a6497bd648d66953 Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Fri, 27 Mar 2026 10:41:40 +0530 Subject: [PATCH 5/5] add user-agent for httpurlconnection --- utils/src/main/java/com/cloud/utils/HttpUtils.java | 5 +++++ utils/src/main/java/com/cloud/utils/UriUtils.java | 3 +++ utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java | 2 ++ 3 files changed, 10 insertions(+) diff --git a/utils/src/main/java/com/cloud/utils/HttpUtils.java b/utils/src/main/java/com/cloud/utils/HttpUtils.java index 9f998efe0994..b7d95f8133be 100644 --- a/utils/src/main/java/com/cloud/utils/HttpUtils.java +++ b/utils/src/main/java/com/cloud/utils/HttpUtils.java @@ -19,6 +19,8 @@ package com.cloud.utils; +import static com.cloud.utils.UriUtils.USER_AGENT; + import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -33,6 +35,8 @@ import java.net.URL; import java.util.Map; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; + public class HttpUtils { protected static Logger LOGGER = LogManager.getLogger(HttpUtils.class); @@ -161,6 +165,7 @@ public static boolean downloadFileWithProgress(final String fileURL, final Strin try { URL url = new URL(fileURL); httpConn = (HttpURLConnection) url.openConnection(); + httpConn.setRequestProperty(USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); int responseCode = httpConn.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { int contentLength = httpConn.getContentLength(); diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java index eba6f88201b0..a4654091e0cb 100644 --- a/utils/src/main/java/com/cloud/utils/UriUtils.java +++ b/utils/src/main/java/com/cloud/utils/UriUtils.java @@ -67,12 +67,14 @@ import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; public class UriUtils { protected static Logger LOGGER = LogManager.getLogger(UriUtils.class); + public static final String USER_AGENT = "User-Agent"; public static String formNfsUri(String host, String path) { try { @@ -227,6 +229,7 @@ public static long getRemoteSize(String url, Boolean followRedirect) { URI uri = new URI(url); httpConn = (HttpURLConnection)uri.toURL().openConnection(); httpConn.setRequestMethod(method); + httpConn.setRequestProperty(USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); httpConn.setConnectTimeout(2000); httpConn.setReadTimeout(5000); httpConn.setInstanceFollowRedirects(Boolean.TRUE.equals(followRedirect)); diff --git a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java index 34d748b0708e..a4a49825d1f6 100644 --- a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java +++ b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java @@ -35,6 +35,7 @@ import com.cloud.utils.NumbersUtil; import com.cloud.utils.UriUtils; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; public final class QCOW2Utils { protected static Logger LOGGER = LogManager.getLogger(QCOW2Utils.class); @@ -119,6 +120,7 @@ public static long getVirtualSizeFromUrl(String urlStr, boolean followRedirects) try { URI url = new URI(urlStr); httpConn = (HttpURLConnection)url.toURL().openConnection(); + httpConn.setRequestProperty(UriUtils.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); httpConn.setInstanceFollowRedirects(followRedirects); return getVirtualSize(httpConn.getInputStream(), UriUtils.isUrlForCompressedFile(urlStr)); } catch (URISyntaxException e) {