From e4759080fe337103164934b1e417a63c22f662ed Mon Sep 17 00:00:00 2001 From: Maximilian Geberl Date: Thu, 20 Aug 2026 10:05:00 +0200 Subject: [PATCH 1/3] Add metrics for API calls --- .../main.go | 6 + go.mod | 4 +- pkg/metrics/http.go | 117 ++++++++++ pkg/metrics/http_test.go | 216 ++++++++++++++++++ pkg/metrics/metrics.go | 53 +++++ pkg/metrics/suite_test.go | 13 ++ pkg/stackit/client/factory.go | 8 +- 7 files changed, 413 insertions(+), 4 deletions(-) create mode 100644 pkg/metrics/http.go create mode 100644 pkg/metrics/http_test.go create mode 100644 pkg/metrics/metrics.go create mode 100644 pkg/metrics/suite_test.go diff --git a/cmd/gardener-extension-provider-stackit/main.go b/cmd/gardener-extension-provider-stackit/main.go index 0327d110..c70d47c4 100644 --- a/cmd/gardener-extension-provider-stackit/main.go +++ b/cmd/gardener-extension-provider-stackit/main.go @@ -8,12 +8,18 @@ import ( "os" "github.com/gardener/gardener/pkg/logger" + "github.com/stackitcloud/gardener-extension-provider-stackit/v2/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager/signals" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" "github.com/stackitcloud/gardener-extension-provider-stackit/v2/cmd/gardener-extension-provider-stackit/app" ) +func init() { + ctrlmetrics.Registry.MustRegister(metrics.NewExporter()) +} + func main() { log.SetLogger(logger.MustNewZapLogger(logger.InfoLevel, logger.FormatJSON)) setupLogger := log.Log.WithName("setup") diff --git a/go.mod b/go.mod index 261cb745..1d9c1c2b 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,8 @@ require ( github.com/pelletier/go-toml/v2 v2.4.3 github.com/pkg/errors v0.9.1 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.93.1 + github.com/prometheus/client_golang v1.24.1 + github.com/prometheus/client_model v0.6.2 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stackitcloud/stackit-sdk-go/core v0.26.0 @@ -178,8 +180,6 @@ require ( github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/alertmanager v0.33.1 // indirect - github.com/prometheus/client_golang v1.24.1 // indirect - github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/exporter-toolkit v0.16.0 // indirect github.com/prometheus/procfs v0.21.1 // indirect diff --git a/pkg/metrics/http.go b/pkg/metrics/http.go new file mode 100644 index 00000000..03f352ad --- /dev/null +++ b/pkg/metrics/http.go @@ -0,0 +1,117 @@ +package metrics + +import ( + "net/http" + "runtime" + "strconv" + "strings" + "time" + "unicode" + + "github.com/prometheus/client_golang/prometheus" +) + +const UnknownOperation = "UnknownOperation" + +func NewHTTPClient(componentName string) *http.Client { + return WrapHTTPClient(http.DefaultClient, componentName) +} + +func WrapHTTPClient(client *http.Client, componentName string) *http.Client { + wrappedClient := *client + + baseTransport := client.Transport + if baseTransport == nil { + baseTransport = http.DefaultTransport + } + + // Chain your instrumented round tripper + wrappedClient.Transport = &InstrumentedRoundTripper{ + base: baseTransport, + componentName: componentName, + } + + return &wrappedClient +} + +type InstrumentedRoundTripper struct { + base http.RoundTripper + componentName string +} + +func (rt *InstrumentedRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + startTime := time.Now() + response, err := rt.base.RoundTrip(request) + duration := time.Since(startTime) + + statusCode := "network_error" + if response != nil { + statusCode = strconv.Itoa(response.StatusCode) + } + + labels := prometheus.Labels{ + componentLabel: rt.componentName, + hostLabel: request.Host, + methodLabel: request.Method, + operationLabel: getSDKOperationName(), + codeLabel: statusCode, + } + + HTTPRequestDurationHistogram.With(labels).Observe(duration.Seconds()) + HTTPRequestCount.With(labels).Inc() + + isHTTPError := response != nil && response.StatusCode >= 400 + isNetworkError := err != nil + + if isHTTPError || isNetworkError { + HTTPErrorCount.With(labels).Inc() + } + + return response, err +} + +// getSDKOperationName returns the name of the STACKIT SDK function. To do this the function gets the last 10 callers and checks +// for functions from the stackitcloud/stackit-sdk-go. It fall back to UnknownOperation if no function was found. +func getSDKOperationName() string { + pc := make([]uintptr, 10) + + // Skip 3 because the first 3 are always Callers, getSDKOperationName, RoundTrip. + n := runtime.Callers(3, pc) + if n == 0 { + return UnknownOperation + } + + frames := runtime.CallersFrames(pc[:n]) + moreFrames := true + for moreFrames { + var frame runtime.Frame + frame, moreFrames = frames.Next() + + if !strings.Contains(frame.Function, "stackitcloud/stackit-sdk-go") { + continue + } + + parts := strings.Split(frame.Function, ".") + if len(parts) > 0 { + funcName := parts[len(parts)-1] + + // Skip function names with 0 len + // Skip Execute, because there is a function with more detailed name + // Skip RoundTrip, because this only the RoundTrip for the AuthFlow + if funcName == "" || + funcName == "Execute" || + funcName == "RoundTrip" { + continue + } + + // Skip Private functions + if !unicode.IsUpper(rune(funcName[0])) { + continue + } + + return strings.TrimSuffix(funcName, "Execute") + } + } + + return UnknownOperation +} diff --git a/pkg/metrics/http_test.go b/pkg/metrics/http_test.go new file mode 100644 index 00000000..a9fa6b85 --- /dev/null +++ b/pkg/metrics/http_test.go @@ -0,0 +1,216 @@ +package metrics + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + dto "github.com/prometheus/client_model/go" + sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" + iaas "github.com/stackitcloud/stackit-sdk-go/services/iaas/v2api" +) + +var _ = Describe("Metrics", func() { + Describe("getSDKOperationName", func() { + var ( + server *httptest.Server + host string + component = "test" + iaasClient *iaas.APIClient + ) + + BeforeEach(func() { + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + url, err := url.Parse(server.URL) + Expect(err).NotTo(HaveOccurred()) + host = url.Host + + HTTPRequestCount.Reset() + HTTPErrorCount.Reset() + HTTPRequestDurationHistogram.Reset() + + iaasClient, err = iaas.NewAPIClient( + sdkconfig.WithHTTPClient(NewHTTPClient(component)), + sdkconfig.WithEndpoint(server.URL), + sdkconfig.WithoutAuthentication(), + ) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + server.Close() + }) + + It("should return DeleteVolume as operation", func() { + err := iaasClient.DefaultAPI.DeleteVolume(context.TODO(), uuid.New().String(), "", uuid.New().String()).Execute() + Expect(err).NotTo(HaveOccurred()) + + labels := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: "DELETE", + operationLabel: "DeleteVolume", + codeLabel: "200", + } + + Expect(testutil.ToFloat64(HTTPRequestCount.With(labels))).To(Equal(float64(1))) + }) + + It("should return DeleteServer as operation", func() { + err := iaasClient.DefaultAPI.DeleteServer(context.TODO(), uuid.New().String(), "", uuid.New().String()).Execute() + Expect(err).NotTo(HaveOccurred()) + + labels := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: "DELETE", + operationLabel: "DeleteServer", + codeLabel: "200", + } + + Expect(testutil.ToFloat64(HTTPRequestCount.With(labels))).To(Equal(float64(1))) + }) + }) + + Describe("InstrumentedRoundTripper", func() { + var ( + server *httptest.Server + httpClient *http.Client + host string + component = "test" + ) + + BeforeEach(func() { + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/404": + w.WriteHeader(http.StatusNotFound) + case "/500": + w.WriteHeader(http.StatusInternalServerError) + case "/400": + w.WriteHeader(http.StatusBadRequest) + default: + w.WriteHeader(http.StatusOK) + } + })) + url, err := url.Parse(server.URL) + Expect(err).NotTo(HaveOccurred()) + host = url.Host + httpClient = NewHTTPClient(component) + + HTTPRequestCount.Reset() + HTTPErrorCount.Reset() + HTTPRequestDurationHistogram.Reset() + }) + + AfterEach(func() { + server.Close() + }) + + It("increments HTTPRequestCount for responses", func() { + labels := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: "GET", + operationLabel: UnknownOperation, + codeLabel: "200", + } + + response, err := httpClient.Get(server.URL + "/request-count-test") + Expect(err).NotTo(HaveOccurred()) + defer response.Body.Close() + + Expect(testutil.ToFloat64(HTTPRequestCount.With(labels))).To(Equal(float64(1))) + }) + + It("records HTTPRequestDurationHistogram observations for responses", func() { + labels := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: "GET", + operationLabel: UnknownOperation, + codeLabel: "200", + } + + response, err := httpClient.Get(server.URL + "/request-duration-test") + Expect(err).NotTo(HaveOccurred()) + defer response.Body.Close() + + Expect(histogramSampleCount(HTTPRequestDurationHistogram.With(labels))).To(Equal(uint64(1))) + }) + + It("increments HTTPErrorCount for error responses (400, 404, 500)", func() { + labels400 := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: http.MethodGet, + operationLabel: UnknownOperation, + codeLabel: "400", + } + labels404 := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: http.MethodGet, + operationLabel: UnknownOperation, + codeLabel: "404", + } + labels500 := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: http.MethodPost, + operationLabel: UnknownOperation, + codeLabel: "500", + } + + response1, err := httpClient.Get(server.URL + "/400") + Expect(err).NotTo(HaveOccurred()) + defer response1.Body.Close() + + response2, err := httpClient.Get(server.URL + "/404") + Expect(err).NotTo(HaveOccurred()) + defer response2.Body.Close() + + response3, err := httpClient.Post(server.URL+"/500", "application/json", nil) + Expect(err).NotTo(HaveOccurred()) + defer response3.Body.Close() + + Expect(testutil.ToFloat64(HTTPErrorCount.With(labels400))).To(Equal(float64(1))) + Expect(testutil.ToFloat64(HTTPErrorCount.With(labels404))).To(Equal(float64(1))) + Expect(testutil.ToFloat64(HTTPErrorCount.With(labels500))).To(Equal(float64(1))) + }) + + It("does not increment HTTPErrorCount for successful responses", func() { + labels := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: http.MethodGet, + operationLabel: UnknownOperation, + codeLabel: "200", + } + + response, err := httpClient.Get(server.URL) + Expect(err).NotTo(HaveOccurred()) + defer response.Body.Close() + + Expect(testutil.ToFloat64(HTTPErrorCount.With(labels))).To(Equal(float64(0))) + }) + }) +}) + +func histogramSampleCount(observer prometheus.Observer) uint64 { + metric, ok := observer.(prometheus.Metric) + Expect(ok).To(BeTrue()) + + dtoMetric := &dto.Metric{} + Expect(metric.Write(dtoMetric)).To(Succeed()) + + return dtoMetric.GetHistogram().GetSampleCount() +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 00000000..d6ad9db2 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,53 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const ( + metricPrefix = "stackit_api" + componentLabel = "component" + hostLabel = "host" + methodLabel = "method" + operationLabel = "operation" + codeLabel = "status_code" +) + +var ( + HTTPRequestCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: "http_requests_total", + Help: "The number of requests to external APIs", + }, []string{componentLabel, hostLabel, methodLabel, operationLabel, codeLabel}) + + HTTPErrorCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: "http_errors_total", + Help: "Number of HTTP errors returned by external APIs", + }, []string{componentLabel, hostLabel, methodLabel, operationLabel, codeLabel}) + + HTTPRequestDurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metricPrefix, + Name: "http_request_duration_seconds", + Help: "The response times of external API requests", + }, []string{componentLabel, hostLabel, methodLabel, operationLabel, codeLabel}) +) + +type Exporter struct { +} + +func NewExporter() *Exporter { + return &Exporter{} +} + +func (e *Exporter) Describe(descs chan<- *prometheus.Desc) { + HTTPRequestCount.Describe(descs) + HTTPErrorCount.Describe(descs) + HTTPRequestDurationHistogram.Describe(descs) +} + +func (e *Exporter) Collect(metrics chan<- prometheus.Metric) { + HTTPRequestCount.Collect(metrics) + HTTPErrorCount.Collect(metrics) + HTTPRequestDurationHistogram.Collect(metrics) +} diff --git a/pkg/metrics/suite_test.go b/pkg/metrics/suite_test.go new file mode 100644 index 00000000..42aa7818 --- /dev/null +++ b/pkg/metrics/suite_test.go @@ -0,0 +1,13 @@ +package metrics + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMetrics(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Metrics Provider Suite") +} diff --git a/pkg/stackit/client/factory.go b/pkg/stackit/client/factory.go index a085f4cb..e4049ea6 100644 --- a/pkg/stackit/client/factory.go +++ b/pkg/stackit/client/factory.go @@ -8,6 +8,7 @@ import ( "net/http" extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller" + "github.com/stackitcloud/gardener-extension-provider-stackit/v2/pkg/metrics" sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" @@ -136,13 +137,16 @@ func clientOptions(endpoints stackitv1alpha1.APIEndpoints, credentials *stackit. result = append(result, sdkconfig.WithTokenEndpoint(*endpoints.TokenEndpoint)) } + httpClient := http.DefaultClient if caBundle != "" { - customHttpClient, err := newHTTPClientWithCustomCA([]byte(caBundle)) + var err error + httpClient, err = newHTTPClientWithCustomCA([]byte(caBundle)) if err != nil { return nil, err } - result = append(result, sdkconfig.WithHTTPClient(customHttpClient)) } + result = append(result, sdkconfig.WithHTTPClient(metrics.WrapHTTPClient(httpClient, "gardener-extension-provider-stackit"))) + return result, nil } From 779d97800d34c6cc393384d6501cee139599b989 Mon Sep 17 00:00:00 2001 From: Maximilian Geberl Date: Fri, 21 Aug 2026 15:10:09 +0200 Subject: [PATCH 2/3] format --- cmd/gardener-extension-provider-stackit/main.go | 2 +- pkg/stackit/client/factory.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/gardener-extension-provider-stackit/main.go b/cmd/gardener-extension-provider-stackit/main.go index c70d47c4..8f330d51 100644 --- a/cmd/gardener-extension-provider-stackit/main.go +++ b/cmd/gardener-extension-provider-stackit/main.go @@ -8,12 +8,12 @@ import ( "os" "github.com/gardener/gardener/pkg/logger" - "github.com/stackitcloud/gardener-extension-provider-stackit/v2/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager/signals" ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" "github.com/stackitcloud/gardener-extension-provider-stackit/v2/cmd/gardener-extension-provider-stackit/app" + "github.com/stackitcloud/gardener-extension-provider-stackit/v2/pkg/metrics" ) func init() { diff --git a/pkg/stackit/client/factory.go b/pkg/stackit/client/factory.go index e4049ea6..6cd98dee 100644 --- a/pkg/stackit/client/factory.go +++ b/pkg/stackit/client/factory.go @@ -8,7 +8,6 @@ import ( "net/http" extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller" - "github.com/stackitcloud/gardener-extension-provider-stackit/v2/pkg/metrics" sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" @@ -16,6 +15,7 @@ import ( "github.com/stackitcloud/gardener-extension-provider-stackit/v2/pkg/apis/stackit/helper" stackitv1alpha1 "github.com/stackitcloud/gardener-extension-provider-stackit/v2/pkg/apis/stackit/v1alpha1" + "github.com/stackitcloud/gardener-extension-provider-stackit/v2/pkg/metrics" "github.com/stackitcloud/gardener-extension-provider-stackit/v2/pkg/stackit" ) From c3e0d262e1fbf59ad4964cf2eb7a38a1cb9934ff Mon Sep 17 00:00:00 2001 From: Maximilian Geberl Date: Fri, 21 Aug 2026 15:10:28 +0200 Subject: [PATCH 3/3] Add PR Review suggestions from cloud-provider-stackit --- pkg/metrics/http.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pkg/metrics/http.go b/pkg/metrics/http.go index 03f352ad..02de268f 100644 --- a/pkg/metrics/http.go +++ b/pkg/metrics/http.go @@ -7,6 +7,7 @@ import ( "strings" "time" "unicode" + "unicode/utf8" "github.com/prometheus/client_golang/prometheus" ) @@ -18,6 +19,9 @@ func NewHTTPClient(componentName string) *http.Client { } func WrapHTTPClient(client *http.Client, componentName string) *http.Client { + if client == nil { + return nil + } wrappedClient := *client baseTransport := client.Transport @@ -49,9 +53,15 @@ func (rt *InstrumentedRoundTripper) RoundTrip(request *http.Request) (*http.Resp statusCode = strconv.Itoa(response.StatusCode) } + // request.Host is optional so we can fallback to request.URL.Host (if available) + host := request.Host + if host == "" && request.URL != nil { + host = request.URL.Host + } + labels := prometheus.Labels{ componentLabel: rt.componentName, - hostLabel: request.Host, + hostLabel: host, methodLabel: request.Method, operationLabel: getSDKOperationName(), codeLabel: statusCode, @@ -105,7 +115,8 @@ func getSDKOperationName() string { } // Skip Private functions - if !unicode.IsUpper(rune(funcName[0])) { + r, _ := utf8.DecodeRuneInString(funcName) + if !unicode.IsUpper(r) { continue }