diff --git a/cmd/stackit-csi-plugin/main.go b/cmd/stackit-csi-plugin/main.go index 302aee97..c07eed50 100644 --- a/cmd/stackit-csi-plugin/main.go +++ b/cmd/stackit-csi-plugin/main.go @@ -126,9 +126,8 @@ func handle(ctx context.Context) { klog.Fatal(err) } - iaasHTTPClient := metrics.NewInstrumentedHTTPClient(metrics.APINameIaaS) iaasOpts := []sdkconfig.ConfigurationOption{ - sdkconfig.WithHTTPClient(iaasHTTPClient), + sdkconfig.WithHTTPClient(metrics.NewHTTPClient("stackit-csi-plugin")), } if cfg.Global.APIEndpoints.IaasAPI != "" { diff --git a/pkg/ccm/stackit.go b/pkg/ccm/stackit.go index 2e610380..f3417d09 100644 --- a/pkg/ccm/stackit.go +++ b/pkg/ccm/stackit.go @@ -117,9 +117,8 @@ func BuildObservability() (*MetricsRemoteWrite, error) { // NewCloudControllerManager creates a new instance of the stackit struct from a stackitconfig struct func NewCloudControllerManager(cfg *stackitconfig.CCMConfig, obs *MetricsRemoteWrite) (*CloudControllerManager, error) { - lbHTTPClient := metrics.NewInstrumentedHTTPClient(metrics.APINameLoadBalancer) lbOpts := []sdkconfig.ConfigurationOption{ - sdkconfig.WithHTTPClient(lbHTTPClient), + sdkconfig.WithHTTPClient(metrics.NewHTTPClient("cloud-controller-manager")), } if cfg.Global.APIEndpoints.LoadBalancerAPI != "" { @@ -139,9 +138,8 @@ func NewCloudControllerManager(cfg *stackitconfig.CCMConfig, obs *MetricsRemoteW return nil, fmt.Errorf("failed to create lb client: %v", err) } - iaasHTTPClient := metrics.NewInstrumentedHTTPClient(metrics.APINameIaaS) iaasOpts := []sdkconfig.ConfigurationOption{ - sdkconfig.WithHTTPClient(iaasHTTPClient), + sdkconfig.WithHTTPClient(metrics.NewHTTPClient("cloud-controller-manager")), } if cfg.Global.APIEndpoints.IaasAPI != "" { diff --git a/pkg/metrics/http.go b/pkg/metrics/http.go index 8fba2dc2..02de268f 100644 --- a/pkg/metrics/http.go +++ b/pkg/metrics/http.go @@ -1,32 +1,49 @@ package metrics import ( - "fmt" "net/http" + "runtime" "strconv" "strings" "time" + "unicode" + "unicode/utf8" "github.com/prometheus/client_golang/prometheus" ) -func NewInstrumentedHTTPClient(api string) *http.Client { - return &http.Client{ - Transport: &InstrumentedRoundTripper{ - api: api, - base: http.DefaultTransport, - }, +const UnknownOperation = "UnknownOperation" + +func NewHTTPClient(componentName string) *http.Client { + return WrapHTTPClient(http.DefaultClient, componentName) +} + +func WrapHTTPClient(client *http.Client, componentName string) *http.Client { + if client == nil { + return nil + } + wrappedClient := *client + + baseTransport := client.Transport + if baseTransport == nil { + baseTransport = http.DefaultTransport + } + + // Chain your instrumented round tripper + wrappedClient.Transport = &InstrumentedRoundTripper{ + base: baseTransport, + componentName: componentName, } + + return &wrappedClient } type InstrumentedRoundTripper struct { - api string - base http.RoundTripper + base http.RoundTripper + componentName string } func (rt *InstrumentedRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { - operation := operationFromRequest(request) - startTime := time.Now() response, err := rt.base.RoundTrip(request) duration := time.Since(startTime) @@ -36,10 +53,17 @@ func (rt *InstrumentedRoundTripper) RoundTrip(request *http.Request) (*http.Resp statusCode = strconv.Itoa(response.StatusCode) } + // request.Host is optional so we can fallback to request.URL.Host (if available) + host := request.Host + if host == "" && request.URL != nil { + host = request.URL.Host + } + labels := prometheus.Labels{ - apiLabel: rt.api, + componentLabel: rt.componentName, + hostLabel: host, methodLabel: request.Method, - operationLabel: operation, + operationLabel: getSDKOperationName(), codeLabel: statusCode, } @@ -56,30 +80,49 @@ func (rt *InstrumentedRoundTripper) RoundTrip(request *http.Request) (*http.Resp return response, err } -func operationFromRequest(request *http.Request) string { - verb := strings.ToLower(request.Method) +// getSDKOperationName returns the name of the STACKIT SDK function. To do this the function gets the last 10 callers and checks +// for functions from the stackitcloud/stackit-sdk-go. It fall back to UnknownOperation if no function was found. +func getSDKOperationName() string { + pc := make([]uintptr, 10) - pathElements := strings.Split(request.URL.Path, "/") - if len(pathElements) <= 1 { - return fmt.Sprintf("%s_%s", verb, request.URL.Path) + // Skip 3 because the first 3 are always Callers, getSDKOperationName, RoundTrip. + n := runtime.Callers(3, pc) + if n == 0 { + return UnknownOperation } - // since the path starts with a '/', the first path element is the empty string - pathElements = pathElements[1:] - - // the subject is always the last or the second to last element: - // .../subject -> even number of path elements - // .../subject/ -> odd number of path elements - var subject string - if len(pathElements) == 1 { - // edge case - subject = pathElements[0] - } else if len(pathElements)%2 == 0 { - // even - subject = pathElements[len(pathElements)-1] - } else { - // odd - subject = pathElements[len(pathElements)-2] + "_instance" + + frames := runtime.CallersFrames(pc[:n]) + moreFrames := true + for moreFrames { + var frame runtime.Frame + frame, moreFrames = frames.Next() + + if !strings.Contains(frame.Function, "stackitcloud/stackit-sdk-go") { + continue + } + + parts := strings.Split(frame.Function, ".") + if len(parts) > 0 { + funcName := parts[len(parts)-1] + + // Skip function names with 0 len + // Skip Execute, because there is a function with more detailed name + // Skip RoundTrip, because this only the RoundTrip for the AuthFlow + if funcName == "" || + funcName == "Execute" || + funcName == "RoundTrip" { + continue + } + + // Skip Private functions + r, _ := utf8.DecodeRuneInString(funcName) + if !unicode.IsUpper(r) { + continue + } + + return strings.TrimSuffix(funcName, "Execute") + } } - return fmt.Sprintf("%s_%s", verb, subject) + return UnknownOperation } diff --git a/pkg/metrics/http_test.go b/pkg/metrics/http_test.go index 1bf3fccc..a9fa6b85 100644 --- a/pkg/metrics/http_test.go +++ b/pkg/metrics/http_test.go @@ -1,163 +1,206 @@ package metrics import ( + "context" "net/http" "net/http/httptest" "net/url" + "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" dto "github.com/prometheus/client_model/go" + sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" + iaas "github.com/stackitcloud/stackit-sdk-go/services/iaas/v2api" ) var _ = Describe("Metrics", func() { - DescribeTable("operationFromRequest", func(method, path, expected string) { - requestURL, _ := url.Parse("https://host" + path) - request := &http.Request{ - Method: method, - URL: requestURL, - } - op := operationFromRequest(request) - Expect(op).To(Equal(expected)) - }, - Entry("post token", "POST", "/token", "post_token"), - Entry("get load-balancers", "GET", "/v2/projects/6-a-4-8-c/regions/eu01/load-balancers", "get_load-balancers"), - Entry("get load-balancers instance", "GET", "/v2/projects/6-a-4-8-c/regions/eu01/load-balancers/id", "get_load-balancers_instance"), - ) - - Describe("InstrumentedRoundTripper", func() { - It("increments HTTPRequestCount for responses", func() { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + Describe("getSDKOperationName", func() { + var ( + server *httptest.Server + host string + component = "test" + iaasClient *iaas.APIClient + ) + + BeforeEach(func() { + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) - defer server.Close() + url, err := url.Parse(server.URL) + Expect(err).NotTo(HaveOccurred()) + host = url.Host + + HTTPRequestCount.Reset() + HTTPErrorCount.Reset() + HTTPRequestDurationHistogram.Reset() + + iaasClient, err = iaas.NewAPIClient( + sdkconfig.WithHTTPClient(NewHTTPClient(component)), + sdkconfig.WithEndpoint(server.URL), + sdkconfig.WithoutAuthentication(), + ) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + server.Close() + }) + + It("should return DeleteVolume as operation", func() { + err := iaasClient.DefaultAPI.DeleteVolume(context.TODO(), uuid.New().String(), "", uuid.New().String()).Execute() + Expect(err).NotTo(HaveOccurred()) labels := prometheus.Labels{ - apiLabel: "test", - methodLabel: "GET", - operationLabel: "get_request-count-test", + hostLabel: host, + componentLabel: component, + methodLabel: "DELETE", + operationLabel: "DeleteVolume", codeLabel: "200", } - before := testutil.ToFloat64(HTTPRequestCount.With(labels)) - client := NewInstrumentedHTTPClient("test") + Expect(testutil.ToFloat64(HTTPRequestCount.With(labels))).To(Equal(float64(1))) + }) - response, err := client.Get(server.URL + "/request-count-test") + It("should return DeleteServer as operation", func() { + err := iaasClient.DefaultAPI.DeleteServer(context.TODO(), uuid.New().String(), "", uuid.New().String()).Execute() Expect(err).NotTo(HaveOccurred()) - defer response.Body.Close() - after := testutil.ToFloat64(HTTPRequestCount.With(labels)) - Expect(after - before).To(Equal(float64(1))) + labels := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: "DELETE", + operationLabel: "DeleteServer", + codeLabel: "200", + } + + Expect(testutil.ToFloat64(HTTPRequestCount.With(labels))).To(Equal(float64(1))) }) + }) - It("records HTTPRequestDurationHistogram observations for responses", func() { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) + Describe("InstrumentedRoundTripper", func() { + var ( + server *httptest.Server + httpClient *http.Client + host string + component = "test" + ) + + BeforeEach(func() { + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/404": + w.WriteHeader(http.StatusNotFound) + case "/500": + w.WriteHeader(http.StatusInternalServerError) + case "/400": + w.WriteHeader(http.StatusBadRequest) + default: + w.WriteHeader(http.StatusOK) + } })) - defer server.Close() + url, err := url.Parse(server.URL) + Expect(err).NotTo(HaveOccurred()) + host = url.Host + httpClient = NewHTTPClient(component) + HTTPRequestCount.Reset() + HTTPErrorCount.Reset() + HTTPRequestDurationHistogram.Reset() + }) + + AfterEach(func() { + server.Close() + }) + + It("increments HTTPRequestCount for responses", func() { labels := prometheus.Labels{ - apiLabel: "test", + hostLabel: host, + componentLabel: component, methodLabel: "GET", - operationLabel: "get_request-duration-test", + operationLabel: UnknownOperation, codeLabel: "200", } - before := histogramSampleCount(HTTPRequestDurationHistogram.With(labels)) - client := NewInstrumentedHTTPClient("test") + response, err := httpClient.Get(server.URL + "/request-count-test") + Expect(err).NotTo(HaveOccurred()) + defer response.Body.Close() + + Expect(testutil.ToFloat64(HTTPRequestCount.With(labels))).To(Equal(float64(1))) + }) - response, err := client.Get(server.URL + "/request-duration-test") + It("records HTTPRequestDurationHistogram observations for responses", func() { + labels := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: "GET", + operationLabel: UnknownOperation, + codeLabel: "200", + } + + response, err := httpClient.Get(server.URL + "/request-duration-test") Expect(err).NotTo(HaveOccurred()) defer response.Body.Close() - after := histogramSampleCount(HTTPRequestDurationHistogram.With(labels)) - Expect(after - before).To(Equal(uint64(1))) + Expect(histogramSampleCount(HTTPRequestDurationHistogram.With(labels))).To(Equal(uint64(1))) }) It("increments HTTPErrorCount for error responses (400, 404, 500)", func() { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodPost { - w.WriteHeader(http.StatusInternalServerError) - return - } - if r.URL.Path == "/404" { - w.WriteHeader(http.StatusNotFound) - return - } - w.WriteHeader(http.StatusBadRequest) - })) - defer server.Close() - labels400 := prometheus.Labels{ - apiLabel: "test", + hostLabel: host, + componentLabel: component, methodLabel: http.MethodGet, - operationLabel: "get_", + operationLabel: UnknownOperation, codeLabel: "400", } labels404 := prometheus.Labels{ - apiLabel: "test", + hostLabel: host, + componentLabel: component, methodLabel: http.MethodGet, - operationLabel: "get_404", + operationLabel: UnknownOperation, codeLabel: "404", } labels500 := prometheus.Labels{ - apiLabel: "test", + hostLabel: host, + componentLabel: component, methodLabel: http.MethodPost, - operationLabel: "post_", + operationLabel: UnknownOperation, codeLabel: "500", } - before400 := testutil.ToFloat64(HTTPErrorCount.With(labels400)) - before404 := testutil.ToFloat64(HTTPErrorCount.With(labels404)) - before500 := testutil.ToFloat64(HTTPErrorCount.With(labels500)) - client := NewInstrumentedHTTPClient("test") - - response1, err := client.Get(server.URL) + response1, err := httpClient.Get(server.URL + "/400") Expect(err).NotTo(HaveOccurred()) defer response1.Body.Close() - response2, err := client.Get(server.URL + "/404") + response2, err := httpClient.Get(server.URL + "/404") Expect(err).NotTo(HaveOccurred()) defer response2.Body.Close() - response3, err := client.Post(server.URL, "application/json", nil) + response3, err := httpClient.Post(server.URL+"/500", "application/json", nil) Expect(err).NotTo(HaveOccurred()) defer response3.Body.Close() - after400 := testutil.ToFloat64(HTTPErrorCount.With(labels400)) - after404 := testutil.ToFloat64(HTTPErrorCount.With(labels404)) - after500 := testutil.ToFloat64(HTTPErrorCount.With(labels500)) - - Expect(after400 - before400).To(Equal(float64(1))) - Expect(after404 - before404).To(Equal(float64(1))) - Expect(after500 - before500).To(Equal(float64(1))) - Expect((after400 - before400) + (after404 - before404) + (after500 - before500)).To(Equal(float64(3))) + Expect(testutil.ToFloat64(HTTPErrorCount.With(labels400))).To(Equal(float64(1))) + Expect(testutil.ToFloat64(HTTPErrorCount.With(labels404))).To(Equal(float64(1))) + Expect(testutil.ToFloat64(HTTPErrorCount.With(labels500))).To(Equal(float64(1))) }) It("does not increment HTTPErrorCount for successful responses", func() { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer server.Close() - labels := prometheus.Labels{ - apiLabel: "test", + hostLabel: host, + componentLabel: component, methodLabel: http.MethodGet, - operationLabel: "get_", + operationLabel: UnknownOperation, codeLabel: "200", } - before := testutil.ToFloat64(HTTPErrorCount.With(labels)) - - client := NewInstrumentedHTTPClient("test") - response, err := client.Get(server.URL) + response, err := httpClient.Get(server.URL) Expect(err).NotTo(HaveOccurred()) defer response.Body.Close() - after := testutil.ToFloat64(HTTPErrorCount.With(labels)) - Expect(after - before).To(Equal(float64(0))) + Expect(testutil.ToFloat64(HTTPErrorCount.With(labels))).To(Equal(float64(0))) }) }) }) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 9655c5e0..d6ad9db2 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -5,64 +5,48 @@ import ( ) const ( - cloudProviderMetricPrefix = "cloud_provider_stackit" - apiLabel = "api" - methodLabel = "method" - codeLabel = "status_code" - operationLabel = "op" - - APINameLoadBalancer = "loadbalancer" - APINameIaaS = "iaas" + metricPrefix = "stackit_api" + componentLabel = "component" + hostLabel = "host" + methodLabel = "method" + operationLabel = "operation" + codeLabel = "status_code" ) var ( HTTPRequestCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: cloudProviderMetricPrefix, - Name: "http_requests_total", - Help: "The number of requests to external APIs", - ConstLabels: nil, - }, []string{apiLabel, methodLabel, operationLabel, codeLabel}) + Namespace: metricPrefix, + Name: "http_requests_total", + Help: "The number of requests to external APIs", + }, []string{componentLabel, hostLabel, methodLabel, operationLabel, codeLabel}) HTTPErrorCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: cloudProviderMetricPrefix, - Name: "http_errors_total", - Help: "Number of HTTP errors returned by external APIs", - ConstLabels: nil, - }, []string{apiLabel, methodLabel, operationLabel, codeLabel}) + Namespace: metricPrefix, + Name: "http_errors_total", + Help: "Number of HTTP errors returned by external APIs", + }, []string{componentLabel, hostLabel, methodLabel, operationLabel, codeLabel}) HTTPRequestDurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: cloudProviderMetricPrefix, - Name: "http_request_duration_seconds", - Help: "The response times of external API requests", - ConstLabels: nil, - Buckets: nil, - }, []string{apiLabel, methodLabel, operationLabel, codeLabel}) + Namespace: metricPrefix, + Name: "http_request_duration_seconds", + Help: "The response times of external API requests", + }, []string{componentLabel, hostLabel, methodLabel, operationLabel, codeLabel}) ) type Exporter struct { } func NewExporter() *Exporter { - e := &Exporter{} - - return e + return &Exporter{} } func (e *Exporter) Describe(descs chan<- *prometheus.Desc) { - e.describeCloudProvider(descs) -} - -func (e *Exporter) Collect(metrics chan<- prometheus.Metric) { - e.collectCloudProvider(metrics) -} - -func (e *Exporter) describeCloudProvider(descs chan<- *prometheus.Desc) { HTTPRequestCount.Describe(descs) HTTPErrorCount.Describe(descs) HTTPRequestDurationHistogram.Describe(descs) } -func (e *Exporter) collectCloudProvider(metrics chan<- prometheus.Metric) { +func (e *Exporter) Collect(metrics chan<- prometheus.Metric) { HTTPRequestCount.Collect(metrics) HTTPErrorCount.Collect(metrics) HTTPRequestDurationHistogram.Collect(metrics)