diff --git a/cmd/application-load-balancer-controller/main.go b/cmd/application-load-balancer-controller/main.go index 390f4e7..3425e28 100644 --- a/cmd/application-load-balancer-controller/main.go +++ b/cmd/application-load-balancer-controller/main.go @@ -5,11 +5,13 @@ import ( "os" "github.com/stackitcloud/application-load-balancer-controller/pkg/controller/ingress" + "github.com/stackitcloud/application-load-balancer-controller/pkg/metrics" albclient "github.com/stackitcloud/application-load-balancer-controller/pkg/stackit" stackitconfig "github.com/stackitcloud/application-load-balancer-controller/pkg/stackit/config" sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" albsdk "github.com/stackitcloud/stackit-sdk-go/services/alb/v2api" certsdk "github.com/stackitcloud/stackit-sdk-go/services/certificates/v2api" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" @@ -32,6 +34,10 @@ type options struct { cloudConfig string } +func init() { + ctrlmetrics.Registry.MustRegister(metrics.NewExporter()) +} + // nolint:funlen // This function isn't awfully complex. func main() { var opts options @@ -75,6 +81,7 @@ func main() { } albOpts := []sdkconfig.ConfigurationOption{ sdkconfig.WithUserAgent("application-load-balancer-controller"), + sdkconfig.WithHTTPClient(metrics.NewHTTPClient("application-load-balancer-controller")), } if config.Global.APIEndpoints.ApplicationLoadBalancerAPI != "" { albOpts = append(albOpts, sdkconfig.WithEndpoint(config.Global.APIEndpoints.ApplicationLoadBalancerAPI)) @@ -82,6 +89,7 @@ func main() { certOpts := []sdkconfig.ConfigurationOption{ sdkconfig.WithUserAgent("application-load-balancer-controller"), + sdkconfig.WithHTTPClient(metrics.NewHTTPClient("application-load-balancer-controller")), } if config.Global.APIEndpoints.ApplicationLoadBalancerCertificateAPI != "" { certOpts = append(certOpts, sdkconfig.WithEndpoint(config.Global.APIEndpoints.ApplicationLoadBalancerCertificateAPI)) diff --git a/go.mod b/go.mod index 6ea1f30..f8ad356 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,8 @@ require ( github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.32.1 github.com/onsi/gomega v1.42.1 + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 github.com/stackitcloud/stackit-sdk-go/core v0.26.0 github.com/stackitcloud/stackit-sdk-go/services/alb v0.17.1 github.com/stackitcloud/stackit-sdk-go/services/certificates v1.9.1 @@ -38,13 +40,12 @@ require ( github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect - github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect github.com/spf13/pflag v1.0.9 // indirect diff --git a/pkg/metrics/http.go b/pkg/metrics/http.go new file mode 100644 index 0000000..02de268 --- /dev/null +++ b/pkg/metrics/http.go @@ -0,0 +1,128 @@ +package metrics + +import ( + "net/http" + "runtime" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/prometheus/client_golang/prometheus" +) + +const UnknownOperation = "UnknownOperation" + +func NewHTTPClient(componentName string) *http.Client { + return WrapHTTPClient(http.DefaultClient, componentName) +} + +func WrapHTTPClient(client *http.Client, componentName string) *http.Client { + if client == nil { + return nil + } + wrappedClient := *client + + baseTransport := client.Transport + if baseTransport == nil { + baseTransport = http.DefaultTransport + } + + // Chain your instrumented round tripper + wrappedClient.Transport = &InstrumentedRoundTripper{ + base: baseTransport, + componentName: componentName, + } + + return &wrappedClient +} + +type InstrumentedRoundTripper struct { + base http.RoundTripper + componentName string +} + +func (rt *InstrumentedRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + startTime := time.Now() + response, err := rt.base.RoundTrip(request) + duration := time.Since(startTime) + + statusCode := "network_error" + if response != nil { + statusCode = strconv.Itoa(response.StatusCode) + } + + // request.Host is optional so we can fallback to request.URL.Host (if available) + host := request.Host + if host == "" && request.URL != nil { + host = request.URL.Host + } + + labels := prometheus.Labels{ + componentLabel: rt.componentName, + hostLabel: host, + methodLabel: request.Method, + operationLabel: getSDKOperationName(), + codeLabel: statusCode, + } + + HTTPRequestDurationHistogram.With(labels).Observe(duration.Seconds()) + HTTPRequestCount.With(labels).Inc() + + isHTTPError := response != nil && response.StatusCode >= 400 + isNetworkError := err != nil + + if isHTTPError || isNetworkError { + HTTPErrorCount.With(labels).Inc() + } + + return response, err +} + +// getSDKOperationName returns the name of the STACKIT SDK function. To do this the function gets the last 10 callers and checks +// for functions from the stackitcloud/stackit-sdk-go. It fall back to UnknownOperation if no function was found. +func getSDKOperationName() string { + pc := make([]uintptr, 10) + + // Skip 3 because the first 3 are always Callers, getSDKOperationName, RoundTrip. + n := runtime.Callers(3, pc) + if n == 0 { + return UnknownOperation + } + + frames := runtime.CallersFrames(pc[:n]) + moreFrames := true + for moreFrames { + var frame runtime.Frame + frame, moreFrames = frames.Next() + + if !strings.Contains(frame.Function, "stackitcloud/stackit-sdk-go") { + continue + } + + parts := strings.Split(frame.Function, ".") + if len(parts) > 0 { + funcName := parts[len(parts)-1] + + // Skip function names with 0 len + // Skip Execute, because there is a function with more detailed name + // Skip RoundTrip, because this only the RoundTrip for the AuthFlow + if funcName == "" || + funcName == "Execute" || + funcName == "RoundTrip" { + continue + } + + // Skip Private functions + r, _ := utf8.DecodeRuneInString(funcName) + if !unicode.IsUpper(r) { + continue + } + + return strings.TrimSuffix(funcName, "Execute") + } + } + + return UnknownOperation +} diff --git a/pkg/metrics/http_test.go b/pkg/metrics/http_test.go new file mode 100644 index 0000000..df8ed6b --- /dev/null +++ b/pkg/metrics/http_test.go @@ -0,0 +1,216 @@ +package metrics + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + dto "github.com/prometheus/client_model/go" + sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" + alb "github.com/stackitcloud/stackit-sdk-go/services/alb/v2api" +) + +var _ = Describe("Metrics", func() { + Describe("getSDKOperationName", func() { + var ( + server *httptest.Server + host string + component = "test" + loadbalancerClient *alb.APIClient + ) + + BeforeEach(func() { + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + url, err := url.Parse(server.URL) + Expect(err).NotTo(HaveOccurred()) + host = url.Host + + HTTPRequestCount.Reset() + HTTPErrorCount.Reset() + HTTPRequestDurationHistogram.Reset() + + loadbalancerClient, err = alb.NewAPIClient( + sdkconfig.WithHTTPClient(NewHTTPClient(component)), + sdkconfig.WithEndpoint(server.URL), + sdkconfig.WithoutAuthentication(), + ) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + server.Close() + }) + + It("should return DeleteLoadBalancer as operation", func() { + _, err := loadbalancerClient.DefaultAPI.DeleteLoadBalancer(context.TODO(), uuid.New().String(), "", uuid.New().String()).Execute() + Expect(err).NotTo(HaveOccurred()) + + labels := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: "DELETE", + operationLabel: "DeleteLoadBalancer", + codeLabel: "200", + } + + Expect(testutil.ToFloat64(HTTPRequestCount.With(labels))).To(Equal(float64(1))) + }) + + It("should return DeleteCredentials as operation", func() { + _, err := loadbalancerClient.DefaultAPI.DeleteCredentials(context.TODO(), uuid.New().String(), "", uuid.New().String()).Execute() + Expect(err).NotTo(HaveOccurred()) + + labels := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: "DELETE", + operationLabel: "DeleteCredentials", + codeLabel: "200", + } + + Expect(testutil.ToFloat64(HTTPRequestCount.With(labels))).To(Equal(float64(1))) + }) + }) + + Describe("InstrumentedRoundTripper", func() { + var ( + server *httptest.Server + httpClient *http.Client + host string + component = "test" + ) + + BeforeEach(func() { + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/404": + w.WriteHeader(http.StatusNotFound) + case "/500": + w.WriteHeader(http.StatusInternalServerError) + case "/400": + w.WriteHeader(http.StatusBadRequest) + default: + w.WriteHeader(http.StatusOK) + } + })) + url, err := url.Parse(server.URL) + Expect(err).NotTo(HaveOccurred()) + host = url.Host + httpClient = NewHTTPClient(component) + + HTTPRequestCount.Reset() + HTTPErrorCount.Reset() + HTTPRequestDurationHistogram.Reset() + }) + + AfterEach(func() { + server.Close() + }) + + It("increments HTTPRequestCount for responses", func() { + labels := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: "GET", + operationLabel: UnknownOperation, + codeLabel: "200", + } + + response, err := httpClient.Get(server.URL + "/request-count-test") + Expect(err).NotTo(HaveOccurred()) + defer response.Body.Close() + + Expect(testutil.ToFloat64(HTTPRequestCount.With(labels))).To(Equal(float64(1))) + }) + + It("records HTTPRequestDurationHistogram observations for responses", func() { + labels := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: "GET", + operationLabel: UnknownOperation, + codeLabel: "200", + } + + response, err := httpClient.Get(server.URL + "/request-duration-test") + Expect(err).NotTo(HaveOccurred()) + defer response.Body.Close() + + Expect(histogramSampleCount(HTTPRequestDurationHistogram.With(labels))).To(Equal(uint64(1))) + }) + + It("increments HTTPErrorCount for error responses (400, 404, 500)", func() { + labels400 := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: http.MethodGet, + operationLabel: UnknownOperation, + codeLabel: "400", + } + labels404 := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: http.MethodGet, + operationLabel: UnknownOperation, + codeLabel: "404", + } + labels500 := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: http.MethodPost, + operationLabel: UnknownOperation, + codeLabel: "500", + } + + response1, err := httpClient.Get(server.URL + "/400") + Expect(err).NotTo(HaveOccurred()) + defer response1.Body.Close() + + response2, err := httpClient.Get(server.URL + "/404") + Expect(err).NotTo(HaveOccurred()) + defer response2.Body.Close() + + response3, err := httpClient.Post(server.URL+"/500", "application/json", nil) + Expect(err).NotTo(HaveOccurred()) + defer response3.Body.Close() + + Expect(testutil.ToFloat64(HTTPErrorCount.With(labels400))).To(Equal(float64(1))) + Expect(testutil.ToFloat64(HTTPErrorCount.With(labels404))).To(Equal(float64(1))) + Expect(testutil.ToFloat64(HTTPErrorCount.With(labels500))).To(Equal(float64(1))) + }) + + It("does not increment HTTPErrorCount for successful responses", func() { + labels := prometheus.Labels{ + hostLabel: host, + componentLabel: component, + methodLabel: http.MethodGet, + operationLabel: UnknownOperation, + codeLabel: "200", + } + + response, err := httpClient.Get(server.URL) + Expect(err).NotTo(HaveOccurred()) + defer response.Body.Close() + + Expect(testutil.ToFloat64(HTTPErrorCount.With(labels))).To(Equal(float64(0))) + }) + }) +}) + +func histogramSampleCount(observer prometheus.Observer) uint64 { + metric, ok := observer.(prometheus.Metric) + Expect(ok).To(BeTrue()) + + dtoMetric := &dto.Metric{} + Expect(metric.Write(dtoMetric)).To(Succeed()) + + return dtoMetric.GetHistogram().GetSampleCount() +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000..d6ad9db --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,53 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const ( + metricPrefix = "stackit_api" + componentLabel = "component" + hostLabel = "host" + methodLabel = "method" + operationLabel = "operation" + codeLabel = "status_code" +) + +var ( + HTTPRequestCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: "http_requests_total", + Help: "The number of requests to external APIs", + }, []string{componentLabel, hostLabel, methodLabel, operationLabel, codeLabel}) + + HTTPErrorCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: "http_errors_total", + Help: "Number of HTTP errors returned by external APIs", + }, []string{componentLabel, hostLabel, methodLabel, operationLabel, codeLabel}) + + HTTPRequestDurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metricPrefix, + Name: "http_request_duration_seconds", + Help: "The response times of external API requests", + }, []string{componentLabel, hostLabel, methodLabel, operationLabel, codeLabel}) +) + +type Exporter struct { +} + +func NewExporter() *Exporter { + return &Exporter{} +} + +func (e *Exporter) Describe(descs chan<- *prometheus.Desc) { + HTTPRequestCount.Describe(descs) + HTTPErrorCount.Describe(descs) + HTTPRequestDurationHistogram.Describe(descs) +} + +func (e *Exporter) Collect(metrics chan<- prometheus.Metric) { + HTTPRequestCount.Collect(metrics) + HTTPErrorCount.Collect(metrics) + HTTPRequestDurationHistogram.Collect(metrics) +} diff --git a/pkg/metrics/suite_test.go b/pkg/metrics/suite_test.go new file mode 100644 index 0000000..42aa781 --- /dev/null +++ b/pkg/metrics/suite_test.go @@ -0,0 +1,13 @@ +package metrics + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMetrics(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Metrics Provider Suite") +}