Add metrics for API calls

cmd/gardener-extension-provider-stackit/main.go

@@ -10,10 +10,16 @@ import (
 	"github.com/gardener/gardener/pkg/logger"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	"github.com/stackitcloud/gardener-extension-provider-stackit/v2/cmd/gardener-extension-provider-stackit/app"
+	"github.com/stackitcloud/gardener-extension-provider-stackit/v2/pkg/metrics"
 )
 
+func init() {
+	ctrlmetrics.Registry.MustRegister(metrics.NewExporter())
+}
+
 func main() {
 	log.SetLogger(logger.MustNewZapLogger(logger.InfoLevel, logger.FormatJSON))
 	setupLogger := log.Log.WithName("setup")

go.mod

@@ -21,6 +21,8 @@ require (
 	github.com/pelletier/go-toml/v2 v2.4.3
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.93.1
+	github.com/prometheus/client_golang v1.24.1
+	github.com/prometheus/client_model v0.6.2
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/pflag v1.0.10
 	github.com/stackitcloud/stackit-sdk-go/core v0.26.0
@@ -178,8 +180,6 @@ require (
 	github.com/pierrec/lz4/v4 v4.1.26 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/alertmanager v0.33.1 // indirect
-	github.com/prometheus/client_golang v1.24.1 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.70.1 // indirect
 	github.com/prometheus/exporter-toolkit v0.16.0 // indirect
 	github.com/prometheus/procfs v0.21.1 // indirect

pkg/metrics/http.go

@@ -0,0 +1,128 @@
+package metrics
+
+import (
+	"net/http"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+	"unicode"
+	"unicode/utf8"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const UnknownOperation = "UnknownOperation"
+
+func NewHTTPClient(componentName string) *http.Client {
+	return WrapHTTPClient(http.DefaultClient, componentName)
+}
+
+func WrapHTTPClient(client *http.Client, componentName string) *http.Client {
+	if client == nil {
+		return nil
+	}
+	wrappedClient := *client
+
+	baseTransport := client.Transport
+	if baseTransport == nil {
+		baseTransport = http.DefaultTransport
+	}
+
+	// Chain your instrumented round tripper
+	wrappedClient.Transport = &InstrumentedRoundTripper{
+		base:          baseTransport,
+		componentName: componentName,
+	}
+
+	return &wrappedClient
+}
+
+type InstrumentedRoundTripper struct {
+	base          http.RoundTripper
+	componentName string
+}
+
+func (rt *InstrumentedRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) {
+	startTime := time.Now()
+	response, err := rt.base.RoundTrip(request)
+	duration := time.Since(startTime)
+
+	statusCode := "network_error"
+	if response != nil {
+		statusCode = strconv.Itoa(response.StatusCode)
+	}
+
+	// request.Host is optional so we can fallback to request.URL.Host (if available)
+	host := request.Host
+	if host == "" && request.URL != nil {
+		host = request.URL.Host
+	}
+
+	labels := prometheus.Labels{
+		componentLabel: rt.componentName,
+		hostLabel:      host,
+		methodLabel:    request.Method,
+		operationLabel: getSDKOperationName(),
+		codeLabel:      statusCode,
+	}
+
+	HTTPRequestDurationHistogram.With(labels).Observe(duration.Seconds())
+	HTTPRequestCount.With(labels).Inc()
+
+	isHTTPError := response != nil && response.StatusCode >= 400
+	isNetworkError := err != nil
+
+	if isHTTPError || isNetworkError {
+		HTTPErrorCount.With(labels).Inc()
+	}
+
+	return response, err
+}
+
+// getSDKOperationName returns the name of the STACKIT SDK function. To do this the function gets the last 10 callers and checks
+// for functions from the stackitcloud/stackit-sdk-go. It fall back to UnknownOperation if no function was found.
+func getSDKOperationName() string {
+	pc := make([]uintptr, 10)
+
+	// Skip 3 because the first 3 are always Callers, getSDKOperationName, RoundTrip.
+	n := runtime.Callers(3, pc)
+	if n == 0 {
+		return UnknownOperation
+	}
+
+	frames := runtime.CallersFrames(pc[:n])
+	moreFrames := true
+	for moreFrames {
+		var frame runtime.Frame
+		frame, moreFrames = frames.Next()
+
+		if !strings.Contains(frame.Function, "stackitcloud/stackit-sdk-go") {
+			continue
+		}
+
+		parts := strings.Split(frame.Function, ".")
+		if len(parts) > 0 {
+			funcName := parts[len(parts)-1]
+
+			// Skip function names with 0 len
+			// Skip Execute, because there is a function with more detailed name
+			// Skip RoundTrip, because this only the RoundTrip for the AuthFlow
+			if funcName == "" ||
+				funcName == "Execute" ||
+				funcName == "RoundTrip" {
+				continue
+			}
+
+			// Skip Private functions
+			r, _ := utf8.DecodeRuneInString(funcName)
+			if !unicode.IsUpper(r) {
+				continue
+			}
+
+			return strings.TrimSuffix(funcName, "Execute")
+		}
+	}
+
+	return UnknownOperation
+}

pkg/metrics/http_test.go

@@ -0,0 +1,216 @@
+package metrics
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+
+	"github.com/google/uuid"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	dto "github.com/prometheus/client_model/go"
+	sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config"
+	iaas "github.com/stackitcloud/stackit-sdk-go/services/iaas/v2api"
+)
+
+var _ = Describe("Metrics", func() {
+	Describe("getSDKOperationName", func() {
+		var (
+			server     *httptest.Server
+			host       string
+			component  = "test"
+			iaasClient *iaas.APIClient
+		)
+
+		BeforeEach(func() {
+			server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+				w.WriteHeader(http.StatusOK)
+			}))
+			url, err := url.Parse(server.URL)
+			Expect(err).NotTo(HaveOccurred())
+			host = url.Host
+
+			HTTPRequestCount.Reset()
+			HTTPErrorCount.Reset()
+			HTTPRequestDurationHistogram.Reset()
+
+			iaasClient, err = iaas.NewAPIClient(
+				sdkconfig.WithHTTPClient(NewHTTPClient(component)),
+				sdkconfig.WithEndpoint(server.URL),
+				sdkconfig.WithoutAuthentication(),
+			)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			server.Close()
+		})
+
+		It("should return DeleteVolume as operation", func() {
+			err := iaasClient.DefaultAPI.DeleteVolume(context.TODO(), uuid.New().String(), "", uuid.New().String()).Execute()
+			Expect(err).NotTo(HaveOccurred())
+
+			labels := prometheus.Labels{
+				hostLabel:      host,
+				componentLabel: component,
+				methodLabel:    "DELETE",
+				operationLabel: "DeleteVolume",
+				codeLabel:      "200",
+			}
+
+			Expect(testutil.ToFloat64(HTTPRequestCount.With(labels))).To(Equal(float64(1)))
+		})
+
+		It("should return DeleteServer as operation", func() {
+			err := iaasClient.DefaultAPI.DeleteServer(context.TODO(), uuid.New().String(), "", uuid.New().String()).Execute()
+			Expect(err).NotTo(HaveOccurred())
+
+			labels := prometheus.Labels{
+				hostLabel:      host,
+				componentLabel: component,
+				methodLabel:    "DELETE",
+				operationLabel: "DeleteServer",
+				codeLabel:      "200",
+			}
+
+			Expect(testutil.ToFloat64(HTTPRequestCount.With(labels))).To(Equal(float64(1)))
+		})
+	})
+
+	Describe("InstrumentedRoundTripper", func() {
+		var (
+			server     *httptest.Server
+			httpClient *http.Client
+			host       string
+			component  = "test"
+		)
+
+		BeforeEach(func() {
+			server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				switch r.URL.Path {
+				case "/404":
+					w.WriteHeader(http.StatusNotFound)
+				case "/500":
+					w.WriteHeader(http.StatusInternalServerError)
+				case "/400":
+					w.WriteHeader(http.StatusBadRequest)
+				default:
+					w.WriteHeader(http.StatusOK)
+				}
+			}))
+			url, err := url.Parse(server.URL)
+			Expect(err).NotTo(HaveOccurred())
+			host = url.Host
+			httpClient = NewHTTPClient(component)
+
+			HTTPRequestCount.Reset()
+			HTTPErrorCount.Reset()
+			HTTPRequestDurationHistogram.Reset()
+		})
+
+		AfterEach(func() {
+			server.Close()
+		})
+
+		It("increments HTTPRequestCount for responses", func() {
+			labels := prometheus.Labels{
+				hostLabel:      host,
+				componentLabel: component,
+				methodLabel:    "GET",
+				operationLabel: UnknownOperation,
+				codeLabel:      "200",
+			}
+
+			response, err := httpClient.Get(server.URL + "/request-count-test")
+			Expect(err).NotTo(HaveOccurred())
+			defer response.Body.Close()
+
+			Expect(testutil.ToFloat64(HTTPRequestCount.With(labels))).To(Equal(float64(1)))
+		})
+
+		It("records HTTPRequestDurationHistogram observations for responses", func() {
+			labels := prometheus.Labels{
+				hostLabel:      host,
+				componentLabel: component,
+				methodLabel:    "GET",
+				operationLabel: UnknownOperation,
+				codeLabel:      "200",
+			}
+
+			response, err := httpClient.Get(server.URL + "/request-duration-test")
+			Expect(err).NotTo(HaveOccurred())
+			defer response.Body.Close()
+
+			Expect(histogramSampleCount(HTTPRequestDurationHistogram.With(labels))).To(Equal(uint64(1)))
+		})
+
+		It("increments HTTPErrorCount for error responses (400, 404, 500)", func() {
+			labels400 := prometheus.Labels{
+				hostLabel:      host,
+				componentLabel: component,
+				methodLabel:    http.MethodGet,
+				operationLabel: UnknownOperation,
+				codeLabel:      "400",
+			}
+			labels404 := prometheus.Labels{
+				hostLabel:      host,
+				componentLabel: component,
+				methodLabel:    http.MethodGet,
+				operationLabel: UnknownOperation,
+				codeLabel:      "404",
+			}
+			labels500 := prometheus.Labels{
+				hostLabel:      host,
+				componentLabel: component,
+				methodLabel:    http.MethodPost,
+				operationLabel: UnknownOperation,
+				codeLabel:      "500",
+			}
+
+			response1, err := httpClient.Get(server.URL + "/400")
+			Expect(err).NotTo(HaveOccurred())
+			defer response1.Body.Close()
+
+			response2, err := httpClient.Get(server.URL + "/404")
+			Expect(err).NotTo(HaveOccurred())
+			defer response2.Body.Close()
+
+			response3, err := httpClient.Post(server.URL+"/500", "application/json", nil)
+			Expect(err).NotTo(HaveOccurred())
+			defer response3.Body.Close()
+
+			Expect(testutil.ToFloat64(HTTPErrorCount.With(labels400))).To(Equal(float64(1)))
+			Expect(testutil.ToFloat64(HTTPErrorCount.With(labels404))).To(Equal(float64(1)))
+			Expect(testutil.ToFloat64(HTTPErrorCount.With(labels500))).To(Equal(float64(1)))
+		})
+
+		It("does not increment HTTPErrorCount for successful responses", func() {
+			labels := prometheus.Labels{
+				hostLabel:      host,
+				componentLabel: component,
+				methodLabel:    http.MethodGet,
+				operationLabel: UnknownOperation,
+				codeLabel:      "200",
+			}
+
+			response, err := httpClient.Get(server.URL)
+			Expect(err).NotTo(HaveOccurred())
+			defer response.Body.Close()
+
+			Expect(testutil.ToFloat64(HTTPErrorCount.With(labels))).To(Equal(float64(0)))
+		})
+	})
+})
+
+func histogramSampleCount(observer prometheus.Observer) uint64 {
+	metric, ok := observer.(prometheus.Metric)
+	Expect(ok).To(BeTrue())
+
+	dtoMetric := &dto.Metric{}
+	Expect(metric.Write(dtoMetric)).To(Succeed())
+
+	return dtoMetric.GetHistogram().GetSampleCount()
+}