Move configs-db integration tests to ./integration

.github/workflows-doc.md

@@ -17,7 +17,6 @@ test-build-deploy.yml specifies a workflow that runs all Cortex continuous integ
 |------------------------|-------------------------------------------------------------------------------------------------------------------------------|------|
 | lint                   | Runs linting and ensures vendor directory, protos and generated documentation are consistent.                                 | CI   |
 | test                   | Runs units tests on Cassandra testing framework.                                                                              | CI   |
-| integration-configs-db | Integration tests for database configurations.                                                                                | CI   |
 | integration            | Runs integration tests after upgrading golang, pulling necessary docker images and downloading necessary module dependencies. | CI   |
 | Security/CodeQL        | CodeQL is a semantic code analysis engine used for automating security checks.                                                | CI   |
 | build                  | Builds and saves an up-to-date Cortex image and website.                                                                      | CI   |
@@ -62,7 +61,7 @@ As of October 2020, GitHub Actions do not persist between different jobs in the
 | Artifact                      | Stored In | Used By                                     | Purpose of Storing Artifact |
 |-------------------------------|-----------|---------------------------------------------|-----------------------------|
 | website public                | build     | deploy_website                              | share data between jobs     |
-| Docker Images                 | build     | deploy, integration, integrations-config-db | share data between jobs     |
+| Docker Images                 | build     | deploy, integration                         | share data between jobs     |
 
 *Note:* Docker Images are zipped before uploading as a workaround. The images contain characters that are illegal in the upload-artifact action.
 ```yaml

.github/workflows/test-build-deploy.yml

@@ -327,6 +327,8 @@ jobs:
           elif [ "$TEST_TAGS" = "integration_query_fuzz" ]; then
             retry docker pull quay.io/cortexproject/cortex:v1.20.1
             retry docker pull quay.io/prometheus/prometheus:v3.8.1
+          elif [ "$TEST_TAGS" = "integration_configs_db" ]; then
+            retry docker pull postgres:9.6.16
           fi
           retry docker pull memcached:1.6.1
           retry docker pull redis:7.0.4-alpine
@@ -346,38 +348,8 @@ jobs:
         env:
           IMAGE_PREFIX: ${{ secrets.IMAGE_PREFIX }}
 
-  integration-configs-db:
-    needs: [build, lint]
-    runs-on: ${{ matrix.runner }}
-    timeout-minutes: 20
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - runner: ubuntu-24.04
-            arch: amd64
-          - runner: ubuntu-24.04-arm
-            arch: arm64
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-      - name: Download Docker Images Artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
-        with:
-          name: Docker Images
-      - name: Extract Docker Images Archive
-        run: tar -xvf images.tar -C /
-      - name: Run Integration Configs Tests
-        timeout-minutes: 15
-        # Github Actions does not support TTY in their default runners yet
-        run: |
-          touch build-image/.uptodate
-          MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations
-          echo "Running configs integration tests on ${{ matrix.arch }}"
-          make BUILD_IMAGE=quay.io/cortexproject/build-image:master-5f643d518c TTY='' configs-integration-test
-
   deploy:
-    needs: [build, test, lint, integration, integration-configs-db]
+    needs: [build, test, lint, integration]
     if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
     runs-on: ubuntu-24.04
     container:

Makefile

@@ -104,7 +104,6 @@ pkg/alertmanager/alertspb/alerts.pb.go: pkg/alertmanager/alertspb/alerts.proto
 all: $(UPTODATE_FILES)
 test: protos
 mod-check: protos
-configs-integration-test: protos
 lint: protos
 build-image/$(UPTODATE): build-image/*
 
@@ -133,22 +132,6 @@ exes $(EXES) protos $(PROTO_GOS) lint test cover shell mod-check check-protos do
 	@echo ">>>> Entering build container: $@"
 	@$(SUDO) time docker run --rm $(TTY) -i $(GOVOLUMES) $(BUILD_IMAGE) $@;
 
-configs-integration-test: build-image/$(UPTODATE)
-	@mkdir -p $(shell pwd)/.pkg
-	@mkdir -p $(shell pwd)/.cache
-	@DB_CONTAINER="$$(docker run -d -e 'POSTGRES_DB=configs_test' postgres:9.6.16)"; \
-	echo ; \
-	echo ">>>> Entering build container: $@"; \
-	$(SUDO) docker run --rm $(TTY) -i $(GOVOLUMES) \
-		-v $(shell pwd)/cmd/cortex/migrations:/migrations:z \
-		--workdir /go/src/github.com/cortexproject/cortex \
-		--link "$$DB_CONTAINER":configs-db.cortex.local \
-		-e DB_ADDR=configs-db.cortex.local \
-		$(BUILD_IMAGE) $@; \
-	status=$$?; \
-	docker rm -f "$$DB_CONTAINER"; \
-	exit $$status
-
 else
 
 exes: $(EXES)
@@ -175,7 +158,7 @@ lint:
 	golangci-lint run
 
 	# Ensure no blocklisted package is imported.
-	GOFLAGS="-tags=requires_docker,integration,integration_alertmanager,integration_backward_compatibility,integration_memberlist,integration_querier,integration_ruler,integration_query_fuzz,integration_remote_write_v2" faillint -paths "github.com/bmizerany/assert=github.com/stretchr/testify/assert,\
+	GOFLAGS="-tags=requires_docker,integration,integration_alertmanager,integration_backward_compatibility,integration_configs_db,integration_memberlist,integration_querier,integration_ruler,integration_query_fuzz,integration_remote_write_v2" faillint -paths "github.com/bmizerany/assert=github.com/stretchr/testify/assert,\
 		golang.org/x/net/context=context,\
 		sync/atomic=go.uber.org/atomic,\
 		github.com/prometheus/client_golang/prometheus.{MultiError}=github.com/prometheus/prometheus/tsdb/errors.{NewMulti},\
@@ -229,9 +212,6 @@ cover:
 shell:
 	bash
 
-configs-integration-test:
-	/bin/bash -c "go test -v -tags 'netgo integration slicelabels' -timeout 10m ./pkg/configs/... ./pkg/ruler/..."
-
 mod-check:
 	GO111MODULE=on go mod download
 	GO111MODULE=on go mod verify

integration/configs_db_test.go

@@ -0,0 +1,246 @@
+//go:build integration_configs_db
+
+package integration
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cortexproject/cortex/integration/e2e"
+	e2edb "github.com/cortexproject/cortex/integration/e2e/db"
+	"github.com/cortexproject/cortex/integration/e2ecortex"
+	"github.com/cortexproject/cortex/pkg/configs/userconfig"
+)
+
+// configsDBSetup starts Postgres + a configs-target Cortex service, returning the
+// scenario and the configs service. The caller can build a ConfigsClient against
+// configs.HTTPEndpoint().
+func configsDBSetup(t *testing.T) (*e2e.Scenario, *e2ecortex.CortexService) {
+	t.Helper()
+
+	s, err := e2e.NewScenario(networkName)
+	require.NoError(t, err)
+	t.Cleanup(s.Close)
+
+	postgres := e2edb.NewPostgres(e2edb.PostgresHostName)
+	require.NoError(t, s.StartAndWaitReady(postgres))
+
+	configs := e2ecortex.NewConfigs("configs", map[string]string{
+		"-configs.database.uri": fmt.Sprintf(
+			"postgres://%s@%s:%d/%s?sslmode=disable",
+			e2edb.PostgresUser,
+			e2e.NetworkContainerHost(networkName, e2edb.PostgresHostName),
+			e2edb.PostgresPort,
+			e2edb.PostgresDB,
+		),
+	}, "")
+	require.NoError(t, s.StartAndWaitReady(configs))
+
+	return s, configs
+}
+
+func makeUserID(prefix string, n int) string {
+	return fmt.Sprintf("%s-%d", prefix, n)
+}
+
+func makeRulesConfig(payload string) userconfig.Config {
+	return userconfig.Config{
+		RulesConfig: userconfig.RulesConfig{
+			FormatVersion: userconfig.RuleFormatV2,
+			Files: map[string]string{
+				"rules.yaml": payload,
+			},
+		},
+	}
+}
+
+func makeAlertmanagerConfig(receiver string) userconfig.Config {
+	return userconfig.Config{
+		AlertmanagerConfig: fmt.Sprintf(`route:
+  receiver: %s
+receivers:
+- name: %s
+`, receiver, receiver),
+		RulesConfig: userconfig.RulesConfig{FormatVersion: userconfig.RuleFormatV2},
+	}
+}
+
+// TestConfigsDB_MigrationsRunOnStartup proves the configs service successfully
+// runs the SQL migrations under cmd/cortex/migrations against an empty database.
+// configsDBSetup already waits for the /ready endpoint, so reaching this point
+// is the assertion.
+func TestConfigsDB_MigrationsRunOnStartup(t *testing.T) {
+	_, _ = configsDBSetup(t)
+}
+
+func TestConfigsDB_AnonymousReturns401(t *testing.T) {
+	_, configs := configsDBSetup(t)
+
+	client := e2ecortex.NewConfigsClient(configs.HTTPEndpoint(), "")
+	_, code, err := client.GetRulesConfig(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusUnauthorized, code)
+}
+
+func TestConfigsDB_GetMissingReturns404(t *testing.T) {
+	_, configs := configsDBSetup(t)
+
+	client := e2ecortex.NewConfigsClient(configs.HTTPEndpoint(), "user-missing")
+	_, code, err := client.GetRulesConfig(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusNotFound, code)
+}
+
+func TestConfigsDB_PostAndGetRulesConfig(t *testing.T) {
+	_, configs := configsDBSetup(t)
+
+	client := e2ecortex.NewConfigsClient(configs.HTTPEndpoint(), "tenant-rules")
+	cfg := makeRulesConfig("groups: []")
+
+	code, body, err := client.PostRulesConfig(context.Background(), cfg)
+	require.NoError(t, err)
+	require.Equalf(t, http.StatusNoContent, code, "unexpected status, body: %s", body)
+
+	view, code, err := client.GetRulesConfig(context.Background())
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, code)
+	require.NotNil(t, view)
+	assert.Equal(t, cfg.RulesConfig.Files, view.Config.RulesConfig.Files)
+}
+
+func TestConfigsDB_PostAndGetAlertmanagerConfig(t *testing.T) {
+	_, configs := configsDBSetup(t)
+
+	client := e2ecortex.NewConfigsClient(configs.HTTPEndpoint(), "tenant-am")
+	cfg := makeAlertmanagerConfig("dummy")
+
+	code, body, err := client.PostAlertmanagerConfig(context.Background(), cfg)
+	require.NoError(t, err)
+	require.Equalf(t, http.StatusNoContent, code, "unexpected status, body: %s", body)
+
+	view, code, err := client.GetAlertmanagerConfig(context.Background())
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, code)
+	require.NotNil(t, view)
+	assert.Equal(t, cfg.AlertmanagerConfig, view.Config.AlertmanagerConfig)
+}
+
+func TestConfigsDB_MultiTenantIsolation(t *testing.T) {
+	_, configs := configsDBSetup(t)
+
+	tenants := []string{makeUserID("iso", 1), makeUserID("iso", 2)}
+	configs1 := makeRulesConfig("groups: [{name: t1}]")
+	configs2 := makeRulesConfig("groups: [{name: t2}]")
+
+	c1 := e2ecortex.NewConfigsClient(configs.HTTPEndpoint(), tenants[0])
+	c2 := e2ecortex.NewConfigsClient(configs.HTTPEndpoint(), tenants[1])
+
+	code, body, err := c1.PostRulesConfig(context.Background(), configs1)
+	require.NoError(t, err)
+	require.Equalf(t, http.StatusNoContent, code, "unexpected status, body: %s", body)
+
+	code, body, err = c2.PostRulesConfig(context.Background(), configs2)
+	require.NoError(t, err)
+	require.Equalf(t, http.StatusNoContent, code, "unexpected status, body: %s", body)
+
+	view1, code, err := c1.GetRulesConfig(context.Background())
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, code)
+	assert.Equal(t, configs1.RulesConfig.Files, view1.Config.RulesConfig.Files)
+
+	view2, code, err := c2.GetRulesConfig(context.Background())
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, code)
+	assert.Equal(t, configs2.RulesConfig.Files, view2.Config.RulesConfig.Files)
+}
+
+func TestConfigsDB_GetAllConfigsReturnsLatest(t *testing.T) {
+	_, configs := configsDBSetup(t)
+
+	tenant := makeUserID("latest", 1)
+	client := e2ecortex.NewConfigsClient(configs.HTTPEndpoint(), tenant)
+
+	older := makeRulesConfig("groups: [{name: older}]")
+	newer := makeRulesConfig("groups: [{name: newer}]")
+	for _, cfg := range []userconfig.Config{older, older, newer} {
+		code, body, err := client.PostRulesConfig(context.Background(), cfg)
+		require.NoError(t, err)
+		require.Equalf(t, http.StatusNoContent, code, "unexpected status, body: %s", body)
+	}
+
+	admin := e2ecortex.NewConfigsClient(configs.HTTPEndpoint(), "")
+	all, code, err := admin.GetAllRulesConfigs(context.Background())
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, code)
+
+	view, ok := all[tenant]
+	require.True(t, ok, "tenant %q not present in admin response", tenant)
+	assert.Equal(t, newer.RulesConfig.Files, view.Config.RulesConfig.Files)
+}
+
+// TestConfigsDB_GetAllConfigsEmpty proves the admin GetAllConfigs SQL returns an
+// empty result set (not an error) against a freshly-migrated, empty database.
+func TestConfigsDB_GetAllConfigsEmpty(t *testing.T) {
+	_, configs := configsDBSetup(t)
+
+	admin := e2ecortex.NewConfigsClient(configs.HTTPEndpoint(), "")
+	all, code, err := admin.GetAllRulesConfigs(context.Background())
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, code)
+	assert.Empty(t, all)
+}
+
+// TestConfigsDB_PostAnonymousReturns401 covers the write path's tenant-auth check,
+// mirroring the read-path check in TestConfigsDB_AnonymousReturns401.
+func TestConfigsDB_PostAnonymousReturns401(t *testing.T) {
+	_, configs := configsDBSetup(t)
+
+	client := e2ecortex.NewConfigsClient(configs.HTTPEndpoint(), "")
+	code, _, err := client.PostRulesConfig(context.Background(), makeRulesConfig("groups: []"))
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusUnauthorized, code)
+}
+
+// TestConfigsDB_GetConfigsSince proves the GetConfigs(since) SQL filter returns
+// only configs whose version ID is greater than the supplied cursor. This is the
+// one Postgres-specific query path not exercised by the other tests.
+func TestConfigsDB_GetConfigsSince(t *testing.T) {
+	_, configs := configsDBSetup(t)
+	ctx := context.Background()
+	endpoint := configs.HTTPEndpoint()
+
+	// Post one config per tenant, in order, capturing each assigned version ID.
+	post := func(tenant, payload string) userconfig.ID {
+		c := e2ecortex.NewConfigsClient(endpoint, tenant)
+		code, body, err := c.PostRulesConfig(ctx, makeRulesConfig(payload))
+		require.NoError(t, err)
+		require.Equalf(t, http.StatusNoContent, code, "unexpected status, body: %s", body)
+		view, code, err := c.GetRulesConfig(ctx)
+		require.NoError(t, err)
+		require.Equal(t, http.StatusOK, code)
+		return view.ID
+	}
+
+	user1, user2, user3 := makeUserID("since", 1), makeUserID("since", 2), makeUserID("since", 3)
+	post(user1, "groups: [{name: first}]")
+	id2 := post(user2, "groups: [{name: second}]")
+	post(user3, "groups: [{name: third}]")
+
+	// since=id2 must exclude user1 and user2 (IDs <= id2) and include only user3.
+	admin := e2ecortex.NewConfigsClient(endpoint, "")
+	all, code, err := admin.GetAllRulesConfigsSince(ctx, id2)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, code)
+
+	_, has1 := all[user1]
+	_, has2 := all[user2]
+	_, has3 := all[user3]
+	assert.False(t, has1, "user1 (ID < since) should be excluded")
+	assert.False(t, has2, "user2 (ID == since) should be excluded")
+	assert.True(t, has3, "user3 (ID > since) should be included")
+}

integration/e2e/db/db.go

@@ -14,6 +14,11 @@ import (
 const (
 	MinioAccessKey = "Cheescake"
 	MinioSecretKey = "supersecret"
+
+	PostgresDB       = "configs_test"
+	PostgresUser     = "postgres"
+	PostgresPort     = 5432
+	PostgresHostName = "configs-db"
 )
 
 // NewMinio returns minio server, used as a local replacement for S3.
@@ -69,6 +74,24 @@ func NewETCD() *e2e.HTTPService {
 		9000, // Metrics
 	)
 }
+
+// NewPostgres returns a postgres server suitable for tests that need a real database
+// (e.g. the configs API). The default database, user and port match the values exposed
+// as Postgres* constants in this package.
+func NewPostgres(name string) *e2e.ConcreteService {
+	s := e2e.NewConcreteService(
+		name,
+		images.Postgres,
+		nil,
+		e2e.NewCmdReadinessProbe(e2e.NewCommand("pg_isready", "-U", PostgresUser, "-d", PostgresDB)),
+		PostgresPort,
+	)
+	s.SetEnvVars(map[string]string{
+		"POSTGRES_DB":               PostgresDB,
+		"POSTGRES_HOST_AUTH_METHOD": "trust",
+	})
+	return s
+}
 func NewPrometheus(image string, flags map[string]string) *e2e.HTTPService {
 	return NewPrometheusWithName("prometheus", image, flags)
 }