From db538d3090411f93cebf1531b74775d652692c94 Mon Sep 17 00:00:00 2001 From: DoDiODev Date: Tue, 28 Jul 2026 10:19:23 +0200 Subject: [PATCH] fix(jira): add missing _raw_data_* columns to _tool_jira_sprint_reports The Sprint Report migration 20260722_add_sprint_report_table.go creates _tool_jira_sprint_reports from a struct that does not embed common.NoPKModel, while the runtime model models.JiraSprintReport does. The columns _raw_data_params / _raw_data_table / _raw_data_id / _raw_data_remark (plus created_at, updated_at) were therefore never created, so the ApiExtractor cleanup query WHERE _raw_data_table = ? AND _raw_data_params = ? made the extractSprintReport subtask fail with "Error 1054 (42S22): Unknown column '_raw_data_table' in 'where clause'". Add a new, additive migration that re-runs AutoMigrateTables on a struct embedding archived.NoPKModel. The original migration is left untouched: migration scripts are append-only, and editing it would not repair databases that already recorded its version. Add two schema-drift regression guards that run the REAL migration scripts instead of AutoMigrate-ing the runtime model, which would hide this class of drift: * plugins/jira/e2e/migration_schema_test.go - Jira-specific guard. * plugins/schema_e2e/migration_schema_test.go - cross-plugin guard for every built-in Go plugin, including TestAllGoPluginsListed so the guard stays complete when a new plugin is added. Both guards run the migrations against a dedicated, empty database created by the new helper e2ehelper.NewIsolatedMigrationDb: the shared e2e database is polluted by the other e2e tests, which AutoMigrate tables without recording anything in _devlake_migration_history, so running the real scripts against it fails with errors such as "Table 'cicd_pipeline_commits' already exists". The cross-plugin guard immediately uncovered three pre-existing drifts of the same class, each fixed with its own additive migration: * _tool_taiga_scope_configs - missing type_mappings * _tool_teambition_scope_configs - missing id, created_at, updated_at * _tool_testmo_scope_configs - missing connection_id, name The teambition table has no primary key at all, and its missing `id` is an auto-increment primary key, which AutoMigrate cannot append to an existing table (MySQL: "Incorrect table definition; there can be only one auto column and it must be defined as a key"). That column is therefore added with explicit DDL, which also keeps the ids of existing rows and the sequence/counter in sync on both MySQL and PostgreSQL. Finally, exclude plugins/schema_e2e from scripts/build-plugins.sh: it is not a plugin and has no main package, which broke `make build-plugin` with "-buildmode=plugin requires exactly one main package". Signed-off-by: DoDiODev --- backend/helpers/e2ehelper/migration_db.go | 126 ++++++++++ .../plugins/jira/e2e/migration_schema_test.go | 110 ++++++++ ...7_add_raw_data_columns_to_sprint_report.go | 65 +++++ .../jira/models/migrationscripts/register.go | 1 + .../schema_e2e/migration_schema_test.go | 238 ++++++++++++++++++ ...260727_add_missing_scope_config_columns.go | 61 +++++ .../taiga/models/migrationscripts/register.go | 1 + ...260727_add_missing_scope_config_columns.go | 90 +++++++ .../models/migrationscripts/register.go | 1 + ...260727_add_missing_scope_config_columns.go | 61 +++++ .../models/migrationscripts/register.go | 1 + backend/scripts/build-plugins.sh | 3 +- 12 files changed, 757 insertions(+), 1 deletion(-) create mode 100644 backend/helpers/e2ehelper/migration_db.go create mode 100644 backend/plugins/jira/e2e/migration_schema_test.go create mode 100644 backend/plugins/jira/models/migrationscripts/20260727_add_raw_data_columns_to_sprint_report.go create mode 100644 backend/plugins/schema_e2e/migration_schema_test.go create mode 100644 backend/plugins/taiga/models/migrationscripts/20260727_add_missing_scope_config_columns.go create mode 100644 backend/plugins/teambition/models/migrationscripts/20260727_add_missing_scope_config_columns.go create mode 100644 backend/plugins/testmo/models/migrationscripts/20260727_add_missing_scope_config_columns.go diff --git a/backend/helpers/e2ehelper/migration_db.go b/backend/helpers/e2ehelper/migration_db.go new file mode 100644 index 00000000000..3d7fe2d49e5 --- /dev/null +++ b/backend/helpers/e2ehelper/migration_db.go @@ -0,0 +1,126 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2ehelper + +import ( + "fmt" + "net/url" + "strings" + "testing" + + "github.com/apache/incubator-devlake/core/config" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/impls/dalgorm" + "github.com/apache/incubator-devlake/impls/logruslog" + "gorm.io/gorm" +) + +// NewIsolatedMigrationDb creates a dedicated, empty database next to the one +// referenced by E2E_DB_URL (`_`) and returns a connection to it. +// +// Tests that execute the REAL migration scripts must not share the regular e2e +// database: the other plugin e2e tests seed/AutoMigrate tables (domain layer +// included) without recording anything in `_devlake_migration_history`, so a +// subsequent migration run fails with errors such as +// "Table 'cicd_pipeline_commits' already exists" when it tries to create or +// rename a table that is already there. +// +// The database is dropped again when the test finishes. If E2E_DB_URL is not +// set the test is skipped. +func NewIsolatedMigrationDb(t *testing.T, suffix string) *gorm.DB { + cfg := config.GetConfig() + e2eDbUrl := cfg.GetString("E2E_DB_URL") + if e2eDbUrl == "" { + t.Skip("E2E_DB_URL is not set; skipping migration schema check") + } + u, err := url.Parse(e2eDbUrl) + if err != nil { + t.Fatalf("unable to parse E2E_DB_URL: %v", err) + } + isolatedName := fmt.Sprintf("%s_%s", strings.TrimPrefix(u.Path, "/"), suffix) + quotedName := quoteDbName(u.Scheme, isolatedName) + + gormConf := &gorm.Config{SkipDefaultTransaction: true} + adminDb, err := runner.MakeDbConnection(e2eDbUrl, gormConf) + if err != nil { + t.Fatalf("unable to connect to E2E_DB_URL: %v", err) + } + if err = adminDb.Exec("DROP DATABASE IF EXISTS " + quotedName).Error; err != nil { + t.Fatalf("unable to drop leftover database %s: %v", isolatedName, err) + } + if err = adminDb.Exec("CREATE DATABASE " + quotedName).Error; err != nil { + t.Fatalf("unable to create database %s: %v", isolatedName, err) + } + closeDb(adminDb) + + isolatedUrl := *u + isolatedUrl.Path = "/" + isolatedName + db, err := runner.MakeDbConnection(isolatedUrl.String(), gormConf) + if err != nil { + t.Fatalf("unable to connect to %s: %v", isolatedName, err) + } + + // migration scripts and models read DB_URL from the global config, keep it + // consistent with the connection we hand out and restore it afterwards. + previousDbUrl := cfg.GetString("DB_URL") + cfg.Set("DB_URL", isolatedUrl.String()) + + // Some migration scripts refuse to run without an encryption secret + // (e.g. jira 20220716: "jira v0.11 invalid encKey"). CI does not + // necessarily provide one, so fall back to a deterministic test value. + // dalgorm.Init registers the `encdec` GORM serializer used by connection + // models - without it migrations fail with "invalid serializer type encdec" + // (runner.CreateBasicRes does not register it, only CreateAppBasicRes does). + if cfg.GetString(plugin.EncodeKeyEnvStr) == "" { + cfg.Set(plugin.EncodeKeyEnvStr, "devlake-e2e-test-encryption-secret") + } + dalgorm.Init(cfg.GetString(plugin.EncodeKeyEnvStr)) + + t.Cleanup(func() { + cfg.Set("DB_URL", previousDbUrl) + closeDb(db) + cleanupDb, cleanupErr := runner.MakeDbConnection(e2eDbUrl, gormConf) + if cleanupErr != nil { + t.Logf("unable to connect for dropping %s: %v", isolatedName, cleanupErr) + return + } + defer closeDb(cleanupDb) + if dropErr := cleanupDb.Exec("DROP DATABASE IF EXISTS " + quotedName).Error; dropErr != nil { + t.Logf("unable to drop database %s: %v", isolatedName, dropErr) + } + }) + + logruslog.Global.Info("running migrations against isolated database %s", isolatedName) + return db +} + +func quoteDbName(scheme string, name string) string { + // database names are derived from E2E_DB_URL + a constant suffix, but quote + // them anyway to stay safe with reserved words. + if strings.EqualFold(scheme, "mysql") { + return "`" + strings.ReplaceAll(name, "`", "") + "`" + } + return `"` + strings.ReplaceAll(name, `"`, "") + `"` +} + +func closeDb(db *gorm.DB) { + if sqlDb, err := db.DB(); err == nil { + _ = sqlDb.Close() + } +} diff --git a/backend/plugins/jira/e2e/migration_schema_test.go b/backend/plugins/jira/e2e/migration_schema_test.go new file mode 100644 index 00000000000..619aee6e415 --- /dev/null +++ b/backend/plugins/jira/e2e/migration_schema_test.go @@ -0,0 +1,110 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "sync" + "testing" + + "github.com/apache/incubator-devlake/core/config" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/migration" + coreMigration "github.com/apache/incubator-devlake/core/models/migrationscripts" + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/impls/dalgorm" + "github.com/apache/incubator-devlake/impls/logruslog" + "github.com/apache/incubator-devlake/plugins/jira/impl" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm/schema" +) + +// TestMigrationSchema guards against schema drift between Jira's migration +// scripts and its runtime GORM models. +// +// Regression test for the Sprint Report bug (introduced by PR #8967/#9010): +// the migration that created `_tool_jira_sprint_reports` used a struct that did +// NOT embed common.NoPKModel, so the `_raw_data_table` / `_raw_data_params` / +// `_raw_data_id` / `_raw_data_remark` columns were missing. The runtime model +// DID embed common.NoPKModel, so the ApiExtractor's cleanup query +// (`WHERE _raw_data_table = ? AND _raw_data_params = ?`) failed at runtime with +// "Error 1054: Unknown column '_raw_data_table' in 'where clause'". +// +// The test runs the REAL migration scripts (framework + jira) to build the +// schema exactly the way a production install would — deliberately NOT via +// AutoMigrate on the runtime model, which would silently hide such drift — and +// then asserts that every column each runtime model expects actually exists in +// the migrated table. Any future migration that forgets to embed +// common.NoPKModel (or otherwise omits a column) will fail this test. +// +// The migrations run against a dedicated, empty database (see +// e2ehelper.NewIsolatedMigrationDb) because the shared e2e database is polluted +// by the other e2e tests, which AutoMigrate tables without recording anything +// in `_devlake_migration_history`. +// +// Requires E2E_DB_URL (runs under `make e2e-test` / `make e2e-test-go-plugins`). +func TestMigrationSchema(t *testing.T) { + var pluginInstance impl.Jira + + db := e2ehelper.NewIsolatedMigrationDb(t, "jira_migration_schema") + dalInstance := dalgorm.NewDalgorm(db) + + // Apply the real migration scripts so the schema matches a production install. + basicRes := runner.CreateBasicRes(config.GetConfig(), logruslog.Global, db) + migrator, err := migration.NewMigrator(basicRes) + require.NoError(t, err) + migrator.Register(coreMigration.All(), "Framework") + migrator.Register(pluginInstance.MigrationScripts(), "jira") + require.NoError(t, migrator.Execute()) + + keepAll := func(dal.ColumnMeta) bool { return true } + + for _, table := range pluginInstance.GetTablesInfo() { + table := table + t.Run(table.TableName(), func(t *testing.T) { + // Columns the runtime GORM model expects. + sch, err := schema.Parse(table, &sync.Map{}, schema.NamingStrategy{}) + require.NoErrorf(t, err, "unable to parse schema for %T", table) + + // Columns that actually exist in the migrated table. + actualColumns, err := dal.GetColumnNames(dalInstance, table, keepAll) + if err != nil || len(actualColumns) == 0 { + // No migration creates this table (e.g. API response models + // that are listed in GetTablesInfo but never persisted) — + // there is no schema to drift from. + t.Skipf("table %q not present after migrations", table.TableName()) + } + existing := make(map[string]struct{}, len(actualColumns)) + for _, c := range actualColumns { + existing[c] = struct{}{} + } + + for _, field := range sch.Fields { + if field.DBName == "" || field.IgnoreMigration { + continue + } + _, ok := existing[field.DBName] + assert.Truef(t, ok, + "table %q is missing column %q expected by model %T — "+ + "did a migration script forget to embed common.NoPKModel (raw-data columns) or add the field?", + table.TableName(), field.DBName, table) + } + }) + } +} diff --git a/backend/plugins/jira/models/migrationscripts/20260727_add_raw_data_columns_to_sprint_report.go b/backend/plugins/jira/models/migrationscripts/20260727_add_raw_data_columns_to_sprint_report.go new file mode 100644 index 00000000000..5ee4d671f70 --- /dev/null +++ b/backend/plugins/jira/models/migrationscripts/20260727_add_raw_data_columns_to_sprint_report.go @@ -0,0 +1,65 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// jiraSprintReport20260727 mirrors the JiraSprintReport model. The original +// migration (20260722) created _tool_jira_sprint_reports without embedding +// common.NoPKModel, so the _raw_data_table / _raw_data_params / _raw_data_id / +// _raw_data_remark columns (and created_at / updated_at) were missing. The +// runtime model expects them, which made the ApiExtractor's cleanup query +// (WHERE _raw_data_table = ? AND _raw_data_params = ?) fail with +// "Unknown column '_raw_data_table' in 'where clause'". Re-running +// AutoMigrateTables adds the missing columns without dropping existing data. +type jiraSprintReport20260727 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + BoardId uint64 `gorm:"primaryKey"` + SprintId uint64 `gorm:"primaryKey"` + IssueId uint64 `gorm:"primaryKey"` + + IssueKey string `gorm:"type:varchar(255)"` + Bucket string `gorm:"type:varchar(32);index"` + Done bool + StoryPointsAtSprintStart *float64 + StoryPointsAtSprintEnd *float64 +} + +func (jiraSprintReport20260727) TableName() string { + return "_tool_jira_sprint_reports" +} + +type addRawDataColumnsToSprintReport struct{} + +func (script *addRawDataColumnsToSprintReport) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &jiraSprintReport20260727{}) +} + +func (*addRawDataColumnsToSprintReport) Version() uint64 { + return 20260727000000 +} + +func (*addRawDataColumnsToSprintReport) Name() string { + return "add missing _raw_data_* columns to _tool_jira_sprint_reports" +} diff --git a/backend/plugins/jira/models/migrationscripts/register.go b/backend/plugins/jira/models/migrationscripts/register.go index 90a3317bd14..e71dee8ea14 100644 --- a/backend/plugins/jira/models/migrationscripts/register.go +++ b/backend/plugins/jira/models/migrationscripts/register.go @@ -59,5 +59,6 @@ func All() []plugin.MigrationScript { new(changeFixVersionsToText20260707), new(addExtraJQLToScopeConfig), new(addSprintReportTable), + new(addRawDataColumnsToSprintReport), } } diff --git a/backend/plugins/schema_e2e/migration_schema_test.go b/backend/plugins/schema_e2e/migration_schema_test.go new file mode 100644 index 00000000000..f1572b4de85 --- /dev/null +++ b/backend/plugins/schema_e2e/migration_schema_test.go @@ -0,0 +1,238 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package schema_e2e contains a cross-plugin regression guard that runs the +// REAL migration scripts of every built-in Go plugin and then asserts that the +// resulting database schema still matches what each runtime GORM model expects. +// +// It lives in an `e2e` package on purpose: it needs a real database +// (E2E_DB_URL) and is therefore only executed by `make e2e-test-go-plugins` +// (scripts/e2e-test-go-plugins.sh selects packages whose import path contains +// "e2e"), and excluded from the DB-less unit test run +// (scripts/unit-test-go.sh skips paths matching "e2e"). +package schema_e2e + +import ( + "os" + "path/filepath" + "sync" + "testing" + + "github.com/apache/incubator-devlake/core/config" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/migration" + coreMigration "github.com/apache/incubator-devlake/core/models/migrationscripts" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/impls/dalgorm" + "github.com/apache/incubator-devlake/impls/logruslog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm/schema" + + ae "github.com/apache/incubator-devlake/plugins/ae/impl" + argocd "github.com/apache/incubator-devlake/plugins/argocd/impl" + asana "github.com/apache/incubator-devlake/plugins/asana/impl" + azuredevops "github.com/apache/incubator-devlake/plugins/azuredevops_go/impl" + bamboo "github.com/apache/incubator-devlake/plugins/bamboo/impl" + bitbucket "github.com/apache/incubator-devlake/plugins/bitbucket/impl" + bitbucket_server "github.com/apache/incubator-devlake/plugins/bitbucket_server/impl" + circleci "github.com/apache/incubator-devlake/plugins/circleci/impl" + claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" + customize "github.com/apache/incubator-devlake/plugins/customize/impl" + dbt "github.com/apache/incubator-devlake/plugins/dbt/impl" + dora "github.com/apache/incubator-devlake/plugins/dora/impl" + feishu "github.com/apache/incubator-devlake/plugins/feishu/impl" + copilot "github.com/apache/incubator-devlake/plugins/gh-copilot/impl" + gitee "github.com/apache/incubator-devlake/plugins/gitee/impl" + gitextractor "github.com/apache/incubator-devlake/plugins/gitextractor/impl" + github "github.com/apache/incubator-devlake/plugins/github/impl" + githubGraphql "github.com/apache/incubator-devlake/plugins/github_graphql/impl" + gitlab "github.com/apache/incubator-devlake/plugins/gitlab/impl" + icla "github.com/apache/incubator-devlake/plugins/icla/impl" + issueTrace "github.com/apache/incubator-devlake/plugins/issue_trace/impl" + jenkins "github.com/apache/incubator-devlake/plugins/jenkins/impl" + jira "github.com/apache/incubator-devlake/plugins/jira/impl" + linear "github.com/apache/incubator-devlake/plugins/linear/impl" + linker "github.com/apache/incubator-devlake/plugins/linker/impl" + opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" + org "github.com/apache/incubator-devlake/plugins/org/impl" + pagerduty "github.com/apache/incubator-devlake/plugins/pagerduty/impl" + q_dev "github.com/apache/incubator-devlake/plugins/q_dev/impl" + refdiff "github.com/apache/incubator-devlake/plugins/refdiff/impl" + rootly "github.com/apache/incubator-devlake/plugins/rootly/impl" + slack "github.com/apache/incubator-devlake/plugins/slack/impl" + sonarqube "github.com/apache/incubator-devlake/plugins/sonarqube/impl" + starrocks "github.com/apache/incubator-devlake/plugins/starrocks/impl" + taiga "github.com/apache/incubator-devlake/plugins/taiga/impl" + tapd "github.com/apache/incubator-devlake/plugins/tapd/impl" + teambition "github.com/apache/incubator-devlake/plugins/teambition/impl" + tempo "github.com/apache/incubator-devlake/plugins/tempo/impl" + testmo "github.com/apache/incubator-devlake/plugins/testmo/impl" + trello "github.com/apache/incubator-devlake/plugins/trello/impl" + webhook "github.com/apache/incubator-devlake/plugins/webhook/impl" + zentao "github.com/apache/incubator-devlake/plugins/zentao/impl" +) + +// allGoPlugins lists EVERY built-in Go plugin. Keep it in sync with the plugin +// directories under backend/plugins/ (the TestAllGoPluginsListed guard below +// fails if a new plugin's `impl` package is added but not registered here). +func allGoPlugins() []plugin.PluginMeta { + return []plugin.PluginMeta{ + ae.AE{}, + argocd.ArgoCD{}, + asana.Asana{}, + azuredevops.Azuredevops{}, + bamboo.Bamboo{}, + bitbucket.Bitbucket{}, + bitbucket_server.BitbucketServer{}, + circleci.Circleci{}, + claudeCode.ClaudeCode{}, + customize.Customize{}, + dbt.Dbt{}, + dora.Dora{}, + feishu.Feishu{}, + copilot.GhCopilot{}, + gitee.Gitee{}, + gitextractor.GitExtractor{}, + github.Github{}, + githubGraphql.GithubGraphql{}, + gitlab.Gitlab{}, + icla.Icla{}, + issueTrace.IssueTrace{}, + jenkins.Jenkins{}, + jira.Jira{}, + linear.Linear{}, + linker.Linker{}, + opsgenie.Opsgenie{}, + org.Org{}, + pagerduty.PagerDuty{}, + q_dev.QDev{}, + refdiff.RefDiff{}, + rootly.Rootly{}, + slack.Slack{}, + sonarqube.Sonarqube{}, + starrocks.StarRocks{}, + taiga.Taiga{}, + tapd.Tapd{}, + teambition.Teambition{}, + tempo.Tempo{}, + testmo.Testmo{}, + trello.Trello{}, + webhook.Webhook{}, + zentao.Zentao{}, + } +} + +// TestAllGoPluginsListed guarantees allGoPlugins() stays complete: it counts the +// plugin directories that ship an `impl` package and fails if that number does +// not match the registered list. This makes the schema-drift guard below +// automatically cover any newly added plugin. +func TestAllGoPluginsListed(t *testing.T) { + entries, err := os.ReadDir("..") + require.NoError(t, err) + dirsWithImpl := 0 + for _, e := range entries { + if !e.IsDir() { + continue + } + if info, statErr := os.Stat(filepath.Join("..", e.Name(), "impl")); statErr == nil && info.IsDir() { + dirsWithImpl++ + } + } + assert.Equalf(t, dirsWithImpl, len(allGoPlugins()), + "number of plugin dirs with an impl/ package (%d) != registered plugins (%d); "+ + "add the new plugin to allGoPlugins() in plugins/schema_e2e/migration_schema_test.go", + dirsWithImpl, len(allGoPlugins())) +} + +// TestMigrationSchemaMatchesModels applies the real framework + plugin migration +// scripts and then verifies, for every plugin model, that each column the +// runtime GORM model declares actually exists in the migrated table. +// +// This is a cross-plugin generalization of the Jira Sprint Report regression: +// a migration created `_tool_jira_sprint_reports` without embedding +// common.NoPKModel, so the `_raw_data_*` columns were missing and the +// ApiExtractor cleanup query failed at runtime with +// "Unknown column '_raw_data_table' in 'where clause'". +// +// Tables that no migration creates (e.g. models materialized lazily at runtime) +// are skipped, so the check specifically targets *drift* between an existing +// table and its model — which is exactly the failure mode above. +// +// The migrations run against a dedicated, empty database (see +// e2ehelper.NewIsolatedMigrationDb) because the shared e2e database is polluted +// by the other e2e tests, which AutoMigrate tables without recording anything +// in `_devlake_migration_history`. +func TestMigrationSchemaMatchesModels(t *testing.T) { + db := e2ehelper.NewIsolatedMigrationDb(t, "schema_drift") + dalInstance := dalgorm.NewDalgorm(db) + basicRes := runner.CreateBasicRes(config.GetConfig(), logruslog.Global, db) + + // Apply the migrations exactly the way the server does on startup. + migrator, migErr := migration.NewMigrator(basicRes) + require.NoError(t, migErr) + migrator.Register(coreMigration.All(), "Framework") + for _, p := range allGoPlugins() { + if migratable, ok := p.(plugin.PluginMigration); ok { + migrator.Register(migratable.MigrationScripts(), p.Name()) + } + } + require.NoError(t, migrator.Execute()) + + keepAll := func(dal.ColumnMeta) bool { return true } + + for _, p := range allGoPlugins() { + modeler, ok := p.(plugin.PluginModel) + if !ok { + continue + } + p := p + t.Run(p.Name(), func(t *testing.T) { + for _, table := range modeler.GetTablesInfo() { + table := table + // Columns that actually exist in the migrated table. + actualColumns, colErr := dal.GetColumnNames(dalInstance, table, keepAll) + if colErr != nil || len(actualColumns) == 0 { + // No migration created this table (e.g. runtime-only / + // dynamic model) — nothing to validate for drift. + t.Logf("skip %q: table not present after migrations", table.TableName()) + continue + } + existing := make(map[string]struct{}, len(actualColumns)) + for _, c := range actualColumns { + existing[c] = struct{}{} + } + + // Columns the runtime GORM model expects. + sch, parseErr := schema.Parse(table, &sync.Map{}, schema.NamingStrategy{}) + require.NoErrorf(t, parseErr, "unable to parse schema for %T", table) + for _, field := range sch.Fields { + if field.DBName == "" || field.IgnoreMigration { + continue + } + _, present := existing[field.DBName] + assert.Truef(t, present, + "[%s] table %q is missing column %q expected by model %T — "+ + "did a migration script forget to embed common.NoPKModel (raw-data columns) or add the field?", + p.Name(), table.TableName(), field.DBName, table) + } + } + }) + } +} diff --git a/backend/plugins/taiga/models/migrationscripts/20260727_add_missing_scope_config_columns.go b/backend/plugins/taiga/models/migrationscripts/20260727_add_missing_scope_config_columns.go new file mode 100644 index 00000000000..84fa66cda41 --- /dev/null +++ b/backend/plugins/taiga/models/migrationscripts/20260727_add_missing_scope_config_columns.go @@ -0,0 +1,61 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// taigaScopeConfig20260727 mirrors models.TaigaScopeConfig. The initial +// migration created `_tool_taiga_scope_configs` without the `type_mappings` +// column, while the runtime model declares it — every read/write of the model +// would fail with "Unknown column 'type_mappings'". +// +// The `uniqueIndex` on `name` is safe to add here: the new column is nullable, +// so pre-existing rows are backfilled with NULL, and both MySQL and PostgreSQL +// allow duplicate NULLs in a unique index (verified against both engines). +type taigaScopeConfig20260727 struct { + archived.Model + Entities []string `gorm:"type:json;serializer:json" json:"entities"` + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `json:"name" gorm:"type:varchar(255);uniqueIndex"` + TypeMappings map[string]json.RawMessage `json:"typeMappings" gorm:"type:json;serializer:json"` +} + +func (taigaScopeConfig20260727) TableName() string { + return "_tool_taiga_scope_configs" +} + +type addMissingScopeConfigColumns struct{} + +func (script *addMissingScopeConfigColumns) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &taigaScopeConfig20260727{}) +} + +func (*addMissingScopeConfigColumns) Version() uint64 { + return 20260727000001 +} + +func (*addMissingScopeConfigColumns) Name() string { + return "add missing type_mappings column to _tool_taiga_scope_configs" +} diff --git a/backend/plugins/taiga/models/migrationscripts/register.go b/backend/plugins/taiga/models/migrationscripts/register.go index d2cfd08e269..da91884aa1f 100644 --- a/backend/plugins/taiga/models/migrationscripts/register.go +++ b/backend/plugins/taiga/models/migrationscripts/register.go @@ -26,5 +26,6 @@ func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(addInitTables20250220), new(addTaskIssueEpicTables20260306), + new(addMissingScopeConfigColumns), } } diff --git a/backend/plugins/teambition/models/migrationscripts/20260727_add_missing_scope_config_columns.go b/backend/plugins/teambition/models/migrationscripts/20260727_add_missing_scope_config_columns.go new file mode 100644 index 00000000000..0b29b9ca044 --- /dev/null +++ b/backend/plugins/teambition/models/migrationscripts/20260727_add_missing_scope_config_columns.go @@ -0,0 +1,90 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +const teambitionScopeConfigTable20260727 = "_tool_teambition_scope_configs" + +// teambitionScopeConfig20260727 mirrors models.TeambitionScopeConfig. The +// migration that created `_tool_teambition_scope_configs` did not include the +// columns of the embedded common.Model (`id`, `created_at`, `updated_at`), +// which the runtime model expects. +type teambitionScopeConfig20260727 struct { + archived.Model + Entities []string `gorm:"type:json;serializer:json" json:"entities"` + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `json:"name" gorm:"type:varchar(255);uniqueIndex"` + TypeMappings map[string]string `json:"typeMappings" gorm:"serializer:json"` + StatusMappings map[string]string `json:"statusMappings" gorm:"serializer:json"` + BugDueDateField string `json:"bugDueDateField" gorm:"column:bug_due_date_field"` + TaskDueDateField string `json:"taskDueDateField" gorm:"column:task_due_date_field"` + StoryDueDateField string `json:"storyDueDateField" gorm:"column:story_due_date_field"` +} + +func (teambitionScopeConfig20260727) TableName() string { + return teambitionScopeConfigTable20260727 +} + +type addMissingScopeConfigColumns struct{} + +// Up adds the columns of the embedded common.Model that the runtime model +// expects. +// +// `id` is an auto-increment primary key, which GORM's AutoMigrate cannot append +// to an existing table: it emits a plain `ADD COLUMN ... AUTO_INCREMENT`, which +// MySQL rejects with "Incorrect table definition; there can be only one auto +// column and it must be defined as a key". The column is therefore added with +// explicit DDL (the table has no primary key so far), letting the database +// backfill ids for existing rows and keep the sequence/counter in sync. The +// remaining columns (`created_at`, `updated_at`) and the indexes are then +// created by AutoMigrate as usual. +func (script *addMissingScopeConfigColumns) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if !db.HasColumn(teambitionScopeConfigTable20260727, "id") { + ddl := fmt.Sprintf( + "ALTER TABLE %s ADD COLUMN id BIGSERIAL PRIMARY KEY", + teambitionScopeConfigTable20260727, + ) + if db.Dialect() == "mysql" { + ddl = fmt.Sprintf( + "ALTER TABLE %s ADD COLUMN id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY", + teambitionScopeConfigTable20260727, + ) + } + if err := db.Exec(ddl); err != nil { + return err + } + } + return migrationhelper.AutoMigrateTables(basicRes, &teambitionScopeConfig20260727{}) +} + +func (*addMissingScopeConfigColumns) Version() uint64 { + return 20260727000001 +} + +func (*addMissingScopeConfigColumns) Name() string { + return "add missing id/created_at/updated_at columns to _tool_teambition_scope_configs" +} diff --git a/backend/plugins/teambition/models/migrationscripts/register.go b/backend/plugins/teambition/models/migrationscripts/register.go index f9914e7a12c..d761a15fb70 100644 --- a/backend/plugins/teambition/models/migrationscripts/register.go +++ b/backend/plugins/teambition/models/migrationscripts/register.go @@ -26,5 +26,6 @@ func All() []plugin.MigrationScript { new(reCreateTeambitionConnections), new(addScopeConfigId), new(addAppIdBack), + new(addMissingScopeConfigColumns), } } diff --git a/backend/plugins/testmo/models/migrationscripts/20260727_add_missing_scope_config_columns.go b/backend/plugins/testmo/models/migrationscripts/20260727_add_missing_scope_config_columns.go new file mode 100644 index 00000000000..11048f60a74 --- /dev/null +++ b/backend/plugins/testmo/models/migrationscripts/20260727_add_missing_scope_config_columns.go @@ -0,0 +1,61 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// testmoScopeConfig20260727 mirrors models.TestmoScopeConfig. The migration +// that created `_tool_testmo_scope_configs` omitted the `connection_id` and +// `name` columns of the embedded common.ScopeConfig, which the runtime model +// expects. +// +// The `uniqueIndex` on `name` is safe to add here: the new column is nullable, +// so pre-existing rows are backfilled with NULL, and both MySQL and PostgreSQL +// allow duplicate NULLs in a unique index (verified against both engines). +type testmoScopeConfig20260727 struct { + archived.Model + Entities []string `gorm:"type:json;serializer:json" json:"entities"` + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `json:"name" gorm:"type:varchar(255);uniqueIndex"` + AcceptanceTestPattern string `json:"acceptanceTestPattern" gorm:"type:varchar(255)"` + SmokeTestPattern string `json:"smokeTestPattern" gorm:"type:varchar(255)"` + TeamPattern string `json:"teamPattern" gorm:"type:varchar(255)"` +} + +func (testmoScopeConfig20260727) TableName() string { + return "_tool_testmo_scope_configs" +} + +type addMissingScopeConfigColumns struct{} + +func (script *addMissingScopeConfigColumns) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &testmoScopeConfig20260727{}) +} + +func (*addMissingScopeConfigColumns) Version() uint64 { + return 20260727000001 +} + +func (*addMissingScopeConfigColumns) Name() string { + return "add missing connection_id/name columns to _tool_testmo_scope_configs" +} diff --git a/backend/plugins/testmo/models/migrationscripts/register.go b/backend/plugins/testmo/models/migrationscripts/register.go index 7843d4b84e1..95f2ae48b41 100644 --- a/backend/plugins/testmo/models/migrationscripts/register.go +++ b/backend/plugins/testmo/models/migrationscripts/register.go @@ -25,5 +25,6 @@ func All() []plugin.MigrationScript { new(addScopeConfigIdToProjects), new(replaceTestsWithRuns), new(fixRawTableNamesAndSchemas), + new(addMissingScopeConfigColumns), } } diff --git a/backend/scripts/build-plugins.sh b/backend/scripts/build-plugins.sh index 03fb7b4f9fb..e0ef94bb9b7 100755 --- a/backend/scripts/build-plugins.sh +++ b/backend/scripts/build-plugins.sh @@ -52,7 +52,8 @@ fi if [ -z "$DEVLAKE_PLUGINS" ]; then echo "Building all plugins" - PLUGINS=$(find $PLUGIN_SRC_DIR/* -maxdepth 0 -type d -not -name core -not -name helper -not -name logs -not -empty) + # schema_e2e is not a plugin, it only holds the cross-plugin schema-drift e2e test + PLUGINS=$(find $PLUGIN_SRC_DIR/* -maxdepth 0 -type d -not -name core -not -name helper -not -name logs -not -name schema_e2e -not -empty) else echo "Building the following plugins: $PLUGIN" PLUGINS=