Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/workflows/replica-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,20 @@ jobs:
- name: Run tests
run: script/docker-gh-ost-replica-tests run

- name: Set artifact name
if: failure()
run: |
ARTIFACT_NAME=$(echo "${{ matrix.image }}" | tr '/:' '-')
echo "ARTIFACT_NAME=test-logs-${ARTIFACT_NAME}" >> $GITHUB_ENV

- name: Upload test logs on failure
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This makes it much easier to debug failures

if: failure()
uses: actions/upload-artifact@v4
with:
name: ${{ env.ARTIFACT_NAME }}
path: /tmp/gh-ost-test.*
retention-days: 7
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Open to shorter retention if desired


- name: Teardown environment
if: always()
run: script/docker-gh-ost-replica-tests down
37 changes: 29 additions & 8 deletions go/logic/applier.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,21 @@ func NewApplier(migrationContext *base.MigrationContext) *Applier {
}
}

// compileMigrationKeyWarningRegex compiles a regex pattern that matches duplicate key warnings
// for the migration's unique key. Duplicate warnings are formatted differently across MySQL versions,
// hence the optional table name prefix. Metacharacters in table/index names are escaped to avoid
// regex syntax errors.
func (this *Applier) compileMigrationKeyWarningRegex() (*regexp.Regexp, error) {
escapedTable := regexp.QuoteMeta(this.migrationContext.GetGhostTableName())
escapedKey := regexp.QuoteMeta(this.migrationContext.UniqueKey.NameInGhostTable)
migrationUniqueKeyPattern := fmt.Sprintf(`for key '(%s\.)?%s'`, escapedTable, escapedKey)
migrationKeyRegex, err := regexp.Compile(migrationUniqueKeyPattern)
if err != nil {
return nil, fmt.Errorf("failed to compile migration key pattern: %w", err)
}
return migrationKeyRegex, nil
}

func (this *Applier) InitDBConnections() (err error) {
applierUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName)
uriWithMulti := fmt.Sprintf("%s&multiStatements=true", applierUri)
Expand Down Expand Up @@ -928,6 +943,12 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected
return nil, err
}

// Compile regex once before loop to avoid performance penalty and handle errors properly
migrationKeyRegex, err := this.compileMigrationKeyWarningRegex()
if err != nil {
return nil, err
}

var sqlWarnings []string
for rows.Next() {
var level, message string
Expand All @@ -936,10 +957,7 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected
this.migrationContext.Log.Warningf("Failed to read SHOW WARNINGS row")
continue
}
// Duplicate warnings are formatted differently across mysql versions, hence the optional table name prefix
migrationUniqueKeyExpression := fmt.Sprintf("for key '(%s\\.)?%s'", this.migrationContext.GetGhostTableName(), this.migrationContext.UniqueKey.NameInGhostTable)
matched, _ := regexp.MatchString(migrationUniqueKeyExpression, message)
if strings.Contains(message, "Duplicate entry") && matched {
if strings.Contains(message, "Duplicate entry") && migrationKeyRegex.MatchString(message) {
continue
}
sqlWarnings = append(sqlWarnings, fmt.Sprintf("%s: %s (%d)", level, message, code))
Expand Down Expand Up @@ -1570,6 +1588,12 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent))
return rollback(err)
}

// Compile regex once before loop to avoid performance penalty and handle errors properly
migrationKeyRegex, err := this.compileMigrationKeyWarningRegex()
if err != nil {
return rollback(err)
}

var sqlWarnings []string
for rows.Next() {
var level, message string
Expand All @@ -1578,10 +1602,7 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent))
this.migrationContext.Log.Warningf("Failed to read SHOW WARNINGS row")
continue
}
// Duplicate warnings are formatted differently across mysql versions, hence the optional table name prefix
migrationUniqueKeyExpression := fmt.Sprintf("for key '(%s\\.)?%s'", this.migrationContext.GetGhostTableName(), this.migrationContext.UniqueKey.NameInGhostTable)
matched, _ := regexp.MatchString(migrationUniqueKeyExpression, message)
if strings.Contains(message, "Duplicate entry") && matched {
if strings.Contains(message, "Duplicate entry") && migrationKeyRegex.MatchString(message) {
// Duplicate entry on migration unique key is expected during binlog replay
// (row was already copied during bulk copy phase)
continue
Expand Down
Loading