Skip to content

Improve ergonomics of data migration test#10164

Merged
smklein merged 38 commits intomainfrom
data-migration-erg
Mar 30, 2026
Merged

Improve ergonomics of data migration test#10164
smklein merged 38 commits intomainfrom
data-migration-erg

Conversation

@smklein
Copy link
Copy Markdown
Collaborator

@smklein smklein commented Mar 26, 2026

  • Splits up each data migration into a separate file
  • Uses migration name instead of migration semver version, to reduce conflicts

This PR should make it easier to concurrently merge data migrations, but it also should make it easier to "remove old data migrations", identified by #8271.

@@ -0,0 +1,88 @@
// This Source Code Form is subject to the terms of the Mozilla Public
Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI! Of all the files in data_migrations.rs to read, this is the most important one to review.

The others are "Just moving" test functions out of schema.rs

Copy link
Copy Markdown
Member

@hawkw hawkw left a comment

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is awesome. I had a couple small suggestions, but I think this will make writing these tests much less annoying!

Comment on lines +48 to +85
macro_rules! register {
($map:ident, $versions:ident, $mod:ident) => {
let name = stringify!($mod).replace('_', "-");
let version = $versions
.get(name.as_str())
.unwrap_or_else(|| {
panic!("migration {name:?} not found in KNOWN_VERSIONS")
})
.clone();
$map.insert(version, $mod::checks());
};
}

pub(crate) fn get_migration_checks() -> BTreeMap<Version, DataMigrationFns> {
let versions: HashMap<&str, Version> = KNOWN_VERSIONS
.iter()
.map(|v| (v.relative_path(), v.semver().clone()))
.collect();
let mut map = BTreeMap::new();

register!(map, versions, zone_image_resolver_inventory);
register!(map, versions, vpc_firewall_icmp);
register!(map, versions, boot_partitions_inventory);
register!(map, versions, fix_leaked_bp_oximeter_read_policy_rows);
register!(map, versions, route_config_rib_priority);
register!(map, versions, inv_clear_mupdate_override);
register!(map, versions, populate_db_metadata_nexus);
register!(map, versions, positive_quotas);
register!(map, versions, disk_types);
register!(map, versions, one_big_ereport_table);
register!(map, versions, blueprint_sled_config_subnet);
register!(map, versions, blueprint_sled_last_used_ip);
register!(map, versions, audit_log_credential_id);
register!(map, versions, fix_session_token_column_order);
register!(map, versions, bgp_unnumbered_peers);
register!(map, versions, bgp_config_max_paths_not_null);
register!(map, versions, ereport_everyone_gets_a_slot);
register!(map, versions, rename_default_igw_ip_pool);
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit, take it or leave it: since register is evaluated inside this function exclusively, i think the map and versions identifiers don't actually need to be passed in?

Suggested change
macro_rules! register {
($map:ident, $versions:ident, $mod:ident) => {
let name = stringify!($mod).replace('_', "-");
let version = $versions
.get(name.as_str())
.unwrap_or_else(|| {
panic!("migration {name:?} not found in KNOWN_VERSIONS")
})
.clone();
$map.insert(version, $mod::checks());
};
}
pub(crate) fn get_migration_checks() -> BTreeMap<Version, DataMigrationFns> {
let versions: HashMap<&str, Version> = KNOWN_VERSIONS
.iter()
.map(|v| (v.relative_path(), v.semver().clone()))
.collect();
let mut map = BTreeMap::new();
register!(map, versions, zone_image_resolver_inventory);
register!(map, versions, vpc_firewall_icmp);
register!(map, versions, boot_partitions_inventory);
register!(map, versions, fix_leaked_bp_oximeter_read_policy_rows);
register!(map, versions, route_config_rib_priority);
register!(map, versions, inv_clear_mupdate_override);
register!(map, versions, populate_db_metadata_nexus);
register!(map, versions, positive_quotas);
register!(map, versions, disk_types);
register!(map, versions, one_big_ereport_table);
register!(map, versions, blueprint_sled_config_subnet);
register!(map, versions, blueprint_sled_last_used_ip);
register!(map, versions, audit_log_credential_id);
register!(map, versions, fix_session_token_column_order);
register!(map, versions, bgp_unnumbered_peers);
register!(map, versions, bgp_config_max_paths_not_null);
register!(map, versions, ereport_everyone_gets_a_slot);
register!(map, versions, rename_default_igw_ip_pool);
macro_rules! register {
($mod:ident) => {
let name = stringify!($mod).replace('_', "-");
let version = versions
.get(name.as_str())
.unwrap_or_else(|| {
panic!("migration {name:?} not found in KNOWN_VERSIONS")
})
.clone();
map.insert(version, $mod::checks());
};
}
pub(crate) fn get_migration_checks() -> BTreeMap<Version, DataMigrationFns> {
let versions: HashMap<&str, Version> = KNOWN_VERSIONS
.iter()
.map(|v| (v.relative_path(), v.semver().clone()))
.collect();
let mut map = BTreeMap::new();
register!(zone_image_resolver_inventory);
register!(vpc_firewall_icmp);
register!(boot_partitions_inventory);
register!(fix_leaked_bp_oximeter_read_policy_rows);
register!(route_config_rib_priority);
register!(inv_clear_mupdate_override);
register!(populate_db_metadata_nexus);
register!(positive_quotas);
register!(disk_types);
register!(one_big_ereport_table);
register!(blueprint_sled_config_subnet);
register!(blueprint_sled_last_used_ip);
register!(audit_log_credential_id);
register!(fix_session_token_column_order);
register!(bgp_unnumbered_peers);
register!(bgp_config_max_paths_not_null);
register!(ereport_everyone_gets_a_slot);
register!(rename_default_igw_ip_pool);

which might be slightly nicer for the user?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, updated. Gotta move the macro inside the function for this to work, but it does (done in faf7e1b)

Comment on lines +15 to +17
//! When advancing the schema baseline, delete the files whose migration
//! names are older than the new baseline, and remove the corresponding
//! `mod` and `register!` lines below.
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it might be nice if this explained a bit more about what "advancing the schema baseline" means. When would I need to do this?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Explained things a bit, and linked to docs in faf7e1b

Base automatically changed from schema-optimization to main March 27, 2026 22:10
@smklein smklein merged commit e2fe401 into main Mar 30, 2026
16 checks passed
@smklein smklein deleted the data-migration-erg branch March 30, 2026 14:59
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

2 participants