diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67641d15e54..4c7e59656ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -651,6 +651,9 @@ jobs: with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # check-editions compares against the merge base, so it needs real history. + fetch-depth: 0 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -660,6 +663,16 @@ jobs: run: | cargo run --profile ci -p xtask -- generate-fbs cargo run --profile ci -p xtask -- generate-proto + - name: "regenerate the edition records" + run: | + cargo run --profile ci -p xtask -- generate-editions + - name: "check frozen edition records never change" + # Independent of the regeneration above: a stale record must not mask a frozen one + # being edited, nor the other way round. + if: "!cancelled()" + run: | + BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}" + cargo run --profile ci -p xtask -- check-editions --base "$BASE" - name: "regenerate FFI header file" run: | cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi diff --git a/Cargo.lock b/Cargo.lock index c05aae6923e..14ee67f7ab6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3651,6 +3651,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "git2" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" +dependencies = [ + "bitflags 2.13.1", + "libc", + "libgit2-sys", + "log", +] + [[package]] name = "glob" version = "0.3.4" @@ -5344,6 +5356,18 @@ dependencies = [ "cc", ] +[[package]] +name = "libgit2-sys" +version = "0.18.7+1.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23c7391e4b9f4ffab1a624223cc1d7385ff9a678f490768add717de7ea2f4d89" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + [[package]] name = "libloading" version = "0.8.9" @@ -5429,6 +5453,18 @@ dependencies = [ "escape8259", ] +[[package]] +name = "libz-sys" +version = "1.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "line-clipping" version = "0.3.8" @@ -9510,6 +9546,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -11268,7 +11310,10 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", + "git2", "prost-build", + "toml", + "vortex-edition", "xshell", ] diff --git a/Cargo.toml b/Cargo.toml index 36aa5b2ac9e..64d787d890a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -166,6 +166,7 @@ geo-types = "0.7.19" geoarrow = "0.8.0" geoarrow-cast = "0.8.0" get_dir = "0.5.0" +git2 = { version = "0.21", default-features = false } glob = "0.3.2" goldenfile = "1" half = { version = "2.7.1", features = ["std", "num-traits"] } @@ -274,6 +275,7 @@ tokio-stream = "0.1.17" tokio-util = "0.7.17" vortex-array-macros = { version = "0.1.0", path = "./vortex-array-macros" } # Pull these into non-public crates to support DF 58 +toml = "0.9" tpchgen = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" } tpchgen-arrow = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" } tracing = { version = "0.1.41", default-features = false } diff --git a/docs/specs/editions.md b/docs/specs/editions.md index 6dd87cd250d..77f13070d67 100644 --- a/docs/specs/editions.md +++ b/docs/specs/editions.md @@ -62,6 +62,19 @@ edition; each encoding's registry entry records the edition it joined in. In the encoding may be *deprecated*, meaning writers stop emitting it — but readers keep decoding it indefinitely, so deprecation never invalidates existing files. +## The `unstable` family + +Alongside `core` there is an `unstable` family, holding encodings that are still being +evaluated. It is the exception to everything above: every `unstable` edition is a permanent +draft, so the family never freezes and carries no read-compatibility guarantee at all. A file +written with these encodings is readable only by a build that knows them, and a future release +may stop supporting one. + +Because of that, the writer only emits them when you opt in — the default session enables the +newest `unstable` edition solely when the `unstable_encodings` cargo feature is selected. +Encodings graduate by being declared in a new `core` edition, which is where they pick up the +read-forever guarantee. + ## Edition registry Coming soon.. diff --git a/vortex/src/editions/core/mod.rs b/vortex-edition/src/declarations/core/mod.rs similarity index 56% rename from vortex/src/editions/core/mod.rs rename to vortex-edition/src/declarations/core/mod.rs index 4d05f11750c..c39585741f6 100644 --- a/vortex/src/editions/core/mod.rs +++ b/vortex-edition/src/declarations/core/mod.rs @@ -6,6 +6,17 @@ //! One module per edition, each declaring the edition and the encodings that join the //! family at it; members of earlier editions are inherited and never restated. +use crate::EditionFamily; + +/// The `core` family: what the default writer may emit. +pub static FAMILY: EditionFamily = EditionFamily { + name: "core", + doc: "The encodings the default file writer emits. Every core edition freezes, and a \ +frozen edition carries a read-forever guarantee: a file written with it stays readable by \ +every later Vortex release. New encodings join by being declared in a new edition; an \ +edition that has frozen never changes again.", +}; + pub mod v2025_05; pub mod v2025_06; pub mod v2025_10; diff --git a/vortex/src/editions/core/v2025_05.rs b/vortex-edition/src/declarations/core/v2025_05.rs similarity index 92% rename from vortex/src/editions/core/v2025_05.rs rename to vortex-edition/src/declarations/core/v2025_05.rs index a25a6f971a4..115193619ba 100644 --- a/vortex/src/editions/core/v2025_05.rs +++ b/vortex-edition/src/declarations/core/v2025_05.rs @@ -3,9 +3,9 @@ //! The baseline `core` edition: stable encodings writable by Vortex 0.36.0. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The first edition of the `core` family, matching the first stable Vortex file release. pub const CORE_2025_05_0: EditionId = EditionId::new("core", 2025, 5, 0); diff --git a/vortex/src/editions/core/v2025_06.rs b/vortex-edition/src/declarations/core/v2025_06.rs similarity index 86% rename from vortex/src/editions/core/v2025_06.rs rename to vortex-edition/src/declarations/core/v2025_06.rs index 015325b8429..7ebd3505799 100644 --- a/vortex/src/editions/core/v2025_06.rs +++ b/vortex-edition/src/declarations/core/v2025_06.rs @@ -3,9 +3,9 @@ //! The `core` edition adding stable encodings released through June 2025. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The June 2025 edition of the `core` family. pub const CORE_2025_06_0: EditionId = EditionId::new("core", 2025, 6, 0); diff --git a/vortex/src/editions/core/v2025_10.rs b/vortex-edition/src/declarations/core/v2025_10.rs similarity index 87% rename from vortex/src/editions/core/v2025_10.rs rename to vortex-edition/src/declarations/core/v2025_10.rs index ae21026b595..6124c9e94b3 100644 --- a/vortex/src/editions/core/v2025_10.rs +++ b/vortex-edition/src/declarations/core/v2025_10.rs @@ -3,9 +3,9 @@ //! The `core` edition adding stable encodings released through October 2025. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The October 2025 edition of the `core` family. pub const CORE_2025_10_0: EditionId = EditionId::new("core", 2025, 10, 0); diff --git a/vortex/src/editions/core/v2026_07.rs b/vortex-edition/src/declarations/core/v2026_07.rs similarity index 85% rename from vortex/src/editions/core/v2026_07.rs rename to vortex-edition/src/declarations/core/v2026_07.rs index 820c68cf7dd..78e0814d5a4 100644 --- a/vortex/src/editions/core/v2026_07.rs +++ b/vortex-edition/src/declarations/core/v2026_07.rs @@ -3,9 +3,9 @@ //! The `core` edition adding stable encodings released through July 2026. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The July 2026 edition of the `core` family. pub const CORE_2026_07_0: EditionId = EditionId::new("core", 2026, 7, 0); diff --git a/vortex/src/editions/core/v2026_08.rs b/vortex-edition/src/declarations/core/v2026_08.rs similarity index 85% rename from vortex/src/editions/core/v2026_08.rs rename to vortex-edition/src/declarations/core/v2026_08.rs index 4d47cfbe027..904dfb4ef73 100644 --- a/vortex/src/editions/core/v2026_08.rs +++ b/vortex-edition/src/declarations/core/v2026_08.rs @@ -3,9 +3,9 @@ //! The August 2026 core edition adding the canonical Map encoding. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The August 2026 core edition containing canonical Map arrays. pub const CORE_2026_08: EditionId = EditionId::new("core", 2026, 8, 0); diff --git a/vortex-edition/src/declarations/mod.rs b/vortex-edition/src/declarations/mod.rs new file mode 100644 index 00000000000..07dd540d0ba --- /dev/null +++ b/vortex-edition/src/declarations/mod.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The first-party Vortex edition declarations, one module per edition. +//! +//! These are plain constants naming encodings by id, so they depend on nothing but the types +//! in this crate. That keeps them cheap to read: tooling that only needs to know what an +//! edition contains — `cargo run -p xtask -- generate-editions`, for one — can depend on +//! this crate alone rather than on the whole of `vortex`. +//! +//! The `vortex` facade re-exports everything here and owns the session wiring: registering +//! the declarations and selecting which of them the default writer may emit. + +pub mod core; +pub mod unstable; + +use crate::EditionDeclaration; +use crate::EditionFamily; + +/// The first-party edition families. Every family must be declared before its editions. +pub static EDITION_FAMILIES: &[&EditionFamily] = &[&core::FAMILY, &unstable::FAMILY]; + +/// The first-party Vortex edition declarations. +pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ + &core::v2025_05::DECLARATION, + &core::v2025_06::DECLARATION, + &core::v2025_10::DECLARATION, + &core::v2026_07::DECLARATION, + &core::v2026_08::DECLARATION, + &unstable::v2025_05::DECLARATION, + &unstable::v2026_02::DECLARATION, + &unstable::v2026_04::DECLARATION, + &unstable::v2026_06::DECLARATION, +]; diff --git a/vortex-edition/src/declarations/unstable/mod.rs b/vortex-edition/src/declarations/unstable/mod.rs new file mode 100644 index 00000000000..96ea60e4cd9 --- /dev/null +++ b/vortex-edition/src/declarations/unstable/mod.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The `unstable` edition family: opt-in encodings without a frozen compatibility guarantee. +//! +//! One module per draft edition, each declaring the encodings that join the family at it. +//! Members of earlier editions are inherited and never restated. + +use crate::EditionFamily; + +/// The `unstable` family: opt-in encodings with no compatibility guarantee. +pub static FAMILY: EditionFamily = EditionFamily { + name: "unstable", + doc: "Opt-in encodings that are still being evaluated. Every unstable edition stays a \ +draft, so the family never freezes and carries no compatibility guarantee: a file written \ +with these encodings is readable only by a build that knows them, and a later release may \ +stop supporting one. The writer emits them only when the `unstable_encodings` feature is \ +selected. An encoding graduates by joining a core edition.", +}; + +pub mod v2025_05; +pub mod v2026_02; +pub mod v2026_04; +pub mod v2026_06; + +pub use v2025_05::UNSTABLE_2025_05_0; +pub use v2026_02::UNSTABLE_2026_02_0; +pub use v2026_04::UNSTABLE_2026_04_0; +pub use v2026_06::UNSTABLE_2026_06_0; diff --git a/vortex/src/editions/unstable/v2025_05.rs b/vortex-edition/src/declarations/unstable/v2025_05.rs similarity index 85% rename from vortex/src/editions/unstable/v2025_05.rs rename to vortex-edition/src/declarations/unstable/v2025_05.rs index 2cc6accdc38..1a992fd9937 100644 --- a/vortex/src/editions/unstable/v2025_05.rs +++ b/vortex-edition/src/declarations/unstable/v2025_05.rs @@ -3,9 +3,9 @@ //! The May 2025 `unstable` encoding cohort. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The May 2025 draft edition of the `unstable` family. pub const UNSTABLE_2025_05_0: EditionId = EditionId::new("unstable", 2025, 5, 0); diff --git a/vortex/src/editions/unstable/v2026_02.rs b/vortex-edition/src/declarations/unstable/v2026_02.rs similarity index 85% rename from vortex/src/editions/unstable/v2026_02.rs rename to vortex-edition/src/declarations/unstable/v2026_02.rs index e992cdee5ad..8e5faeeb28f 100644 --- a/vortex/src/editions/unstable/v2026_02.rs +++ b/vortex-edition/src/declarations/unstable/v2026_02.rs @@ -3,9 +3,9 @@ //! The February 2026 `unstable` encoding cohort. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The February 2026 draft edition of the `unstable` family. pub const UNSTABLE_2026_02_0: EditionId = EditionId::new("unstable", 2026, 2, 0); diff --git a/vortex/src/editions/unstable/v2026_04.rs b/vortex-edition/src/declarations/unstable/v2026_04.rs similarity index 88% rename from vortex/src/editions/unstable/v2026_04.rs rename to vortex-edition/src/declarations/unstable/v2026_04.rs index 90955f01d04..d64bc07529e 100644 --- a/vortex/src/editions/unstable/v2026_04.rs +++ b/vortex-edition/src/declarations/unstable/v2026_04.rs @@ -3,9 +3,9 @@ //! The April 2026 `unstable` encoding cohort. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The April 2026 draft edition of the `unstable` family. pub const UNSTABLE_2026_04_0: EditionId = EditionId::new("unstable", 2026, 4, 0); diff --git a/vortex/src/editions/unstable/v2026_06.rs b/vortex-edition/src/declarations/unstable/v2026_06.rs similarity index 85% rename from vortex/src/editions/unstable/v2026_06.rs rename to vortex-edition/src/declarations/unstable/v2026_06.rs index acbd739b656..560b3b5bf02 100644 --- a/vortex/src/editions/unstable/v2026_06.rs +++ b/vortex-edition/src/declarations/unstable/v2026_06.rs @@ -3,9 +3,9 @@ //! The June 2026 `unstable` encoding cohort. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The June 2026 draft edition of the `unstable` family. pub const UNSTABLE_2026_06_0: EditionId = EditionId::new("unstable", 2026, 6, 0); diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index a8eaf8e8287..48e01ea7662 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -22,6 +22,7 @@ //! and enables them on the default session. See the published spec at //! . +pub mod declarations; mod session; pub mod test_harness; #[cfg(test)] @@ -33,6 +34,8 @@ use std::fmt::Debug; use std::fmt::Display; use std::fmt::Formatter; +pub use declarations::EDITION_DECLARATIONS; +pub use declarations::EDITION_FAMILIES; pub use session::EditionSession; pub use session::EditionSessionExt; pub use session::EnabledEditions; @@ -109,6 +112,41 @@ impl Display for EditionId { } } +/// A family of editions: an independently versioned, additive group of encodings, registered +/// with [`EditionSession::declare_family`]. +/// +/// Every [`EditionId`] names one. Declaring the family is what makes the name real: +/// [`EditionSession::validate`] rejects an edition whose family was never declared, so a typo +/// cannot quietly mint a family of one. +#[derive(Clone, Copy, Debug)] +pub struct EditionFamily { + /// The family name, matching the [`EditionId::family`] of its editions, e.g. `core`. + pub name: &'static str, + /// What the family is for. Exported into the family's record, so a few sentences at + /// most: the long form belongs in the published spec. + pub doc: &'static str, +} + +impl EditionFamily { + /// Validate the family's form: a non-empty lowercase name and a non-empty doc. Checked + /// for every declared family by [`EditionSession::validate`]. + pub fn validate(&self) -> Result<(), EditionError> { + if self.name.is_empty() || !self.name.chars().all(|c| c.is_ascii_lowercase()) { + return Err(EditionError::new(format!( + "edition family {:?} must have a non-empty lowercase name, e.g. `core`", + self.name + ))); + } + if self.doc.trim().is_empty() { + return Err(EditionError::new(format!( + "edition family {} must document what it is for", + self.name + ))); + } + Ok(()) + } +} + /// An edition: a named set of encodings with a read-compatibility guarantee, registered with /// [`EditionSession::declare_edition`]. The set itself is computed from the registered /// [`EditionInclusion`]s by [`EditionSession::encodings_in`]. diff --git a/vortex-edition/src/session.rs b/vortex-edition/src/session.rs index e682590ba8f..2e1f5d0d9aa 100644 --- a/vortex-edition/src/session.rs +++ b/vortex-edition/src/session.rs @@ -17,6 +17,7 @@ use vortex_session::registry::Id; use crate::Edition; use crate::EditionDeclaration; use crate::EditionError; +use crate::EditionFamily; use crate::EditionId; use crate::EditionInclusion; use crate::parse_release; @@ -35,6 +36,8 @@ pub struct EditionSession { #[derive(Debug, Default)] struct Inner { + /// Keyed by family name. + families: BTreeMap, /// Keyed by the display form of the edition id. editions: BTreeMap, /// Keyed by interned encoding id; ordered by the id's string form. @@ -90,6 +93,31 @@ impl EditionSession { Ok(()) } + /// Declare an edition family. Errors if a family with the same name is already + /// declared. Every family must be declared before [`EditionSession::validate`] will + /// accept editions belonging to it. + pub fn declare_family(&self, family: &EditionFamily) -> Result<(), EditionError> { + let mut inner = self.inner.write(); + if inner.families.contains_key(family.name) { + return Err(EditionError::new(format!( + "duplicate edition family {}", + family.name + ))); + } + inner.families.insert(family.name.to_string(), *family); + Ok(()) + } + + /// All declared families, sorted by name. + pub fn families(&self) -> Vec { + self.inner.read().families.values().copied().collect() + } + + /// Find a declared family by name. + pub fn find_family(&self, name: &str) -> Option { + self.inner.read().families.get(name).copied() + } + /// Declare an edition. Errors if an edition with the same id is already declared. pub fn declare_edition(&self, edition: Edition) -> Result<(), EditionError> { let mut inner = self.inner.write(); @@ -150,15 +178,26 @@ impl EditionSession { .collect() } - /// Validate all registered declarations. Errors on inclusions referencing undeclared - /// editions, editions out of chronological order within a family (unversioned drafts + /// Validate all registered declarations. Errors on editions in undeclared families, + /// inclusions referencing undeclared editions, editions out of chronological order within a family (unversioned drafts /// must be newest), malformed version strings, and members requiring a release newer /// than their edition declares. pub fn validate(&self) -> Result<(), EditionError> { let editions = self.editions(); + for family in self.families() { + family.validate()?; + } + for edition in &editions { edition.id.validate()?; + if self.find_family(edition.id.family).is_none() { + return Err(EditionError::new(format!( + "edition {} belongs to undeclared family {}; declare the family before \ + its editions", + edition.id, edition.id.family, + ))); + } if let Some(version) = edition.min_vortex_version && parse_release(version).is_none() { diff --git a/vortex-edition/src/tests.rs b/vortex-edition/src/tests.rs index 09e1345615a..e023359760e 100644 --- a/vortex-edition/src/tests.rs +++ b/vortex-edition/src/tests.rs @@ -5,12 +5,23 @@ use vortex_session::VortexSession; use crate::Edition; use crate::EditionDeclaration; +use crate::EditionFamily; use crate::EditionId; use crate::EditionInclusion; use crate::EditionSession; use crate::EditionSessionExt; use crate::EnabledEditions; +static TEST_FAMILY: EditionFamily = EditionFamily { + name: "test", + doc: "A family used by the unit tests.", +}; + +static OTHER_FAMILY: EditionFamily = EditionFamily { + name: "other", + doc: "A second family, for checking that families stay independent.", +}; + const FIRST: EditionId = EditionId::new("test", 2026, 1, 0); const SECOND: EditionId = EditionId::new("test", 2026, 7, 0); @@ -33,6 +44,9 @@ static DECLARATIONS: &[EditionDeclaration] = &[ fn session() -> EditionSession { let editions = EditionSession::empty(); + editions + .declare_family(&TEST_FAMILY) + .unwrap_or_else(|e| panic!("declaring the test family: {e}")); for declaration in DECLARATIONS { editions .declare(declaration) @@ -98,6 +112,7 @@ fn drafts_and_current() { // Freezing the first edition makes it current; the second stays a draft. let editions = EditionSession::empty(); + editions.declare_family(&TEST_FAMILY).unwrap(); editions .declare_edition(Edition { id: FIRST, @@ -179,8 +194,11 @@ fn enabled_editions_are_independent_across_families() -> Result<(), crate::Editi }; let session = VortexSession::empty().with::(); + session.editions().declare_family(&TEST_FAMILY)?; + session.editions().declare_family(&OTHER_FAMILY)?; session.register_edition(&DECLARATIONS[0])?; session.register_edition(&OTHER_DECLARATION)?; + session.editions().validate()?; session.enable_edition(FIRST)?; session.enable_edition(OTHER)?; @@ -274,3 +292,31 @@ fn edition_ids_order_within_family_only() { fn edition_id_display() { assert_eq!(FIRST.to_string(), "test2026.01.0"); } + +#[test] +fn families_must_be_declared_before_their_editions() -> Result<(), crate::EditionError> { + // An edition whose family was never declared: the name would otherwise be whatever the + // declaration happened to spell, and a typo would mint a family of one. + let editions = EditionSession::empty(); + editions.declare(&DECLARATIONS[0])?; + assert!(editions.validate().is_err()); + + editions.declare_family(&TEST_FAMILY)?; + editions.validate()?; + + // Declaring the same family twice is an error, as it is for editions. + assert!(editions.declare_family(&TEST_FAMILY).is_err()); + Ok(()) +} + +#[test] +fn families_must_document_themselves() { + let editions = EditionSession::empty(); + editions + .declare_family(&EditionFamily { + name: "undocumented", + doc: " ", + }) + .unwrap(); + assert!(editions.validate().is_err()); +} diff --git a/vortex/editions/core/core2025.05.0.toml b/vortex/editions/core/core2025.05.0.toml new file mode 100644 index 00000000000..eac393d6ddf --- /dev/null +++ b/vortex/editions/core/core2025.05.0.toml @@ -0,0 +1,64 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI. + +edition = "core2025.05.0" +family = "core" +min_vortex_version = "0.36.0" + +# The encodings that join the family at this edition. +added = [ + "fastlanes.bitpacked", + "fastlanes.for", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fsst", + "vortex.list", + "vortex.null", + "vortex.primitive", + "vortex.runend", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.bitpacked", + "fastlanes.for", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fsst", + "vortex.list", + "vortex.null", + "vortex.primitive", + "vortex.runend", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", +] diff --git a/vortex/editions/core/core2025.06.0.toml b/vortex/editions/core/core2025.06.0.toml new file mode 100644 index 00000000000..a4e63311f4c --- /dev/null +++ b/vortex/editions/core/core2025.06.0.toml @@ -0,0 +1,47 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI. + +edition = "core2025.06.0" +family = "core" +min_vortex_version = "0.40.0" + +# The encodings that join the family at this edition. +added = [ + "vortex.pco", + "vortex.sequence", + "vortex.zstd", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.bitpacked", + "fastlanes.for", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fsst", + "vortex.list", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", +] diff --git a/vortex/editions/core/core2025.10.0.toml b/vortex/editions/core/core2025.10.0.toml new file mode 100644 index 00000000000..4e3bbba2d69 --- /dev/null +++ b/vortex/editions/core/core2025.10.0.toml @@ -0,0 +1,52 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI. + +edition = "core2025.10.0" +family = "core" +min_vortex_version = "0.54.0" + +# The encodings that join the family at this edition. +added = [ + "fastlanes.rle", + "vortex.fixed_size_list", + "vortex.listview", + "vortex.masked", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.masked", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", +] diff --git a/vortex/editions/core/core2026.07.0.toml b/vortex/editions/core/core2026.07.0.toml new file mode 100644 index 00000000000..e49b6f0bf26 --- /dev/null +++ b/vortex/editions/core/core2026.07.0.toml @@ -0,0 +1,50 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI. + +edition = "core2026.07.0" +family = "core" +min_vortex_version = "0.65.0" + +# The encodings that join the family at this edition. +added = [ + "vortex.variant", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.masked", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.variant", + "vortex.zigzag", + "vortex.zstd", +] diff --git a/vortex/editions/core/core2026.08.0.toml b/vortex/editions/core/core2026.08.0.toml new file mode 100644 index 00000000000..82db60f548d --- /dev/null +++ b/vortex/editions/core/core2026.08.0.toml @@ -0,0 +1,51 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI. + +edition = "core2026.08.0" +family = "core" +min_vortex_version = "0.84.0" + +# The encodings that join the family at this edition. +added = [ + "vortex.map", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.map", + "vortex.masked", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.variant", + "vortex.zigzag", + "vortex.zstd", +] diff --git a/vortex/editions/core/family.toml b/vortex/editions/core/family.toml new file mode 100644 index 00000000000..ebff1159dba --- /dev/null +++ b/vortex/editions/core/family.toml @@ -0,0 +1,12 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This describes the family the editions beside it belong to. + +name = "core" + +doc = """ +The encodings the default file writer emits. Every core edition freezes, and a frozen +edition carries a read-forever guarantee: a file written with it stays readable by every +later Vortex release. New encodings join by being declared in a new edition; an edition that +has frozen never changes again. +""" diff --git a/vortex/editions/unstable/family.toml b/vortex/editions/unstable/family.toml new file mode 100644 index 00000000000..19c6b159176 --- /dev/null +++ b/vortex/editions/unstable/family.toml @@ -0,0 +1,13 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This describes the family the editions beside it belong to. + +name = "unstable" + +doc = """ +Opt-in encodings that are still being evaluated. Every unstable edition stays a draft, so +the family never freezes and carries no compatibility guarantee: a file written with these +encodings is readable only by a build that knows them, and a later release may stop +supporting one. The writer emits them only when the `unstable_encodings` feature is +selected. An encoding graduates by joining a core edition. +""" diff --git a/vortex/editions/unstable/unstable2025.05.0.toml b/vortex/editions/unstable/unstable2025.05.0.toml new file mode 100644 index 00000000000..b2b2ddb8325 --- /dev/null +++ b/vortex/editions/unstable/unstable2025.05.0.toml @@ -0,0 +1,19 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again. + +edition = "unstable2025.05.0" +family = "unstable" + +# The encodings that join the family at this edition. +added = [ + "fastlanes.delta", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.delta", +] diff --git a/vortex/editions/unstable/unstable2026.02.0.toml b/vortex/editions/unstable/unstable2026.02.0.toml new file mode 100644 index 00000000000..00f1b0bef6f --- /dev/null +++ b/vortex/editions/unstable/unstable2026.02.0.toml @@ -0,0 +1,20 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again. + +edition = "unstable2026.02.0" +family = "unstable" + +# The encodings that join the family at this edition. +added = [ + "vortex.zstd_buffers", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.delta", + "vortex.zstd_buffers", +] diff --git a/vortex/editions/unstable/unstable2026.04.0.toml b/vortex/editions/unstable/unstable2026.04.0.toml new file mode 100644 index 00000000000..7e721892046 --- /dev/null +++ b/vortex/editions/unstable/unstable2026.04.0.toml @@ -0,0 +1,31 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again. + +edition = "unstable2026.04.0" +family = "unstable" + +# The encodings that join the family at this edition. +added = [ + "vortex.parquet.variant", + "vortex.patched", + "vortex.tensor.cosine_similarity", + "vortex.tensor.inner_product", + "vortex.tensor.l2_norm", + "vortex.tensor.normalized", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.delta", + "vortex.parquet.variant", + "vortex.patched", + "vortex.tensor.cosine_similarity", + "vortex.tensor.inner_product", + "vortex.tensor.l2_norm", + "vortex.tensor.normalized", + "vortex.zstd_buffers", +] diff --git a/vortex/editions/unstable/unstable2026.06.0.toml b/vortex/editions/unstable/unstable2026.06.0.toml new file mode 100644 index 00000000000..c890c13627d --- /dev/null +++ b/vortex/editions/unstable/unstable2026.06.0.toml @@ -0,0 +1,27 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again. + +edition = "unstable2026.06.0" +family = "unstable" + +# The encodings that join the family at this edition. +added = [ + "vortex.onpair", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.delta", + "vortex.onpair", + "vortex.parquet.variant", + "vortex.patched", + "vortex.tensor.cosine_similarity", + "vortex.tensor.inner_product", + "vortex.tensor.l2_norm", + "vortex.tensor.normalized", + "vortex.zstd_buffers", +] diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index 88e4f351d0d..2538a270901 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -3,42 +3,45 @@ //! The Vortex edition declarations. //! -//! [`vortex_edition`] provides the types, session variables, and test harness. The actual -//! first-party declarations live here, one module per edition. The default session first -//! registers them with [`crate::editions::register_default_editions`] and then selects its write -//! policy with [`crate::editions::enable_default_editions`]. +//! [`vortex_edition`] provides the types, session variables, test harness, and the +//! first-party declarations themselves. This module re-exports them and owns the session +//! wiring: the default session first registers them with +//! [`crate::editions::register_default_editions`] and then selects its write policy with +//! [`crate::editions::enable_default_editions`]. //! //! The default file writer resolves the session's enabled editions at write time. The //! facade enables the newest frozen `core` edition, [`crate::editions::CORE_2026_08`], and //! additionally enables the latest unstable edition when the `unstable_encodings` feature is //! selected. -pub mod core; #[cfg(test)] mod tests; -pub mod unstable; +pub use vortex_edition::EDITION_DECLARATIONS; +pub use vortex_edition::EDITION_FAMILIES; pub use vortex_edition::Edition; pub use vortex_edition::EditionDeclaration; +pub use vortex_edition::EditionFamily; pub use vortex_edition::EditionId; pub use vortex_edition::EditionInclusion; pub use vortex_edition::EditionSession; pub use vortex_edition::EditionSessionExt; pub use vortex_edition::EnabledEditions; +pub use vortex_edition::declarations::core; +pub use vortex_edition::declarations::core::CORE_2025_05_0; +pub use vortex_edition::declarations::core::CORE_2025_06_0; +pub use vortex_edition::declarations::core::CORE_2025_10_0; +pub use vortex_edition::declarations::core::CORE_2026_07_0; +pub use vortex_edition::declarations::core::CORE_2026_08; +pub use vortex_edition::declarations::unstable; +pub use vortex_edition::declarations::unstable::UNSTABLE_2025_05_0; +pub use vortex_edition::declarations::unstable::UNSTABLE_2026_02_0; +pub use vortex_edition::declarations::unstable::UNSTABLE_2026_04_0; +pub use vortex_edition::declarations::unstable::UNSTABLE_2026_06_0; use vortex_error::VortexExpect; use vortex_error::vortex_err; use vortex_session::VortexSession; -pub use self::core::CORE_2025_05_0; -pub use self::core::CORE_2025_06_0; -pub use self::core::CORE_2025_10_0; -pub use self::core::CORE_2026_07_0; -pub use self::core::CORE_2026_08; -pub use self::unstable::UNSTABLE_2025_05_0; -pub use self::unstable::UNSTABLE_2026_02_0; -pub use self::unstable::UNSTABLE_2026_04_0; -pub use self::unstable::UNSTABLE_2026_06_0; - /// The `core` edition enabled for writing by the default Vortex session. pub const DEFAULT_CORE_EDITION: EditionId = CORE_2026_08; @@ -46,21 +49,16 @@ pub const DEFAULT_CORE_EDITION: EditionId = CORE_2026_08; /// `unstable_encodings` feature is selected. pub const DEFAULT_UNSTABLE_EDITION: EditionId = UNSTABLE_2026_06_0; -/// The first-party Vortex edition declarations. -pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ - &core::v2025_05::DECLARATION, - &core::v2025_06::DECLARATION, - &core::v2025_10::DECLARATION, - &core::v2026_07::DECLARATION, - &core::v2026_08::DECLARATION, - &unstable::v2025_05::DECLARATION, - &unstable::v2026_02::DECLARATION, - &unstable::v2026_04::DECLARATION, - &unstable::v2026_06::DECLARATION, -]; - -/// Register the Vortex edition declarations with the session's [`EditionSession`]. +/// Register the Vortex edition families and declarations with the session's +/// [`EditionSession`]. pub fn register_default_editions(session: &VortexSession) { + for family in EDITION_FAMILIES { + session + .editions() + .declare_family(family) + .map_err(|e| vortex_err!("{e}")) + .vortex_expect("edition families are valid"); + } for declaration in EDITION_DECLARATIONS { session .register_edition(declaration) diff --git a/vortex/src/editions/tests.rs b/vortex/src/editions/tests.rs index b8608f4427a..8a7bd100362 100644 --- a/vortex/src/editions/tests.rs +++ b/vortex/src/editions/tests.rs @@ -47,6 +47,9 @@ use super::UNSTABLE_2026_06_0; fn session() -> Result { let session = EditionSession::empty(); + for family in super::EDITION_FAMILIES { + session.declare_family(family)?; + } for declaration in EDITION_DECLARATIONS { session.declare(declaration)?; } @@ -62,55 +65,6 @@ fn every_declared_edition_validates() -> Result<(), EditionError> { Ok(()) } -/// The full encoding set of the newest frozen `core` edition. This set is frozen: the only -/// way it may change is by declaring a *new* edition, so a failure here means a frozen -/// declaration was edited. -#[test] -fn core_2026_07_encoding_set_is_pinned() { - let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); - let encodings = session.encodings_in(&CORE_2026_07_0); - let ids: Vec<&str> = encodings - .iter() - .map(|inclusion| inclusion.encoding_id.as_str()) - .collect(); - assert_eq!( - ids, - [ - "fastlanes.bitpacked", - "fastlanes.for", - "fastlanes.rle", - "vortex.alp", - "vortex.alprd", - "vortex.bool", - "vortex.bytebool", - "vortex.chunked", - "vortex.constant", - "vortex.datetimeparts", - "vortex.decimal", - "vortex.decimal_byte_parts", - "vortex.dict", - "vortex.ext", - "vortex.fixed_size_list", - "vortex.fsst", - "vortex.list", - "vortex.listview", - "vortex.masked", - "vortex.null", - "vortex.pco", - "vortex.primitive", - "vortex.runend", - "vortex.sequence", - "vortex.sparse", - "vortex.struct", - "vortex.varbin", - "vortex.varbinview", - "vortex.variant", - "vortex.zigzag", - "vortex.zstd", - ] - ); -} - #[test] fn encodings_in_editions_unions_families() { let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); diff --git a/vortex/src/editions/unstable/mod.rs b/vortex/src/editions/unstable/mod.rs deleted file mode 100644 index 5544ba45c0f..00000000000 --- a/vortex/src/editions/unstable/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The `unstable` edition family: opt-in encodings without a frozen compatibility guarantee. -//! -//! One module per draft edition, each declaring the encodings that join the family at it. -//! Members of earlier editions are inherited and never restated. - -pub mod v2025_05; -pub mod v2026_02; -pub mod v2026_04; -pub mod v2026_06; - -pub use v2025_05::UNSTABLE_2025_05_0; -pub use v2026_02::UNSTABLE_2026_02_0; -pub use v2026_04::UNSTABLE_2026_04_0; -pub use v2026_06::UNSTABLE_2026_06_0; diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index eae2db43413..fedd46b7dd8 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -22,7 +22,10 @@ test = false [dependencies] anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } +git2 = { workspace = true } prost-build = { workspace = true } +toml = { workspace = true } +vortex-edition = { workspace = true } xshell = { workspace = true } [lints] diff --git a/xtask/src/check_editions.rs b/xtask/src/check_editions.rs new file mode 100644 index 00000000000..a02179bc70d --- /dev/null +++ b/xtask/src/check_editions.rs @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Check that frozen edition records under `vortex/editions` never change. +//! +//! A record's mutability follows its edition. A draft is still being assembled, so its record +//! may change, be renamed, or be dropped. Freezing — recording a `min_vortex_version` — turns +//! the record into a read-forever contract, and from then on it may never change again. +//! Whether a record was frozen is read from the base revision, so a change cannot unfreeze an +//! edition and edit it in the same diff. +//! +//! A newly added record must also be newer than every edition already recorded for its +//! family: editions are only ever added going forward. Records are grouped by family, so +//! `vortex/editions/core/core2025.05.0.toml` must sit under the family its name declares. The +//! `family.toml` beside them documents the family rather than pinning a contract, so it is +//! exempt. +//! +//! Both revisions are read out of the object database, so the check sees committed state only +//! and never the working tree. + +use std::collections::BTreeMap; +use std::path::Path; + +use anyhow::Context; +use anyhow::anyhow; +use anyhow::bail; +use git2::Commit; +use git2::Delta; +use git2::DiffFindOptions; +use git2::Repository; +use git2::TreeWalkMode; +use git2::TreeWalkResult; +use toml::Table; + +use crate::generate_editions::FAMILY_FILE; +use crate::generate_editions::RECORD_DIR; + +/// A record carries this key exactly when the edition it records is frozen. +const FROZEN_MARKER: &str = "min_vortex_version"; + +const REMEDY: &str = "\ +A frozen edition is immutable. To add encodings, declare a NEW edition in + vortex-edition/src/declarations// and regenerate the records with + `cargo run -p xtask -- generate-editions`."; + +/// An edition's position in its family's chronology, from `..`. +type Chronology = (u16, u8, u8); + +/// Split a record file name into its family and its place in that family's chronology. +fn parse_name(name: &str) -> anyhow::Result<(&str, Chronology)> { + let malformed = || { + anyhow!( + "{RECORD_DIR}/{name} is not a valid record name. Records are named after the \ + edition they record, e.g. `core/core2026.08.0.toml`." + ) + }; + + let stem = name.strip_suffix(".toml").ok_or_else(malformed)?; + let split = stem + .find(|c: char| c.is_ascii_digit()) + .ok_or_else(malformed)?; + let (family, version) = stem.split_at(split); + if family.is_empty() || !family.chars().all(|c| c.is_ascii_lowercase()) { + return Err(malformed()); + } + + let parts: Vec<&str> = version.split('.').collect(); + let [year, month, version] = parts.as_slice() else { + return Err(malformed()); + }; + if year.len() != 4 || month.len() != 2 { + return Err(malformed()); + } + let chronology = ( + year.parse().map_err(|_| malformed())?, + month.parse().map_err(|_| malformed())?, + version.parse().map_err(|_| malformed())?, + ); + Ok((family, chronology)) +} + +/// Parse a record out of a commit's tree, or `None` when it holds no such file. +fn read_record(repo: &Repository, commit: &Commit, path: &str) -> anyhow::Result> { + let Ok(entry) = commit.tree()?.get_path(Path::new(path)) else { + return Ok(None); + }; + let blob = entry.to_object(repo)?.peel_to_blob()?; + let text = std::str::from_utf8(blob.content()) + .with_context(|| format!("{path} at {} is not UTF-8", commit.id()))?; + Ok(Some(text.parse::().with_context(|| { + format!("{path} at {} is not valid TOML", commit.id()) + })?)) +} + +/// The newest edition already recorded for each family at `commit`. +fn newest_recorded( + repo: &Repository, + commit: &Commit, +) -> anyhow::Result> { + let mut newest = BTreeMap::new(); + let Ok(entry) = commit.tree()?.get_path(Path::new(RECORD_DIR)) else { + return Ok(newest); + }; + let records = entry.to_object(repo)?.peel_to_tree()?; + + let mut malformed = None; + records.walk(TreeWalkMode::PreOrder, |_, entry| { + let Ok(name) = entry.name() else { + return TreeWalkResult::Ok; + }; + if !name.ends_with(".toml") || name == FAMILY_FILE { + return TreeWalkResult::Ok; + } + match parse_name(name) { + Ok((family, chronology)) => { + let slot = newest.entry(family.to_string()).or_insert(chronology); + *slot = (*slot).max(chronology); + TreeWalkResult::Ok + } + Err(error) => { + malformed = Some(error); + TreeWalkResult::Abort + } + } + })?; + match malformed { + Some(error) => Err(error), + None => Ok(newest), + } +} + +/// A frozen record may not change at all; name the fields that did. +fn check_modification(before: &Table, after: &Table, name: &str) -> Vec { + let mut changed: Vec<&str> = before + .keys() + .chain(after.keys()) + .map(String::as_str) + .filter(|key| before.get(*key) != after.get(*key)) + .collect(); + changed.sort_unstable(); + changed.dedup(); + + if changed.is_empty() { + return vec![]; + } + if changed.contains(&FROZEN_MARKER) && !after.contains_key(FROZEN_MARKER) { + return vec![format!( + "unfreezes {name}; an edition that recorded a {FROZEN_MARKER} carries a \ + read-forever guarantee and may never return to draft" + )]; + } + vec![format!( + "modifies the frozen record {name}: {}", + changed.join(", ") + )] +} + +/// A new record must extend its family's chronology, and be filed under that family. +fn check_addition( + path: &str, + record: &Table, + newest: &BTreeMap, +) -> anyhow::Result> { + let mut errors = Vec::new(); + let name = Path::new(path) + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("{path} has no file name"))?; + let (family, chronology) = parse_name(name)?; + + if let Some(previous) = newest.get(family) + && chronology <= *previous + { + errors.push(format!( + "adds {name}, which is not newer than the {family} edition already recorded \ + ({family}{}.{:02}.{}). Editions may only be added going forward.", + previous.0, previous.1, previous.2, + )); + } + + // A record's family decides which chronology it extends, so the directory it sits in has + // to agree with the family its name declares. + let directory = Path::new(path) + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + .unwrap_or_default(); + if directory != family { + errors.push(format!( + "adds {name} under {directory}/, but it records a {family} edition; records are \ + grouped by family" + )); + } + + // The file name is the edition's identity, so it has to agree with the content. + match record.get("edition").and_then(|edition| edition.as_str()) { + None => errors.push(format!("adds {name}, which has no `edition` field")), + Some(edition) if edition != name.trim_end_matches(".toml") => errors.push(format!( + "adds {name}, which records edition {edition:?}; the file name must be the \ + edition id" + )), + Some(_) => {} + } + Ok(errors) +} + +fn under_record_dir(path: Option<&Path>) -> bool { + path.is_some_and(|path| path.starts_with(RECORD_DIR)) +} + +fn is_family_record(path: Option<&Path>) -> bool { + path.is_some_and(|path| path.file_name().is_some_and(|name| name == FAMILY_FILE)) +} + +fn path_str(path: Option<&Path>) -> String { + path.map(|path| path.display().to_string()) + .unwrap_or_default() +} + +pub fn check_editions(base: &str) -> anyhow::Result<()> { + let repo = Repository::discover(".").context("opening the repository")?; + let base_tip = repo + .revparse_single(base) + .with_context(|| format!("cannot resolve {base:?} in this repository"))? + .peel_to_commit()?; + let head = repo.head()?.peel_to_commit()?; + + let merge_base = repo.merge_base(base_tip.id(), head.id()).with_context(|| { + format!( + "{base} and HEAD have no common ancestor. The checkout is probably too shallow; \ + this check needs `fetch-depth: 0`." + ) + })?; + let base_commit = repo.find_commit(merge_base)?; + + let mut diff = repo.diff_tree_to_tree(Some(&base_commit.tree()?), Some(&head.tree()?), None)?; + diff.find_similar(Some(DiffFindOptions::new().renames(true)))?; + + let newest = newest_recorded(&repo, &base_commit)?; + let mut errors = Vec::new(); + let mut added = Vec::new(); + + for delta in diff.deltas() { + let (old_path, new_path) = (delta.old_file().path(), delta.new_file().path()); + if !under_record_dir(old_path) && !under_record_dir(new_path) { + continue; + } + // The family record is documentation rather than a contract, so it stays editable. + if is_family_record(new_path) || is_family_record(old_path) { + continue; + } + + if delta.status() == Delta::Added { + added.push(path_str(new_path)); + continue; + } + + // Frozen-ness comes from the base revision, so a diff cannot unfreeze an edition and + // then edit it. A draft's record is free to change, move, or go away with the draft. + let old = path_str(old_path); + let Some(before) = read_record(&repo, &base_commit, &old)? else { + continue; + }; + if !before.contains_key(FROZEN_MARKER) { + continue; + } + + let new = path_str(new_path); + if delta.status() == Delta::Modified { + let after = read_record(&repo, &head, &new)?.unwrap_or_default(); + let name = Path::new(&new) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(&new); + errors.extend(check_modification(&before, &after, name)); + } else { + let verb = match delta.status() { + Delta::Deleted => "deletes", + Delta::Renamed => "renames", + Delta::Copied => "copies", + Delta::Typechange => "retypes", + _ => "changes", + }; + let moved = if old == new { + old.clone() + } else { + format!("{old} -> {new}") + }; + errors.push(format!("{verb} the frozen record {moved}")); + } + } + + added.sort(); + for path in &added { + if let Some(record) = read_record(&repo, &head, path)? { + errors.extend(check_addition(path, &record, &newest)?); + } + } + + if errors.is_empty() { + println!("{RECORD_DIR} preserves every frozen record against {base}."); + return Ok(()); + } + + let listed = errors + .iter() + .map(|error| format!(" - it {error}")) + .collect::>() + .join("\n"); + bail!("This change breaks the edition records in {RECORD_DIR}:\n\n{listed}\n\n{REMEDY}"); +} diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs new file mode 100644 index 00000000000..d4c4cd0cf8f --- /dev/null +++ b/xtask/src/generate_editions.rs @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Export the edition records under `vortex/editions`. +//! +//! Every declared edition gets one TOML file recording what it contains: the identifier, the +//! minimum Vortex version whose reader supports it once frozen, and its full encoding set. +//! Records are grouped by family — `vortex/editions/core/core2025.05.0.toml` — mirroring the +//! declarations in `vortex-edition/src/declarations`, since families version independently. +//! +//! A record's mutability follows its edition. A draft is still being assembled, so its record +//! changes with it. Freezing — recording a `min_vortex_version` — turns the record into a +//! contract carrying a read-forever guarantee, and from then on it may never change again. CI +//! enforces that against git history in `.github/scripts/check_edition_records.py`; this +//! exporter enforces the two rules that history cannot see, refusing to delete a record or to +//! unfreeze one. + +use std::collections::BTreeSet; +use std::fs; +use std::path::Path; +use std::path::PathBuf; + +use anyhow::Context; +use anyhow::anyhow; +use vortex_edition::EDITION_DECLARATIONS; +use vortex_edition::EDITION_FAMILIES; +use vortex_edition::Edition; +use vortex_edition::EditionFamily; +use vortex_edition::EditionSession; + +const GENERATED_BY: &str = "# Generated by `cargo run -p xtask -- generate-editions`.\n#"; + +const FROZEN_NOTE: &str = "\ +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI."; + +const DRAFT_NOTE: &str = "\ +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again."; + +/// The file recording what a family is, beside that family's editions. +pub const FAMILY_FILE: &str = "family.toml"; + +/// The edition records, relative to the repository root. +pub const RECORD_DIR: &str = "vortex/editions"; + +/// Render a family's record: its name and what it is for. Unlike an edition record this is +/// documentation, not a contract, so it stays editable. +fn family_record(family: &EditionFamily) -> String { + let mut lines = vec![ + GENERATED_BY.to_string(), + "# This describes the family the editions beside it belong to.".to_string(), + String::new(), + format!("name = \"{}\"", family.name), + String::new(), + "doc = \"\"\"".to_string(), + ]; + lines.extend(wrap(family.doc, 92)); + lines.extend(["\"\"\"".to_string(), String::new()]); + lines.join("\n") +} + +/// Wrap prose to a column, so a long doc reads as a paragraph rather than one endless line. +fn wrap(text: &str, width: usize) -> Vec { + let mut lines = Vec::new(); + let mut line = String::new(); + for word in text.split_whitespace() { + if !line.is_empty() && line.len() + 1 + word.len() > width { + lines.push(std::mem::take(&mut line)); + } + if !line.is_empty() { + line.push(' '); + } + line.push_str(word); + } + if !line.is_empty() { + lines.push(line); + } + lines +} + +/// Render one edition's record. Deterministic: every list is sorted by encoding id, so the +/// generated bytes depend only on the declarations. +fn record(session: &EditionSession, edition: &Edition) -> String { + let inclusions = session.encodings_in(&edition.id); + let members: BTreeSet<&str> = inclusions + .iter() + .map(|inclusion| inclusion.encoding_id.as_str()) + .collect(); + let added: BTreeSet<&str> = inclusions + .iter() + .filter(|inclusion| inclusion.since == edition.id) + .map(|inclusion| inclusion.encoding_id.as_str()) + .collect(); + + let list = |ids: &BTreeSet<&str>| -> Vec { + ids.iter().map(|id| format!(" \"{id}\",")).collect() + }; + + let note = if edition.is_draft() { + DRAFT_NOTE + } else { + FROZEN_NOTE + }; + let mut lines = vec![ + GENERATED_BY.to_string(), + note.to_string(), + String::new(), + format!("edition = \"{}\"", edition.id), + format!("family = \"{}\"", edition.id.family), + ]; + if let Some(min_vortex_version) = edition.min_vortex_version { + lines.push(format!("min_vortex_version = \"{min_vortex_version}\"")); + } + lines.extend([ + String::new(), + "# The encodings that join the family at this edition.".to_string(), + "added = [".to_string(), + ]); + lines.extend(list(&added)); + lines.extend([ + "]".to_string(), + String::new(), + "# The edition's full membership: the encodings above, plus every member of earlier" + .to_string(), + "# editions of the family.".to_string(), + "encodings = [".to_string(), + ]); + lines.extend(list(&members)); + lines.extend(["]".to_string(), String::new()]); + lines.join("\n") +} + +/// The records present on disk, as `family/edition.toml` paths relative to the record +/// directory. A record filed under the wrong family reads as a stray, which is what it is. +fn existing_records(dir: &Path) -> anyhow::Result> { + let mut records = BTreeSet::new(); + if !dir.exists() { + return Ok(records); + } + for family in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { + let family = family?.path(); + if !family.is_dir() { + continue; + } + for entry in + fs::read_dir(&family).with_context(|| format!("reading {}", family.display()))? + { + let path = entry?.path(); + if path + .extension() + .is_some_and(|extension| extension == "toml") + && let Ok(relative) = path.strip_prefix(dir) + && let Some(relative) = relative.to_str() + { + records.insert(relative.to_string()); + } + } + } + Ok(records) +} + +/// A record carries a `min_vortex_version` exactly when the edition it records is frozen. +fn records_a_frozen_edition(contents: &str) -> bool { + contents + .lines() + .any(|line| line.starts_with("min_vortex_version = ")) +} + +pub fn generate_editions() -> anyhow::Result<()> { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join(RECORD_DIR); + fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; + + let session = EditionSession::empty(); + for family in EDITION_FAMILIES { + session + .declare_family(family) + .map_err(|error| anyhow!("declaring edition families: {error}"))?; + } + for declaration in EDITION_DECLARATIONS { + session + .declare(declaration) + .map_err(|error| anyhow!("declaring editions: {error}"))?; + } + session + .validate() + .map_err(|error| anyhow!("validating editions: {error}"))?; + + let mut expected = BTreeSet::new(); + for family in session.families() { + let relative = format!("{}/{FAMILY_FILE}", family.name); + let path = dir.join(&relative); + expected.insert(relative); + fs::create_dir_all(dir.join(family.name)) + .with_context(|| format!("creating the {} record directory", family.name))?; + fs::write(&path, family_record(&family)) + .with_context(|| format!("writing {}", path.display()))?; + } + + for edition in session.editions() { + let relative = format!("{}/{}.toml", edition.id.family, edition.id); + let path = dir.join(&relative); + expected.insert(relative); + fs::create_dir_all(dir.join(edition.id.family)) + .with_context(|| format!("creating the {} record directory", edition.id.family))?; + + // Freezing is permanent, and the record on disk is the only memory of it. Refusing + // here means unfreezing cannot be laundered through the exporter. + if edition.is_draft() + && let Ok(previous) = fs::read_to_string(&path) + && records_a_frozen_edition(&previous) + { + return Err(anyhow!( + "{} is recorded as frozen but its declaration is now a draft.\n\ + An edition that recorded a min_vortex_version carries a read-forever \ + guarantee and may never return to draft.", + edition.id, + )); + } + + fs::write(&path, record(&session, &edition)) + .with_context(|| format!("writing {}", path.display()))?; + } + + let strays: Vec = existing_records(&dir)? + .difference(&expected) + .cloned() + .collect(); + if !strays.is_empty() { + return Err(anyhow!( + "{} has records with no declared edition behind them: {strays:?}.\n\ + A frozen edition may never be deleted; its declaration must stay in \ + `EDITION_DECLARATIONS`.", + dir.display(), + )); + } + + Ok(()) +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 1155ee3246a..8cc582be233 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,11 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +mod check_editions; +mod generate_editions; mod generate_fbs; mod generate_proto; use clap::Parser; +use crate::check_editions::check_editions; +use crate::generate_editions::generate_editions; use crate::generate_fbs::generate_fbs; use crate::generate_proto::generate_proto; @@ -17,19 +21,31 @@ struct Xtask { #[derive(clap::Subcommand)] enum Commands { + /// Subcommand to check that frozen edition records never change. + #[command(name = "check-editions")] + CheckEditions { + /// The revision to compare against. + #[arg(long, default_value = "origin/develop")] + base: String, + }, + /// Subcommand to regenerate the edition records under `vortex/editions`. + #[command(name = "generate-editions")] + Editions, /// Subcommand to regenerate flatbuffers language bindings for the Rust project. #[command(name = "generate-fbs")] - GenerateFlatbuffers, + Flatbuffers, /// Subcommand to regenerate protobuf language bindings for the Rust project. #[command(name = "generate-proto")] - GenerateProto, + Proto, } fn main() -> anyhow::Result<()> { let cli = Xtask::parse(); match cli.command { - Commands::GenerateFlatbuffers => generate_fbs()?, - Commands::GenerateProto => generate_proto()?, + Commands::CheckEditions { base } => check_editions(&base)?, + Commands::Editions => generate_editions()?, + Commands::Flatbuffers => generate_fbs()?, + Commands::Proto => generate_proto()?, } Ok(()) }