diff --git a/README.md b/README.md index 226bef3d..a2e66bbd 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,8 @@ Reads `.rvt` projects and `.rfa` families. Testing covers sample projects saved The tested projects include basic and advanced architecture, structure and MEP samples, plus several larger multi-discipline models. +The stored class schema determines the decoding layout, including element row and ID widths. The saving year in `BasicFileInfo` is a fallback when the schema release cannot be recognized. Diagnostics report both years when they differ; an older saving year does not override a recognized newer schema. + ## Limitations - Version 0.4.0 is a preview. The API can change before 1.0. diff --git a/src/RvtIO/Parsing/ElemTableLayout.cs b/src/RvtIO/Parsing/ElemTableLayout.cs index 3519955f..741a9896 100644 --- a/src/RvtIO/Parsing/ElemTableLayout.cs +++ b/src/RvtIO/Parsing/ElemTableLayout.cs @@ -23,13 +23,13 @@ internal static class ElemTableLayout internal static int? IdWidth(FormatsSchema schema) => schema.Get("Identifier") is { Fields: [{ } field] } && LeafWidth(field.Type) is int width && Agrees(SchemaRelease.IdWidth(schema.Release), width) ? width : null; - /// True when the schema declares an element id width that the release in force does not have (a file saved by an older - /// release than its schema): the layout of that release is not the layout of the bytes. + /// True when the declared element id width contradicts the recognized release. Such a schema must not fall back + /// to a layout with a different width. internal static bool DisagreesWithRelease(FormatsSchema schema) => schema.Get("Identifier") is { Fields: [{ } field] } && LeafWidth(field.Type) is int width && !Agrees(SchemaRelease.IdWidth(schema.Release), width); - /// The widths the schema declares must be those of the release in force (the older of the saved and the schema release); - /// a file whose two releases disagree on the layout is not decoded by a layout it does not have. + /// Declared widths must agree with the recognized schema release, or with the saving release when the schema release + /// is unknown. Contradictory evidence is not decoded with a layout it does not have. private static bool Agrees(int? releaseWidth, int declared) => releaseWidth is not int expected || expected == declared; /// The rows of a decoded Global/ElemTable stream, or null unless the stream is this schema's ElemTable: the class diff --git a/src/RvtIO/Parsing/ElemTableReader.cs b/src/RvtIO/Parsing/ElemTableReader.cs index 414e5fc9..22bab043 100644 --- a/src/RvtIO/Parsing/ElemTableReader.cs +++ b/src/RvtIO/Parsing/ElemTableReader.cs @@ -59,7 +59,7 @@ public static Dictionary Read(RootStorage root, { RecordStream(diagnostics, "Global/ElemTable", data.Length, 1, 1, 0, 0, 0, exactTable.Length); // Every version but R22: the layout comes from the file's own schema, not from a constant. The shipped R22 layout - // is used only for a file whose release in force (the older of the saved and the schema release) is 2022 or unknown. + // is used only for a file whose schema release (falling back to the saving release) is 2022 or unknown. var formats = FormatsSchema.TryRead(root); bool shipped = exactTable.Length >= 2 && BitConverter.ToUInt16(exactTable, 0) == ElementExportIds.ElemTableClassId && SchemaRelease.AllowsShipped(formats?.Release ?? SchemaRelease.Of(root, null), 2022); diff --git a/src/RvtIO/Parsing/SchemaRelease.cs b/src/RvtIO/Parsing/SchemaRelease.cs index 1527826f..e0835346 100644 --- a/src/RvtIO/Parsing/SchemaRelease.cs +++ b/src/RvtIO/Parsing/SchemaRelease.cs @@ -40,13 +40,13 @@ internal static class SchemaRelease internal static int? Saved(string? revitVersion) => int.TryParse(revitVersion, out int year) && year is >= 2000 and <= 2100 ? year : null; - /// The older of the two releases, which decides every layout and row-size choice; the one that is known when only one - /// is, null when neither is. + /// The stored schema decides layouts and row sizes. The saving release is only a fallback when the schema release + /// is unknown; an older or misread BasicFileInfo year must not override the layout of the bytes. internal static int? Effective(int? saved, int? schema) - => saved is int s && schema is int c ? Math.Min(s, c) : saved ?? schema; + => schema ?? saved; - /// The release that decides the layouts of an open file: the older of the saved release and the schema's release - /// (), null when neither is known. Every layout and row-size choice reads this one value. + /// The release that decides the layouts of an open file, preferring over the saving + /// release. Every layout and row-size choice reads this one value. internal static int? Of(OpenMcdf.RootStorage root, int? schemaRelease) => Effective(Saved(BasicFileInfoReader.Read(root).GetValueOrDefault("RevitVersion")), schemaRelease); @@ -64,7 +64,7 @@ internal static class SchemaRelease internal static string Describe(int? saved, int? schema) { string Show(int? year) => year?.ToString() ?? "unknown"; - string note = saved is int s && schema is int c && s != c ? $", the layout of release {Math.Min(s, c)} is used" : ""; + string note = saved is int s && schema is int c && s != c ? $", the layout of release {Effective(s, c)} is used" : ""; return $"Release: saved by {Show(saved)}, schema of {Show(schema)}{note}"; } } diff --git a/tests/RvtIO.Tests/Fixtures/README.md b/tests/RvtIO.Tests/Fixtures/README.md index 2818da9c..799a8d92 100644 --- a/tests/RvtIO.Tests/Fixtures/README.md +++ b/tests/RvtIO.Tests/Fixtures/README.md @@ -33,3 +33,32 @@ solid and the host frame, not element names or a type-name size. This covers unfilled extruded openings in straight walls. Curved hosts, curtain hosts and the separately cached quantities of certified cut-pair bodies are not covered by this fixture. + +# Schema release selection regression (#31) + +`SchemaReleaseTests.SyntheticFilesKeepTheDeclaredTableAndRoutingWidths` builds +small compound files with paged schema and element-table streams. It covers +both directions of a saving/schema year mismatch across the 32/64-bit ID +boundary, matching years, an unknown schema release and a missing saving year. +It requires no downloaded files. + +`SchemaReleaseExportTests` additionally reads public Autodesk samples and copies +whose labelled `BasicFileInfo` `Format` year is changed. Only that saving-year +field is normalized when comparing the full model, IFC and compact JSON with +properties; all remaining content must match. The 2024 sample is tested both +with the native wide-schema reader and with `R22Layout` geometry decoding. +The tests are skipped unless the following variables name downloaded files: + +| Variable | Public sample | SHA-256 | +| --- | --- | --- | +| `RVTIO_SCHEMA_SAMPLE_2022` | [Revit 2022 structural basic sample](https://revit.downloads.autodesk.com/download/2022RVT_RTM/Docs/InProd/rstbasicsampleproject.rvt) | `054a6e5803098dece0f7aa8d5248a74915696daed1a56492b95ce54d0f1de1de` | +| `RVTIO_SCHEMA_SAMPLE_2024` | [Snowdon Towers Sample Site](https://revit.downloads.autodesk.com/download/2024RVT_RTM/Docs/InProd/Samples/Snowdon%20Towers%20Sample%20Site.rvt) | `6bf4e490959c687e9ee851662c2f0f202ef486b00bf95d61c05f3fee34a2ebe5` | + +The repository contains no Revit files. To run these checks in PowerShell after +downloading the samples outside the checkout: + +```powershell +$env:RVTIO_SCHEMA_SAMPLE_2022 = 'C:\Samples\rstbasicsampleproject.rvt' +$env:RVTIO_SCHEMA_SAMPLE_2024 = 'C:\Samples\Snowdon Towers Sample Site.rvt' +dotnet test tests/RvtIO.Tests/RvtIO.Tests.csproj --filter 'FullyQualifiedName~SchemaRelease' +``` diff --git a/tests/RvtIO.Tests/FormatsSchemaTests.cs b/tests/RvtIO.Tests/FormatsSchemaTests.cs index a39b1051..e1b81340 100644 --- a/tests/RvtIO.Tests/FormatsSchemaTests.cs +++ b/tests/RvtIO.Tests/FormatsSchemaTests.cs @@ -40,7 +40,7 @@ public SchemaBuilder Class(string name, int baseId, uint version, params (string /// The classes the ElemTable layout needs, in the order and shape of Revit 2022 (28-byte rows, 4-byte ids) or /// Revit 2024 and later (40-byte rows, 8-byte ids, m_history declared first). ElemTable is id 19. - private static byte[] ElemTableSchema(bool wideIds) + internal static byte[] ElemTableSchema(bool wideIds, int? markerId = null) { byte idType = wideIds ? (byte)0x0B : (byte)0x04; string idName = wideIds ? "m_id64" : "m_id"; @@ -52,11 +52,17 @@ private static byte[] ElemTableSchema(bool wideIds) .Class("ElementHistory", 0, 8, ("m_originalElementId", 0x0E, 0, 14), ("m_creationDate", 0x0E, 0, 15), ("m_lastModificationDate", 0x0E, 0, 15), ("m_lastUserModificationDate", 0x0E, 0, 15)) // 16 .Class("PartitionId", 0, 1, ("m_id", 0x04, 0, 0)); // 17 - return (wideIds + (wideIds ? b.Class("ElemRec", 0, 10, ("m_history", 0x0E, 0, 16), ("m_id", 0x0E, 0, 13), ("m_OwningElementId", 0x0E, 0, 13), ("m_partitionId", 0x0E, 0, 17)) : b.Class("ElemRec", 0, 10, ("m_id", 0x0E, 0, 13), ("m_history", 0x0E, 0, 16), ("m_partitionId", 0x0E, 0, 17), ("m_OwningElementId", 0x0E, 0, 13))) // 18 - .Class("ElemTable", 0, 9, ("m_elemArr", 0x0E, 0x50, 18), ("m_bExpandAllOnLoad", 0x01, 0, 0)) // 19 - .Build(); + .Class("ElemTable", 0, 9, ("m_elemArr", 0x0E, 0x50, 18), ("m_bExpandAllOnLoad", 0x01, 0, 0)); // 19 + if (markerId is int marker) + { + for (int id = 20; id < marker; id++) b.Class("Filler" + id, 0, 1); + foreach (string name in new[] { "ContentMarker", "ContentKey", "SegmentMarker", "SegmentCheckback", "SignatureMarker" }) + b.Class(name, 0, 1); + } + return b.Build(); } [Theory] @@ -140,7 +146,7 @@ public void InMemoryPointerMembersOfElemRecTakeNoBytes() Assert.Equal(40, ElemTableLayout.RowLength(schema)); } - private static byte[] Rows(bool wideIds, params (long Id, long Original, int Created, int Partition, long Owner)[] rows) + internal static byte[] Rows(bool wideIds, params (long Id, long Original, int Created, int Partition, long Owner)[] rows) { var s = new MemoryStream(); var w = new BinaryWriter(s); w.Write((ushort)19); w.Write((uint)rows.Length); diff --git a/tests/RvtIO.Tests/SchemaReleaseExportTests.cs b/tests/RvtIO.Tests/SchemaReleaseExportTests.cs new file mode 100644 index 00000000..af6eeecb --- /dev/null +++ b/tests/RvtIO.Tests/SchemaReleaseExportTests.cs @@ -0,0 +1,77 @@ +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; +using RvtIO.Compact; +using RvtIO.Ifc; +using Xunit; + +namespace RvtIO.Tests; + +/// Full reads and exports of public Autodesk samples with only the BasicFileInfo Format year changed. +/// Download URLs and hashes are recorded in Fixtures/README.md; the repository contains no Revit files. +public class SchemaReleaseExportTests +{ + [SkippableTheory] + [InlineData("RVTIO_SCHEMA_SAMPLE_2022", 2022, 2025, false)] + [InlineData("RVTIO_SCHEMA_SAMPLE_2024", 2024, 2022, false)] + [InlineData("RVTIO_SCHEMA_SAMPLE_2024", 2024, 2022, true)] + public void ChangingTheSavingYearPreservesTheFullReadAndExports(string variable, int schemaYear, int savingYear, bool r22Layout) + { + string original = TestFiles.FileFromEnvironment(variable); + string copy = SchemaReleaseTests.CopyWithSavedRelease(original, savingYear); + try + { + var options = new RvtReaderOptions { R22Layout = r22Layout, SystemFamilyNameLocale = "ENU" }; + var originalReader = new RvtFileReader(options); + var expected = originalReader.Read(original); + var copyReader = new RvtFileReader(options); + var actual = copyReader.Read(copy); + Assert.Equal(schemaYear.ToString(), expected.Metadata.RevitVersion); + Assert.Equal(savingYear.ToString(), actual.Metadata.RevitVersion); + Assert.True(expected.ElemTableIsExact); + Assert.True(actual.ElemTableIsExact); + Assert.NotEmpty(expected.Elements); + Assert.Equal(expected.Elements.Select(e => e.Id), actual.Elements.Select(e => e.Id)); + Assert.Equal(expected.GeometryUndecoded, actual.GeometryUndecoded); + Assert.Equal(expected.WideLayoutNotRewritten, actual.WideLayoutNotRewritten); + + // The changed saving year is observable metadata (also used in the IFC application header). + // Normalize that one field; all other metadata, element data and export content must match. + actual.Metadata.RevitVersion = expected.Metadata.RevitVersion; + var json = new JsonSerializerOptions { NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals }; + Assert.Equal(Hash(s => JsonSerializer.Serialize(s, expected, json)), Hash(s => JsonSerializer.Serialize(s, actual, json))); + + var timestamp = DateTimeOffset.FromUnixTimeSeconds(0); + var originalIfc = new IfcExporter { Timestamp = timestamp }; + var copyIfc = new IfcExporter { Timestamp = timestamp }; + Assert.Equal(Hash(s => + { + using var writer = new StreamWriter(s, leaveOpen: true); + originalIfc.Export(expected, writer); + }), Hash(s => + { + using var writer = new StreamWriter(s, leaveOpen: true); + copyIfc.Export(actual, writer); + })); + Assert.True(originalIfc.ExportedProducts > 0); + Assert.Equal(originalIfc.ExportedProducts, copyIfc.ExportedProducts); + + Parsing.ValidatedElementNames.Apply(expected, originalReader.LastParsingState!); + Parsing.ValidatedElementNames.Apply(actual, copyReader.LastParsingState!); + var expectedIdentity = new CompactIdentity(CompactUniverse.Select(expected)); + var actualIdentity = new CompactIdentity(CompactUniverse.Select(actual)); + var expectedSource = CompactPropertySources.Create(expected, originalReader.LastParsingState, true, identity: expectedIdentity); + var actualSource = CompactPropertySources.Create(actual, copyReader.LastParsingState, true, identity: actualIdentity); + Assert.Equal(Hash(s => CompactJsonExporter.Export(expected, s, expectedSource, identity: expectedIdentity)), + Hash(s => CompactJsonExporter.Export(actual, s, actualSource, identity: actualIdentity))); + } + finally { File.Delete(copy); } + } + + private static string Hash(Action write) + { + using var hash = SHA256.Create(); + using (var stream = new CryptoStream(Stream.Null, hash, CryptoStreamMode.Write)) write(stream); + return Convert.ToHexString(hash.Hash!); + } +} diff --git a/tests/RvtIO.Tests/SchemaReleaseTests.cs b/tests/RvtIO.Tests/SchemaReleaseTests.cs index d25c8199..f3809aec 100644 --- a/tests/RvtIO.Tests/SchemaReleaseTests.cs +++ b/tests/RvtIO.Tests/SchemaReleaseTests.cs @@ -1,5 +1,6 @@ using OpenMcdf; using RvtIO.Parsing; +using RvtIO.Writing; using Xunit; namespace RvtIO.Tests; @@ -19,12 +20,12 @@ public void EvidenceMustAgreeOrTheReleaseIsUnknown(int? marker, int classCount, [Theory] [InlineData(2025, 2022, 2022)] // saved by a newer release, schema of an older one - [InlineData(2022, 2025, 2022)] + [InlineData(2022, 2025, 2025)] // an older saving year cannot change a known schema's layout [InlineData(2024, 2024, 2024)] [InlineData(2024, null, 2024)] [InlineData(null, 2022, 2022)] [InlineData(null, null, null)] - public void TheOlderReleaseDecidesTheLayout(int? saved, int? schema, int? expected) + public void TheSchemaReleaseDecidesTheLayoutWithTheSavingReleaseAsFallback(int? saved, int? schema, int? expected) => Assert.Equal(expected, SchemaRelease.Effective(saved, schema)); [Theory] @@ -39,6 +40,7 @@ public void TheSavedReleaseIsAFourDigitYear(string? text, int? expected) public void TheDiagnosticsLineNamesBothReleases() { Assert.Equal("Release: saved by 2025, schema of 2022, the layout of release 2022 is used", SchemaRelease.Describe(2025, 2022)); + Assert.Equal("Release: saved by 2022, schema of 2025, the layout of release 2025 is used", SchemaRelease.Describe(2022, 2025)); Assert.Equal("Release: saved by 2024, schema of 2024", SchemaRelease.Describe(2024, 2024)); Assert.Equal("Release: saved by unknown, schema of unknown", SchemaRelease.Describe(null, null)); } @@ -60,27 +62,29 @@ public void SampleFilesAgreeAndDecodeWithTheirOwnRowSize(string name, int releas Assert.NotNull(ElemTableLayout.TryParse(PagedGzipStream.Read(root, "Global", "ElemTable")!, schema)); } - /// A copy of the sample whose BasicFileInfo names another saving release (the same four digits in every UTF-16 place), so the - /// saved release and the schema release of the file differ. - private static string CopyWithSavedRelease(string sample, int savedRelease) + /// Change only the labelled saving year. The class schema, build, paths and all model streams stay untouched. + internal static string CopyWithSavedRelease(string sample, int savedRelease) { string copy = Path.Combine(Path.GetTempPath(), $"schema-release-{Guid.NewGuid():N}.rvt"); File.Copy(sample, copy); using (var root = RootStorage.Open(copy, FileMode.Open)) { var info = BasicFileInfoReader.Read(root); - string from = info["RevitVersion"], to = savedRelease.ToString(); using var stream = root.OpenStream("BasicFileInfo"); var data = new byte[stream.Length]; stream.ReadExactly(data); - var oldBytes = System.Text.Encoding.Unicode.GetBytes(from); - var newBytes = System.Text.Encoding.Unicode.GetBytes(to); - for (int at = data.AsSpan().IndexOf(oldBytes); at >= 0; ) + bool replaced = false; + foreach (string separator in new[] { ": ", ":\t" }) { - newBytes.CopyTo(data, at); - int next = data.AsSpan(at + oldBytes.Length).IndexOf(oldBytes); - at = next < 0 ? -1 : at + oldBytes.Length + next; + var label = System.Text.Encoding.Unicode.GetBytes("Format" + separator); + int at = data.AsSpan().IndexOf(label); + if (at < 0) continue; + Assert.Equal(info["RevitVersion"], System.Text.Encoding.Unicode.GetString(data, at + label.Length, 8)); + System.Text.Encoding.Unicode.GetBytes(savedRelease.ToString()).CopyTo(data, at + label.Length); + replaced = true; + break; } + Assert.True(replaced, "The sample must have a labelled Format year in BasicFileInfo."); stream.Position = 0; stream.Write(data, 0, data.Length); stream.Flush(); @@ -88,14 +92,84 @@ private static string CopyWithSavedRelease(string sample, int savedRelease) return copy; } - /// Every layout choice reads the release in force, the older of the saved and the schema release: a Revit 2025 file that - /// says it was saved by 2022 is read by the 2022 layout (4-byte ids), which its 8-byte rows do not have, so its element table is - /// not decoded; a Revit 2022 file that says it was saved by 2025 keeps the 2022 layout it has. + [Theory] + [InlineData(false, 817, 2025, 2022)] + [InlineData(true, 891, 2022, 2024)] + [InlineData(true, 913, 2020, 2025)] + [InlineData(false, 817, 2020, 2022)] + [InlineData(false, 817, 2022, 2022)] + [InlineData(true, 891, 2024, 2024)] + [InlineData(true, 500, 2024, 2024)] // unknown schema release: use the saving release + [InlineData(true, 891, null, 2024)] // no saving year: use the schema + public void SyntheticFilesKeepTheDeclaredTableAndRoutingWidths(bool wide, int marker, int? saved, int expected) + { + string path = TestFiles.Scratch($"schema-layout-{Guid.NewGuid():N}.rvt"); + try + { + WriteSyntheticFile(path, wide, marker, saved); + using (var root = RootStorage.OpenRead(path)) + { + var formats = FormatsSchema.TryRead(root)!; + var schema = FileSchema.Read(root)!; + Assert.Equal(expected, formats.Release); + Assert.Equal(expected, schema.Release); + Assert.Equal(wide ? 40 : 28, ElemTableLayout.RowLength(formats)); + Assert.Equal(wide ? 8 : 4, ElemTableLayout.IdWidth(formats)); + Assert.Equal(wide ? 8 : 4, WideElementRecords.IdWidth(schema)); + Assert.Equal(wide ? 8 : 4, RecordRouting.Begin(formats, out _)!.IdWidth); + var rows = ElemTableLayout.TryParse(PagedGzipStream.Read(root, "Global", "ElemTable")!, formats)!; + Assert.Equal(new ElemTableLayout.Row(700, 500, 64, 7, 1, 65), rows[1]); + var elements = ElemTableReader.Read(root, out var worksets, out var dates, out bool exact); + Assert.True(exact); + Assert.Equal(new uint[] { 0, 700 }, elements.Keys.Order().ToArray()); + Assert.Equal(7u, worksets[700]); + Assert.Equal(64, dates[700]); + } + } + finally { File.Delete(path); } + } + + [Theory] + [InlineData(false, 891, 2022)] // known wide schema release, but narrow declared fields + [InlineData(true, 817, 2024)] // known narrow schema release, but wide declared fields + [InlineData(true, 500, 2022)] // unknown schema release: declared fields contradict the saving-year fallback + public void ContradictoryDeclaredWidthsRemainUndecoded(bool wide, int marker, int saved) + { + string path = TestFiles.Scratch($"schema-layout-{Guid.NewGuid():N}.rvt"); + try + { + WriteSyntheticFile(path, wide, marker, saved); + using var root = RootStorage.OpenRead(path); + var formats = FormatsSchema.TryRead(root)!; + Assert.True(ElemTableLayout.DisagreesWithRelease(formats)); + Assert.Null(ElemTableLayout.IdWidth(formats)); + Assert.Null(ElemTableLayout.RowLength(formats)); + Assert.Null(WideElementRecords.IdWidth(FileSchema.Read(root)!)); + Assert.Null(RecordRouting.Begin(formats, out _)); + Assert.Empty(ElemTableReader.Read(root, out _, out _, out bool exact)); + Assert.False(exact); + } + finally { File.Delete(path); } + } + + private static void WriteSyntheticFile(string path, bool wide, int marker, int? saved) + { + using var root = RootStorage.Create(path, OpenMcdf.Version.V3); + using var formats = root.CreateStorage("Formats").CreateStream("Latest"); + formats.Write(PageCodec.EncodePadded(GzipMember.Write(FormatsSchemaTests.ElemTableSchema(wide, marker)))); + using var table = root.CreateStorage("Global").CreateStream("ElemTable"); + table.Write(PageCodec.EncodePadded(GzipMember.Write(FormatsSchemaTests.Rows(wide, + (0, 0, 0, 0, -1), (700, 500, 64, 7, 1))))); + using var info = root.CreateStream("BasicFileInfo"); + info.Write(RvtWriterTests.BuildBasicFileInfoBytes(saved)); + } + + /// Every layout choice keeps the schema's layout when the saving release is changed in either direction. [SkippableTheory] [InlineData("Test_2025.rvt", 2025, 2025, true)] // saved == schema: decoded as before - [InlineData("Test_2025.rvt", 2022, 2022, false)] // saved by an older release than the schema: the 2022 layout is in force + [InlineData("Test_2025.rvt", 2022, 2025, true)] // saved by an older release: retain the schema's 8-byte ids [InlineData("TESTFILE.rvt", 2025, 2022, true)] // saved by a newer release: the schema's older layout is in force - [InlineData("TESTFILE.rvt", 2020, 2020, true)] // same 4-byte family + [InlineData("TESTFILE.rvt", 2020, 2022, true)] // same 4-byte family public void TheReleaseInForceDecidesTheElementTableLayout(string name, int saved, int inForce, bool decodes) { string copy = CopyWithSavedRelease(TestFiles.Find(name), saved);