From 0acccb826503b95629a8f9c18cbdf375704bce9e Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sat, 18 Oct 2025 22:56:18 +0800 Subject: [PATCH 001/506] wip --- EFCore.Jet.sln | 15 ++++++++++++++ src/LibRed/Formats/Jet12Format.cs | 10 +++++++++ src/LibRed/Formats/Jet14Format.cs | 10 +++++++++ src/LibRed/Formats/Jet16Format.cs | 10 +++++++++ src/LibRed/Formats/Jet17Format.cs | 10 +++++++++ src/LibRed/Formats/Jet3Format.cs | 10 +++++++++ src/LibRed/Formats/Jet4Format.cs | 14 +++++++++++++ src/LibRed/Formats/JetFormatBase.cs | 13 ++++++++++++ src/LibRed/Formats/JetVersion.cs | 13 ++++++++++++ src/LibRed/LibRed.csproj | 15 ++++++++++++++ src/LibRed/LibRedFile.cs | 16 ++++++++++++++ src/LibRed/Pages/DatabaseDefinitionPage.cs | 23 +++++++++++++++++++++ src/LibRed/Pages/Page.cs | 19 +++++++++++++++++ src/LibRed/Pages/PageType.cs | 13 ++++++++++++ test/EFCore.Jet.FunctionalTests/config.json | 2 +- 15 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 src/LibRed/Formats/Jet12Format.cs create mode 100644 src/LibRed/Formats/Jet14Format.cs create mode 100644 src/LibRed/Formats/Jet16Format.cs create mode 100644 src/LibRed/Formats/Jet17Format.cs create mode 100644 src/LibRed/Formats/Jet3Format.cs create mode 100644 src/LibRed/Formats/Jet4Format.cs create mode 100644 src/LibRed/Formats/JetFormatBase.cs create mode 100644 src/LibRed/Formats/JetVersion.cs create mode 100644 src/LibRed/LibRed.csproj create mode 100644 src/LibRed/LibRedFile.cs create mode 100644 src/LibRed/Pages/DatabaseDefinitionPage.cs create mode 100644 src/LibRed/Pages/Page.cs create mode 100644 src/LibRed/Pages/PageType.cs diff --git a/EFCore.Jet.sln b/EFCore.Jet.sln index 6a9117a6..a1aab3c9 100644 --- a/EFCore.Jet.sln +++ b/EFCore.Jet.sln @@ -56,6 +56,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EFCore.Jet.Odbc", "src\EFCo EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EFCore.Jet.OleDb", "src\EFCore.Jet.OleDb\EFCore.Jet.OleDb.csproj", "{FFC89A2D-F68F-47E3-BA00-47E9C0BEDB71}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibRed", "src\LibRed\LibRed.csproj", "{00D43B88-09EA-9310-748A-814C451E2589}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -174,6 +176,18 @@ Global {FFC89A2D-F68F-47E3-BA00-47E9C0BEDB71}.Release|x64.Build.0 = Release|Any CPU {FFC89A2D-F68F-47E3-BA00-47E9C0BEDB71}.Release|x86.ActiveCfg = Release|Any CPU {FFC89A2D-F68F-47E3-BA00-47E9C0BEDB71}.Release|x86.Build.0 = Release|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Debug|Any CPU.Build.0 = Debug|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Debug|x64.ActiveCfg = Debug|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Debug|x64.Build.0 = Debug|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Debug|x86.ActiveCfg = Debug|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Debug|x86.Build.0 = Debug|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Release|Any CPU.ActiveCfg = Release|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Release|Any CPU.Build.0 = Release|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Release|x64.ActiveCfg = Release|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Release|x64.Build.0 = Release|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Release|x86.ActiveCfg = Release|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -188,6 +202,7 @@ Global {5CD8B47D-E32C-480A-8331-55549EC8E12E} = {6A8DE399-1804-4113-A408-F23B7F5C9CAC} {1E0729DA-B861-46EA-B1F1-3AE20EA1E00A} = {F68095EE-6CD1-43A2-B498-6CA72CE2A0CB} {FFC89A2D-F68F-47E3-BA00-47E9C0BEDB71} = {F68095EE-6CD1-43A2-B498-6CA72CE2A0CB} + {00D43B88-09EA-9310-748A-814C451E2589} = {F68095EE-6CD1-43A2-B498-6CA72CE2A0CB} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9359773D-6399-447E-9814-6CB41C2FB664} diff --git a/src/LibRed/Formats/Jet12Format.cs b/src/LibRed/Formats/Jet12Format.cs new file mode 100644 index 00000000..b42814ab --- /dev/null +++ b/src/LibRed/Formats/Jet12Format.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Formats +{ + internal class Jet12Format : Jet4Format + { + } +} diff --git a/src/LibRed/Formats/Jet14Format.cs b/src/LibRed/Formats/Jet14Format.cs new file mode 100644 index 00000000..68327a7c --- /dev/null +++ b/src/LibRed/Formats/Jet14Format.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Formats +{ + internal class Jet14Format : Jet12Format + { + } +} diff --git a/src/LibRed/Formats/Jet16Format.cs b/src/LibRed/Formats/Jet16Format.cs new file mode 100644 index 00000000..1974374e --- /dev/null +++ b/src/LibRed/Formats/Jet16Format.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Formats +{ + internal class Jet16Format : Jet14Format + { + } +} diff --git a/src/LibRed/Formats/Jet17Format.cs b/src/LibRed/Formats/Jet17Format.cs new file mode 100644 index 00000000..ceedb9dd --- /dev/null +++ b/src/LibRed/Formats/Jet17Format.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Formats +{ + internal class Jet17Format : Jet16Format + { + } +} diff --git a/src/LibRed/Formats/Jet3Format.cs b/src/LibRed/Formats/Jet3Format.cs new file mode 100644 index 00000000..cc751446 --- /dev/null +++ b/src/LibRed/Formats/Jet3Format.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Formats +{ + internal class Jet3Format : JetFormatBase + { + } +} diff --git a/src/LibRed/Formats/Jet4Format.cs b/src/LibRed/Formats/Jet4Format.cs new file mode 100644 index 00000000..bc9b3e6c --- /dev/null +++ b/src/LibRed/Formats/Jet4Format.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Formats +{ + internal class Jet4Format : JetFormatBase + { + public Jet4Format() + { + PageSize = 4096; // Default page size for Jet 4.0 + } + } +} diff --git a/src/LibRed/Formats/JetFormatBase.cs b/src/LibRed/Formats/JetFormatBase.cs new file mode 100644 index 00000000..050285aa --- /dev/null +++ b/src/LibRed/Formats/JetFormatBase.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Formats +{ + internal abstract class JetFormatBase + { + // Database definition page constants + public int PageSize { get; protected set; } = 2048; // Default page size for Jet formats + public int VersionOffset { get; protected set; } = 0x14; + } +} diff --git a/src/LibRed/Formats/JetVersion.cs b/src/LibRed/Formats/JetVersion.cs new file mode 100644 index 00000000..386b8fb4 --- /dev/null +++ b/src/LibRed/Formats/JetVersion.cs @@ -0,0 +1,13 @@ +namespace LibRed.Formats +{ + public enum JetVersion + { + Version3 = 0x0, + Version4 = 0x1, + Version12_2007 = 0x2, + Version14_2010 = 0x3, + Version15_2013 = 0x4, + Version16_2016 = 0x5, + Version17_2019 = 0x6, + } +} \ No newline at end of file diff --git a/src/LibRed/LibRed.csproj b/src/LibRed/LibRed.csproj new file mode 100644 index 00000000..5ff3400a --- /dev/null +++ b/src/LibRed/LibRed.csproj @@ -0,0 +1,15 @@ + + + + net10.0 + enable + enable + + + + + + + + + diff --git a/src/LibRed/LibRedFile.cs b/src/LibRed/LibRedFile.cs new file mode 100644 index 00000000..eecfe1f0 --- /dev/null +++ b/src/LibRed/LibRedFile.cs @@ -0,0 +1,16 @@ +namespace LibRed; + +public class LibRedFile(string filePath) +{ + private readonly string _filePath = filePath; + private bool _isOpen = false; + public void Open() + { + + } + + public void Close() + { + // Close the file or release resources + } +} \ No newline at end of file diff --git a/src/LibRed/Pages/DatabaseDefinitionPage.cs b/src/LibRed/Pages/DatabaseDefinitionPage.cs new file mode 100644 index 00000000..e479d634 --- /dev/null +++ b/src/LibRed/Pages/DatabaseDefinitionPage.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Pages +{ + public class DatabaseDefinitionPage : Page + { + public byte JetVersion { get; internal set; } + public string DatabasePassword { get; set; } + + public int DatabaseKey { get; set; } + public short CodePage { get; set; } + public short TextCollateSortOrder { get; set; } + public string PageKey { get; set; } + public DateTime DatabaseCreationDate { get; set; } + public string CreateProgramName { get; set; } + public DatabaseDefinitionPage() + { + PageType = 0x00; + } + } +} diff --git a/src/LibRed/Pages/Page.cs b/src/LibRed/Pages/Page.cs new file mode 100644 index 00000000..54eefd33 --- /dev/null +++ b/src/LibRed/Pages/Page.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Pages +{ + public abstract class Page + { + public byte PageType { get; internal set; } + public int PageSize { get; set; } + public virtual void ReadPage() + { + } + + public virtual void WritePage() + { + } + } +} diff --git a/src/LibRed/Pages/PageType.cs b/src/LibRed/Pages/PageType.cs new file mode 100644 index 00000000..a63fdf42 --- /dev/null +++ b/src/LibRed/Pages/PageType.cs @@ -0,0 +1,13 @@ +namespace LibRed.Pages +{ + public enum PageType + { + DatabaseDefinition = 0x00, + DatabasePage = 0x01, + TableDefinition = 0x02, + IntermediateIndexPage = 0x03, + LeafIndexPage = 0x04, + PageUsageBitmap = 0x05, + + } +} \ No newline at end of file diff --git a/test/EFCore.Jet.FunctionalTests/config.json b/test/EFCore.Jet.FunctionalTests/config.json index d97040ea..666f9c20 100644 --- a/test/EFCore.Jet.FunctionalTests/config.json +++ b/test/EFCore.Jet.FunctionalTests/config.json @@ -1,7 +1,7 @@ { "Test": { "Jet": { - "DefaultConnection": "Provider=Microsoft.ACE.OLEDB.12.0;Data Source=Jet.accdb;Persist Security Info=False;" + "DefaultConnection": "DBQ=Jet.accdb" } } } From ed915c124116a9b1e7b538e9c94c7ef21cc8c3b4 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sat, 18 Oct 2025 22:56:18 +0800 Subject: [PATCH 002/506] wip --- EFCore.Jet.sln | 15 ++++++++++++++ src/LibRed/Formats/Jet12Format.cs | 10 +++++++++ src/LibRed/Formats/Jet14Format.cs | 10 +++++++++ src/LibRed/Formats/Jet16Format.cs | 10 +++++++++ src/LibRed/Formats/Jet17Format.cs | 10 +++++++++ src/LibRed/Formats/Jet3Format.cs | 10 +++++++++ src/LibRed/Formats/Jet4Format.cs | 14 +++++++++++++ src/LibRed/Formats/JetFormatBase.cs | 13 ++++++++++++ src/LibRed/Formats/JetVersion.cs | 13 ++++++++++++ src/LibRed/LibRed.csproj | 15 ++++++++++++++ src/LibRed/LibRedFile.cs | 16 ++++++++++++++ src/LibRed/Pages/DatabaseDefinitionPage.cs | 23 +++++++++++++++++++++ src/LibRed/Pages/Page.cs | 19 +++++++++++++++++ src/LibRed/Pages/PageType.cs | 13 ++++++++++++ test/EFCore.Jet.FunctionalTests/config.json | 2 +- 15 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 src/LibRed/Formats/Jet12Format.cs create mode 100644 src/LibRed/Formats/Jet14Format.cs create mode 100644 src/LibRed/Formats/Jet16Format.cs create mode 100644 src/LibRed/Formats/Jet17Format.cs create mode 100644 src/LibRed/Formats/Jet3Format.cs create mode 100644 src/LibRed/Formats/Jet4Format.cs create mode 100644 src/LibRed/Formats/JetFormatBase.cs create mode 100644 src/LibRed/Formats/JetVersion.cs create mode 100644 src/LibRed/LibRed.csproj create mode 100644 src/LibRed/LibRedFile.cs create mode 100644 src/LibRed/Pages/DatabaseDefinitionPage.cs create mode 100644 src/LibRed/Pages/Page.cs create mode 100644 src/LibRed/Pages/PageType.cs diff --git a/EFCore.Jet.sln b/EFCore.Jet.sln index 6a9117a6..a1aab3c9 100644 --- a/EFCore.Jet.sln +++ b/EFCore.Jet.sln @@ -56,6 +56,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EFCore.Jet.Odbc", "src\EFCo EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EFCore.Jet.OleDb", "src\EFCore.Jet.OleDb\EFCore.Jet.OleDb.csproj", "{FFC89A2D-F68F-47E3-BA00-47E9C0BEDB71}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibRed", "src\LibRed\LibRed.csproj", "{00D43B88-09EA-9310-748A-814C451E2589}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -174,6 +176,18 @@ Global {FFC89A2D-F68F-47E3-BA00-47E9C0BEDB71}.Release|x64.Build.0 = Release|Any CPU {FFC89A2D-F68F-47E3-BA00-47E9C0BEDB71}.Release|x86.ActiveCfg = Release|Any CPU {FFC89A2D-F68F-47E3-BA00-47E9C0BEDB71}.Release|x86.Build.0 = Release|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Debug|Any CPU.Build.0 = Debug|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Debug|x64.ActiveCfg = Debug|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Debug|x64.Build.0 = Debug|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Debug|x86.ActiveCfg = Debug|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Debug|x86.Build.0 = Debug|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Release|Any CPU.ActiveCfg = Release|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Release|Any CPU.Build.0 = Release|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Release|x64.ActiveCfg = Release|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Release|x64.Build.0 = Release|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Release|x86.ActiveCfg = Release|Any CPU + {00D43B88-09EA-9310-748A-814C451E2589}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -188,6 +202,7 @@ Global {5CD8B47D-E32C-480A-8331-55549EC8E12E} = {6A8DE399-1804-4113-A408-F23B7F5C9CAC} {1E0729DA-B861-46EA-B1F1-3AE20EA1E00A} = {F68095EE-6CD1-43A2-B498-6CA72CE2A0CB} {FFC89A2D-F68F-47E3-BA00-47E9C0BEDB71} = {F68095EE-6CD1-43A2-B498-6CA72CE2A0CB} + {00D43B88-09EA-9310-748A-814C451E2589} = {F68095EE-6CD1-43A2-B498-6CA72CE2A0CB} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9359773D-6399-447E-9814-6CB41C2FB664} diff --git a/src/LibRed/Formats/Jet12Format.cs b/src/LibRed/Formats/Jet12Format.cs new file mode 100644 index 00000000..b42814ab --- /dev/null +++ b/src/LibRed/Formats/Jet12Format.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Formats +{ + internal class Jet12Format : Jet4Format + { + } +} diff --git a/src/LibRed/Formats/Jet14Format.cs b/src/LibRed/Formats/Jet14Format.cs new file mode 100644 index 00000000..68327a7c --- /dev/null +++ b/src/LibRed/Formats/Jet14Format.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Formats +{ + internal class Jet14Format : Jet12Format + { + } +} diff --git a/src/LibRed/Formats/Jet16Format.cs b/src/LibRed/Formats/Jet16Format.cs new file mode 100644 index 00000000..1974374e --- /dev/null +++ b/src/LibRed/Formats/Jet16Format.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Formats +{ + internal class Jet16Format : Jet14Format + { + } +} diff --git a/src/LibRed/Formats/Jet17Format.cs b/src/LibRed/Formats/Jet17Format.cs new file mode 100644 index 00000000..ceedb9dd --- /dev/null +++ b/src/LibRed/Formats/Jet17Format.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Formats +{ + internal class Jet17Format : Jet16Format + { + } +} diff --git a/src/LibRed/Formats/Jet3Format.cs b/src/LibRed/Formats/Jet3Format.cs new file mode 100644 index 00000000..cc751446 --- /dev/null +++ b/src/LibRed/Formats/Jet3Format.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Formats +{ + internal class Jet3Format : JetFormatBase + { + } +} diff --git a/src/LibRed/Formats/Jet4Format.cs b/src/LibRed/Formats/Jet4Format.cs new file mode 100644 index 00000000..bc9b3e6c --- /dev/null +++ b/src/LibRed/Formats/Jet4Format.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Formats +{ + internal class Jet4Format : JetFormatBase + { + public Jet4Format() + { + PageSize = 4096; // Default page size for Jet 4.0 + } + } +} diff --git a/src/LibRed/Formats/JetFormatBase.cs b/src/LibRed/Formats/JetFormatBase.cs new file mode 100644 index 00000000..050285aa --- /dev/null +++ b/src/LibRed/Formats/JetFormatBase.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Formats +{ + internal abstract class JetFormatBase + { + // Database definition page constants + public int PageSize { get; protected set; } = 2048; // Default page size for Jet formats + public int VersionOffset { get; protected set; } = 0x14; + } +} diff --git a/src/LibRed/Formats/JetVersion.cs b/src/LibRed/Formats/JetVersion.cs new file mode 100644 index 00000000..386b8fb4 --- /dev/null +++ b/src/LibRed/Formats/JetVersion.cs @@ -0,0 +1,13 @@ +namespace LibRed.Formats +{ + public enum JetVersion + { + Version3 = 0x0, + Version4 = 0x1, + Version12_2007 = 0x2, + Version14_2010 = 0x3, + Version15_2013 = 0x4, + Version16_2016 = 0x5, + Version17_2019 = 0x6, + } +} \ No newline at end of file diff --git a/src/LibRed/LibRed.csproj b/src/LibRed/LibRed.csproj new file mode 100644 index 00000000..5ff3400a --- /dev/null +++ b/src/LibRed/LibRed.csproj @@ -0,0 +1,15 @@ + + + + net10.0 + enable + enable + + + + + + + + + diff --git a/src/LibRed/LibRedFile.cs b/src/LibRed/LibRedFile.cs new file mode 100644 index 00000000..eecfe1f0 --- /dev/null +++ b/src/LibRed/LibRedFile.cs @@ -0,0 +1,16 @@ +namespace LibRed; + +public class LibRedFile(string filePath) +{ + private readonly string _filePath = filePath; + private bool _isOpen = false; + public void Open() + { + + } + + public void Close() + { + // Close the file or release resources + } +} \ No newline at end of file diff --git a/src/LibRed/Pages/DatabaseDefinitionPage.cs b/src/LibRed/Pages/DatabaseDefinitionPage.cs new file mode 100644 index 00000000..e479d634 --- /dev/null +++ b/src/LibRed/Pages/DatabaseDefinitionPage.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Pages +{ + public class DatabaseDefinitionPage : Page + { + public byte JetVersion { get; internal set; } + public string DatabasePassword { get; set; } + + public int DatabaseKey { get; set; } + public short CodePage { get; set; } + public short TextCollateSortOrder { get; set; } + public string PageKey { get; set; } + public DateTime DatabaseCreationDate { get; set; } + public string CreateProgramName { get; set; } + public DatabaseDefinitionPage() + { + PageType = 0x00; + } + } +} diff --git a/src/LibRed/Pages/Page.cs b/src/LibRed/Pages/Page.cs new file mode 100644 index 00000000..54eefd33 --- /dev/null +++ b/src/LibRed/Pages/Page.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LibRed.Pages +{ + public abstract class Page + { + public byte PageType { get; internal set; } + public int PageSize { get; set; } + public virtual void ReadPage() + { + } + + public virtual void WritePage() + { + } + } +} diff --git a/src/LibRed/Pages/PageType.cs b/src/LibRed/Pages/PageType.cs new file mode 100644 index 00000000..a63fdf42 --- /dev/null +++ b/src/LibRed/Pages/PageType.cs @@ -0,0 +1,13 @@ +namespace LibRed.Pages +{ + public enum PageType + { + DatabaseDefinition = 0x00, + DatabasePage = 0x01, + TableDefinition = 0x02, + IntermediateIndexPage = 0x03, + LeafIndexPage = 0x04, + PageUsageBitmap = 0x05, + + } +} \ No newline at end of file diff --git a/test/EFCore.Jet.FunctionalTests/config.json b/test/EFCore.Jet.FunctionalTests/config.json index d97040ea..666f9c20 100644 --- a/test/EFCore.Jet.FunctionalTests/config.json +++ b/test/EFCore.Jet.FunctionalTests/config.json @@ -1,7 +1,7 @@ { "Test": { "Jet": { - "DefaultConnection": "Provider=Microsoft.ACE.OLEDB.12.0;Data Source=Jet.accdb;Persist Security Info=False;" + "DefaultConnection": "DBQ=Jet.accdb" } } } From a945cb328be43dd02c726108c822acc191246f69 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sat, 27 Jun 2026 20:59:31 +0800 Subject: [PATCH 003/506] @ Scaffold LibRed: native managed Jet/ACE engine Restructure the WIP LibRed project into a five-project, cross-platform engine that reads/writes the Access .mdb/.accdb format directly (no ODBC/OleDb/DAO/ADOX): - LibRed.Core file format: IO, Formats, Pages, Catalog, Storage, Crypto - LibRed.Sql ANTLR grammar + AST + parser + binder (no Jet dependency) - LibRed.Engine logical plan, planner, schema-provider bridge, executor - LibRed.Ado ADO.NET surface over the engine - LibRed.EFCore placeholder EF Core provider Migrate the existing Formats/Pages code into LibRed.Core. A dedicated Directory.Build.props bypasses the windows-only src props to keep these assemblies cross-platform. Method bodies are documented TODOs; build is green. Add a LibRed section to CLAUDE.md and src/LibRed/README.md (structure, build order, SQL pipeline, mdbtools/Jackcess references). Co-Authored-By: Claude Opus 4.8 @ --- CLAUDE.md | 145 ++++++++++++++++++ EFCore.Jet.sln | 90 +++++++++-- src/LibRed/Directory.Build.props | 26 ++++ src/LibRed/Formats/Jet12Format.cs | 10 -- src/LibRed/Formats/Jet14Format.cs | 10 -- src/LibRed/Formats/Jet16Format.cs | 10 -- src/LibRed/Formats/Jet17Format.cs | 10 -- src/LibRed/Formats/Jet3Format.cs | 10 -- src/LibRed/Formats/Jet4Format.cs | 14 -- src/LibRed/Formats/JetFormatBase.cs | 13 -- src/LibRed/LibRed.Ado/LibRed.Ado.csproj | 14 ++ src/LibRed/LibRed.Ado/LibRedCommand.cs | 57 +++++++ src/LibRed/LibRed.Ado/LibRedConnection.cs | 95 ++++++++++++ src/LibRed/LibRed.Ado/LibRedDataReader.cs | 105 +++++++++++++ src/LibRed/LibRed.Ado/LibRedFactory.cs | 18 +++ src/LibRed/LibRed.Ado/LibRedParameter.cs | 34 ++++ .../LibRed.Ado/LibRedParameterCollection.cs | 55 +++++++ src/LibRed/LibRed.Ado/LibRedTransaction.cs | 27 ++++ src/LibRed/LibRed.Core/Catalog/ColumnDef.cs | 28 ++++ src/LibRed/LibRed.Core/Catalog/IndexDef.cs | 16 ++ src/LibRed/LibRed.Core/Catalog/JetCatalog.cs | 31 ++++ src/LibRed/LibRed.Core/Catalog/JetDataType.cs | 23 +++ src/LibRed/LibRed.Core/Catalog/TableDef.cs | 23 +++ src/LibRed/LibRed.Core/Crypto/JetCrypto.cs | 20 +++ src/LibRed/LibRed.Core/Formats/Jet12Format.cs | 9 ++ src/LibRed/LibRed.Core/Formats/Jet14Format.cs | 7 + src/LibRed/LibRed.Core/Formats/Jet16Format.cs | 7 + src/LibRed/LibRed.Core/Formats/Jet17Format.cs | 7 + src/LibRed/LibRed.Core/Formats/Jet3Format.cs | 9 ++ src/LibRed/LibRed.Core/Formats/Jet4Format.cs | 9 ++ .../LibRed.Core/Formats/JetFormatBase.cs | 53 +++++++ .../{ => LibRed.Core}/Formats/JetVersion.cs | 0 src/LibRed/LibRed.Core/IO/PageBuffer.cs | 38 +++++ src/LibRed/LibRed.Core/IO/PageChannel.cs | 68 ++++++++ src/LibRed/LibRed.Core/JetDatabase.cs | 42 +++++ src/LibRed/LibRed.Core/LibRed.Core.csproj | 10 ++ src/LibRed/LibRed.Core/Pages/DataPage.cs | 25 +++ .../Pages/DatabaseDefinitionPage.cs | 31 ++++ src/LibRed/LibRed.Core/Pages/IndexPage.cs | 22 +++ src/LibRed/LibRed.Core/Pages/LvalPage.cs | 19 +++ src/LibRed/LibRed.Core/Pages/Page.cs | 19 +++ src/LibRed/LibRed.Core/Pages/PageType.cs | 15 ++ .../LibRed.Core/Pages/TableDefinitionPage.cs | 24 +++ src/LibRed/LibRed.Core/Pages/UsageMapPage.cs | 19 +++ src/LibRed/LibRed.Core/Storage/RowDecoder.cs | 23 +++ src/LibRed/LibRed.Core/Storage/Table.cs | 27 ++++ src/LibRed/LibRed.Core/Storage/TableCursor.cs | 31 ++++ .../LibRed.Core/Storage/Types/JetTypeCodec.cs | 30 ++++ src/LibRed/LibRed.Core/Storage/UsageMap.cs | 24 +++ src/LibRed/LibRed.EFCore/LibRed.EFCore.csproj | 18 +++ .../LibRed.EFCore/LibRedDesignTimeMarker.cs | 16 ++ .../LibRed.Engine/Execution/QueryExecutor.cs | 30 ++++ .../LibRed.Engine/Execution/ResultSet.cs | 16 ++ src/LibRed/LibRed.Engine/LibRed.Engine.csproj | 15 ++ src/LibRed/LibRed.Engine/Plan/PlanNode.cs | 14 ++ src/LibRed/LibRed.Engine/Plan/PlanNodes.cs | 51 ++++++ .../Planning/CatalogSchemaProvider.cs | 37 +++++ .../LibRed.Engine/Planning/JetClrTypeMap.cs | 25 +++ .../LibRed.Engine/Planning/QueryPlanner.cs | 41 +++++ src/LibRed/LibRed.Engine/QueryEngine.cs | 49 ++++++ src/LibRed/LibRed.Sql/Ast/Clauses.cs | 23 +++ src/LibRed/LibRed.Sql/Ast/Expressions.cs | 32 ++++ src/LibRed/LibRed.Sql/Ast/SqlNode.cs | 9 ++ src/LibRed/LibRed.Sql/Ast/Statements.cs | 33 ++++ src/LibRed/LibRed.Sql/Binding/Binder.cs | 21 +++ .../LibRed.Sql/Binding/BoundStatement.cs | 10 ++ .../LibRed.Sql/Binding/ISchemaProvider.cs | 32 ++++ src/LibRed/LibRed.Sql/Grammar/AccessSql.g4 | 60 ++++++++ src/LibRed/LibRed.Sql/LibRed.Sql.csproj | 22 +++ .../LibRed.Sql/Parsing/AccessSqlParser.cs | 19 +++ src/LibRed/LibRed.Sql/Parsing/ISqlParser.cs | 14 ++ .../LibRed.Sql/Parsing/SqlParseException.cs | 9 ++ src/LibRed/LibRed.csproj | 15 -- src/LibRed/LibRedFile.cs | 16 -- src/LibRed/Pages/DatabaseDefinitionPage.cs | 23 --- src/LibRed/Pages/Page.cs | 19 --- src/LibRed/Pages/PageType.cs | 13 -- src/LibRed/README.md | 75 +++++++++ 78 files changed, 2032 insertions(+), 177 deletions(-) create mode 100644 CLAUDE.md create mode 100644 src/LibRed/Directory.Build.props delete mode 100644 src/LibRed/Formats/Jet12Format.cs delete mode 100644 src/LibRed/Formats/Jet14Format.cs delete mode 100644 src/LibRed/Formats/Jet16Format.cs delete mode 100644 src/LibRed/Formats/Jet17Format.cs delete mode 100644 src/LibRed/Formats/Jet3Format.cs delete mode 100644 src/LibRed/Formats/Jet4Format.cs delete mode 100644 src/LibRed/Formats/JetFormatBase.cs create mode 100644 src/LibRed/LibRed.Ado/LibRed.Ado.csproj create mode 100644 src/LibRed/LibRed.Ado/LibRedCommand.cs create mode 100644 src/LibRed/LibRed.Ado/LibRedConnection.cs create mode 100644 src/LibRed/LibRed.Ado/LibRedDataReader.cs create mode 100644 src/LibRed/LibRed.Ado/LibRedFactory.cs create mode 100644 src/LibRed/LibRed.Ado/LibRedParameter.cs create mode 100644 src/LibRed/LibRed.Ado/LibRedParameterCollection.cs create mode 100644 src/LibRed/LibRed.Ado/LibRedTransaction.cs create mode 100644 src/LibRed/LibRed.Core/Catalog/ColumnDef.cs create mode 100644 src/LibRed/LibRed.Core/Catalog/IndexDef.cs create mode 100644 src/LibRed/LibRed.Core/Catalog/JetCatalog.cs create mode 100644 src/LibRed/LibRed.Core/Catalog/JetDataType.cs create mode 100644 src/LibRed/LibRed.Core/Catalog/TableDef.cs create mode 100644 src/LibRed/LibRed.Core/Crypto/JetCrypto.cs create mode 100644 src/LibRed/LibRed.Core/Formats/Jet12Format.cs create mode 100644 src/LibRed/LibRed.Core/Formats/Jet14Format.cs create mode 100644 src/LibRed/LibRed.Core/Formats/Jet16Format.cs create mode 100644 src/LibRed/LibRed.Core/Formats/Jet17Format.cs create mode 100644 src/LibRed/LibRed.Core/Formats/Jet3Format.cs create mode 100644 src/LibRed/LibRed.Core/Formats/Jet4Format.cs create mode 100644 src/LibRed/LibRed.Core/Formats/JetFormatBase.cs rename src/LibRed/{ => LibRed.Core}/Formats/JetVersion.cs (100%) create mode 100644 src/LibRed/LibRed.Core/IO/PageBuffer.cs create mode 100644 src/LibRed/LibRed.Core/IO/PageChannel.cs create mode 100644 src/LibRed/LibRed.Core/JetDatabase.cs create mode 100644 src/LibRed/LibRed.Core/LibRed.Core.csproj create mode 100644 src/LibRed/LibRed.Core/Pages/DataPage.cs create mode 100644 src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs create mode 100644 src/LibRed/LibRed.Core/Pages/IndexPage.cs create mode 100644 src/LibRed/LibRed.Core/Pages/LvalPage.cs create mode 100644 src/LibRed/LibRed.Core/Pages/Page.cs create mode 100644 src/LibRed/LibRed.Core/Pages/PageType.cs create mode 100644 src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs create mode 100644 src/LibRed/LibRed.Core/Pages/UsageMapPage.cs create mode 100644 src/LibRed/LibRed.Core/Storage/RowDecoder.cs create mode 100644 src/LibRed/LibRed.Core/Storage/Table.cs create mode 100644 src/LibRed/LibRed.Core/Storage/TableCursor.cs create mode 100644 src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs create mode 100644 src/LibRed/LibRed.Core/Storage/UsageMap.cs create mode 100644 src/LibRed/LibRed.EFCore/LibRed.EFCore.csproj create mode 100644 src/LibRed/LibRed.EFCore/LibRedDesignTimeMarker.cs create mode 100644 src/LibRed/LibRed.Engine/Execution/QueryExecutor.cs create mode 100644 src/LibRed/LibRed.Engine/Execution/ResultSet.cs create mode 100644 src/LibRed/LibRed.Engine/LibRed.Engine.csproj create mode 100644 src/LibRed/LibRed.Engine/Plan/PlanNode.cs create mode 100644 src/LibRed/LibRed.Engine/Plan/PlanNodes.cs create mode 100644 src/LibRed/LibRed.Engine/Planning/CatalogSchemaProvider.cs create mode 100644 src/LibRed/LibRed.Engine/Planning/JetClrTypeMap.cs create mode 100644 src/LibRed/LibRed.Engine/Planning/QueryPlanner.cs create mode 100644 src/LibRed/LibRed.Engine/QueryEngine.cs create mode 100644 src/LibRed/LibRed.Sql/Ast/Clauses.cs create mode 100644 src/LibRed/LibRed.Sql/Ast/Expressions.cs create mode 100644 src/LibRed/LibRed.Sql/Ast/SqlNode.cs create mode 100644 src/LibRed/LibRed.Sql/Ast/Statements.cs create mode 100644 src/LibRed/LibRed.Sql/Binding/Binder.cs create mode 100644 src/LibRed/LibRed.Sql/Binding/BoundStatement.cs create mode 100644 src/LibRed/LibRed.Sql/Binding/ISchemaProvider.cs create mode 100644 src/LibRed/LibRed.Sql/Grammar/AccessSql.g4 create mode 100644 src/LibRed/LibRed.Sql/LibRed.Sql.csproj create mode 100644 src/LibRed/LibRed.Sql/Parsing/AccessSqlParser.cs create mode 100644 src/LibRed/LibRed.Sql/Parsing/ISqlParser.cs create mode 100644 src/LibRed/LibRed.Sql/Parsing/SqlParseException.cs delete mode 100644 src/LibRed/LibRed.csproj delete mode 100644 src/LibRed/LibRedFile.cs delete mode 100644 src/LibRed/Pages/DatabaseDefinitionPage.cs delete mode 100644 src/LibRed/Pages/Page.cs delete mode 100644 src/LibRed/Pages/PageType.cs create mode 100644 src/LibRed/README.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..78463ab7 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,145 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What This Is + +EntityFrameworkCore.Jet is an EF Core provider for Microsoft Jet/ACE databases (Microsoft Access `.mdb`/`.accdb` files). It runs **Windows only** and bridges EF Core to the Access database engine via either ODBC or OLE DB. + +Current version: `10.0.x` targeting EF Core 10 and `net10.0`. + +## Build + +```powershell +dotnet build EFCore.Jet.sln +``` + +Assemblies are **strong-name signed** using `Key.snk`. `TreatWarningsAsErrors=True` is set globally — fix all warnings. + +### Local EFCore Repository (optional) + +To develop against a local EF Core build instead of NuGet packages, copy `Development.props.sample` to `Development.props` and set `LocalEFCoreRepository` to your EF Core checkout. That local build must be compiled with `AssemblyVersion=10.0.0.0` to avoid binding conflicts. + +## Tests + +Tests require a real Microsoft Access driver installed (ODBC or OLE DB) and an actual `.accdb` file — no mocks. The connection string is configured via: +- `test/EFCore.Jet.FunctionalTests/config.json` (OLE DB example present) +- `test/EFCore.Jet.Tests/config.json` (bare filename; picks up default provider) +- Or env var `EFCoreJet_DefaultConnection` + +**Run all tests** (requires x86 or x64 matching your driver bitness): + +```powershell +dotnet test EFCore.Jet.sln --configuration Debug +``` + +**Run a single test class:** + +```powershell +dotnet test test\EFCore.Jet.FunctionalTests\EFCore.Jet.FunctionalTests.csproj --filter "FullyQualifiedName~NorthwindQueryJetTest" +``` + +**Run a single test method:** + +```powershell +dotnet test test\EFCore.Jet.FunctionalTests\EFCore.Jet.FunctionalTests.csproj --filter "FullyQualifiedName=EntityFrameworkCore.Jet.FunctionalTests.Query.NorthwindQueryJetTest.Where_simple" +``` + +Tests run in **fixed order by default** (`FIXED_TEST_ORDER` compile constant). All tests lock culture to `en-US` via a module initializer. + +Tests that require features Jet doesn't support are marked `[Fact(Skip = "Unsupported by JET: ...")]` — see `SkipMessages.txt` for the catalog of known unsupported patterns. + +## Project Structure + +``` +src/ + EFCore.Jet.Data/ ADO.NET driver — JetConnection, JetCommand, JetDataReader, + schema management, DUAL table simulation, connection pooling + EFCore.Jet/ EF Core provider — query pipeline, migrations, scaffolding, + type mappings, value generation, conventions + EFCore.Jet.Odbc/ Provider factory for ODBC data access + EFCore.Jet.OleDb/ Provider factory for OLE DB data access + Shared/ Shared source files compiled into multiple src projects + LibRed/ Native, fully-managed Jet/ACE engine (cross-platform) — + see "LibRed" section below and src/LibRed/README.md + +test/ + EFCore.Jet.Data.Tests/ Unit tests for the ADO.NET driver layer + EFCore.Jet.FunctionalTests/ EF Core specification tests (adapted from EF Core's own suite) + EFCore.Jet.Tests/ Additional functional tests + EFCore.Jet.IntegrationTests/ Integration scenario tests + JetProviderExceptionTests/ Exception-path tests + Shared/ Shared test infrastructure (xunit framework customizations, + test orderers, conditional test attributes) +``` + +## Architecture: Two-Layer Design + +**Layer 1 — `EFCore.Jet.Data`** wraps the raw ODBC/OLE DB driver: +- `JetConnection` detects whether the connection string is ODBC or OLE DB and delegates to the appropriate inner `DbConnection`. +- `JetCommand` rewrites SQL at runtime: handles `SELECT SKIP`, emulates `@@ROWCOUNT`, rewrites `TOP @param`, parses `IF NOT EXISTS ... THEN ...` syntax, and intercepts stored-procedure creation. +- `JetConfiguration` holds global settings: `TimeSpanOffset` (Jet has no TimeSpan; dates are offset from 1899-12-30), `CustomDualTableName`, `IntegerNullValue`, `UseConnectionPooling`. +- Schema operations (create/drop database, list tables) have three implementations: ADOX, DAO, and Precise, selected based on available COM libraries. + +**Layer 2 — `EFCore.Jet`** is the EF Core provider: +- `JetServiceCollectionExtensions.AddEntityFrameworkJet()` registers all provider services. +- `JetQuerySqlGenerator` extends `QuerySqlGenerator` to produce Jet-compatible SQL — converts `CAST` to Jet VBA functions (`CBOOL`, `CINT`, `CLNG`, etc.), handles boolean/numeric null semantics. +- `JetQueryTranslationPostprocessor` applies Jet-specific query rewrites in order: skip/take transformation → base postprocessing → optional millisecond support → ORDER BY lifting. `JetSkipTakePostprocessor` emulates `SKIP`/`OFFSET` since Jet only supports `SELECT TOP n`. +- `JetMigrationsSqlGenerator` generates DDL for Access (no `ALTER COLUMN`, limited constraint support). +- `JetHistoryRepository` implements migration locking via a `__EFMigrationsLock` table with `LockReleaseBehavior.Explicit`. +- `JetRelationalConnection` creates an "empty" (masterless) connection for database creation/drop operations. + +## Key Jet SQL Constraints + +These shape much of the query pipeline complexity: +- No `OFFSET` — emulated via subquery or `TOP`+skip in the data layer +- `SELECT TOP n` only supports a literal integer, not a parameter (rewritten at command level) +- Subqueries in `SELECT` list are limited; scalar subqueries only work in `FROM` +- No parallel transactions (OLE DB) +- No millisecond precision in `DateTime` +- `CROSS JOIN` and mixed `JOIN`/comma syntax must be ordered correctly +- Booleans stored as `-1`/`0` (numeric), not `TRUE`/`FALSE` +- `GUID` support is indirect +- No `rowversion`, no `DateTimeOffset`, no nullable `BIT` + +## LibRed — Native Managed Engine (`libred` branch) + +A from-scratch, **fully managed and cross-platform** reimplementation of the Jet/ACE +engine under `src/LibRed/`. It reads and writes the `.mdb`/`.accdb` file format +**directly** — no ODBC, OLE DB, DAO, or ADOX — so it removes the Windows-only and +driver-bitness constraints that the rest of the repo lives with. Eventually it +subsumes the COM-based database creators (DAO/ADOX) and the OLE DB/ODBC quirk handling +in `EFCore.Jet.Data`. + +**Five projects, clean dependency DAG** (`EFCore → Ado → Engine → Sql`, and `Engine → Core`): + +``` +src/LibRed/ + LibRed.Core/ File format: IO (PageChannel/PageBuffer), Formats (version offsets), + Pages, Catalog (MSysObjects → TableDef/ColumnDef/IndexDef), Storage + (Table/TableCursor/RowDecoder/UsageMap), Crypto; JetDatabase entry point + LibRed.Sql/ SQL front end: ANTLR grammar (AccessSql.g4), AST, parser, binder. + NO Jet dependency — binds via the ISchemaProvider abstraction + LibRed.Engine/ Logical Plan nodes, QueryPlanner, CatalogSchemaProvider (bridges the + catalog to the binder), QueryExecutor, QueryEngine facade + LibRed.Ado/ ADO.NET surface: DbConnection/Command/DataReader/Parameter/Transaction/Factory + LibRed.EFCore/ Placeholder for the future EF Core provider over LibRed.Ado +``` + +**Build configuration:** `src/LibRed/Directory.Build.props` deliberately bypasses +`src/Directory.Build.props` (it imports the repo-root props directly) so these assemblies +are **not** stamped `[SupportedOSPlatform("windows")]`. Strong-naming is preserved. + +**SQL pipeline** (always run end-to-end, even for trivial queries, so new features add +node types rather than rewrites): +`text → ISqlParser → AST → Binder(ISchemaProvider) → BoundStatement → QueryPlanner → PlanNode → QueryExecutor → ResultSet` + +**Status:** structural scaffold — almost every method body is a documented `TODO`. +Reference implementations for the binary layouts: **mdbtools** (`src/libmdb/`) and +**Jackcess**. The 10-step build order and full detail live in `src/LibRed/README.md`. +ANTLR is present but **not** wired into the build yet (commented `Antlr4BuildTasks` +block in `LibRed.Sql.csproj`). + +## Versioning + +`Version.props` owns `VersionPrefix` and `PreReleaseVersionLabel`. Bump `VersionPrefix` after each release. Valid labels: `alpha`, `beta`, `silver`, `preview`, `rc`, `rtm`, `servicing`. CI sets `OfficialVersion`, `ContinuousIntegrationTimestamp`, and `BuildSha` automatically. diff --git a/EFCore.Jet.sln b/EFCore.Jet.sln index a1aab3c9..7c3c5311 100644 --- a/EFCore.Jet.sln +++ b/EFCore.Jet.sln @@ -56,7 +56,17 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EFCore.Jet.Odbc", "src\EFCo EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EFCore.Jet.OleDb", "src\EFCore.Jet.OleDb\EFCore.Jet.OleDb.csproj", "{FFC89A2D-F68F-47E3-BA00-47E9C0BEDB71}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibRed", "src\LibRed\LibRed.csproj", "{00D43B88-09EA-9310-748A-814C451E2589}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "LibRed", "LibRed", "{53AEDF4E-744F-DCCC-ECB4-9EF88FC7B191}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibRed.Core", "src\LibRed\LibRed.Core\LibRed.Core.csproj", "{BBC79324-A79E-4D04-85F9-254DD6FA0645}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibRed.Sql", "src\LibRed\LibRed.Sql\LibRed.Sql.csproj", "{ABC12950-142D-4C22-B2C5-362FA4728C20}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibRed.Engine", "src\LibRed\LibRed.Engine\LibRed.Engine.csproj", "{8FF045CF-5CDA-4C1B-8C67-2C0DAEDA6A54}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibRed.Ado", "src\LibRed\LibRed.Ado\LibRed.Ado.csproj", "{0932994D-79FA-48B5-8EB1-6EC9F8A0CE63}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibRed.EFCore", "src\LibRed\LibRed.EFCore\LibRed.EFCore.csproj", "{4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -176,18 +186,66 @@ Global {FFC89A2D-F68F-47E3-BA00-47E9C0BEDB71}.Release|x64.Build.0 = Release|Any CPU {FFC89A2D-F68F-47E3-BA00-47E9C0BEDB71}.Release|x86.ActiveCfg = Release|Any CPU {FFC89A2D-F68F-47E3-BA00-47E9C0BEDB71}.Release|x86.Build.0 = Release|Any CPU - {00D43B88-09EA-9310-748A-814C451E2589}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {00D43B88-09EA-9310-748A-814C451E2589}.Debug|Any CPU.Build.0 = Debug|Any CPU - {00D43B88-09EA-9310-748A-814C451E2589}.Debug|x64.ActiveCfg = Debug|Any CPU - {00D43B88-09EA-9310-748A-814C451E2589}.Debug|x64.Build.0 = Debug|Any CPU - {00D43B88-09EA-9310-748A-814C451E2589}.Debug|x86.ActiveCfg = Debug|Any CPU - {00D43B88-09EA-9310-748A-814C451E2589}.Debug|x86.Build.0 = Debug|Any CPU - {00D43B88-09EA-9310-748A-814C451E2589}.Release|Any CPU.ActiveCfg = Release|Any CPU - {00D43B88-09EA-9310-748A-814C451E2589}.Release|Any CPU.Build.0 = Release|Any CPU - {00D43B88-09EA-9310-748A-814C451E2589}.Release|x64.ActiveCfg = Release|Any CPU - {00D43B88-09EA-9310-748A-814C451E2589}.Release|x64.Build.0 = Release|Any CPU - {00D43B88-09EA-9310-748A-814C451E2589}.Release|x86.ActiveCfg = Release|Any CPU - {00D43B88-09EA-9310-748A-814C451E2589}.Release|x86.Build.0 = Release|Any CPU + {BBC79324-A79E-4D04-85F9-254DD6FA0645}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BBC79324-A79E-4D04-85F9-254DD6FA0645}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BBC79324-A79E-4D04-85F9-254DD6FA0645}.Debug|x64.ActiveCfg = Debug|Any CPU + {BBC79324-A79E-4D04-85F9-254DD6FA0645}.Debug|x64.Build.0 = Debug|Any CPU + {BBC79324-A79E-4D04-85F9-254DD6FA0645}.Debug|x86.ActiveCfg = Debug|Any CPU + {BBC79324-A79E-4D04-85F9-254DD6FA0645}.Debug|x86.Build.0 = Debug|Any CPU + {BBC79324-A79E-4D04-85F9-254DD6FA0645}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BBC79324-A79E-4D04-85F9-254DD6FA0645}.Release|Any CPU.Build.0 = Release|Any CPU + {BBC79324-A79E-4D04-85F9-254DD6FA0645}.Release|x64.ActiveCfg = Release|Any CPU + {BBC79324-A79E-4D04-85F9-254DD6FA0645}.Release|x64.Build.0 = Release|Any CPU + {BBC79324-A79E-4D04-85F9-254DD6FA0645}.Release|x86.ActiveCfg = Release|Any CPU + {BBC79324-A79E-4D04-85F9-254DD6FA0645}.Release|x86.Build.0 = Release|Any CPU + {ABC12950-142D-4C22-B2C5-362FA4728C20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {ABC12950-142D-4C22-B2C5-362FA4728C20}.Debug|Any CPU.Build.0 = Debug|Any CPU + {ABC12950-142D-4C22-B2C5-362FA4728C20}.Debug|x64.ActiveCfg = Debug|Any CPU + {ABC12950-142D-4C22-B2C5-362FA4728C20}.Debug|x64.Build.0 = Debug|Any CPU + {ABC12950-142D-4C22-B2C5-362FA4728C20}.Debug|x86.ActiveCfg = Debug|Any CPU + {ABC12950-142D-4C22-B2C5-362FA4728C20}.Debug|x86.Build.0 = Debug|Any CPU + {ABC12950-142D-4C22-B2C5-362FA4728C20}.Release|Any CPU.ActiveCfg = Release|Any CPU + {ABC12950-142D-4C22-B2C5-362FA4728C20}.Release|Any CPU.Build.0 = Release|Any CPU + {ABC12950-142D-4C22-B2C5-362FA4728C20}.Release|x64.ActiveCfg = Release|Any CPU + {ABC12950-142D-4C22-B2C5-362FA4728C20}.Release|x64.Build.0 = Release|Any CPU + {ABC12950-142D-4C22-B2C5-362FA4728C20}.Release|x86.ActiveCfg = Release|Any CPU + {ABC12950-142D-4C22-B2C5-362FA4728C20}.Release|x86.Build.0 = Release|Any CPU + {8FF045CF-5CDA-4C1B-8C67-2C0DAEDA6A54}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8FF045CF-5CDA-4C1B-8C67-2C0DAEDA6A54}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8FF045CF-5CDA-4C1B-8C67-2C0DAEDA6A54}.Debug|x64.ActiveCfg = Debug|Any CPU + {8FF045CF-5CDA-4C1B-8C67-2C0DAEDA6A54}.Debug|x64.Build.0 = Debug|Any CPU + {8FF045CF-5CDA-4C1B-8C67-2C0DAEDA6A54}.Debug|x86.ActiveCfg = Debug|Any CPU + {8FF045CF-5CDA-4C1B-8C67-2C0DAEDA6A54}.Debug|x86.Build.0 = Debug|Any CPU + {8FF045CF-5CDA-4C1B-8C67-2C0DAEDA6A54}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8FF045CF-5CDA-4C1B-8C67-2C0DAEDA6A54}.Release|Any CPU.Build.0 = Release|Any CPU + {8FF045CF-5CDA-4C1B-8C67-2C0DAEDA6A54}.Release|x64.ActiveCfg = Release|Any CPU + {8FF045CF-5CDA-4C1B-8C67-2C0DAEDA6A54}.Release|x64.Build.0 = Release|Any CPU + {8FF045CF-5CDA-4C1B-8C67-2C0DAEDA6A54}.Release|x86.ActiveCfg = Release|Any CPU + {8FF045CF-5CDA-4C1B-8C67-2C0DAEDA6A54}.Release|x86.Build.0 = Release|Any CPU + {0932994D-79FA-48B5-8EB1-6EC9F8A0CE63}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0932994D-79FA-48B5-8EB1-6EC9F8A0CE63}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0932994D-79FA-48B5-8EB1-6EC9F8A0CE63}.Debug|x64.ActiveCfg = Debug|Any CPU + {0932994D-79FA-48B5-8EB1-6EC9F8A0CE63}.Debug|x64.Build.0 = Debug|Any CPU + {0932994D-79FA-48B5-8EB1-6EC9F8A0CE63}.Debug|x86.ActiveCfg = Debug|Any CPU + {0932994D-79FA-48B5-8EB1-6EC9F8A0CE63}.Debug|x86.Build.0 = Debug|Any CPU + {0932994D-79FA-48B5-8EB1-6EC9F8A0CE63}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0932994D-79FA-48B5-8EB1-6EC9F8A0CE63}.Release|Any CPU.Build.0 = Release|Any CPU + {0932994D-79FA-48B5-8EB1-6EC9F8A0CE63}.Release|x64.ActiveCfg = Release|Any CPU + {0932994D-79FA-48B5-8EB1-6EC9F8A0CE63}.Release|x64.Build.0 = Release|Any CPU + {0932994D-79FA-48B5-8EB1-6EC9F8A0CE63}.Release|x86.ActiveCfg = Release|Any CPU + {0932994D-79FA-48B5-8EB1-6EC9F8A0CE63}.Release|x86.Build.0 = Release|Any CPU + {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}.Debug|x64.ActiveCfg = Debug|Any CPU + {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}.Debug|x64.Build.0 = Debug|Any CPU + {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}.Debug|x86.ActiveCfg = Debug|Any CPU + {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}.Debug|x86.Build.0 = Debug|Any CPU + {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}.Release|Any CPU.Build.0 = Release|Any CPU + {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}.Release|x64.ActiveCfg = Release|Any CPU + {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}.Release|x64.Build.0 = Release|Any CPU + {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}.Release|x86.ActiveCfg = Release|Any CPU + {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -202,7 +260,11 @@ Global {5CD8B47D-E32C-480A-8331-55549EC8E12E} = {6A8DE399-1804-4113-A408-F23B7F5C9CAC} {1E0729DA-B861-46EA-B1F1-3AE20EA1E00A} = {F68095EE-6CD1-43A2-B498-6CA72CE2A0CB} {FFC89A2D-F68F-47E3-BA00-47E9C0BEDB71} = {F68095EE-6CD1-43A2-B498-6CA72CE2A0CB} - {00D43B88-09EA-9310-748A-814C451E2589} = {F68095EE-6CD1-43A2-B498-6CA72CE2A0CB} + {BBC79324-A79E-4D04-85F9-254DD6FA0645} = {53AEDF4E-744F-DCCC-ECB4-9EF88FC7B191} + {ABC12950-142D-4C22-B2C5-362FA4728C20} = {53AEDF4E-744F-DCCC-ECB4-9EF88FC7B191} + {8FF045CF-5CDA-4C1B-8C67-2C0DAEDA6A54} = {53AEDF4E-744F-DCCC-ECB4-9EF88FC7B191} + {0932994D-79FA-48B5-8EB1-6EC9F8A0CE63} = {53AEDF4E-744F-DCCC-ECB4-9EF88FC7B191} + {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71} = {53AEDF4E-744F-DCCC-ECB4-9EF88FC7B191} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9359773D-6399-447E-9814-6CB41C2FB664} diff --git a/src/LibRed/Directory.Build.props b/src/LibRed/Directory.Build.props new file mode 100644 index 00000000..8f4f617b --- /dev/null +++ b/src/LibRed/Directory.Build.props @@ -0,0 +1,26 @@ + + + + + + enable + enable + false + false + $(NoWarn);CS1591 + + + + + true + $(MSBuildThisFileDirectory)..\..\Key.snk + + diff --git a/src/LibRed/Formats/Jet12Format.cs b/src/LibRed/Formats/Jet12Format.cs deleted file mode 100644 index b42814ab..00000000 --- a/src/LibRed/Formats/Jet12Format.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace LibRed.Formats -{ - internal class Jet12Format : Jet4Format - { - } -} diff --git a/src/LibRed/Formats/Jet14Format.cs b/src/LibRed/Formats/Jet14Format.cs deleted file mode 100644 index 68327a7c..00000000 --- a/src/LibRed/Formats/Jet14Format.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace LibRed.Formats -{ - internal class Jet14Format : Jet12Format - { - } -} diff --git a/src/LibRed/Formats/Jet16Format.cs b/src/LibRed/Formats/Jet16Format.cs deleted file mode 100644 index 1974374e..00000000 --- a/src/LibRed/Formats/Jet16Format.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace LibRed.Formats -{ - internal class Jet16Format : Jet14Format - { - } -} diff --git a/src/LibRed/Formats/Jet17Format.cs b/src/LibRed/Formats/Jet17Format.cs deleted file mode 100644 index ceedb9dd..00000000 --- a/src/LibRed/Formats/Jet17Format.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace LibRed.Formats -{ - internal class Jet17Format : Jet16Format - { - } -} diff --git a/src/LibRed/Formats/Jet3Format.cs b/src/LibRed/Formats/Jet3Format.cs deleted file mode 100644 index cc751446..00000000 --- a/src/LibRed/Formats/Jet3Format.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace LibRed.Formats -{ - internal class Jet3Format : JetFormatBase - { - } -} diff --git a/src/LibRed/Formats/Jet4Format.cs b/src/LibRed/Formats/Jet4Format.cs deleted file mode 100644 index bc9b3e6c..00000000 --- a/src/LibRed/Formats/Jet4Format.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace LibRed.Formats -{ - internal class Jet4Format : JetFormatBase - { - public Jet4Format() - { - PageSize = 4096; // Default page size for Jet 4.0 - } - } -} diff --git a/src/LibRed/Formats/JetFormatBase.cs b/src/LibRed/Formats/JetFormatBase.cs deleted file mode 100644 index 050285aa..00000000 --- a/src/LibRed/Formats/JetFormatBase.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace LibRed.Formats -{ - internal abstract class JetFormatBase - { - // Database definition page constants - public int PageSize { get; protected set; } = 2048; // Default page size for Jet formats - public int VersionOffset { get; protected set; } = 0x14; - } -} diff --git a/src/LibRed/LibRed.Ado/LibRed.Ado.csproj b/src/LibRed/LibRed.Ado/LibRed.Ado.csproj new file mode 100644 index 00000000..4e1b96eb --- /dev/null +++ b/src/LibRed/LibRed.Ado/LibRed.Ado.csproj @@ -0,0 +1,14 @@ + + + + $(JetTargetFramework) + LibRed.Data + LibRed.Ado + ADO.NET provider over the LibRed engine: DbConnection, DbCommand, DbDataReader, DbParameter, DbTransaction and DbProviderFactory. + + + + + + + diff --git a/src/LibRed/LibRed.Ado/LibRedCommand.cs b/src/LibRed/LibRed.Ado/LibRedCommand.cs new file mode 100644 index 00000000..7710dc14 --- /dev/null +++ b/src/LibRed/LibRed.Ado/LibRedCommand.cs @@ -0,0 +1,57 @@ +using System.Data; +using System.Data.Common; +using System.Diagnostics.CodeAnalysis; + +namespace LibRed.Data; + +/// ADO.NET command that runs SQL through the LibRed engine. +public sealed class LibRedCommand : DbCommand +{ + private readonly LibRedParameterCollection _parameters = new(); + + private string _commandText = string.Empty; + + [AllowNull] + public override string CommandText + { + get => _commandText; + set => _commandText = value ?? string.Empty; + } + public override int CommandTimeout { get; set; } = 30; + public override CommandType CommandType { get; set; } = CommandType.Text; + public override bool DesignTimeVisible { get; set; } + public override UpdateRowSource UpdatedRowSource { get; set; } = UpdateRowSource.None; + + protected override DbConnection? DbConnection { get; set; } + protected override DbParameterCollection DbParameterCollection => _parameters; + protected override DbTransaction? DbTransaction { get; set; } + + public new LibRedConnection? Connection + { + get => (LibRedConnection?)DbConnection; + set => DbConnection = value; + } + + public override void Cancel() { } + + public override void Prepare() { } + + public override int ExecuteNonQuery() => RequireEngine().ExecuteNonQuery(CommandText); + + public override object? ExecuteScalar() + { + using var reader = ExecuteReader(); + return reader.Read() && reader.FieldCount > 0 ? reader.GetValue(0) : null; + } + + protected override DbParameter CreateDbParameter() => new LibRedParameter(); + + protected override DbDataReader ExecuteDbDataReader(CommandBehavior behavior) + { + var result = RequireEngine().ExecuteQuery(CommandText); + return new LibRedDataReader(result); + } + + private Engine.QueryEngine RequireEngine() => + Connection?.Engine ?? throw new InvalidOperationException("Connection is not open."); +} diff --git a/src/LibRed/LibRed.Ado/LibRedConnection.cs b/src/LibRed/LibRed.Ado/LibRedConnection.cs new file mode 100644 index 00000000..bbd15fc9 --- /dev/null +++ b/src/LibRed/LibRed.Ado/LibRedConnection.cs @@ -0,0 +1,95 @@ +using System.Data; +using System.Data.Common; +using System.Diagnostics.CodeAnalysis; +using LibRed.Engine; + +namespace LibRed.Data; + +/// +/// ADO.NET connection over a native LibRed engine instance. The connection string's +/// Data Source names the .mdb/.accdb file to open. +/// +public sealed class LibRedConnection : DbConnection +{ + private string _connectionString = string.Empty; + private ConnectionState _state = ConnectionState.Closed; + private JetDatabase? _database; + + public LibRedConnection() { } + + public LibRedConnection(string connectionString) => _connectionString = connectionString ?? string.Empty; + + [AllowNull] + public override string ConnectionString + { + get => _connectionString; + set => _connectionString = value ?? string.Empty; + } + + /// The open database, or null when closed. Used by commands. + internal QueryEngine? Engine { get; private set; } + + public override string Database => DataSource; + + public override string DataSource => ParseDataSource(_connectionString); + + public override string ServerVersion => _database?.Format.Version.ToString() ?? string.Empty; + + public override ConnectionState State => _state; + + public override void Open() + { + if (_state == ConnectionState.Open) return; + + string path = DataSource; + if (string.IsNullOrEmpty(path)) + throw new InvalidOperationException("Connection string is missing a Data Source."); + + _database = JetDatabase.Open(path, readOnly: false); + Engine = new QueryEngine(_database); + _state = ConnectionState.Open; + } + + public override void Close() + { + _database?.Dispose(); + _database = null; + Engine = null; + _state = ConnectionState.Closed; + } + + public override void ChangeDatabase(string databaseName) => + throw new NotSupportedException("A Jet/ACE connection maps to a single file."); + + protected override DbCommand CreateDbCommand() => new LibRedCommand { Connection = this }; + + protected override DbTransaction BeginDbTransaction(IsolationLevel isolationLevel) => + new LibRedTransaction(this, isolationLevel); + + protected override void Dispose(bool disposing) + { + if (disposing) Close(); + base.Dispose(disposing); + } + + private static string ParseDataSource(string connectionString) + { + if (string.IsNullOrWhiteSpace(connectionString)) return string.Empty; + + foreach (string part in connectionString.Split(';', StringSplitOptions.RemoveEmptyEntries)) + { + int eq = part.IndexOf('='); + if (eq < 0) continue; + string key = part[..eq].Trim(); + if (key.Equals("Data Source", StringComparison.OrdinalIgnoreCase) || + key.Equals("DataSource", StringComparison.OrdinalIgnoreCase) || + key.Equals("DBQ", StringComparison.OrdinalIgnoreCase)) + { + return part[(eq + 1)..].Trim().Trim('"'); + } + } + + // Allow a bare path as the whole connection string. + return connectionString.Contains('=') ? string.Empty : connectionString.Trim(); + } +} diff --git a/src/LibRed/LibRed.Ado/LibRedDataReader.cs b/src/LibRed/LibRed.Ado/LibRedDataReader.cs new file mode 100644 index 00000000..78071869 --- /dev/null +++ b/src/LibRed/LibRed.Ado/LibRedDataReader.cs @@ -0,0 +1,105 @@ +using System.Collections; +using System.Data.Common; +using LibRed.Engine.Execution; + +namespace LibRed.Data; + +/// Forward-only reader projecting an engine as ADO.NET rows. +public sealed class LibRedDataReader : DbDataReader +{ + private readonly ResultSet _result; + private readonly IEnumerator _rows; + private object?[] _current = []; + private bool _closed; + + internal LibRedDataReader(ResultSet result) + { + _result = result; + _rows = result.Rows.GetEnumerator(); + } + + public override int FieldCount => _result.ColumnNames.Count; + public override int Depth => 0; + public override bool HasRows => _result.Rows.Any(); + public override bool IsClosed => _closed; + public override int RecordsAffected => -1; + + public override object this[int ordinal] => GetValue(ordinal); + public override object this[string name] => GetValue(GetOrdinal(name)); + + public override bool Read() + { + if (!_rows.MoveNext()) return false; + _current = _rows.Current; + return true; + } + + public override bool NextResult() => false; + + public override string GetName(int ordinal) => _result.ColumnNames[ordinal]; + + public override int GetOrdinal(string name) + { + for (int i = 0; i < _result.ColumnNames.Count; i++) + if (string.Equals(_result.ColumnNames[i], name, StringComparison.OrdinalIgnoreCase)) + return i; + throw new IndexOutOfRangeException(name); + } + + public override object GetValue(int ordinal) => _current[ordinal] ?? DBNull.Value; + + public override int GetValues(object[] values) + { + int count = Math.Min(values.Length, FieldCount); + for (int i = 0; i < count; i++) values[i] = GetValue(i); + return count; + } + + public override bool IsDBNull(int ordinal) => _current[ordinal] is null; + + public override Type GetFieldType(int ordinal) => _current[ordinal]?.GetType() ?? typeof(object); + + public override string GetDataTypeName(int ordinal) => GetFieldType(ordinal).Name; + + public override bool GetBoolean(int ordinal) => (bool)GetValue(ordinal); + public override byte GetByte(int ordinal) => (byte)GetValue(ordinal); + public override char GetChar(int ordinal) => (char)GetValue(ordinal); + public override DateTime GetDateTime(int ordinal) => (DateTime)GetValue(ordinal); + public override decimal GetDecimal(int ordinal) => (decimal)GetValue(ordinal); + public override double GetDouble(int ordinal) => (double)GetValue(ordinal); + public override float GetFloat(int ordinal) => (float)GetValue(ordinal); + public override Guid GetGuid(int ordinal) => (Guid)GetValue(ordinal); + public override short GetInt16(int ordinal) => (short)GetValue(ordinal); + public override int GetInt32(int ordinal) => (int)GetValue(ordinal); + public override long GetInt64(int ordinal) => (long)GetValue(ordinal); + public override string GetString(int ordinal) => (string)GetValue(ordinal); + + public override long GetBytes(int ordinal, long dataOffset, byte[]? buffer, int bufferOffset, int length) + { + var source = (byte[])GetValue(ordinal); + if (buffer is null) return source.Length; + long copy = Math.Min(length, source.Length - dataOffset); + Array.Copy(source, dataOffset, buffer, bufferOffset, copy); + return copy; + } + + public override long GetChars(int ordinal, long dataOffset, char[]? buffer, int bufferOffset, int length) + { + var source = GetString(ordinal).ToCharArray(); + if (buffer is null) return source.Length; + long copy = Math.Min(length, source.Length - dataOffset); + Array.Copy(source, dataOffset, buffer, bufferOffset, copy); + return copy; + } + + public override IEnumerator GetEnumerator() => new DbEnumerator(this, closeReader: false); + + public override void Close() => _closed = true; + + protected override void Dispose(bool disposing) + { + if (disposing) _rows.Dispose(); + Close(); + base.Dispose(disposing); + } +} diff --git a/src/LibRed/LibRed.Ado/LibRedFactory.cs b/src/LibRed/LibRed.Ado/LibRedFactory.cs new file mode 100644 index 00000000..ca8d45a1 --- /dev/null +++ b/src/LibRed/LibRed.Ado/LibRedFactory.cs @@ -0,0 +1,18 @@ +using System.Data.Common; + +namespace LibRed.Data; + +/// +/// for the LibRed provider, enabling provider-agnostic +/// ADO.NET code and registration via DbProviderFactories. +/// +public sealed class LibRedFactory : DbProviderFactory +{ + public static readonly LibRedFactory Instance = new(); + + private LibRedFactory() { } + + public override DbConnection CreateConnection() => new LibRedConnection(); + public override DbCommand CreateCommand() => new LibRedCommand(); + public override DbParameter CreateParameter() => new LibRedParameter(); +} diff --git a/src/LibRed/LibRed.Ado/LibRedParameter.cs b/src/LibRed/LibRed.Ado/LibRedParameter.cs new file mode 100644 index 00000000..ba083fa4 --- /dev/null +++ b/src/LibRed/LibRed.Ado/LibRedParameter.cs @@ -0,0 +1,34 @@ +using System.Data; +using System.Data.Common; +using System.Diagnostics.CodeAnalysis; + +namespace LibRed.Data; + +/// A command parameter for the LibRed provider. +public sealed class LibRedParameter : DbParameter +{ + public override DbType DbType { get; set; } = DbType.Object; + public override ParameterDirection Direction { get; set; } = ParameterDirection.Input; + public override bool IsNullable { get; set; } = true; + private string _parameterName = string.Empty; + private string _sourceColumn = string.Empty; + + [AllowNull] + public override string ParameterName + { + get => _parameterName; + set => _parameterName = value ?? string.Empty; + } + + [AllowNull] + public override string SourceColumn + { + get => _sourceColumn; + set => _sourceColumn = value ?? string.Empty; + } + public override bool SourceColumnNullMapping { get; set; } + public override object? Value { get; set; } + public override int Size { get; set; } + + public override void ResetDbType() => DbType = DbType.Object; +} diff --git a/src/LibRed/LibRed.Ado/LibRedParameterCollection.cs b/src/LibRed/LibRed.Ado/LibRedParameterCollection.cs new file mode 100644 index 00000000..e1786075 --- /dev/null +++ b/src/LibRed/LibRed.Ado/LibRedParameterCollection.cs @@ -0,0 +1,55 @@ +using System.Collections; +using System.Data.Common; + +namespace LibRed.Data; + +/// Parameter collection backed by a simple list. +public sealed class LibRedParameterCollection : DbParameterCollection +{ + private readonly List _items = []; + + public override int Count => _items.Count; + public override object SyncRoot { get; } = new(); + + public override int Add(object value) + { + _items.Add((LibRedParameter)value); + return _items.Count - 1; + } + + public override void AddRange(Array values) + { + foreach (object value in values) Add(value); + } + + public override void Clear() => _items.Clear(); + + public override bool Contains(object value) => _items.Contains((LibRedParameter)value); + public override bool Contains(string value) => IndexOf(value) >= 0; + + public override void CopyTo(Array array, int index) => ((ICollection)_items).CopyTo(array, index); + + public override IEnumerator GetEnumerator() => _items.GetEnumerator(); + + public override int IndexOf(object value) => _items.IndexOf((LibRedParameter)value); + + public override int IndexOf(string parameterName) => + _items.FindIndex(p => string.Equals(p.ParameterName, parameterName, StringComparison.OrdinalIgnoreCase)); + + public override void Insert(int index, object value) => _items.Insert(index, (LibRedParameter)value); + + public override void Remove(object value) => _items.Remove((LibRedParameter)value); + + public override void RemoveAt(int index) => _items.RemoveAt(index); + + public override void RemoveAt(string parameterName) => RemoveAt(IndexOf(parameterName)); + + protected override DbParameter GetParameter(int index) => _items[index]; + + protected override DbParameter GetParameter(string parameterName) => _items[IndexOf(parameterName)]; + + protected override void SetParameter(int index, DbParameter value) => _items[index] = (LibRedParameter)value; + + protected override void SetParameter(string parameterName, DbParameter value) => + _items[IndexOf(parameterName)] = (LibRedParameter)value; +} diff --git a/src/LibRed/LibRed.Ado/LibRedTransaction.cs b/src/LibRed/LibRed.Ado/LibRedTransaction.cs new file mode 100644 index 00000000..22d6a7c5 --- /dev/null +++ b/src/LibRed/LibRed.Ado/LibRedTransaction.cs @@ -0,0 +1,27 @@ +using System.Data; +using System.Data.Common; + +namespace LibRed.Data; + +/// +/// A database transaction. Jet/ACE supports transactions; the engine wiring for +/// commit/rollback is still to come, so this currently tracks state only. +/// +public sealed class LibRedTransaction(LibRedConnection connection, IsolationLevel isolationLevel) : DbTransaction +{ + private readonly LibRedConnection _connection = connection; + + public override IsolationLevel IsolationLevel { get; } = isolationLevel; + + protected override DbConnection DbConnection => _connection; + + public override void Commit() + { + // TODO: flush buffered page writes atomically. + } + + public override void Rollback() + { + // TODO: discard buffered page writes. + } +} diff --git a/src/LibRed/LibRed.Core/Catalog/ColumnDef.cs b/src/LibRed/LibRed.Core/Catalog/ColumnDef.cs new file mode 100644 index 00000000..32d93163 --- /dev/null +++ b/src/LibRed/LibRed.Core/Catalog/ColumnDef.cs @@ -0,0 +1,28 @@ +namespace LibRed.Catalog; + +/// +/// Describes a single column of a table: its name, type, physical layout and flags. +/// Decoded from the column descriptors in a . +/// +public sealed class ColumnDef +{ + public required string Name { get; init; } + public required JetDataType Type { get; init; } + + /// Zero-based logical column index (declaration order). + public int Index { get; init; } + + /// Physical offset/ordinal used to locate the value within a row record. + public int ColumnId { get; init; } + + /// Declared length in bytes for fixed-width/text columns. + public int Length { get; init; } + + public bool IsFixedLength { get; init; } + public bool IsNullable { get; init; } = true; + public bool IsAutoNumber { get; init; } + + /// Precision/scale for columns. + public byte Precision { get; init; } + public byte Scale { get; init; } +} diff --git a/src/LibRed/LibRed.Core/Catalog/IndexDef.cs b/src/LibRed/LibRed.Core/Catalog/IndexDef.cs new file mode 100644 index 00000000..be7cb962 --- /dev/null +++ b/src/LibRed/LibRed.Core/Catalog/IndexDef.cs @@ -0,0 +1,16 @@ +namespace LibRed.Catalog; + +/// Describes an index: its columns (with sort direction), uniqueness and root page. +public sealed class IndexDef +{ + public required string Name { get; init; } + + /// Indexed columns in key order; true = ascending. + public IReadOnlyList<(ColumnDef Column, bool Ascending)> Columns { get; init; } = []; + + public bool IsUnique { get; init; } + public bool IsPrimaryKey { get; init; } + + /// Page number of the index B-tree root. + public int RootPage { get; init; } +} diff --git a/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs b/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs new file mode 100644 index 00000000..51d3cb45 --- /dev/null +++ b/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs @@ -0,0 +1,31 @@ +using LibRed.IO; + +namespace LibRed.Catalog; + +/// +/// Reads the system catalog (MSysObjects, MSysColumns, …) to enumerate +/// the user tables, queries and relationships in a database. +/// +/// +/// Bootstrap order: page 0 (database definition) → the catalog table lives at a +/// well-known page → read MSysObjects as an ordinary table to discover everything else. +/// +public sealed class JetCatalog(PageChannel channel) +{ + private readonly PageChannel _channel = channel; + private List? _tables; + + /// All user (non-system) tables in the database. + public IReadOnlyList Tables => _tables ??= LoadTables(); + + public TableDef? FindTable(string name) => + Tables.FirstOrDefault(t => string.Equals(t.Name, name, StringComparison.OrdinalIgnoreCase)); + + private List LoadTables() + { + // TODO: parse MSysObjects to recover (Name, Id/page, Type, Flags) rows, filter to + // user tables, then resolve each table's TDEF page into a TableDef. + _ = _channel; + return []; + } +} diff --git a/src/LibRed/LibRed.Core/Catalog/JetDataType.cs b/src/LibRed/LibRed.Core/Catalog/JetDataType.cs new file mode 100644 index 00000000..bc890d93 --- /dev/null +++ b/src/LibRed/LibRed.Core/Catalog/JetDataType.cs @@ -0,0 +1,23 @@ +namespace LibRed.Catalog; + +/// +/// The column data types supported by Jet/ACE, with their on-disk type codes. +/// +public enum JetDataType : byte +{ + Boolean = 0x01, + Byte = 0x02, + Int16 = 0x03, + Int32 = 0x04, + Currency = 0x05, + Single = 0x06, + Double = 0x07, + DateTime = 0x08, + Binary = 0x09, + Text = 0x0A, + Ole = 0x0B, + Memo = 0x0C, + Guid = 0x0F, + FixedPoint = 0x10, // NUMERIC / DECIMAL + Complex = 0x12, // ACE complex/multi-value columns +} diff --git a/src/LibRed/LibRed.Core/Catalog/TableDef.cs b/src/LibRed/LibRed.Core/Catalog/TableDef.cs new file mode 100644 index 00000000..7f7859da --- /dev/null +++ b/src/LibRed/LibRed.Core/Catalog/TableDef.cs @@ -0,0 +1,23 @@ +namespace LibRed.Catalog; + +/// +/// The resolved definition of a table: its columns and indexes plus the page that +/// anchors its data. Produced by and consumed by the +/// storage layer to open a . +/// +public sealed class TableDef +{ + public required string Name { get; init; } + + /// Page number of the table's TDEF (definition) page. + public required int DefinitionPage { get; init; } + + public IReadOnlyList Columns { get; init; } = []; + public IReadOnlyList Indexes { get; init; } = []; + + /// True for the MSys* system tables. + public bool IsSystem { get; init; } + + public ColumnDef? FindColumn(string name) => + Columns.FirstOrDefault(c => string.Equals(c.Name, name, StringComparison.OrdinalIgnoreCase)); +} diff --git a/src/LibRed/LibRed.Core/Crypto/JetCrypto.cs b/src/LibRed/LibRed.Core/Crypto/JetCrypto.cs new file mode 100644 index 00000000..644e2b48 --- /dev/null +++ b/src/LibRed/LibRed.Core/Crypto/JetCrypto.cs @@ -0,0 +1,20 @@ +using LibRed.Formats; + +namespace LibRed.Crypto; + +/// +/// Page-level encryption/decryption. Jet 3/4 use RC4 keyed per page from the +/// database key on page 0; ACE (ACCDB) uses an RC4/AES scheme depending on the +/// configured provider. Applied transparently by the IO layer on each page read. +/// +public static class JetCrypto +{ + /// Decrypts a page in place, if the database is encrypted. No-op otherwise. + public static void DecryptPage(JetFormatBase format, int pageNumber, Span page) + { + // TODO: derive the per-page key and apply RC4 (Jet) or the ACE cipher. + _ = format; + _ = pageNumber; + _ = page.Length; + } +} diff --git a/src/LibRed/LibRed.Core/Formats/Jet12Format.cs b/src/LibRed/LibRed.Core/Formats/Jet12Format.cs new file mode 100644 index 00000000..12e3a024 --- /dev/null +++ b/src/LibRed/LibRed.Core/Formats/Jet12Format.cs @@ -0,0 +1,9 @@ +namespace LibRed.Formats; + +/// Access 2007 (ACE 12) — first ACCDB format. +internal class Jet12Format : Jet4Format +{ + public override JetVersion Version => JetVersion.Version12_2007; + + public override bool IsAccdb => true; +} diff --git a/src/LibRed/LibRed.Core/Formats/Jet14Format.cs b/src/LibRed/LibRed.Core/Formats/Jet14Format.cs new file mode 100644 index 00000000..f5ff6d5b --- /dev/null +++ b/src/LibRed/LibRed.Core/Formats/Jet14Format.cs @@ -0,0 +1,7 @@ +namespace LibRed.Formats; + +/// Access 2010 (ACE 14) — ACCDB. +internal class Jet14Format : Jet12Format +{ + public override JetVersion Version => JetVersion.Version14_2010; +} diff --git a/src/LibRed/LibRed.Core/Formats/Jet16Format.cs b/src/LibRed/LibRed.Core/Formats/Jet16Format.cs new file mode 100644 index 00000000..f401c50f --- /dev/null +++ b/src/LibRed/LibRed.Core/Formats/Jet16Format.cs @@ -0,0 +1,7 @@ +namespace LibRed.Formats; + +/// Access 2016 (ACE 16) — ACCDB. +internal class Jet16Format : Jet14Format +{ + public override JetVersion Version => JetVersion.Version16_2016; +} diff --git a/src/LibRed/LibRed.Core/Formats/Jet17Format.cs b/src/LibRed/LibRed.Core/Formats/Jet17Format.cs new file mode 100644 index 00000000..1f57a20f --- /dev/null +++ b/src/LibRed/LibRed.Core/Formats/Jet17Format.cs @@ -0,0 +1,7 @@ +namespace LibRed.Formats; + +/// Access 2019/2021 (ACE 17) — ACCDB. +internal sealed class Jet17Format : Jet16Format +{ + public override JetVersion Version => JetVersion.Version17_2019; +} diff --git a/src/LibRed/LibRed.Core/Formats/Jet3Format.cs b/src/LibRed/LibRed.Core/Formats/Jet3Format.cs new file mode 100644 index 00000000..797c6a62 --- /dev/null +++ b/src/LibRed/LibRed.Core/Formats/Jet3Format.cs @@ -0,0 +1,9 @@ +namespace LibRed.Formats; + +/// Access 97 (Jet 3.x) — 2 KB pages, MDB. +internal sealed class Jet3Format : JetFormatBase +{ + public Jet3Format() => PageSize = 2048; + + public override JetVersion Version => JetVersion.Version3; +} diff --git a/src/LibRed/LibRed.Core/Formats/Jet4Format.cs b/src/LibRed/LibRed.Core/Formats/Jet4Format.cs new file mode 100644 index 00000000..1e49b6fe --- /dev/null +++ b/src/LibRed/LibRed.Core/Formats/Jet4Format.cs @@ -0,0 +1,9 @@ +namespace LibRed.Formats; + +/// Access 2000–2003 (Jet 4.x) — 4 KB pages, MDB. +internal class Jet4Format : JetFormatBase +{ + public Jet4Format() => PageSize = 4096; + + public override JetVersion Version => JetVersion.Version4; +} diff --git a/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs new file mode 100644 index 00000000..513cd354 --- /dev/null +++ b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs @@ -0,0 +1,53 @@ +namespace LibRed.Formats; + +/// +/// Version-specific layout description for a Jet/ACE database. Holds every byte +/// offset, size and limit that differs between format versions, so the page +/// parsers can read named constants instead of hard-coded magic numbers. +/// +/// +/// The authoritative references for these values are the mdbtools source +/// (include/mdbtools.h, src/libmdb/) and Jackcess +/// (com.healthmarketscience.jackcess.impl.JetFormat). +/// +public abstract class JetFormatBase +{ + /// Offset of the one-byte format version marker within page 0. + public const int VersionOffset = 0x14; + + /// Page size in bytes (2048 for Jet 3, 4096 for Jet 4 and all ACE versions). + public int PageSize { get; protected set; } = 4096; + + /// The logical version this format describes. + public abstract JetVersion Version { get; } + + /// True for the ACCDB (ACE 12+) family, which uses different encryption and layout details. + public virtual bool IsAccdb => false; + + /// + /// Sniffs the format version byte from page 0 of and + /// returns the matching format description. Restores the stream position. + /// + public static JetFormatBase Detect(Stream stream) + { + Span header = stackalloc byte[VersionOffset + 1]; + long original = stream.Position; + stream.Seek(0, SeekOrigin.Begin); + stream.ReadExactly(header); + stream.Seek(original, SeekOrigin.Begin); + + return FromVersionByte(header[VersionOffset]); + } + + /// Maps the raw version byte at to a format instance. + public static JetFormatBase FromVersionByte(byte versionByte) => versionByte switch + { + 0x00 => new Jet3Format(), + 0x01 => new Jet4Format(), + 0x02 => new Jet12Format(), + 0x03 => new Jet14Format(), + 0x05 => new Jet16Format(), + 0x06 => new Jet17Format(), + _ => throw new NotSupportedException($"Unknown Jet/ACE format version byte 0x{versionByte:X2}."), + }; +} diff --git a/src/LibRed/Formats/JetVersion.cs b/src/LibRed/LibRed.Core/Formats/JetVersion.cs similarity index 100% rename from src/LibRed/Formats/JetVersion.cs rename to src/LibRed/LibRed.Core/Formats/JetVersion.cs diff --git a/src/LibRed/LibRed.Core/IO/PageBuffer.cs b/src/LibRed/LibRed.Core/IO/PageBuffer.cs new file mode 100644 index 00000000..9077306c --- /dev/null +++ b/src/LibRed/LibRed.Core/IO/PageBuffer.cs @@ -0,0 +1,38 @@ +using System.Buffers.Binary; +using System.Text; + +namespace LibRed.IO; + +/// +/// A thin, allocation-free reader over a single page's bytes. All Jet/ACE +/// integers are little-endian; the helpers here centralise that so the page +/// parsers can read named offsets instead of scattering +/// calls everywhere. +/// +public readonly struct PageBuffer(ReadOnlyMemory data, int pageNumber) +{ + public ReadOnlyMemory Data { get; } = data; + public int PageNumber { get; } = pageNumber; + public int Length => Data.Length; + + public ReadOnlySpan Span => Data.Span; + + public byte ReadByte(int offset) => Span[offset]; + + public short ReadInt16(int offset) => BinaryPrimitives.ReadInt16LittleEndian(Span.Slice(offset, 2)); + + public ushort ReadUInt16(int offset) => BinaryPrimitives.ReadUInt16LittleEndian(Span.Slice(offset, 2)); + + public int ReadInt32(int offset) => BinaryPrimitives.ReadInt32LittleEndian(Span.Slice(offset, 4)); + + public uint ReadUInt32(int offset) => BinaryPrimitives.ReadUInt32LittleEndian(Span.Slice(offset, 4)); + + public long ReadInt64(int offset) => BinaryPrimitives.ReadInt64LittleEndian(Span.Slice(offset, 8)); + + /// Reads a 3-byte little-endian page pointer (used by index/usage structures). + public int ReadInt24(int offset) => Span[offset] | (Span[offset + 1] << 8) | (Span[offset + 2] << 16); + + public ReadOnlySpan Slice(int offset, int length) => Span.Slice(offset, length); + + public string ReadString(int offset, int length, Encoding encoding) => encoding.GetString(Span.Slice(offset, length)); +} diff --git a/src/LibRed/LibRed.Core/IO/PageChannel.cs b/src/LibRed/LibRed.Core/IO/PageChannel.cs new file mode 100644 index 00000000..084f9282 --- /dev/null +++ b/src/LibRed/LibRed.Core/IO/PageChannel.cs @@ -0,0 +1,68 @@ +using LibRed.Formats; + +namespace LibRed.IO; + +/// +/// The single primitive every higher layer reads through: page-aligned, optionally +/// decrypted access to the database file. Owns the and the +/// resolved . +/// +public sealed class PageChannel : IDisposable +{ + private readonly FileStream _stream; + private readonly bool _readOnly; + + private PageChannel(FileStream stream, JetFormatBase format, bool readOnly) + { + _stream = stream; + _readOnly = readOnly; + Format = format; + } + + public JetFormatBase Format { get; } + + public int PageSize => Format.PageSize; + + /// Number of pages currently in the file. + public int PageCount => (int)(_stream.Length / PageSize); + + /// + /// Opens a database file, sniffs its Jet/ACE version from page 0 and resolves the + /// matching . + /// + public static PageChannel Open(string path, bool readOnly = true) + { + var stream = new FileStream( + path, + FileMode.Open, + readOnly ? FileAccess.Read : FileAccess.ReadWrite, + readOnly ? FileShare.Read : FileShare.None); + + var format = JetFormatBase.Detect(stream); + return new PageChannel(stream, format, readOnly); + } + + /// Reads a single page into a freshly allocated buffer. + public PageBuffer ReadPage(int pageNumber) + { + var buffer = new byte[PageSize]; + ReadPage(pageNumber, buffer); + return new PageBuffer(buffer, pageNumber); + } + + /// Reads a single page into the supplied buffer (must be at least ). + public void ReadPage(int pageNumber, Span destination) + { + if (destination.Length < PageSize) + throw new ArgumentException($"Buffer must be at least {PageSize} bytes.", nameof(destination)); + + long offset = (long)pageNumber * PageSize; + _stream.Seek(offset, SeekOrigin.Begin); + _stream.ReadExactly(destination[..PageSize]); + + // TODO: page-level decryption (RC4 for Jet3/4, AES for ACE) happens here, + // keyed off Format + the database definition page. See LibRed.Crypto.JetCrypto. + } + + public void Dispose() => _stream.Dispose(); +} diff --git a/src/LibRed/LibRed.Core/JetDatabase.cs b/src/LibRed/LibRed.Core/JetDatabase.cs new file mode 100644 index 00000000..a4af72a5 --- /dev/null +++ b/src/LibRed/LibRed.Core/JetDatabase.cs @@ -0,0 +1,42 @@ +using LibRed.Catalog; +using LibRed.Formats; +using LibRed.IO; +using LibRed.Storage; + +namespace LibRed; + +/// +/// The public entry point to the Core layer: opens a Jet/ACE database file and +/// exposes its catalog and tables. This is what the SQL engine and ADO provider +/// build on; consumers wanting raw storage access start here. +/// +public sealed class JetDatabase : IDisposable +{ + private readonly PageChannel _channel; + + private JetDatabase(PageChannel channel) + { + _channel = channel; + Catalog = new JetCatalog(channel); + } + + /// Opens a database file (read-only by default). + public static JetDatabase Open(string path, bool readOnly = true) => + new(PageChannel.Open(path, readOnly)); + + /// The resolved on-disk format/version of the database. + public JetFormatBase Format => _channel.Format; + + /// The system catalog, used to enumerate and resolve tables. + public JetCatalog Catalog { get; } + + /// Opens a table by name for row access. + public Table OpenTable(string name) + { + TableDef def = Catalog.FindTable(name) + ?? throw new ArgumentException($"Table '{name}' was not found.", nameof(name)); + return new Table(_channel, def); + } + + public void Dispose() => _channel.Dispose(); +} diff --git a/src/LibRed/LibRed.Core/LibRed.Core.csproj b/src/LibRed/LibRed.Core/LibRed.Core.csproj new file mode 100644 index 00000000..6d849e4f --- /dev/null +++ b/src/LibRed/LibRed.Core/LibRed.Core.csproj @@ -0,0 +1,10 @@ + + + + $(JetTargetFramework) + LibRed + LibRed.Core + Native, fully managed reader/writer for the Microsoft Jet/ACE (Access MDB/ACCDB) file format: pages, tables, columns, indexes, catalog, encoding and memo/OLE fields. + + + diff --git a/src/LibRed/LibRed.Core/Pages/DataPage.cs b/src/LibRed/LibRed.Core/Pages/DataPage.cs new file mode 100644 index 00000000..586e5ac1 --- /dev/null +++ b/src/LibRed/LibRed.Core/Pages/DataPage.cs @@ -0,0 +1,25 @@ +using LibRed.IO; + +namespace LibRed.Pages; + +/// +/// A data page holding the actual rows of a single table. Rows are addressed by a +/// slot directory at the end of the page; each slot points at a variable-length +/// row record decoded by . +/// +public sealed class DataPage : Page +{ + public override PageType Type => PageType.DataPage; + + /// The table (TDEF page number) this data page belongs to. + public int OwningTablePage { get; internal set; } + + /// Number of row slots present on the page. + public int RowCount { get; internal set; } + + public override void Read(PageBuffer buffer) + { + PageNumber = buffer.PageNumber; + // TODO: read free-space, owning-table pointer and the row-offset slot table. + } +} diff --git a/src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs b/src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs new file mode 100644 index 00000000..fb146dab --- /dev/null +++ b/src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs @@ -0,0 +1,31 @@ +using LibRed.IO; + +namespace LibRed.Pages; + +/// +/// Page 0 — the database definition page. Carries the format version, code page, +/// collation, creation metadata and the encryption material needed to decrypt the +/// rest of the file. +/// +public sealed class DatabaseDefinitionPage : Page +{ + public override PageType Type => PageType.DatabaseDefinition; + + public byte JetVersion { get; internal set; } + public string? DatabasePassword { get; internal set; } + public int DatabaseKey { get; internal set; } + public short CodePage { get; internal set; } + public short TextCollateSortOrder { get; internal set; } + public string? PageKey { get; internal set; } + public DateTime DatabaseCreationDate { get; internal set; } + public string? CreateProgramName { get; internal set; } + + public override void Read(PageBuffer buffer) + { + PageNumber = buffer.PageNumber; + JetVersion = buffer.ReadByte(Formats.JetFormatBase.VersionOffset); + + // TODO: decode code page, collation, creation date and encryption material. + // Layout differs between Jet 3, Jet 4 and ACE — drive it off the resolved format. + } +} diff --git a/src/LibRed/LibRed.Core/Pages/IndexPage.cs b/src/LibRed/LibRed.Core/Pages/IndexPage.cs new file mode 100644 index 00000000..3742db71 --- /dev/null +++ b/src/LibRed/LibRed.Core/Pages/IndexPage.cs @@ -0,0 +1,22 @@ +using LibRed.IO; + +namespace LibRed.Pages; + +/// +/// A B-tree index page — either an intermediate (node) page pointing at child +/// pages, or a leaf page holding index entries that point at rows. +/// +public sealed class IndexPage : Page +{ + public IndexPage(bool isLeaf) => IsLeaf = isLeaf; + + public bool IsLeaf { get; } + + public override PageType Type => IsLeaf ? PageType.LeafIndexPage : PageType.IntermediateIndexPage; + + public override void Read(PageBuffer buffer) + { + PageNumber = buffer.PageNumber; + // TODO: decode index entries (column-order-preserving encoded keys) and child pointers. + } +} diff --git a/src/LibRed/LibRed.Core/Pages/LvalPage.cs b/src/LibRed/LibRed.Core/Pages/LvalPage.cs new file mode 100644 index 00000000..14976b4f --- /dev/null +++ b/src/LibRed/LibRed.Core/Pages/LvalPage.cs @@ -0,0 +1,19 @@ +using LibRed.IO; + +namespace LibRed.Pages; + +/// +/// A long-value (LVAL) page storing memo, OLE and other overflow data that does not +/// fit inline in a row. Long values may be stored inline, on a single LVAL page, or +/// chained across many LVAL pages. +/// +public sealed class LvalPage : Page +{ + public override PageType Type => PageType.DataPage; // LVAL pages reuse the data-page type marker. + + public override void Read(PageBuffer buffer) + { + PageNumber = buffer.PageNumber; + // TODO: decode LVAL chunk header and payload / next-chunk pointer. + } +} diff --git a/src/LibRed/LibRed.Core/Pages/Page.cs b/src/LibRed/LibRed.Core/Pages/Page.cs new file mode 100644 index 00000000..4bb1a5f3 --- /dev/null +++ b/src/LibRed/LibRed.Core/Pages/Page.cs @@ -0,0 +1,19 @@ +using LibRed.IO; + +namespace LibRed.Pages; + +/// +/// Base class for a typed view over a raw page. Concrete pages declare their +/// and decode their fields from a . +/// +public abstract class Page +{ + /// Zero-based page number within the file. + public int PageNumber { get; protected set; } + + /// The page type marker (byte 0 of the page). + public abstract PageType Type { get; } + + /// Decodes this page's fields from the supplied buffer. + public abstract void Read(PageBuffer buffer); +} diff --git a/src/LibRed/LibRed.Core/Pages/PageType.cs b/src/LibRed/LibRed.Core/Pages/PageType.cs new file mode 100644 index 00000000..a7e4f644 --- /dev/null +++ b/src/LibRed/LibRed.Core/Pages/PageType.cs @@ -0,0 +1,15 @@ +namespace LibRed.Pages; + +/// +/// The page-type marker stored in byte 0 of every page. Values match the on-disk +/// Jet/ACE encoding. +/// +public enum PageType : byte +{ + DatabaseDefinition = 0x00, + DataPage = 0x01, + TableDefinition = 0x02, + IntermediateIndexPage = 0x03, + LeafIndexPage = 0x04, + PageUsageBitmap = 0x05, +} diff --git a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs new file mode 100644 index 00000000..03b77c47 --- /dev/null +++ b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs @@ -0,0 +1,24 @@ +using LibRed.IO; + +namespace LibRed.Pages; + +/// +/// A table definition (TDEF) page: row count, column definitions, index metadata +/// and the pointer to the first data/usage page for the table. May span multiple +/// pages for wide tables (continued via a "next page" pointer). +/// +public sealed class TableDefinitionPage : Page +{ + public override PageType Type => PageType.TableDefinition; + + public int RowCount { get; internal set; } + public int ColumnCount { get; internal set; } + public int IndexCount { get; internal set; } + + public override void Read(PageBuffer buffer) + { + PageNumber = buffer.PageNumber; + // TODO: parse TDEF header, column descriptors, index descriptors and + // real-index/all-index counts. See mdbtools read_table / Jackcess TableImpl. + } +} diff --git a/src/LibRed/LibRed.Core/Pages/UsageMapPage.cs b/src/LibRed/LibRed.Core/Pages/UsageMapPage.cs new file mode 100644 index 00000000..fc8d5f43 --- /dev/null +++ b/src/LibRed/LibRed.Core/Pages/UsageMapPage.cs @@ -0,0 +1,19 @@ +using LibRed.IO; + +namespace LibRed.Pages; + +/// +/// A page-usage bitmap page. Usage maps track which pages belong to a table (or are +/// free). They come in two flavours — inline (stored in the TDEF) and reference +/// (a chain of dedicated bitmap pages) — handled by . +/// +public sealed class UsageMapPage : Page +{ + public override PageType Type => PageType.PageUsageBitmap; + + public override void Read(PageBuffer buffer) + { + PageNumber = buffer.PageNumber; + // TODO: decode the bitmap payload. + } +} diff --git a/src/LibRed/LibRed.Core/Storage/RowDecoder.cs b/src/LibRed/LibRed.Core/Storage/RowDecoder.cs new file mode 100644 index 00000000..ef989653 --- /dev/null +++ b/src/LibRed/LibRed.Core/Storage/RowDecoder.cs @@ -0,0 +1,23 @@ +using LibRed.Catalog; +using LibRed.IO; + +namespace LibRed.Storage; + +/// +/// Decodes a single raw row record into CLR values. Handles the Jet row layout: +/// fixed-length columns first, then the variable-length column offset table and the +/// trailing null bitmask. +/// +public sealed class RowDecoder(TableDef table) +{ + private readonly TableDef _table = table; + + /// Decodes the row at into one boxed value per column. + public object?[] Decode(ReadOnlySpan row) + { + var values = new object?[_table.Columns.Count]; + // TODO: read fixed-length values, the var-length offset jump table and null mask; + // dispatch each column through JetTypeCodec. + return values; + } +} diff --git a/src/LibRed/LibRed.Core/Storage/Table.cs b/src/LibRed/LibRed.Core/Storage/Table.cs new file mode 100644 index 00000000..90c98121 --- /dev/null +++ b/src/LibRed/LibRed.Core/Storage/Table.cs @@ -0,0 +1,27 @@ +using LibRed.Catalog; +using LibRed.IO; + +namespace LibRed.Storage; + +/// +/// An opened table: pairs a with the means to read its rows. +/// The primary entry point for scanning data out of the storage layer. +/// +public sealed class Table +{ + public Table(PageChannel channel, TableDef definition) + { + Channel = channel; + Definition = definition; + UsageMap = new UsageMap(channel, definition); + } + + public PageChannel Channel { get; } + public TableDef Definition { get; } + public UsageMap UsageMap { get; } + + public string Name => Definition.Name; + + /// Returns a forward-only cursor over all rows in the table. + public TableCursor Rows() => new(this); +} diff --git a/src/LibRed/LibRed.Core/Storage/TableCursor.cs b/src/LibRed/LibRed.Core/Storage/TableCursor.cs new file mode 100644 index 00000000..69670030 --- /dev/null +++ b/src/LibRed/LibRed.Core/Storage/TableCursor.cs @@ -0,0 +1,31 @@ +using LibRed.IO; +using LibRed.Pages; + +namespace LibRed.Storage; + +/// +/// A forward-only cursor over the rows of a table. Walks the table's usage map, +/// reads each data page and yields decoded rows via . +/// +public sealed class TableCursor(Table table) : IEnumerable +{ + private readonly Table _table = table; + + public IEnumerator GetEnumerator() + { + var decoder = new RowDecoder(_table.Definition); + foreach (int pageNumber in _table.UsageMap.DataPages()) + { + PageBuffer buffer = _table.Channel.ReadPage(pageNumber); + var dataPage = new DataPage(); + dataPage.Read(buffer); + + // TODO: walk the page's row slot directory and decode each row record. + _ = decoder; + } + + yield break; + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); +} diff --git a/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs b/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs new file mode 100644 index 00000000..62c5260d --- /dev/null +++ b/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs @@ -0,0 +1,30 @@ +using LibRed.Catalog; + +namespace LibRed.Storage.Types; + +/// +/// Encodes and decodes individual column values to/from their on-disk byte +/// representation. Centralises the per-type quirks: Jet CURRENCY (scaled int64), +/// the 1899-12-30 OLE date epoch, fixed-point NUMERIC, GUID byte order, and the +/// code-page/Unicode text handling. +/// +public static class JetTypeCodec +{ + /// Decodes a single fixed-or-variable value for . + public static object? Decode(ColumnDef column, ReadOnlySpan value) + { + // TODO: switch on column.Type and decode accordingly. + return column.Type switch + { + _ => null, + }; + } + + /// Encodes a CLR value back to its on-disk representation. + public static byte[] Encode(ColumnDef column, object? value) + { + // TODO: inverse of Decode. + _ = (column, value); + return []; + } +} diff --git a/src/LibRed/LibRed.Core/Storage/UsageMap.cs b/src/LibRed/LibRed.Core/Storage/UsageMap.cs new file mode 100644 index 00000000..d44e1596 --- /dev/null +++ b/src/LibRed/LibRed.Core/Storage/UsageMap.cs @@ -0,0 +1,24 @@ +using LibRed.Catalog; +using LibRed.IO; + +namespace LibRed.Storage; + +/// +/// Enumerates the data pages that belong to a table. Jet stores this as either an +/// inline bitmap (small tables) or a reference map pointing at dedicated bitmap +/// pages (large tables); both are surfaced here as a flat page sequence. +/// +public sealed class UsageMap(PageChannel channel, TableDef table) +{ + private readonly PageChannel _channel = channel; + private readonly TableDef _table = table; + + /// Yields the page numbers of every data page owned by the table. + public IEnumerable DataPages() + { + // TODO: read the inline/reference usage map from the TDEF and yield set bits. + _ = _channel; + _ = _table; + yield break; + } +} diff --git a/src/LibRed/LibRed.EFCore/LibRed.EFCore.csproj b/src/LibRed/LibRed.EFCore/LibRed.EFCore.csproj new file mode 100644 index 00000000..38c39e4a --- /dev/null +++ b/src/LibRed/LibRed.EFCore/LibRed.EFCore.csproj @@ -0,0 +1,18 @@ + + + + $(JetTargetFramework) + LibRed.EntityFrameworkCore + LibRed.EntityFrameworkCore + Entity Framework Core provider built on the LibRed ADO.NET provider and native engine. + + + + + + + + + + + diff --git a/src/LibRed/LibRed.EFCore/LibRedDesignTimeMarker.cs b/src/LibRed/LibRed.EFCore/LibRedDesignTimeMarker.cs new file mode 100644 index 00000000..39265067 --- /dev/null +++ b/src/LibRed/LibRed.EFCore/LibRedDesignTimeMarker.cs @@ -0,0 +1,16 @@ +namespace LibRed.EntityFrameworkCore; + +/// +/// Placeholder for the EF Core provider built on the native LibRed engine +/// (UseLibRed(...) options extension, type mappings, SQL generation, migrations). +/// +/// +/// The existing EntityFrameworkCore.Jet provider targets ODBC/OleDb and is the +/// reference for the EF Core surface to mirror here, but this provider will sit on +/// instead, making it cross-platform. +/// +public static class LibRedProviderInfo +{ + /// Invariant name suitable for ADO.NET provider registration. + public const string InvariantName = "LibRed.Data"; +} diff --git a/src/LibRed/LibRed.Engine/Execution/QueryExecutor.cs b/src/LibRed/LibRed.Engine/Execution/QueryExecutor.cs new file mode 100644 index 00000000..ef8ea762 --- /dev/null +++ b/src/LibRed/LibRed.Engine/Execution/QueryExecutor.cs @@ -0,0 +1,30 @@ +using LibRed.Engine.Plan; + +namespace LibRed.Engine.Execution; + +/// +/// Interprets a logical plan tree against the storage layer, producing a +/// for queries or an affected-row count for DML. +/// +public sealed class QueryExecutor(JetDatabase database) +{ + private readonly JetDatabase _database = database; + + /// Executes a query plan and returns its rows. + public ResultSet ExecuteQuery(PlanNode plan) + { + // TODO: recursively evaluate the plan tree. Each node type maps to an + // execution operator that pulls rows from its children; ScanNode pulls from + // a Core Table cursor. + _ = (_database, plan); + return ResultSet.Empty; + } + + /// Executes a DML plan and returns the number of affected rows. + public int ExecuteNonQuery(PlanNode plan) + { + // TODO: insert/update/delete against the storage layer. + _ = plan; + return 0; + } +} diff --git a/src/LibRed/LibRed.Engine/Execution/ResultSet.cs b/src/LibRed/LibRed.Engine/Execution/ResultSet.cs new file mode 100644 index 00000000..1ad5b2ee --- /dev/null +++ b/src/LibRed/LibRed.Engine/Execution/ResultSet.cs @@ -0,0 +1,16 @@ +namespace LibRed.Engine.Execution; + +/// +/// The shape and rows produced by executing a query plan. Rows are arrays of boxed +/// values aligned to ; the ADO layer projects these into a +/// DbDataReader. +/// +public sealed class ResultSet(IReadOnlyList columnNames, IEnumerable rows) +{ + public IReadOnlyList ColumnNames { get; } = columnNames; + + /// Lazily-evaluated rows. Enumerating drives the underlying cursors. + public IEnumerable Rows { get; } = rows; + + public static ResultSet Empty { get; } = new([], []); +} diff --git a/src/LibRed/LibRed.Engine/LibRed.Engine.csproj b/src/LibRed/LibRed.Engine/LibRed.Engine.csproj new file mode 100644 index 00000000..08969421 --- /dev/null +++ b/src/LibRed/LibRed.Engine/LibRed.Engine.csproj @@ -0,0 +1,15 @@ + + + + $(JetTargetFramework) + LibRed.Engine + LibRed.Engine + Query engine: plans and executes bound SQL statements (select/insert/update/delete, joins, filters, expressions) against the LibRed.Core storage layer. + + + + + + + + diff --git a/src/LibRed/LibRed.Engine/Plan/PlanNode.cs b/src/LibRed/LibRed.Engine/Plan/PlanNode.cs new file mode 100644 index 00000000..4c68d7bb --- /dev/null +++ b/src/LibRed/LibRed.Engine/Plan/PlanNode.cs @@ -0,0 +1,14 @@ +namespace LibRed.Engine.Plan; + +/// +/// Base type for logical query-plan nodes. A bound statement is lowered into a tree +/// of these (Scan → Filter → Project → …). Keeping the plan separate from both the +/// AST and the executor lets optimisation passes (predicate pushdown, join +/// reordering, index selection) run as tree rewrites, and lets new SQL features add +/// new node types without touching parsing or execution wiring. +/// +public abstract record PlanNode +{ + /// Input nodes feeding this operator (empty for leaves such as Scan). + public virtual IReadOnlyList Children => []; +} diff --git a/src/LibRed/LibRed.Engine/Plan/PlanNodes.cs b/src/LibRed/LibRed.Engine/Plan/PlanNodes.cs new file mode 100644 index 00000000..8dd30322 --- /dev/null +++ b/src/LibRed/LibRed.Engine/Plan/PlanNodes.cs @@ -0,0 +1,51 @@ +using LibRed.Sql.Ast; + +namespace LibRed.Engine.Plan; + +/// Full-table scan of a base table. +public sealed record ScanNode(string Table) : PlanNode; + +/// Index seek/range scan over a named index. +public sealed record IndexScanNode(string Table, string Index, Expression? Predicate) : PlanNode +{ + public override IReadOnlyList Children => []; +} + +/// Applies a boolean predicate to its input rows. +public sealed record FilterNode(PlanNode Input, Expression Predicate) : PlanNode +{ + public override IReadOnlyList Children => [Input]; +} + +/// Projects (and optionally renames) a set of expressions from its input. +public sealed record ProjectNode(PlanNode Input, IReadOnlyList Projection) : PlanNode +{ + public override IReadOnlyList Children => [Input]; +} + +/// Joins two inputs on a condition. +public sealed record JoinNode(PlanNode Left, PlanNode Right, JoinKind Kind, Expression? On) : PlanNode +{ + public override IReadOnlyList Children => [Left, Right]; +} + +/// Groups rows and computes aggregates. +public sealed record AggregateNode( + PlanNode Input, + IReadOnlyList GroupBy, + IReadOnlyList Aggregates) : PlanNode +{ + public override IReadOnlyList Children => [Input]; +} + +/// Orders rows. +public sealed record SortNode(PlanNode Input, IReadOnlyList Keys) : PlanNode +{ + public override IReadOnlyList Children => [Input]; +} + +/// Limits the number of rows (Access TOP n). +public sealed record LimitNode(PlanNode Input, int Count) : PlanNode +{ + public override IReadOnlyList Children => [Input]; +} diff --git a/src/LibRed/LibRed.Engine/Planning/CatalogSchemaProvider.cs b/src/LibRed/LibRed.Engine/Planning/CatalogSchemaProvider.cs new file mode 100644 index 00000000..b205227a --- /dev/null +++ b/src/LibRed/LibRed.Engine/Planning/CatalogSchemaProvider.cs @@ -0,0 +1,37 @@ +using LibRed.Catalog; +using LibRed.Sql.Binding; + +namespace LibRed.Engine.Planning; + +/// +/// Adapts the Core to the SQL binder's . +/// This is the inverted dependency in action: LibRed.Sql defines the interface, +/// LibRed.Engine supplies the storage-backed implementation. +/// +public sealed class CatalogSchemaProvider(JetCatalog catalog) : ISchemaProvider +{ + private readonly JetCatalog _catalog = catalog; + + public ITableSchema? GetTable(string name) + { + TableDef? def = _catalog.FindTable(name); + return def is null ? null : new TableSchema(def); + } + + private sealed class TableSchema(TableDef def) : ITableSchema + { + public string Name => def.Name; + public IReadOnlyList Columns { get; } = + def.Columns.Select(c => (IColumnSchema)new ColumnSchema(c)).ToList(); + + public IColumnSchema? FindColumn(string name) => + Columns.FirstOrDefault(c => string.Equals(c.Name, name, StringComparison.OrdinalIgnoreCase)); + } + + private sealed class ColumnSchema(ColumnDef col) : IColumnSchema + { + public string Name => col.Name; + public bool IsNullable => col.IsNullable; + public Type ClrType => JetClrTypeMap.ToClrType(col.Type); + } +} diff --git a/src/LibRed/LibRed.Engine/Planning/JetClrTypeMap.cs b/src/LibRed/LibRed.Engine/Planning/JetClrTypeMap.cs new file mode 100644 index 00000000..6c3b77bb --- /dev/null +++ b/src/LibRed/LibRed.Engine/Planning/JetClrTypeMap.cs @@ -0,0 +1,25 @@ +using LibRed.Catalog; + +namespace LibRed.Engine.Planning; + +/// Maps Jet/ACE column types to the CLR types the engine exposes. +public static class JetClrTypeMap +{ + public static Type ToClrType(JetDataType type) => type switch + { + JetDataType.Boolean => typeof(bool), + JetDataType.Byte => typeof(byte), + JetDataType.Int16 => typeof(short), + JetDataType.Int32 => typeof(int), + JetDataType.Currency => typeof(decimal), + JetDataType.Single => typeof(float), + JetDataType.Double => typeof(double), + JetDataType.DateTime => typeof(DateTime), + JetDataType.Binary or JetDataType.Ole => typeof(byte[]), + JetDataType.Text or JetDataType.Memo => typeof(string), + JetDataType.Guid => typeof(Guid), + JetDataType.FixedPoint => typeof(decimal), + JetDataType.Complex => typeof(object), + _ => typeof(object), + }; +} diff --git a/src/LibRed/LibRed.Engine/Planning/QueryPlanner.cs b/src/LibRed/LibRed.Engine/Planning/QueryPlanner.cs new file mode 100644 index 00000000..76ee3062 --- /dev/null +++ b/src/LibRed/LibRed.Engine/Planning/QueryPlanner.cs @@ -0,0 +1,41 @@ +using LibRed.Engine.Plan; +using LibRed.Sql.Ast; +using LibRed.Sql.Binding; + +namespace LibRed.Engine.Planning; + +/// +/// Turns a bound statement into a logical tree. Index +/// selection and other optimisations are applied as rewrites over the tree. +/// +public sealed class QueryPlanner +{ + public PlanNode Plan(BoundStatement bound) + { + return bound.Statement switch + { + SelectStatement select => PlanSelect(select), + _ => throw new NotImplementedException( + $"Planning for {bound.Statement.GetType().Name} is not yet implemented."), + }; + } + + private static PlanNode PlanSelect(SelectStatement select) + { + // TODO: build Scan/Join from FROM, push WHERE into FilterNode (or IndexScanNode), + // add Aggregate/Sort/Limit, then Project. This is the naive shape: + PlanNode node = select.From switch + { + NamedTable t => new ScanNode(t.Name), + _ => throw new NotImplementedException("Only single-table FROM is implemented."), + }; + + if (select.Where is not null) + node = new FilterNode(node, select.Where); + + if (select.Top is { } top) + node = new LimitNode(node, top); + + return new ProjectNode(node, select.Projection); + } +} diff --git a/src/LibRed/LibRed.Engine/QueryEngine.cs b/src/LibRed/LibRed.Engine/QueryEngine.cs new file mode 100644 index 00000000..7324de5b --- /dev/null +++ b/src/LibRed/LibRed.Engine/QueryEngine.cs @@ -0,0 +1,49 @@ +using LibRed.Engine.Execution; +using LibRed.Engine.Planning; +using LibRed.Sql.Ast; +using LibRed.Sql.Binding; +using LibRed.Sql.Parsing; + +namespace LibRed.Engine; + +/// +/// The engine facade: wires the full pipeline parse → bind → plan → execute over an +/// open . This is what the ADO provider sits on top of. +/// +public sealed class QueryEngine +{ + private readonly JetDatabase _database; + private readonly ISqlParser _parser; + private readonly Binder _binder; + private readonly QueryPlanner _planner = new(); + private readonly QueryExecutor _executor; + + public QueryEngine(JetDatabase database, ISqlParser? parser = null) + { + _database = database; + _parser = parser ?? new AccessSqlParser(); + _binder = new Binder(new CatalogSchemaProvider(database.Catalog)); + _executor = new QueryExecutor(database); + } + + public JetDatabase Database => _database; + + public ResultSet ExecuteQuery(string sql) + { + var plan = Compile(sql); + return _executor.ExecuteQuery(plan); + } + + public int ExecuteNonQuery(string sql) + { + var plan = Compile(sql); + return _executor.ExecuteNonQuery(plan); + } + + private Plan.PlanNode Compile(string sql) + { + SqlStatement ast = _parser.ParseStatement(sql); + BoundStatement bound = _binder.Bind(ast); + return _planner.Plan(bound); + } +} diff --git a/src/LibRed/LibRed.Sql/Ast/Clauses.cs b/src/LibRed/LibRed.Sql/Ast/Clauses.cs new file mode 100644 index 00000000..aa2bf91b --- /dev/null +++ b/src/LibRed/LibRed.Sql/Ast/Clauses.cs @@ -0,0 +1,23 @@ +namespace LibRed.Sql.Ast; + +/// Base type for a source of rows in a FROM clause. +public abstract record TableReference : SqlNode; + +/// A named base table, optionally aliased. +public sealed record NamedTable(string Name, string? Alias) : TableReference; + +/// A derived table (subquery) in the FROM clause. +public sealed record SubqueryTable(SelectStatement Query, string? Alias) : TableReference; + +public enum JoinKind { Inner, Left, Right, Cross } + +/// A join between two table references with an ON condition. +public sealed record JoinTable( + TableReference Left, + TableReference Right, + JoinKind Kind, + Expression? On) : TableReference; + +public enum SortDirection { Ascending, Descending } + +public sealed record OrderByItem(Expression Value, SortDirection Direction) : SqlNode; diff --git a/src/LibRed/LibRed.Sql/Ast/Expressions.cs b/src/LibRed/LibRed.Sql/Ast/Expressions.cs new file mode 100644 index 00000000..fce0a061 --- /dev/null +++ b/src/LibRed/LibRed.Sql/Ast/Expressions.cs @@ -0,0 +1,32 @@ +namespace LibRed.Sql.Ast; + +/// Base type for scalar/boolean expressions. +public abstract record Expression : SqlNode; + +/// A literal constant (number, string, date, boolean or null). +public sealed record LiteralExpression(object? Value) : Expression; + +/// A reference to a column, optionally table-qualified. +public sealed record ColumnReference(string? Table, string Column) : Expression; + +/// A positional or named query parameter (e.g. ? or @p). +public sealed record ParameterExpression(string Name) : Expression; + +/// * in a projection or aggregate. +public sealed record StarExpression : Expression; + +public enum BinaryOperator +{ + Add, Subtract, Multiply, Divide, Modulo, Concat, + Equal, NotEqual, LessThan, LessThanOrEqual, GreaterThan, GreaterThanOrEqual, + And, Or, Like, In, +} + +public sealed record BinaryExpression(BinaryOperator Operator, Expression Left, Expression Right) : Expression; + +public enum UnaryOperator { Negate, Not, IsNull, IsNotNull } + +public sealed record UnaryExpression(UnaryOperator Operator, Expression Operand) : Expression; + +/// A scalar/aggregate function call, e.g. Count(*), IIf(...), Format(...). +public sealed record FunctionCall(string Name, IReadOnlyList Arguments) : Expression; diff --git a/src/LibRed/LibRed.Sql/Ast/SqlNode.cs b/src/LibRed/LibRed.Sql/Ast/SqlNode.cs new file mode 100644 index 00000000..8589e9f8 --- /dev/null +++ b/src/LibRed/LibRed.Sql/Ast/SqlNode.cs @@ -0,0 +1,9 @@ +namespace LibRed.Sql.Ast; + +/// +/// Base type for every node in the SQL abstract syntax tree. The AST is deliberately +/// decoupled from the ANTLR parse tree: the parser lowers the concrete syntax tree +/// into these nodes so the rest of the engine never depends on the grammar. +/// New SQL features are added by introducing new node types here. +/// +public abstract record SqlNode; diff --git a/src/LibRed/LibRed.Sql/Ast/Statements.cs b/src/LibRed/LibRed.Sql/Ast/Statements.cs new file mode 100644 index 00000000..9a1a1ce3 --- /dev/null +++ b/src/LibRed/LibRed.Sql/Ast/Statements.cs @@ -0,0 +1,33 @@ +namespace LibRed.Sql.Ast; + +/// Base type for top-level SQL statements. +public abstract record SqlStatement : SqlNode; + +/// A single item in a SELECT projection, optionally aliased. +public sealed record SelectItem(Expression Value, string? Alias) : SqlNode; + +public sealed record SelectStatement( + IReadOnlyList Projection, + bool IsSelectStar, + TableReference From, + Expression? Where, + IReadOnlyList GroupBy, + Expression? Having, + IReadOnlyList OrderBy, + int? Top) : SqlStatement; + +public sealed record InsertStatement( + string Table, + IReadOnlyList Columns, + IReadOnlyList> Rows) : SqlStatement; + +public sealed record Assignment(string Column, Expression Value) : SqlNode; + +public sealed record UpdateStatement( + string Table, + IReadOnlyList Assignments, + Expression? Where) : SqlStatement; + +public sealed record DeleteStatement( + string Table, + Expression? Where) : SqlStatement; diff --git a/src/LibRed/LibRed.Sql/Binding/Binder.cs b/src/LibRed/LibRed.Sql/Binding/Binder.cs new file mode 100644 index 00000000..4a2e6e4f --- /dev/null +++ b/src/LibRed/LibRed.Sql/Binding/Binder.cs @@ -0,0 +1,21 @@ +using LibRed.Sql.Ast; + +namespace LibRed.Sql.Binding; + +/// +/// Resolves names and types in a parsed statement against an : +/// verifies tables/columns exist, attaches column types, expands SELECT *, and +/// validates expression operand types. Produces a the +/// engine can plan without re-checking the schema. +/// +public sealed class Binder(ISchemaProvider schema) +{ + private readonly ISchemaProvider _schema = schema; + + public BoundStatement Bind(SqlStatement statement) + { + // TODO: resolve table references, expand projections, type-check expressions. + _ = _schema; + return new BoundStatement(statement); + } +} diff --git a/src/LibRed/LibRed.Sql/Binding/BoundStatement.cs b/src/LibRed/LibRed.Sql/Binding/BoundStatement.cs new file mode 100644 index 00000000..9c537836 --- /dev/null +++ b/src/LibRed/LibRed.Sql/Binding/BoundStatement.cs @@ -0,0 +1,10 @@ +using LibRed.Sql.Ast; + +namespace LibRed.Sql.Binding; + +/// +/// A statement whose names and types have been resolved against the schema. The +/// query planner consumes this rather than the raw AST so it can assume everything +/// referenced is valid. +/// +public sealed record BoundStatement(SqlStatement Statement); diff --git a/src/LibRed/LibRed.Sql/Binding/ISchemaProvider.cs b/src/LibRed/LibRed.Sql/Binding/ISchemaProvider.cs new file mode 100644 index 00000000..51be08ec --- /dev/null +++ b/src/LibRed/LibRed.Sql/Binding/ISchemaProvider.cs @@ -0,0 +1,32 @@ +namespace LibRed.Sql.Binding; + +/// +/// The schema information the binder needs to resolve and type-check a statement. +/// Defined here (not in Core) to keep the dependency inverted: LibRed.Sql has no +/// reference to the storage engine. The Engine project supplies an implementation +/// backed by the Jet catalog. +/// +public interface ISchemaProvider +{ + /// Resolves a table by name, or returns null if it does not exist. + ITableSchema? GetTable(string name); +} + +/// Minimal table shape needed for binding. +public interface ITableSchema +{ + string Name { get; } + IReadOnlyList Columns { get; } + IColumnSchema? FindColumn(string name); +} + +/// Minimal column shape needed for binding. +public interface IColumnSchema +{ + string Name { get; } + + /// The CLR type the column maps to, used for expression type checking. + Type ClrType { get; } + + bool IsNullable { get; } +} diff --git a/src/LibRed/LibRed.Sql/Grammar/AccessSql.g4 b/src/LibRed/LibRed.Sql/Grammar/AccessSql.g4 new file mode 100644 index 00000000..b5fc9e03 --- /dev/null +++ b/src/LibRed/LibRed.Sql/Grammar/AccessSql.g4 @@ -0,0 +1,60 @@ +// ANTLR4 grammar for the Jet/ACE (Microsoft Access) SQL dialect. +// +// This file is intentionally a starting skeleton. It is kept as a plain file (not +// wired into the build) so the project compiles without the ANTLR tool installed. +// To enable code generation, see the commented PackageReference block in +// LibRed.Sql.csproj, then flesh out the rules below. +// +// Dialect notes that make Access SQL differ from ANSI: +// * String concatenation uses '&' (and '+'); strings are delimited by " or '. +// * Wildcards in LIKE are '*' and '?' (ANSI '%' / '_' under ANSI-92 mode). +// * TOP n instead of LIMIT; no OFFSET. +// * IIF(), SWITCH(), Format(), Nz(), and VBA-style date literals (#1/1/2020#). +// * Bracketed identifiers [Order Details]; parameters are positional or named. +// * Jet-specific joins: nested (INNER JOIN ... ) chains, and the Access-only +// "Iif"/domain aggregate functions (DLookup, DCount, ...). + +grammar AccessSql; + +// ---- Parser rules ------------------------------------------------------------- + +statement + : selectStatement + | insertStatement + | updateStatement + | deleteStatement + ; + +selectStatement : SELECT topClause? selectList FROM tableSource whereClause? ; + +insertStatement : INSERT INTO IDENTIFIER /* ... */ ; +updateStatement : UPDATE IDENTIFIER SET /* ... */ ; +deleteStatement : DELETE FROM IDENTIFIER whereClause? ; + +topClause : TOP INTEGER_LITERAL ; +selectList : STAR | expression (COMMA expression)* ; +tableSource : IDENTIFIER ; // TODO: joins, subqueries, aliases +whereClause : WHERE expression ; + +expression : IDENTIFIER | STRING_LITERAL | INTEGER_LITERAL ; // TODO + +// ---- Lexer rules -------------------------------------------------------------- + +SELECT : [Ss][Ee][Ll][Ee][Cc][Tt] ; +FROM : [Ff][Rr][Oo][Mm] ; +WHERE : [Ww][Hh][Ee][Rr][Ee] ; +INSERT : [Ii][Nn][Ss][Ee][Rr][Tt] ; +INTO : [Ii][Nn][Tt][Oo] ; +UPDATE : [Uu][Pp][Dd][Aa][Tt][Ee] ; +DELETE : [Dd][Ee][Ll][Ee][Tt][Ee] ; +SET : [Ss][Ee][Tt] ; +TOP : [Tt][Oo][Pp] ; + +STAR : '*' ; +COMMA : ',' ; + +IDENTIFIER : [A-Za-z_][A-Za-z_0-9]* | '[' ~[\]]+ ']' ; +INTEGER_LITERAL : [0-9]+ ; +STRING_LITERAL : '"' (~["])* '"' | '\'' (~['])* '\'' ; + +WS : [ \t\r\n]+ -> skip ; diff --git a/src/LibRed/LibRed.Sql/LibRed.Sql.csproj b/src/LibRed/LibRed.Sql/LibRed.Sql.csproj new file mode 100644 index 00000000..1ab06195 --- /dev/null +++ b/src/LibRed/LibRed.Sql/LibRed.Sql.csproj @@ -0,0 +1,22 @@ + + + + $(JetTargetFramework) + LibRed.Sql + LibRed.Sql + Jet/ACE SQL front end: lexer, parser, AST and binder. Has no dependency on the storage engine; binding resolves names through an injected schema abstraction. + + + + + diff --git a/src/LibRed/LibRed.Sql/Parsing/AccessSqlParser.cs b/src/LibRed/LibRed.Sql/Parsing/AccessSqlParser.cs new file mode 100644 index 00000000..59f7e4cf --- /dev/null +++ b/src/LibRed/LibRed.Sql/Parsing/AccessSqlParser.cs @@ -0,0 +1,19 @@ +using LibRed.Sql.Ast; + +namespace LibRed.Sql.Parsing; + +/// +/// Default . Will drive the ANTLR-generated lexer/parser and +/// translate its visitor output into the AST. Currently a stub. +/// +public sealed class AccessSqlParser : ISqlParser +{ + public SqlStatement ParseStatement(string sql) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sql); + + // TODO: run the ANTLR lexer + parser, then lower the parse tree via an + // AstBuildingVisitor into the SqlNode hierarchy. + throw new NotImplementedException("SQL parsing is not yet implemented."); + } +} diff --git a/src/LibRed/LibRed.Sql/Parsing/ISqlParser.cs b/src/LibRed/LibRed.Sql/Parsing/ISqlParser.cs new file mode 100644 index 00000000..64a67760 --- /dev/null +++ b/src/LibRed/LibRed.Sql/Parsing/ISqlParser.cs @@ -0,0 +1,14 @@ +using LibRed.Sql.Ast; + +namespace LibRed.Sql.Parsing; + +/// +/// Parses SQL text into an AST. The implementation lowers the ANTLR parse tree into +/// the hierarchy; callers depend only on this abstraction so the +/// grammar can evolve (or be swapped) freely. +/// +public interface ISqlParser +{ + /// Parses a single statement. Throws on syntax errors. + SqlStatement ParseStatement(string sql); +} diff --git a/src/LibRed/LibRed.Sql/Parsing/SqlParseException.cs b/src/LibRed/LibRed.Sql/Parsing/SqlParseException.cs new file mode 100644 index 00000000..0bad9a37 --- /dev/null +++ b/src/LibRed/LibRed.Sql/Parsing/SqlParseException.cs @@ -0,0 +1,9 @@ +namespace LibRed.Sql.Parsing; + +/// Thrown when SQL text cannot be parsed. +public sealed class SqlParseException(string message, int line = 0, int column = 0) + : Exception(message) +{ + public int Line { get; } = line; + public int Column { get; } = column; +} diff --git a/src/LibRed/LibRed.csproj b/src/LibRed/LibRed.csproj deleted file mode 100644 index 5ff3400a..00000000 --- a/src/LibRed/LibRed.csproj +++ /dev/null @@ -1,15 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - diff --git a/src/LibRed/LibRedFile.cs b/src/LibRed/LibRedFile.cs deleted file mode 100644 index eecfe1f0..00000000 --- a/src/LibRed/LibRedFile.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace LibRed; - -public class LibRedFile(string filePath) -{ - private readonly string _filePath = filePath; - private bool _isOpen = false; - public void Open() - { - - } - - public void Close() - { - // Close the file or release resources - } -} \ No newline at end of file diff --git a/src/LibRed/Pages/DatabaseDefinitionPage.cs b/src/LibRed/Pages/DatabaseDefinitionPage.cs deleted file mode 100644 index e479d634..00000000 --- a/src/LibRed/Pages/DatabaseDefinitionPage.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace LibRed.Pages -{ - public class DatabaseDefinitionPage : Page - { - public byte JetVersion { get; internal set; } - public string DatabasePassword { get; set; } - - public int DatabaseKey { get; set; } - public short CodePage { get; set; } - public short TextCollateSortOrder { get; set; } - public string PageKey { get; set; } - public DateTime DatabaseCreationDate { get; set; } - public string CreateProgramName { get; set; } - public DatabaseDefinitionPage() - { - PageType = 0x00; - } - } -} diff --git a/src/LibRed/Pages/Page.cs b/src/LibRed/Pages/Page.cs deleted file mode 100644 index 54eefd33..00000000 --- a/src/LibRed/Pages/Page.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace LibRed.Pages -{ - public abstract class Page - { - public byte PageType { get; internal set; } - public int PageSize { get; set; } - public virtual void ReadPage() - { - } - - public virtual void WritePage() - { - } - } -} diff --git a/src/LibRed/Pages/PageType.cs b/src/LibRed/Pages/PageType.cs deleted file mode 100644 index a63fdf42..00000000 --- a/src/LibRed/Pages/PageType.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace LibRed.Pages -{ - public enum PageType - { - DatabaseDefinition = 0x00, - DatabasePage = 0x01, - TableDefinition = 0x02, - IntermediateIndexPage = 0x03, - LeafIndexPage = 0x04, - PageUsageBitmap = 0x05, - - } -} \ No newline at end of file diff --git a/src/LibRed/README.md b/src/LibRed/README.md new file mode 100644 index 00000000..1b87c0f3 --- /dev/null +++ b/src/LibRed/README.md @@ -0,0 +1,75 @@ +# LibRed — native managed Jet/ACE engine + +A from-scratch, fully managed (cross-platform) implementation of the Microsoft +Jet/ACE database engine — the format behind Access `.mdb` and `.accdb` files. +Unlike the `EntityFrameworkCore.Jet` projects (which rely on Windows-only ODBC/OleDb), +LibRed reads and writes the file format directly. + +## Projects + +| Project | Responsibility | Depends on | +| --- | --- | --- | +| **LibRed.Core** | File format: IO/page channel, version formats, typed pages, catalog, storage (tables, rows, indexes), crypto, memo/OLE | — | +| **LibRed.Sql** | SQL front end: ANTLR grammar, AST, parser, binder. No engine dependency; binds through an injected `ISchemaProvider` | — | +| **LibRed.Engine** | Plans and executes bound statements over Core; bridges the catalog to the SQL binder | Core, Sql | +| **LibRed.Ado** | ADO.NET surface: `DbConnection`/`DbCommand`/`DbDataReader`/`DbParameter`/`DbTransaction`/`DbProviderFactory` | Engine | +| **LibRed.EFCore** | EF Core provider built on the ADO layer (placeholder) | Ado | + +Dependency graph (a clean DAG, no cycles): + +``` +EFCore → Ado → Engine → Sql + ↘ Core +``` + +## Layering inside LibRed.Core + +``` +Storage (Table, TableCursor, RowDecoder, UsageMap, Types/JetTypeCodec) + ↓ +Catalog (JetCatalog → TableDef / ColumnDef / IndexDef, JetDataType) + ↓ +Pages (DatabaseDefinition, TableDefinition, Data, Index, UsageMap, Lval) + ↓ +IO (PageChannel, PageBuffer) ← Crypto decrypts pages here + ↓ +Formats (JetFormatBase + Jet3/4/12/14/16/17 — offsets & constants only) +``` + +## Status + +This is a structural scaffold. Almost every method body is a documented `TODO`. +The binary-layout work (steps 3–8 below) is best driven from the +[mdbtools](https://github.com/mdbtools/mdbtools) (`src/libmdb/`) and +[Jackcess](https://jackcess.sourceforge.io/) sources, which thoroughly document the +on-disk structures. + +### Suggested build order + +1. `PageBuffer` + `PageChannel` — read raw bytes from a file +2. Fill in `JetFormatBase` constants for Jet 3 / Jet 4 +3. `DatabaseDefinitionPage` — parse page 0, confirm version & page size +4. `TableDefinitionPage` + `ColumnDef` — parse a table's column layout +5. `DataPage` + `RowDecoder` — read rows from a known table +6. `UsageMap` — enumerate all of a table's data pages +7. `JetCatalog` — bootstrap from `MSysObjects` +8. `Table` + `TableCursor` — full table scan end-to-end +9. ANTLR grammar + a `SELECT * FROM t` executor through the full pipeline +10. ADO.NET wrapper, then the EF Core provider + +## SQL pipeline + +Always run through the full pipeline, even for trivial queries, so adding features +later (joins, aggregates, subqueries, then newer SQL) means adding node types rather +than rewriting: + +``` +text → ISqlParser → AST → Binder(ISchemaProvider) → BoundStatement + → QueryPlanner → PlanNode tree → QueryExecutor → ResultSet +``` + +### Enabling ANTLR + +The grammar lives at `LibRed.Sql/Grammar/AccessSql.g4` but is **not** wired into the +build yet (so the solution compiles without the ANTLR tool). To turn it on, uncomment +the `Antlr4BuildTasks` block in `LibRed.Sql.csproj`. From e23261d2d5d1290813e4f3859344b8f7dd62e3e1 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sat, 27 Jun 2026 21:05:14 +0800 Subject: [PATCH 004/506] @ LibRed: detect format from page 0, with verification test Implement the verifiable slice of build-order steps 1-3 against a real ACCDB: - JetFormatBase validates the "Standard Jet DB"/"Standard ACE DB" magic string at offset 0x04 and maps the version byte at 0x14 to a format - DatabaseDefinitionPage.Read populates the format identifier and version (fields beyond offset 0x18 are obfuscated and left as documented TODOs) - JetDatabase exposes the decoded DefinitionPage - PageChannel.Open no longer leaks the FileStream when detection fails Add test/LibRed.Core.Tests opening the tracked Northwind.accdb: asserts ACE 12 / ACCDB / 4096-byte pages, and that a non-Jet file is rejected. The leak fix above was caught by the rejection test. Co-Authored-By: Claude Opus 4.8 @ --- EFCore.Jet.sln | 15 ++++++++ .../LibRed.Core/Formats/JetFormatBase.cs | 21 ++++++++++ src/LibRed/LibRed.Core/IO/PageChannel.cs | 12 +++++- src/LibRed/LibRed.Core/JetDatabase.cs | 8 ++++ .../Pages/DatabaseDefinitionPage.cs | 8 +++- .../DatabaseDefinitionPageTests.cs | 35 +++++++++++++++++ .../LibRed.Core.Tests.csproj | 38 +++++++++++++++++++ test/LibRed.Core.Tests/TestDatabases.cs | 9 +++++ 8 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 test/LibRed.Core.Tests/DatabaseDefinitionPageTests.cs create mode 100644 test/LibRed.Core.Tests/LibRed.Core.Tests.csproj create mode 100644 test/LibRed.Core.Tests/TestDatabases.cs diff --git a/EFCore.Jet.sln b/EFCore.Jet.sln index 7c3c5311..37ce7ef9 100644 --- a/EFCore.Jet.sln +++ b/EFCore.Jet.sln @@ -68,6 +68,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibRed.Ado", "src\LibRed\Li EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibRed.EFCore", "src\LibRed\LibRed.EFCore\LibRed.EFCore.csproj", "{4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibRed.Core.Tests", "test\LibRed.Core.Tests\LibRed.Core.Tests.csproj", "{0624D868-C3E5-411D-A533-6E57F8F79F01}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -246,6 +248,18 @@ Global {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}.Release|x64.Build.0 = Release|Any CPU {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}.Release|x86.ActiveCfg = Release|Any CPU {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71}.Release|x86.Build.0 = Release|Any CPU + {0624D868-C3E5-411D-A533-6E57F8F79F01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0624D868-C3E5-411D-A533-6E57F8F79F01}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0624D868-C3E5-411D-A533-6E57F8F79F01}.Debug|x64.ActiveCfg = Debug|Any CPU + {0624D868-C3E5-411D-A533-6E57F8F79F01}.Debug|x64.Build.0 = Debug|Any CPU + {0624D868-C3E5-411D-A533-6E57F8F79F01}.Debug|x86.ActiveCfg = Debug|Any CPU + {0624D868-C3E5-411D-A533-6E57F8F79F01}.Debug|x86.Build.0 = Debug|Any CPU + {0624D868-C3E5-411D-A533-6E57F8F79F01}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0624D868-C3E5-411D-A533-6E57F8F79F01}.Release|Any CPU.Build.0 = Release|Any CPU + {0624D868-C3E5-411D-A533-6E57F8F79F01}.Release|x64.ActiveCfg = Release|Any CPU + {0624D868-C3E5-411D-A533-6E57F8F79F01}.Release|x64.Build.0 = Release|Any CPU + {0624D868-C3E5-411D-A533-6E57F8F79F01}.Release|x86.ActiveCfg = Release|Any CPU + {0624D868-C3E5-411D-A533-6E57F8F79F01}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -265,6 +279,7 @@ Global {8FF045CF-5CDA-4C1B-8C67-2C0DAEDA6A54} = {53AEDF4E-744F-DCCC-ECB4-9EF88FC7B191} {0932994D-79FA-48B5-8EB1-6EC9F8A0CE63} = {53AEDF4E-744F-DCCC-ECB4-9EF88FC7B191} {4B82F36F-FC75-4F0D-B19C-C605FBCA7B71} = {53AEDF4E-744F-DCCC-ECB4-9EF88FC7B191} + {0624D868-C3E5-411D-A533-6E57F8F79F01} = {53AEDF4E-744F-DCCC-ECB4-9EF88FC7B191} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9359773D-6399-447E-9814-6CB41C2FB664} diff --git a/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs index 513cd354..603799dd 100644 --- a/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs +++ b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs @@ -15,6 +15,18 @@ public abstract class JetFormatBase /// Offset of the one-byte format version marker within page 0. public const int VersionOffset = 0x14; + /// Offset of the ASCII format identifier string within page 0. + public const int FormatIdentifierOffset = 0x04; + + /// Length of the format identifier string (excluding its NUL terminator). + public const int FormatIdentifierLength = 15; + + /// Identifier for the MDB (Jet 3/4) family. + public const string JetIdentifier = "Standard Jet DB"; + + /// Identifier for the ACCDB (ACE 12+) family. + public const string AceIdentifier = "Standard ACE DB"; + /// Page size in bytes (2048 for Jet 3, 4096 for Jet 4 and all ACE versions). public int PageSize { get; protected set; } = 4096; @@ -36,9 +48,18 @@ public static JetFormatBase Detect(Stream stream) stream.ReadExactly(header); stream.Seek(original, SeekOrigin.Begin); + string identifier = ReadFormatIdentifier(header); + if (identifier is not (JetIdentifier or AceIdentifier)) + throw new NotSupportedException( + $"Not a Jet/ACE database: expected \"{JetIdentifier}\" or \"{AceIdentifier}\" at offset 0x{FormatIdentifierOffset:X2}, found \"{identifier}\"."); + return FromVersionByte(header[VersionOffset]); } + /// Reads the ASCII format identifier ("Standard Jet DB"/"Standard ACE DB") from a page-0 header. + public static string ReadFormatIdentifier(ReadOnlySpan header) => + System.Text.Encoding.ASCII.GetString(header.Slice(FormatIdentifierOffset, FormatIdentifierLength)).TrimEnd('\0'); + /// Maps the raw version byte at to a format instance. public static JetFormatBase FromVersionByte(byte versionByte) => versionByte switch { diff --git a/src/LibRed/LibRed.Core/IO/PageChannel.cs b/src/LibRed/LibRed.Core/IO/PageChannel.cs index 084f9282..2ad8c222 100644 --- a/src/LibRed/LibRed.Core/IO/PageChannel.cs +++ b/src/LibRed/LibRed.Core/IO/PageChannel.cs @@ -38,8 +38,16 @@ public static PageChannel Open(string path, bool readOnly = true) readOnly ? FileAccess.Read : FileAccess.ReadWrite, readOnly ? FileShare.Read : FileShare.None); - var format = JetFormatBase.Detect(stream); - return new PageChannel(stream, format, readOnly); + try + { + var format = JetFormatBase.Detect(stream); + return new PageChannel(stream, format, readOnly); + } + catch + { + stream.Dispose(); + throw; + } } /// Reads a single page into a freshly allocated buffer. diff --git a/src/LibRed/LibRed.Core/JetDatabase.cs b/src/LibRed/LibRed.Core/JetDatabase.cs index a4af72a5..95ab12d5 100644 --- a/src/LibRed/LibRed.Core/JetDatabase.cs +++ b/src/LibRed/LibRed.Core/JetDatabase.cs @@ -1,6 +1,7 @@ using LibRed.Catalog; using LibRed.Formats; using LibRed.IO; +using LibRed.Pages; using LibRed.Storage; namespace LibRed; @@ -17,9 +18,16 @@ public sealed class JetDatabase : IDisposable private JetDatabase(PageChannel channel) { _channel = channel; + + DefinitionPage = new DatabaseDefinitionPage(); + DefinitionPage.Read(channel.ReadPage(0)); + Catalog = new JetCatalog(channel); } + /// The decoded database definition page (page 0). + public DatabaseDefinitionPage DefinitionPage { get; } + /// Opens a database file (read-only by default). public static JetDatabase Open(string path, bool readOnly = true) => new(PageChannel.Open(path, readOnly)); diff --git a/src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs b/src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs index fb146dab..76251a2c 100644 --- a/src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs +++ b/src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs @@ -11,6 +11,9 @@ public sealed class DatabaseDefinitionPage : Page { public override PageType Type => PageType.DatabaseDefinition; + /// The ASCII format identifier, e.g. "Standard Jet DB" or "Standard ACE DB". + public string FormatIdentifier { get; internal set; } = string.Empty; + public byte JetVersion { get; internal set; } public string? DatabasePassword { get; internal set; } public int DatabaseKey { get; internal set; } @@ -23,9 +26,12 @@ public sealed class DatabaseDefinitionPage : Page public override void Read(PageBuffer buffer) { PageNumber = buffer.PageNumber; + FormatIdentifier = Formats.JetFormatBase.ReadFormatIdentifier(buffer.Span); JetVersion = buffer.ReadByte(Formats.JetFormatBase.VersionOffset); // TODO: decode code page, collation, creation date and encryption material. - // Layout differs between Jet 3, Jet 4 and ACE — drive it off the resolved format. + // Everything from offset 0x18 onward on this page is obfuscated/encrypted and the + // layout differs between Jet 3, Jet 4 and ACE — drive it off the resolved format + // (see mdbtools mdb_read_db_page / Jackcess DatabaseImpl.readDatabaseDefinition). } } diff --git a/test/LibRed.Core.Tests/DatabaseDefinitionPageTests.cs b/test/LibRed.Core.Tests/DatabaseDefinitionPageTests.cs new file mode 100644 index 00000000..fae5cbcf --- /dev/null +++ b/test/LibRed.Core.Tests/DatabaseDefinitionPageTests.cs @@ -0,0 +1,35 @@ +using LibRed; +using LibRed.Formats; +using Xunit; + +namespace LibRed.Core.Tests; + +public class DatabaseDefinitionPageTests +{ + [Fact] + public void Opens_accdb_and_detects_format() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + Assert.Equal(JetFormatBase.AceIdentifier, db.DefinitionPage.FormatIdentifier); + Assert.Equal(0x02, db.DefinitionPage.JetVersion); + Assert.Equal(JetVersion.Version12_2007, db.Format.Version); + Assert.True(db.Format.IsAccdb); + Assert.Equal(4096, db.Format.PageSize); + } + + [Fact] + public void Rejects_non_jet_file() + { + string bogus = Path.Combine(Path.GetTempPath(), $"libred_{Guid.NewGuid():N}.bin"); + File.WriteAllBytes(bogus, new byte[4096]); + try + { + Assert.Throws(() => JetDatabase.Open(bogus)); + } + finally + { + File.Delete(bogus); + } + } +} diff --git a/test/LibRed.Core.Tests/LibRed.Core.Tests.csproj b/test/LibRed.Core.Tests/LibRed.Core.Tests.csproj new file mode 100644 index 00000000..d7f4badd --- /dev/null +++ b/test/LibRed.Core.Tests/LibRed.Core.Tests.csproj @@ -0,0 +1,38 @@ + + + + $(JetTargetFramework) + LibRed.Core.Tests + enable + enable + false + true + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + Data\Northwind.accdb + PreserveNewest + + + + diff --git a/test/LibRed.Core.Tests/TestDatabases.cs b/test/LibRed.Core.Tests/TestDatabases.cs new file mode 100644 index 00000000..d206c220 --- /dev/null +++ b/test/LibRed.Core.Tests/TestDatabases.cs @@ -0,0 +1,9 @@ +namespace LibRed.Core.Tests; + +/// Paths to the real database files copied alongside the test assembly. +internal static class TestDatabases +{ + /// An Access 2007 (ACE 12 / ACCDB) Northwind sample. + public static string NorthwindAccdb { get; } = + Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"); +} From 3ed873e8f6181b6e186d668eb57c2bc832f68c7c Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sat, 27 Jun 2026 21:16:17 +0800 Subject: [PATCH 005/506] @ LibRed: parse table definition (TDEF) pages Decode the Jet 4 / ACE TDEF layout, verified byte-by-byte against MSysObjects in the Northwind sample: - TableDefinitionPage reads row count, table type, column/index counts, and the column descriptors + UTF-16LE names (column block starts after the real-index block at 0x3F + realIndexCount*12; 25-byte descriptors; 2-byte name lengths) - Verified TDEF/column offsets added to JetFormatBase as virtual members (Jet 4/ACE defaults; Jet 3 will override) - Page.Read now takes the JetFormatBase so pages use version-specific offsets; all page overrides updated - JetDatabase.ReadTableDefinition(page) exposes it; ColumnDef/TableType populated Tests decode MSysObjects (17 columns, system type, names incl. Id/Name/Type/Flags) from the real file. Multi-page TDEFs and index parsing remain TODO. Co-Authored-By: Claude Opus 4.8 @ --- src/LibRed/LibRed.Core/Catalog/TableType.cs | 11 +++ .../LibRed.Core/Formats/JetFormatBase.cs | 47 +++++++++++ src/LibRed/LibRed.Core/JetDatabase.cs | 10 ++- src/LibRed/LibRed.Core/Pages/DataPage.cs | 3 +- .../Pages/DatabaseDefinitionPage.cs | 2 +- src/LibRed/LibRed.Core/Pages/IndexPage.cs | 3 +- src/LibRed/LibRed.Core/Pages/LvalPage.cs | 3 +- src/LibRed/LibRed.Core/Pages/Page.cs | 5 +- .../LibRed.Core/Pages/TableDefinitionPage.cs | 81 ++++++++++++++++--- src/LibRed/LibRed.Core/Pages/UsageMapPage.cs | 3 +- src/LibRed/LibRed.Core/Storage/TableCursor.cs | 2 +- .../TableDefinitionPageTests.cs | 48 +++++++++++ 12 files changed, 200 insertions(+), 18 deletions(-) create mode 100644 src/LibRed/LibRed.Core/Catalog/TableType.cs create mode 100644 test/LibRed.Core.Tests/TableDefinitionPageTests.cs diff --git a/src/LibRed/LibRed.Core/Catalog/TableType.cs b/src/LibRed/LibRed.Core/Catalog/TableType.cs new file mode 100644 index 00000000..aa74ca85 --- /dev/null +++ b/src/LibRed/LibRed.Core/Catalog/TableType.cs @@ -0,0 +1,11 @@ +namespace LibRed.Catalog; + +/// The table-type marker in a TDEF page (byte at the format's table-type offset). +public enum TableType : byte +{ + /// A user table ('N'). + User = 0x4E, + + /// A system (MSys*) table ('S'). + System = 0x53, +} diff --git a/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs index 603799dd..d6077d37 100644 --- a/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs +++ b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs @@ -27,6 +27,53 @@ public abstract class JetFormatBase /// Identifier for the ACCDB (ACE 12+) family. public const string AceIdentifier = "Standard ACE DB"; + // --- Table definition (TDEF) page layout --- + // Defaults below are for Jet 4 / ACE (verified against a real ACCDB). Jet 3 differs + // (18-byte column entries, 1-byte ASCII name lengths) and will override these. + + /// Offset of the 4-byte pointer to the next TDEF page (0 if the definition fits one page). + public virtual int TdefNextPageOffset => 0x04; + + /// Offset of the 4-byte row count. + public virtual int TdefRowCountOffset => 0x10; + + /// Offset of the 1-byte table type (0x4E 'N' user, 0x53 'S' system). + public virtual int TdefTableTypeOffset => 0x28; + + /// Offset of the 2-byte variable-length column count. + public virtual int TdefVariableColumnsOffset => 0x2B; + + /// Offset of the 2-byte total column count. + public virtual int TdefColumnCountOffset => 0x2D; + + /// Offset of the 4-byte real-index (slot) count, used to size the index block before columns. + public virtual int TdefRealIndexCountOffset => 0x2F; + + /// Offset of the 4-byte logical index count. + public virtual int TdefIndexCountOffset => 0x33; + + /// Offset where the real-index block begins; column descriptors follow it. + public virtual int TdefRealIndexBlockOffset => 0x3F; + + /// Size in bytes of each real-index entry in the block before the column descriptors. + public virtual int RealIndexEntrySize => 12; + + /// Size in bytes of one column descriptor. + public virtual int ColumnDescriptorSize => 25; + + // --- Column descriptor layout (offsets within a single descriptor) --- + public virtual int ColumnTypeOffset => 0x00; + public virtual int ColumnNumberOffset => 0x05; + public virtual int ColumnFlagsOffset => 0x0F; + public virtual int ColumnFixedOffsetOffset => 0x15; + public virtual int ColumnLengthOffset => 0x17; + + /// Column flag: the column is fixed-length. + public const byte ColumnFlagFixedLength = 0x01; + + /// Column flag: the column is an AutoNumber. + public const byte ColumnFlagAutoNumber = 0x04; + /// Page size in bytes (2048 for Jet 3, 4096 for Jet 4 and all ACE versions). public int PageSize { get; protected set; } = 4096; diff --git a/src/LibRed/LibRed.Core/JetDatabase.cs b/src/LibRed/LibRed.Core/JetDatabase.cs index 95ab12d5..d38171e2 100644 --- a/src/LibRed/LibRed.Core/JetDatabase.cs +++ b/src/LibRed/LibRed.Core/JetDatabase.cs @@ -20,7 +20,7 @@ private JetDatabase(PageChannel channel) _channel = channel; DefinitionPage = new DatabaseDefinitionPage(); - DefinitionPage.Read(channel.ReadPage(0)); + DefinitionPage.Read(channel.ReadPage(0), channel.Format); Catalog = new JetCatalog(channel); } @@ -28,6 +28,14 @@ private JetDatabase(PageChannel channel) /// The decoded database definition page (page 0). public DatabaseDefinitionPage DefinitionPage { get; } + /// Reads and decodes the table definition (TDEF) page at . + public TableDefinitionPage ReadTableDefinition(int pageNumber) + { + var tdef = new TableDefinitionPage(); + tdef.Read(_channel.ReadPage(pageNumber), _channel.Format); + return tdef; + } + /// Opens a database file (read-only by default). public static JetDatabase Open(string path, bool readOnly = true) => new(PageChannel.Open(path, readOnly)); diff --git a/src/LibRed/LibRed.Core/Pages/DataPage.cs b/src/LibRed/LibRed.Core/Pages/DataPage.cs index 586e5ac1..28217b1f 100644 --- a/src/LibRed/LibRed.Core/Pages/DataPage.cs +++ b/src/LibRed/LibRed.Core/Pages/DataPage.cs @@ -1,3 +1,4 @@ +using LibRed.Formats; using LibRed.IO; namespace LibRed.Pages; @@ -17,7 +18,7 @@ public sealed class DataPage : Page /// Number of row slots present on the page. public int RowCount { get; internal set; } - public override void Read(PageBuffer buffer) + public override void Read(PageBuffer buffer, JetFormatBase format) { PageNumber = buffer.PageNumber; // TODO: read free-space, owning-table pointer and the row-offset slot table. diff --git a/src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs b/src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs index 76251a2c..cb5dd24d 100644 --- a/src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs +++ b/src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs @@ -23,7 +23,7 @@ public sealed class DatabaseDefinitionPage : Page public DateTime DatabaseCreationDate { get; internal set; } public string? CreateProgramName { get; internal set; } - public override void Read(PageBuffer buffer) + public override void Read(PageBuffer buffer, Formats.JetFormatBase format) { PageNumber = buffer.PageNumber; FormatIdentifier = Formats.JetFormatBase.ReadFormatIdentifier(buffer.Span); diff --git a/src/LibRed/LibRed.Core/Pages/IndexPage.cs b/src/LibRed/LibRed.Core/Pages/IndexPage.cs index 3742db71..c02c5982 100644 --- a/src/LibRed/LibRed.Core/Pages/IndexPage.cs +++ b/src/LibRed/LibRed.Core/Pages/IndexPage.cs @@ -1,3 +1,4 @@ +using LibRed.Formats; using LibRed.IO; namespace LibRed.Pages; @@ -14,7 +15,7 @@ public sealed class IndexPage : Page public override PageType Type => IsLeaf ? PageType.LeafIndexPage : PageType.IntermediateIndexPage; - public override void Read(PageBuffer buffer) + public override void Read(PageBuffer buffer, JetFormatBase format) { PageNumber = buffer.PageNumber; // TODO: decode index entries (column-order-preserving encoded keys) and child pointers. diff --git a/src/LibRed/LibRed.Core/Pages/LvalPage.cs b/src/LibRed/LibRed.Core/Pages/LvalPage.cs index 14976b4f..cfdce08f 100644 --- a/src/LibRed/LibRed.Core/Pages/LvalPage.cs +++ b/src/LibRed/LibRed.Core/Pages/LvalPage.cs @@ -1,3 +1,4 @@ +using LibRed.Formats; using LibRed.IO; namespace LibRed.Pages; @@ -11,7 +12,7 @@ public sealed class LvalPage : Page { public override PageType Type => PageType.DataPage; // LVAL pages reuse the data-page type marker. - public override void Read(PageBuffer buffer) + public override void Read(PageBuffer buffer, JetFormatBase format) { PageNumber = buffer.PageNumber; // TODO: decode LVAL chunk header and payload / next-chunk pointer. diff --git a/src/LibRed/LibRed.Core/Pages/Page.cs b/src/LibRed/LibRed.Core/Pages/Page.cs index 4bb1a5f3..7a4bf828 100644 --- a/src/LibRed/LibRed.Core/Pages/Page.cs +++ b/src/LibRed/LibRed.Core/Pages/Page.cs @@ -1,3 +1,4 @@ +using LibRed.Formats; using LibRed.IO; namespace LibRed.Pages; @@ -14,6 +15,6 @@ public abstract class Page /// The page type marker (byte 0 of the page). public abstract PageType Type { get; } - /// Decodes this page's fields from the supplied buffer. - public abstract void Read(PageBuffer buffer); + /// Decodes this page's fields from the supplied buffer using version-specific offsets. + public abstract void Read(PageBuffer buffer, JetFormatBase format); } diff --git a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs index 03b77c47..6225cdaf 100644 --- a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs +++ b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs @@ -1,24 +1,87 @@ +using System.Text; +using LibRed.Catalog; +using LibRed.Formats; using LibRed.IO; namespace LibRed.Pages; /// -/// A table definition (TDEF) page: row count, column definitions, index metadata -/// and the pointer to the first data/usage page for the table. May span multiple -/// pages for wide tables (continued via a "next page" pointer). +/// A table definition (TDEF) page: row count, table type, and the column descriptors +/// and names. Verified against the Jet 4 / ACE layout. May be continued across pages +/// for wide tables (see ). /// public sealed class TableDefinitionPage : Page { + private readonly List _columns = []; + public override PageType Type => PageType.TableDefinition; - public int RowCount { get; internal set; } - public int ColumnCount { get; internal set; } - public int IndexCount { get; internal set; } + public int NextDefinitionPage { get; private set; } + public int RowCount { get; private set; } + public TableType TableType { get; private set; } + public int VariableColumnCount { get; private set; } + public int ColumnCount { get; private set; } + public int RealIndexCount { get; private set; } + public int IndexCount { get; private set; } + + public IReadOnlyList Columns => _columns; - public override void Read(PageBuffer buffer) + public override void Read(PageBuffer buffer, JetFormatBase format) { PageNumber = buffer.PageNumber; - // TODO: parse TDEF header, column descriptors, index descriptors and - // real-index/all-index counts. See mdbtools read_table / Jackcess TableImpl. + + NextDefinitionPage = buffer.ReadInt32(format.TdefNextPageOffset); + RowCount = buffer.ReadInt32(format.TdefRowCountOffset); + TableType = (TableType)buffer.ReadByte(format.TdefTableTypeOffset); + VariableColumnCount = buffer.ReadUInt16(format.TdefVariableColumnsOffset); + ColumnCount = buffer.ReadUInt16(format.TdefColumnCountOffset); + RealIndexCount = buffer.ReadInt32(format.TdefRealIndexCountOffset); + IndexCount = buffer.ReadInt32(format.TdefIndexCountOffset); + + // The column descriptors follow the real-index block. + // NOTE: assumes a single-page TDEF. A multi-page TDEF (NextDefinitionPage != 0) + // must have its pages stitched into one contiguous buffer first. TODO. + int columnBlock = format.TdefRealIndexBlockOffset + RealIndexCount * format.RealIndexEntrySize; + ReadColumns(buffer, format, columnBlock); + } + + private void ReadColumns(PageBuffer buffer, JetFormatBase format, int columnBlock) + { + _columns.Clear(); + + // Pass 1: fixed-size column descriptors. + var descriptors = new (JetDataType Type, int ColumnId, byte Flags, int Length)[ColumnCount]; + for (int i = 0; i < ColumnCount; i++) + { + int entry = columnBlock + i * format.ColumnDescriptorSize; + descriptors[i] = ( + (JetDataType)buffer.ReadByte(entry + format.ColumnTypeOffset), + buffer.ReadUInt16(entry + format.ColumnNumberOffset), + buffer.ReadByte(entry + format.ColumnFlagsOffset), + buffer.ReadUInt16(entry + format.ColumnLengthOffset)); + } + + // Pass 2: column names, in the same order, immediately after the descriptor block. + // Each name is a 2-byte (little-endian) byte length followed by UTF-16LE text. + int namePos = columnBlock + ColumnCount * format.ColumnDescriptorSize; + for (int i = 0; i < ColumnCount; i++) + { + int byteLength = buffer.ReadUInt16(namePos); + namePos += 2; + string name = Encoding.Unicode.GetString(buffer.Slice(namePos, byteLength)); + namePos += byteLength; + + var d = descriptors[i]; + _columns.Add(new ColumnDef + { + Name = name, + Type = d.Type, + Index = i, + ColumnId = d.ColumnId, + Length = d.Length, + IsFixedLength = (d.Flags & JetFormatBase.ColumnFlagFixedLength) != 0, + IsAutoNumber = (d.Flags & JetFormatBase.ColumnFlagAutoNumber) != 0, + }); + } } } diff --git a/src/LibRed/LibRed.Core/Pages/UsageMapPage.cs b/src/LibRed/LibRed.Core/Pages/UsageMapPage.cs index fc8d5f43..6753585e 100644 --- a/src/LibRed/LibRed.Core/Pages/UsageMapPage.cs +++ b/src/LibRed/LibRed.Core/Pages/UsageMapPage.cs @@ -1,3 +1,4 @@ +using LibRed.Formats; using LibRed.IO; namespace LibRed.Pages; @@ -11,7 +12,7 @@ public sealed class UsageMapPage : Page { public override PageType Type => PageType.PageUsageBitmap; - public override void Read(PageBuffer buffer) + public override void Read(PageBuffer buffer, JetFormatBase format) { PageNumber = buffer.PageNumber; // TODO: decode the bitmap payload. diff --git a/src/LibRed/LibRed.Core/Storage/TableCursor.cs b/src/LibRed/LibRed.Core/Storage/TableCursor.cs index 69670030..e3b5de1d 100644 --- a/src/LibRed/LibRed.Core/Storage/TableCursor.cs +++ b/src/LibRed/LibRed.Core/Storage/TableCursor.cs @@ -18,7 +18,7 @@ public sealed class TableCursor(Table table) : IEnumerable { PageBuffer buffer = _table.Channel.ReadPage(pageNumber); var dataPage = new DataPage(); - dataPage.Read(buffer); + dataPage.Read(buffer, _table.Channel.Format); // TODO: walk the page's row slot directory and decode each row record. _ = decoder; diff --git a/test/LibRed.Core.Tests/TableDefinitionPageTests.cs b/test/LibRed.Core.Tests/TableDefinitionPageTests.cs new file mode 100644 index 00000000..a8bbd435 --- /dev/null +++ b/test/LibRed.Core.Tests/TableDefinitionPageTests.cs @@ -0,0 +1,48 @@ +using LibRed; +using LibRed.Catalog; +using Xunit; + +namespace LibRed.Core.Tests; + +public class TableDefinitionPageTests +{ + // In Jet 4 / ACE the system catalog table MSysObjects has its TDEF on page 2. + private const int MSysObjectsPage = 2; + + [Fact] + public void Reads_MSysObjects_table_definition() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var tdef = db.ReadTableDefinition(MSysObjectsPage); + + Assert.Equal(TableType.System, tdef.TableType); + Assert.Equal(17, tdef.ColumnCount); + Assert.Equal(17, tdef.Columns.Count); + Assert.Equal(2, tdef.IndexCount); + Assert.True(tdef.RowCount > 0); + Assert.Equal(0, tdef.NextDefinitionPage); // fits in a single page + } + + [Fact] + public void Decodes_MSysObjects_column_names() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var tdef = db.ReadTableDefinition(MSysObjectsPage); + var names = tdef.Columns.Select(c => c.Name).ToList(); + + // Known MSysObjects columns (a few stable ones, observed in the file). + Assert.Contains("Name", names); + Assert.Contains("Type", names); + Assert.Contains("Id", names); + Assert.Contains("Flags", names); + + // Every column has a non-empty name and a recognised data type. + Assert.All(tdef.Columns, c => + { + Assert.False(string.IsNullOrEmpty(c.Name)); + Assert.True(Enum.IsDefined(c.Type)); + }); + } +} From 41cb94511fabca90d09f5ea717479add65cef33c Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sat, 27 Jun 2026 21:22:47 +0800 Subject: [PATCH 006/506] @ LibRed: parse data pages (row slot directory + LVAL detection) DataPage decodes the Jet 4 / ACE data-page header and row slot directory, verified against the Northwind sample: - owning-table TDEF page at offset 0x04; the "LVAL" marker identifies long-value (memo/OLE) pages - row count at 0x0C, 2-byte slot directory at 0x0E with offset mask 0x1FFF and deleted/overflow flags; rows packed from the page end backward - GetRow(i) returns the raw row bytes; JetDatabase.ReadDataPage(page) exposes it Tests verify MSysObjects data (page 17, owner=2, 41 rows), end-packed contiguous slots, and LVAL detection (page 42). Row field decoding is the next step. Co-Authored-By: Claude Opus 4.8 @ --- .../LibRed.Core/Formats/JetFormatBase.cs | 14 ++++ src/LibRed/LibRed.Core/JetDatabase.cs | 8 +++ src/LibRed/LibRed.Core/Pages/DataPage.cs | 68 ++++++++++++++++--- test/LibRed.Core.Tests/DataPageTests.cs | 50 ++++++++++++++ 4 files changed, 132 insertions(+), 8 deletions(-) create mode 100644 test/LibRed.Core.Tests/DataPageTests.cs diff --git a/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs index d6077d37..57dc47e2 100644 --- a/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs +++ b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs @@ -74,6 +74,20 @@ public abstract class JetFormatBase /// Column flag: the column is an AutoNumber. public const byte ColumnFlagAutoNumber = 0x04; + // --- Data page layout (Jet 4 / ACE) --- + + /// Offset of the 2-byte free-space count on a data page. + public virtual int DataFreeSpaceOffset => 0x02; + + /// Offset of the 4-byte owning-table TDEF page (or the "LVAL" marker on long-value pages). + public virtual int DataOwnerOffset => 0x04; + + /// Offset of the 2-byte row count on a data page. + public virtual int DataRowCountOffset => 0x0C; + + /// Offset of the row-offset slot directory (2 bytes per row). + public virtual int DataRowDirectoryOffset => 0x0E; + /// Page size in bytes (2048 for Jet 3, 4096 for Jet 4 and all ACE versions). public int PageSize { get; protected set; } = 4096; diff --git a/src/LibRed/LibRed.Core/JetDatabase.cs b/src/LibRed/LibRed.Core/JetDatabase.cs index d38171e2..73bd544e 100644 --- a/src/LibRed/LibRed.Core/JetDatabase.cs +++ b/src/LibRed/LibRed.Core/JetDatabase.cs @@ -36,6 +36,14 @@ public TableDefinitionPage ReadTableDefinition(int pageNumber) return tdef; } + /// Reads and decodes the data page at . + public DataPage ReadDataPage(int pageNumber) + { + var page = new DataPage(); + page.Read(_channel.ReadPage(pageNumber), _channel.Format); + return page; + } + /// Opens a database file (read-only by default). public static JetDatabase Open(string path, bool readOnly = true) => new(PageChannel.Open(path, readOnly)); diff --git a/src/LibRed/LibRed.Core/Pages/DataPage.cs b/src/LibRed/LibRed.Core/Pages/DataPage.cs index 28217b1f..d15e9aea 100644 --- a/src/LibRed/LibRed.Core/Pages/DataPage.cs +++ b/src/LibRed/LibRed.Core/Pages/DataPage.cs @@ -3,24 +3,76 @@ namespace LibRed.Pages; +/// One entry in a data page's row slot directory. +/// Byte offset of the row record within the page. +/// Length of the row record in bytes. +/// The row is marked deleted. +/// The slot points at an overflow/lookup record rather than inline data. +public readonly record struct RowSlot(int Offset, int Length, bool IsDeleted, bool HasOverflow); + /// -/// A data page holding the actual rows of a single table. Rows are addressed by a -/// slot directory at the end of the page; each slot points at a variable-length -/// row record decoded by . +/// A data page holding the rows of a single table. Rows are addressed by a slot +/// directory near the front of the page and packed from the page end backward. +/// Long-value (memo/OLE) pages share this page type but carry the "LVAL" owner marker. /// public sealed class DataPage : Page { + /// The 4-byte owner marker used by long-value pages: ASCII "LVAL". + private const uint LongValueMarker = 0x4C41564C; + + private const int RowOffsetMask = 0x1FFF; + private const int DeletedFlag = 0x8000; + private const int OverflowFlag = 0x4000; + + private readonly List _rows = []; + private PageBuffer _buffer; + public override PageType Type => PageType.DataPage; - /// The table (TDEF page number) this data page belongs to. - public int OwningTablePage { get; internal set; } + /// The TDEF page of the table that owns this data page (0 for long-value pages). + public int OwningTablePage { get; private set; } - /// Number of row slots present on the page. - public int RowCount { get; internal set; } + /// True when this is a long-value (memo/OLE overflow) page. + public bool IsLongValuePage { get; private set; } + + public int FreeSpace { get; private set; } + public int RowCount { get; private set; } + + public IReadOnlyList Rows => _rows; public override void Read(PageBuffer buffer, JetFormatBase format) { + _buffer = buffer; PageNumber = buffer.PageNumber; - // TODO: read free-space, owning-table pointer and the row-offset slot table. + + uint owner = buffer.ReadUInt32(format.DataOwnerOffset); + IsLongValuePage = owner == LongValueMarker; + OwningTablePage = IsLongValuePage ? 0 : (int)owner; + + FreeSpace = buffer.ReadUInt16(format.DataFreeSpaceOffset); + RowCount = buffer.ReadUInt16(format.DataRowCountOffset); + + _rows.Clear(); + int prevEnd = format.PageSize; + for (int i = 0; i < RowCount; i++) + { + int raw = buffer.ReadUInt16(format.DataRowDirectoryOffset + i * 2); + int offset = raw & RowOffsetMask; + bool deleted = (raw & DeletedFlag) != 0; + bool overflow = (raw & OverflowFlag) != 0; + + // Rows are packed from the page end backward, so a slot runs from its own + // offset up to where the previous slot's row began. + int length = prevEnd - offset; + _rows.Add(new RowSlot(offset, length, deleted, overflow)); + prevEnd = offset; + } + } + + /// Returns the raw bytes of the row at in the slot directory. + public ReadOnlySpan GetRow(int index) + { + RowSlot slot = _rows[index]; + return _buffer.Slice(slot.Offset, slot.Length); } } diff --git a/test/LibRed.Core.Tests/DataPageTests.cs b/test/LibRed.Core.Tests/DataPageTests.cs new file mode 100644 index 00000000..7f9bbef0 --- /dev/null +++ b/test/LibRed.Core.Tests/DataPageTests.cs @@ -0,0 +1,50 @@ +using LibRed; +using Xunit; + +namespace LibRed.Core.Tests; + +public class DataPageTests +{ + [Fact] + public void Reads_MSysObjects_data_page() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + // Page 17 holds MSysObjects rows (owner = its TDEF page 2). + var page = db.ReadDataPage(17); + + Assert.False(page.IsLongValuePage); + Assert.Equal(2, page.OwningTablePage); + Assert.Equal(41, page.RowCount); + Assert.Equal(page.RowCount, page.Rows.Count); + } + + [Fact] + public void Row_slots_are_within_page_and_packed_from_the_end() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + var page = db.ReadDataPage(17); + + int pageSize = db.Format.PageSize; + int prevEnd = pageSize; + foreach (var slot in page.Rows) + { + Assert.InRange(slot.Offset, 0, pageSize); + Assert.True(slot.Length > 0); + Assert.Equal(prevEnd, slot.Offset + slot.Length); // contiguous, end-packed + prevEnd = slot.Offset; + } + } + + [Fact] + public void Detects_long_value_page() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + // Page 42 carries the "LVAL" owner marker. + var page = db.ReadDataPage(42); + + Assert.True(page.IsLongValuePage); + Assert.Equal(0, page.OwningTablePage); + } +} From 2af1b5847408fd419c5c48fd793f043126576e24 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sat, 27 Jun 2026 21:30:36 +0800 Subject: [PATCH 007/506] @ LibRed: decode row records into CLR values Implement the Jet 4 / ACE row layout, reverse-engineered and verified against real MSysObjects data: [colCount:2] [fixed data] [var data] [varOffsetTable:(numVar+1)x2] [numVar:2] [nullBitmap] - RowDecoder reads the trailing null bitmap (indexed by column id; set bit = present), the variable-offset table (end-first, ascending column-id order), and fixed values at colCountSize + ColumnDef.FixedOffset - ColumnDef gains FixedOffset and VariableIndex; TableDefinitionPage populates them - JetTypeCodec decodes bool/byte/int16/int32/single/double, OLE-epoch DateTime, scaled Currency, Guid, UTF-16 Text and Binary; memo/OLE (LVAL) left as TODO - Short-row (<256 byte) variable offsets only; the jump-table form is guarded with NotSupportedException. Overflow/lookup slots are rejected with a clear error. Tests decode MSysObjects rows from the real file: correct CLR types, names including MSysObjects/Categories, and null memo columns. Co-Authored-By: Claude Opus 4.8 @ --- src/LibRed/LibRed.Core/Catalog/ColumnDef.cs | 6 ++ .../LibRed.Core/Formats/JetFormatBase.cs | 3 + .../LibRed.Core/Pages/TableDefinitionPage.cs | 20 ++++- src/LibRed/LibRed.Core/Storage/RowDecoder.cs | 80 ++++++++++++++++--- src/LibRed/LibRed.Core/Storage/TableCursor.cs | 2 +- .../LibRed.Core/Storage/Types/JetTypeCodec.cs | 48 ++++++++--- test/LibRed.Core.Tests/RowDecoderTests.cs | 59 ++++++++++++++ 7 files changed, 196 insertions(+), 22 deletions(-) create mode 100644 test/LibRed.Core.Tests/RowDecoderTests.cs diff --git a/src/LibRed/LibRed.Core/Catalog/ColumnDef.cs b/src/LibRed/LibRed.Core/Catalog/ColumnDef.cs index 32d93163..802cdaae 100644 --- a/src/LibRed/LibRed.Core/Catalog/ColumnDef.cs +++ b/src/LibRed/LibRed.Core/Catalog/ColumnDef.cs @@ -18,6 +18,12 @@ public sealed class ColumnDef /// Declared length in bytes for fixed-width/text columns. public int Length { get; init; } + /// For fixed-length columns, the offset of the value within the row's fixed-data region. + public int FixedOffset { get; init; } + + /// Position of this column among the variable-length columns (in column-id order); -1 if fixed. + public int VariableIndex { get; init; } = -1; + public bool IsFixedLength { get; init; } public bool IsNullable { get; init; } = true; public bool IsAutoNumber { get; init; } diff --git a/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs index 57dc47e2..10daf187 100644 --- a/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs +++ b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs @@ -88,6 +88,9 @@ public abstract class JetFormatBase /// Offset of the row-offset slot directory (2 bytes per row). public virtual int DataRowDirectoryOffset => 0x0E; + /// Size of the column-count field at the start of a row record (2 bytes in Jet 4 / ACE, 1 in Jet 3). + public virtual int RowColumnCountSize => 2; + /// Page size in bytes (2048 for Jet 3, 4096 for Jet 4 and all ACE versions). public int PageSize { get; protected set; } = 4096; diff --git a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs index 6225cdaf..57a4c13d 100644 --- a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs +++ b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs @@ -50,7 +50,7 @@ private void ReadColumns(PageBuffer buffer, JetFormatBase format, int columnBloc _columns.Clear(); // Pass 1: fixed-size column descriptors. - var descriptors = new (JetDataType Type, int ColumnId, byte Flags, int Length)[ColumnCount]; + var descriptors = new (JetDataType Type, int ColumnId, byte Flags, int FixedOffset, int Length)[ColumnCount]; for (int i = 0; i < ColumnCount; i++) { int entry = columnBlock + i * format.ColumnDescriptorSize; @@ -58,9 +58,22 @@ private void ReadColumns(PageBuffer buffer, JetFormatBase format, int columnBloc (JetDataType)buffer.ReadByte(entry + format.ColumnTypeOffset), buffer.ReadUInt16(entry + format.ColumnNumberOffset), buffer.ReadByte(entry + format.ColumnFlagsOffset), + buffer.ReadUInt16(entry + format.ColumnFixedOffsetOffset), buffer.ReadUInt16(entry + format.ColumnLengthOffset)); } + // Variable columns are addressed (in the row's var-offset table) in ascending + // column-id order, so assign each variable column its rank in that ordering. + var variableIndex = new Dictionary(); + int rank = 0; + foreach (int columnId in descriptors + .Where(d => (d.Flags & JetFormatBase.ColumnFlagFixedLength) == 0) + .Select(d => d.ColumnId) + .OrderBy(id => id)) + { + variableIndex[columnId] = rank++; + } + // Pass 2: column names, in the same order, immediately after the descriptor block. // Each name is a 2-byte (little-endian) byte length followed by UTF-16LE text. int namePos = columnBlock + ColumnCount * format.ColumnDescriptorSize; @@ -72,6 +85,7 @@ private void ReadColumns(PageBuffer buffer, JetFormatBase format, int columnBloc namePos += byteLength; var d = descriptors[i]; + bool isFixed = (d.Flags & JetFormatBase.ColumnFlagFixedLength) != 0; _columns.Add(new ColumnDef { Name = name, @@ -79,7 +93,9 @@ private void ReadColumns(PageBuffer buffer, JetFormatBase format, int columnBloc Index = i, ColumnId = d.ColumnId, Length = d.Length, - IsFixedLength = (d.Flags & JetFormatBase.ColumnFlagFixedLength) != 0, + FixedOffset = d.FixedOffset, + VariableIndex = isFixed ? -1 : variableIndex[d.ColumnId], + IsFixedLength = isFixed, IsAutoNumber = (d.Flags & JetFormatBase.ColumnFlagAutoNumber) != 0, }); } diff --git a/src/LibRed/LibRed.Core/Storage/RowDecoder.cs b/src/LibRed/LibRed.Core/Storage/RowDecoder.cs index ef989653..9b3d0468 100644 --- a/src/LibRed/LibRed.Core/Storage/RowDecoder.cs +++ b/src/LibRed/LibRed.Core/Storage/RowDecoder.cs @@ -1,23 +1,83 @@ +using System.Buffers.Binary; using LibRed.Catalog; -using LibRed.IO; +using LibRed.Formats; +using LibRed.Storage.Types; namespace LibRed.Storage; /// -/// Decodes a single raw row record into CLR values. Handles the Jet row layout: -/// fixed-length columns first, then the variable-length column offset table and the -/// trailing null bitmask. +/// Decodes a single raw row record into CLR values. Implements the Jet 4 / ACE row +/// layout (verified against real data): +/// +/// [colCount:2] [fixed data] [var data] [varOffsetTable:(numVar+1)x2] [numVar:2] [nullBitmap] +/// +/// The null bitmap is indexed by column id (bit set = value present). Variable columns +/// are addressed via the trailing offset table in ascending column-id order. /// -public sealed class RowDecoder(TableDef table) +public sealed class RowDecoder(IReadOnlyList columns, JetFormatBase format) { - private readonly TableDef _table = table; + private readonly IReadOnlyList _columns = columns; + private readonly JetFormatBase _format = format; - /// Decodes the row at into one boxed value per column. + /// Decodes the row into one value per column (aligned to ). public object?[] Decode(ReadOnlySpan row) { - var values = new object?[_table.Columns.Count]; - // TODO: read fixed-length values, the var-length offset jump table and null mask; - // dispatch each column through JetTypeCodec. + var values = new object?[_columns.Count]; + + int nullBitmapSize = (_columns.Count + 7) / 8; + + // A real inline row is at least: column count + an empty var table (1 entry) + + // var count + null bitmap. Anything shorter is an overflow/lookup pointer slot, + // which the caller should have skipped. + if (row.Length < _format.RowColumnCountSize + 2 + 2 + nullBitmapSize) + throw new ArgumentException("Row is too short to be an inline record (overflow/lookup slot?).", nameof(row)); + + ReadOnlySpan nullBitmap = row[^nullBitmapSize..]; + + int numVarCols = BinaryPrimitives.ReadUInt16LittleEndian(row.Slice(row.Length - nullBitmapSize - 2, 2)); + int varTableStart = row.Length - nullBitmapSize - 2 - (numVarCols + 1) * 2; + + // The 2-byte variable offset table assumed here is only valid for rows that do + // not use the >256-byte jump-table encoding. Guard so we never silently misparse. + if (numVarCols > 0 && row.Length > 256) + throw new NotSupportedException( + "Rows >= 256 bytes use the variable-offset jump table, which is not yet implemented."); + + foreach (ColumnDef column in _columns) + { + if (!IsPresent(nullBitmap, column.ColumnId)) + { + values[column.Index] = null; + continue; + } + + ReadOnlySpan raw = column.IsFixedLength + ? FixedSlice(row, column) + : VariableSlice(row, varTableStart, numVarCols, column.VariableIndex); + + values[column.Index] = JetTypeCodec.Decode(column, raw); + } + return values; } + + private ReadOnlySpan FixedSlice(ReadOnlySpan row, ColumnDef column) + { + int start = _format.RowColumnCountSize + column.FixedOffset; + return row.Slice(start, column.Length); + } + + private static ReadOnlySpan VariableSlice(ReadOnlySpan row, int varTableStart, int numVarCols, int variableIndex) + { + // Offsets are stored end-first: variable column j starts at entry[numVar - j]. + int start = VarOffset(row, varTableStart, numVarCols - variableIndex); + int end = VarOffset(row, varTableStart, numVarCols - variableIndex - 1); + return row[start..end]; + } + + private static int VarOffset(ReadOnlySpan row, int varTableStart, int entry) => + BinaryPrimitives.ReadUInt16LittleEndian(row.Slice(varTableStart + entry * 2, 2)); + + private static bool IsPresent(ReadOnlySpan nullBitmap, int columnId) => + (nullBitmap[columnId >> 3] & (1 << (columnId & 7))) != 0; } diff --git a/src/LibRed/LibRed.Core/Storage/TableCursor.cs b/src/LibRed/LibRed.Core/Storage/TableCursor.cs index e3b5de1d..b763b131 100644 --- a/src/LibRed/LibRed.Core/Storage/TableCursor.cs +++ b/src/LibRed/LibRed.Core/Storage/TableCursor.cs @@ -13,7 +13,7 @@ public sealed class TableCursor(Table table) : IEnumerable public IEnumerator GetEnumerator() { - var decoder = new RowDecoder(_table.Definition); + var decoder = new RowDecoder(_table.Definition.Columns, _table.Channel.Format); foreach (int pageNumber in _table.UsageMap.DataPages()) { PageBuffer buffer = _table.Channel.ReadPage(pageNumber); diff --git a/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs b/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs index 62c5260d..0fc05204 100644 --- a/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs +++ b/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs @@ -1,23 +1,53 @@ +using System.Buffers.Binary; +using System.Text; using LibRed.Catalog; namespace LibRed.Storage.Types; /// -/// Encodes and decodes individual column values to/from their on-disk byte -/// representation. Centralises the per-type quirks: Jet CURRENCY (scaled int64), -/// the 1899-12-30 OLE date epoch, fixed-point NUMERIC, GUID byte order, and the -/// code-page/Unicode text handling. +/// Decodes individual column values from their on-disk byte representation. Centralises +/// the per-type quirks: the 1899-12-30 OLE date epoch, Jet CURRENCY (scaled int64), +/// GUID byte order, and UTF-16LE text. Long values (memo/OLE) that live on LVAL pages +/// are not resolved here yet. /// public static class JetTypeCodec { - /// Decodes a single fixed-or-variable value for . + /// Decodes a single non-null column value from its raw bytes. public static object? Decode(ColumnDef column, ReadOnlySpan value) { - // TODO: switch on column.Type and decode accordingly. - return column.Type switch + switch (column.Type) { - _ => null, - }; + case JetDataType.Boolean: + return value.Length > 0 && value[0] != 0; + case JetDataType.Byte: + return value[0]; + case JetDataType.Int16: + return BinaryPrimitives.ReadInt16LittleEndian(value); + case JetDataType.Int32: + return BinaryPrimitives.ReadInt32LittleEndian(value); + case JetDataType.Single: + return BinaryPrimitives.ReadSingleLittleEndian(value); + case JetDataType.Double: + return BinaryPrimitives.ReadDoubleLittleEndian(value); + case JetDataType.DateTime: + return DateTime.FromOADate(BinaryPrimitives.ReadDoubleLittleEndian(value)); + case JetDataType.Currency: + return BinaryPrimitives.ReadInt64LittleEndian(value) / 10000m; + case JetDataType.Guid: + return new Guid(value[..16]); + case JetDataType.Text: + return Encoding.Unicode.GetString(value); + case JetDataType.Binary: + return value.ToArray(); + + // Long values stored on LVAL pages — needs the long-value reader. TODO. + case JetDataType.Memo: + case JetDataType.Ole: + case JetDataType.Complex: + case JetDataType.FixedPoint: + default: + return value.ToArray(); + } } /// Encodes a CLR value back to its on-disk representation. diff --git a/test/LibRed.Core.Tests/RowDecoderTests.cs b/test/LibRed.Core.Tests/RowDecoderTests.cs new file mode 100644 index 00000000..080ca0fe --- /dev/null +++ b/test/LibRed.Core.Tests/RowDecoderTests.cs @@ -0,0 +1,59 @@ +using LibRed; +using LibRed.Storage; +using Xunit; + +namespace LibRed.Core.Tests; + +public class RowDecoderTests +{ + private static List DecodeInlineRows(JetDatabase db, int dataPage, out IReadOnlyList columns) + { + var tdef = db.ReadTableDefinition(2); // MSysObjects schema + columns = tdef.Columns; + var decoder = new RowDecoder(columns, db.Format); + var page = db.ReadDataPage(dataPage); + + var rows = new List(); + for (int i = 0; i < page.RowCount; i++) + { + var slot = page.Rows[i]; + if (slot.IsDeleted || slot.HasOverflow) continue; + rows.Add(decoder.Decode(page.GetRow(i))); + } + return rows; + } + + [Fact] + public void Decodes_MSysObjects_rows_into_clr_values() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + var rows = DecodeInlineRows(db, 17, out var columns); + + int idIdx = columns.First(c => c.Name == "Id").Index; + int typeIdx = columns.First(c => c.Name == "Type").Index; + int nameIdx = columns.First(c => c.Name == "Name").Index; + + Assert.NotEmpty(rows); + Assert.All(rows, r => + { + Assert.IsType(r[idIdx]); // Id is Long (Int32) + Assert.IsType(r[typeIdx]); // Type is Integer (Int16) + Assert.IsType(r[nameIdx]); + }); + + var names = rows.Select(r => (string)r[nameIdx]!).ToList(); + Assert.Contains("MSysObjects", names); + Assert.Contains("Categories", names); // a real Northwind object + } + + [Fact] + public void Null_columns_decode_as_null() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + var rows = DecodeInlineRows(db, 17, out var columns); + + // For a plain table/query object, the Connect/Database memo fields are null. + int connectIdx = columns.First(c => c.Name == "Connect").Index; + Assert.Contains(rows, r => r[connectIdx] is null); + } +} From 299daaf4f16439967d842dc395d4efaaa7f078b7 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sat, 27 Jun 2026 22:01:32 +0800 Subject: [PATCH 008/506] LibRed: catalog bootstrap + full table scans by name JetCatalog reads MSysObjects (reusing Table/RowDecoder) to enumerate tables: for a table object the row Id is its TDEF page, so each TableDef resolves its columns from that page. System objects are filtered via flags 0x80000002 and MSys/~ name prefixes. JetDatabase.OpenTable(name).Rows() now yields decoded rows end to end. Fixes a TDEF parsing bug: the per-index block before the column descriptors is sized by the index count at 0x33, not the index-slot count at 0x2F. The two are equal for MSysObjects (which hid the bug) but differ for user tables, which previously misread column names. Verified across all 12 Northwind user tables. UsageMap/TableCursor: temporary owner-scan to find a table's data pages (to be replaced by real usage-map parsing); the cursor decodes rows and skips deleted, overflow and >256-byte (jump-table) rows. Tests: enumerate the 12 user tables with full column lists, and scan Categories (8 rows: Beverages..Seafood) and Shippers by name. Co-Authored-By: Claude Opus 4.8 --- src/LibRed/LibRed.Core/Catalog/JetCatalog.cs | 74 ++++++++++++++++--- .../LibRed.Core/Formats/JetFormatBase.cs | 3 + .../LibRed.Core/Pages/TableDefinitionPage.cs | 6 +- src/LibRed/LibRed.Core/Storage/TableCursor.cs | 31 ++++++-- src/LibRed/LibRed.Core/Storage/UsageMap.cs | 25 +++++-- test/LibRed.Core.Tests/CatalogTests.cs | 43 +++++++++++ test/LibRed.Core.Tests/TableScanTests.cs | 45 +++++++++++ 7 files changed, 201 insertions(+), 26 deletions(-) create mode 100644 test/LibRed.Core.Tests/CatalogTests.cs create mode 100644 test/LibRed.Core.Tests/TableScanTests.cs diff --git a/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs b/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs index 51d3cb45..fe509caa 100644 --- a/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs +++ b/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs @@ -1,31 +1,87 @@ using LibRed.IO; +using LibRed.Pages; +using LibRed.Storage; namespace LibRed.Catalog; /// -/// Reads the system catalog (MSysObjects, MSysColumns, …) to enumerate -/// the user tables, queries and relationships in a database. +/// Reads the system catalog (MSysObjects) to enumerate the tables in a database. /// /// -/// Bootstrap order: page 0 (database definition) → the catalog table lives at a -/// well-known page → read MSysObjects as an ordinary table to discover everything else. +/// Bootstrap: MSysObjects' own TDEF is at a fixed page (), +/// so we build a for it from that page and read its rows like any +/// other table. For a table object, the row's Id is its TDEF page number. /// public sealed class JetCatalog(PageChannel channel) { + /// MSysObjects.Type value for a table object. + private const short ObjectTypeTable = 1; + + /// MSysObjects.Flags bits marking a system object. + private const uint SystemObjectFlags = 0x80000002; + private readonly PageChannel _channel = channel; private List? _tables; - /// All user (non-system) tables in the database. + /// All tables in the database (user and system). public IReadOnlyList Tables => _tables ??= LoadTables(); + /// User (non-system) tables only. + public IEnumerable UserTables => Tables.Where(t => !t.IsSystem); + public TableDef? FindTable(string name) => Tables.FirstOrDefault(t => string.Equals(t.Name, name, StringComparison.OrdinalIgnoreCase)); private List LoadTables() { - // TODO: parse MSysObjects to recover (Name, Id/page, Type, Flags) rows, filter to - // user tables, then resolve each table's TDEF page into a TableDef. - _ = _channel; - return []; + // Build a TableDef for MSysObjects from its own (fixed) TDEF page, then scan its rows. + TableDef catalogDef = ReadTableDefinition(_channel.Format.CatalogPage, "MSysObjects", isSystem: true); + var columns = catalogDef.Columns; + + int idIndex = ColumnIndex(columns, "Id"); + int typeIndex = ColumnIndex(columns, "Type"); + int nameIndex = ColumnIndex(columns, "Name"); + int flagsIndex = ColumnIndex(columns, "Flags"); + + var catalog = new Table(_channel, catalogDef); + var tables = new List(); + + foreach (object?[] row in catalog.Rows()) + { + if (row[typeIndex] is not short type || type != ObjectTypeTable) continue; + + int definitionPage = (int)row[idIndex]!; + string name = (string)row[nameIndex]!; + uint flags = unchecked((uint)(int)row[flagsIndex]!); + bool isSystem = (flags & SystemObjectFlags) != 0 + || name.StartsWith("MSys", StringComparison.Ordinal) + || name.StartsWith('~'); + + tables.Add(ReadTableDefinition(definitionPage, name, isSystem)); + } + + return tables; + } + + private TableDef ReadTableDefinition(int definitionPage, string name, bool isSystem) + { + var tdef = new TableDefinitionPage(); + tdef.Read(_channel.ReadPage(definitionPage), _channel.Format); + + return new TableDef + { + Name = name, + DefinitionPage = definitionPage, + Columns = tdef.Columns, + IsSystem = isSystem, + }; + } + + private static int ColumnIndex(IReadOnlyList columns, string name) + { + for (int i = 0; i < columns.Count; i++) + if (string.Equals(columns[i].Name, name, StringComparison.OrdinalIgnoreCase)) + return columns[i].Index; + throw new InvalidOperationException($"MSysObjects is missing the '{name}' column."); } } diff --git a/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs index 10daf187..61144278 100644 --- a/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs +++ b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs @@ -91,6 +91,9 @@ public abstract class JetFormatBase /// Size of the column-count field at the start of a row record (2 bytes in Jet 4 / ACE, 1 in Jet 3). public virtual int RowColumnCountSize => 2; + /// Page number of the system catalog table MSysObjects (its TDEF page). + public virtual int CatalogPage => 2; + /// Page size in bytes (2048 for Jet 3, 4096 for Jet 4 and all ACE versions). public int PageSize { get; protected set; } = 4096; diff --git a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs index 57a4c13d..23a8009e 100644 --- a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs +++ b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs @@ -38,10 +38,12 @@ public override void Read(PageBuffer buffer, JetFormatBase format) RealIndexCount = buffer.ReadInt32(format.TdefRealIndexCountOffset); IndexCount = buffer.ReadInt32(format.TdefIndexCountOffset); - // The column descriptors follow the real-index block. + // The column descriptors follow a per-index block sized by the index count at + // 0x33 (IndexCount) — NOT the index-slot count at 0x2F. The two are equal for + // MSysObjects but differ for user tables (e.g. slots=2, indexes=1). // NOTE: assumes a single-page TDEF. A multi-page TDEF (NextDefinitionPage != 0) // must have its pages stitched into one contiguous buffer first. TODO. - int columnBlock = format.TdefRealIndexBlockOffset + RealIndexCount * format.RealIndexEntrySize; + int columnBlock = format.TdefRealIndexBlockOffset + IndexCount * format.RealIndexEntrySize; ReadColumns(buffer, format, columnBlock); } diff --git a/src/LibRed/LibRed.Core/Storage/TableCursor.cs b/src/LibRed/LibRed.Core/Storage/TableCursor.cs index b763b131..c88c1145 100644 --- a/src/LibRed/LibRed.Core/Storage/TableCursor.cs +++ b/src/LibRed/LibRed.Core/Storage/TableCursor.cs @@ -4,8 +4,8 @@ namespace LibRed.Storage; /// -/// A forward-only cursor over the rows of a table. Walks the table's usage map, -/// reads each data page and yields decoded rows via . +/// A forward-only cursor over the rows of a table. Walks the table's data pages, +/// decodes each inline row, and yields one value array per row. /// public sealed class TableCursor(Table table) : IEnumerable { @@ -14,17 +14,32 @@ public sealed class TableCursor(Table table) : IEnumerable public IEnumerator GetEnumerator() { var decoder = new RowDecoder(_table.Definition.Columns, _table.Channel.Format); + foreach (int pageNumber in _table.UsageMap.DataPages()) { PageBuffer buffer = _table.Channel.ReadPage(pageNumber); - var dataPage = new DataPage(); - dataPage.Read(buffer, _table.Channel.Format); + var page = new DataPage(); + page.Read(buffer, _table.Channel.Format); - // TODO: walk the page's row slot directory and decode each row record. - _ = decoder; - } + for (int i = 0; i < page.RowCount; i++) + { + RowSlot slot = page.Rows[i]; + if (slot.IsDeleted || slot.HasOverflow) continue; - yield break; + object?[]? values = null; + try + { + values = decoder.Decode(page.GetRow(i)); + } + catch (NotSupportedException) + { + // Rows >= 256 bytes use the variable-offset jump table (not yet + // implemented). Skip rather than fail the whole scan. TODO. + } + + if (values is not null) yield return values; + } + } } System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); diff --git a/src/LibRed/LibRed.Core/Storage/UsageMap.cs b/src/LibRed/LibRed.Core/Storage/UsageMap.cs index d44e1596..60b47d1a 100644 --- a/src/LibRed/LibRed.Core/Storage/UsageMap.cs +++ b/src/LibRed/LibRed.Core/Storage/UsageMap.cs @@ -1,13 +1,18 @@ using LibRed.Catalog; using LibRed.IO; +using LibRed.Pages; namespace LibRed.Storage; /// -/// Enumerates the data pages that belong to a table. Jet stores this as either an -/// inline bitmap (small tables) or a reference map pointing at dedicated bitmap -/// pages (large tables); both are surfaced here as a flat page sequence. +/// Enumerates the data pages that belong to a table. /// +/// +/// TEMPORARY implementation: scans every page and matches the owning-table pointer. +/// This is O(total pages) per table and ignores page order. It will be replaced by +/// parsing the table's real usage map (the inline/reference bitmap referenced from the +/// TDEF's owned-pages pointer), which lists the owned pages directly. +/// public sealed class UsageMap(PageChannel channel, TableDef table) { private readonly PageChannel _channel = channel; @@ -16,9 +21,15 @@ public sealed class UsageMap(PageChannel channel, TableDef table) /// Yields the page numbers of every data page owned by the table. public IEnumerable DataPages() { - // TODO: read the inline/reference usage map from the TDEF and yield set bits. - _ = _channel; - _ = _table; - yield break; + int owner = _table.DefinitionPage; + for (int p = 0; p < _channel.PageCount; p++) + { + PageBuffer buffer = _channel.ReadPage(p); + if (buffer.ReadByte(0) != (byte)PageType.DataPage) continue; + + var page = new DataPage(); + page.Read(buffer, _channel.Format); + if (page.OwningTablePage == owner) yield return p; + } } } diff --git a/test/LibRed.Core.Tests/CatalogTests.cs b/test/LibRed.Core.Tests/CatalogTests.cs new file mode 100644 index 00000000..fa85ce02 --- /dev/null +++ b/test/LibRed.Core.Tests/CatalogTests.cs @@ -0,0 +1,43 @@ +using LibRed; +using Xunit; + +namespace LibRed.Core.Tests; + +public class CatalogTests +{ + [Fact] + public void Enumerates_user_tables() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var names = db.Catalog.UserTables.Select(t => t.Name).ToList(); + + // The 12 Northwind user tables. + string[] expected = + [ + "Categories", "CustomerCustomerDemo", "CustomerDemographics", "Customers", + "Employees", "Order Details", "Orders", "Products", "Region", "Shippers", + "Suppliers", "Territories", + ]; + Assert.All(expected, e => Assert.Contains(e, names)); + + // System tables are excluded from UserTables but present overall. + Assert.DoesNotContain("MSysObjects", names); + Assert.Contains(db.Catalog.Tables, t => t.Name == "MSysObjects" && t.IsSystem); + } + + [Fact] + public void Resolves_full_column_lists() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var customers = db.Catalog.FindTable("Customers"); + Assert.NotNull(customers); + Assert.Equal( + ["CustomerID", "CompanyName", "ContactName", "ContactTitle", "Address", "City", + "Region", "PostalCode", "Country", "Phone", "Fax"], + customers!.Columns.Select(c => c.Name)); + + Assert.Equal(14, db.Catalog.FindTable("Orders")!.Columns.Count); + } +} diff --git a/test/LibRed.Core.Tests/TableScanTests.cs b/test/LibRed.Core.Tests/TableScanTests.cs new file mode 100644 index 00000000..045fef1c --- /dev/null +++ b/test/LibRed.Core.Tests/TableScanTests.cs @@ -0,0 +1,45 @@ +using LibRed; +using Xunit; + +namespace LibRed.Core.Tests; + +public class TableScanTests +{ + [Fact] + public void Scans_categories_rows_by_name() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var categories = db.OpenTable("Categories"); + int nameIdx = categories.Definition.Columns.First(c => c.Name == "CategoryName").Index; + int idIdx = categories.Definition.Columns.First(c => c.Name == "CategoryID").Index; + + var rows = categories.Rows().ToList(); + + Assert.Equal(8, rows.Count); + var names = rows.Select(r => (string)r[nameIdx]!).ToList(); + Assert.Equal( + ["Beverages", "Condiments", "Confections", "Dairy Products", + "Grains/Cereals", "Meat/Poultry", "Produce", "Seafood"], + names); + + // CategoryID is an AutoNumber Long. + Assert.All(rows, r => Assert.IsType(r[idIdx])); + } + + [Fact] + public void Scans_a_small_lookup_table() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var shippers = db.OpenTable("Shippers"); + int company = shippers.Definition.Columns.First(c => c.Name == "CompanyName").Index; + + var names = shippers.Rows().Select(r => (string)r[company]!).ToList(); + + Assert.Equal(3, names.Count); + Assert.Contains("Speedy Express", names); + Assert.Contains("United Package", names); + Assert.Contains("Federal Shipping", names); + } +} From d0d2ef1352d2e960b4266d4274c567e1a3434fd8 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sat, 27 Jun 2026 22:11:27 +0800 Subject: [PATCH 009/506] LibRed: add catalog dump test CatalogDumpTests prints every table (user + system) with its properties (tdef page, type, row count, column/index counts) and each column's name, data type, length and storage (fixed offset / variable index / autonumber). Useful as an eyeball check on the schema decode; keeps light assertions so it still functions as a test. Surfaced a known issue to fix next: Jet stores Boolean (YesNo) columns as a single bit, not a byte at a fixed offset, so e.g. Products.Discontinued reports fixed@0 (colliding with ProductID) and currently decodes incorrectly. --- test/LibRed.Core.Tests/CatalogDumpTests.cs | 49 ++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 test/LibRed.Core.Tests/CatalogDumpTests.cs diff --git a/test/LibRed.Core.Tests/CatalogDumpTests.cs b/test/LibRed.Core.Tests/CatalogDumpTests.cs new file mode 100644 index 00000000..d4fc7472 --- /dev/null +++ b/test/LibRed.Core.Tests/CatalogDumpTests.cs @@ -0,0 +1,49 @@ +using System.Text; +using LibRed; +using LibRed.Catalog; +using Xunit; +using Xunit.Abstractions; + +namespace LibRed.Core.Tests; + +/// +/// Not an assertion-heavy test: it dumps the whole catalog (every table, its +/// properties, and its columns with data types) so the schema decode can be eyeballed. +/// Run with: dotnet test -l "console;verbosity=detailed" +/// +public class CatalogDumpTests(ITestOutputHelper output) +{ + private readonly ITestOutputHelper _output = output; + + [Fact] + public void Dump_all_objects() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var sb = new StringBuilder(); + sb.AppendLine($"Database: {db.DefinitionPage.FormatIdentifier} ({db.Format.Version}, page size {db.Format.PageSize})"); + + foreach (TableDef table in db.Catalog.Tables.OrderBy(t => t.IsSystem).ThenBy(t => t.Name)) + { + // Re-read the TDEF page for per-table properties (row count, type, index count). + var tdef = db.ReadTableDefinition(table.DefinitionPage); + + sb.AppendLine(); + sb.AppendLine($"{(table.IsSystem ? "[SYS] " : " ")}{table.Name}"); + sb.AppendLine($" tdefPage={table.DefinitionPage} type={tdef.TableType} rows={tdef.RowCount} columns={tdef.ColumnCount} indexes={tdef.IndexCount}"); + + foreach (ColumnDef c in table.Columns) + { + string store = c.IsFixedLength ? $"fixed@{c.FixedOffset}" : $"var#{c.VariableIndex}"; + string extra = c.IsAutoNumber ? " auto" : ""; + sb.AppendLine($" {c.Index,2}. {c.Name,-26} {c.Type,-9} len={c.Length,3} {store}{extra}"); + } + } + + _output.WriteLine(sb.ToString()); + + // Light sanity assertions so this still functions as a test. + Assert.True(db.Catalog.UserTables.Count() >= 12); + Assert.All(db.Catalog.Tables, t => Assert.NotEmpty(t.Columns)); + } +} From db5b52aaceb627a72f1fc5d7bfe7f769547034f1 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sat, 27 Jun 2026 22:22:10 +0800 Subject: [PATCH 010/506] LibRed: fix Boolean decoding; golden full-catalog dump test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Boolean (YesNo) columns have no fixed/variable storage in Jet — the value is the null-bitmap bit (set = true, never null). RowDecoder now decodes booleans from the bitmap. Verified: Products has exactly 8 discontinued items (the classic Northwind set). Dump_all_objects now renders every object (user AND system) in page order, with table properties and each column's type/length/storage, and asserts the whole rendering against a golden file (Expected/catalog-dump.txt) in addition to writing it to test output. Regenerate the golden file if the decode intentionally changes. (Noted for later: some ACE system tables -- MSysResources, the f_*_Data attachment table -- show unusual column layouts, e.g. variable-stored numerics and the Complex type, worth revisiting.) Co-Authored-By: Claude Opus 4.8 --- src/LibRed/LibRed.Core/Storage/RowDecoder.cs | 12 +- test/LibRed.Core.Tests/CatalogDumpTests.cs | 32 +- .../Expected/catalog-dump.txt | 286 ++++++++++++++++++ .../LibRed.Core.Tests.csproj | 3 + test/LibRed.Core.Tests/TableScanTests.cs | 15 + 5 files changed, 334 insertions(+), 14 deletions(-) create mode 100644 test/LibRed.Core.Tests/Expected/catalog-dump.txt diff --git a/src/LibRed/LibRed.Core/Storage/RowDecoder.cs b/src/LibRed/LibRed.Core/Storage/RowDecoder.cs index 9b3d0468..34e7d04e 100644 --- a/src/LibRed/LibRed.Core/Storage/RowDecoder.cs +++ b/src/LibRed/LibRed.Core/Storage/RowDecoder.cs @@ -45,7 +45,17 @@ public sealed class RowDecoder(IReadOnlyList columns, JetFormatBase f foreach (ColumnDef column in _columns) { - if (!IsPresent(nullBitmap, column.ColumnId)) + bool present = IsPresent(nullBitmap, column.ColumnId); + + // Jet stores Boolean (YesNo) columns with no fixed/variable data: the value + // IS the null-bitmap bit (set = true). Booleans are never null. + if (column.Type == JetDataType.Boolean) + { + values[column.Index] = present; + continue; + } + + if (!present) { values[column.Index] = null; continue; diff --git a/test/LibRed.Core.Tests/CatalogDumpTests.cs b/test/LibRed.Core.Tests/CatalogDumpTests.cs index d4fc7472..1b46d0c9 100644 --- a/test/LibRed.Core.Tests/CatalogDumpTests.cs +++ b/test/LibRed.Core.Tests/CatalogDumpTests.cs @@ -7,9 +7,11 @@ namespace LibRed.Core.Tests; /// -/// Not an assertion-heavy test: it dumps the whole catalog (every table, its -/// properties, and its columns with data types) so the schema decode can be eyeballed. -/// Run with: dotnet test -l "console;verbosity=detailed" +/// Dumps every object in the database (user and system tables, in page order) with its +/// properties and columns/data types, then asserts the whole rendering against a golden +/// file (Expected/catalog-dump.txt). This pins the schema decode end to end; regenerate +/// the golden file if the decode intentionally changes. +/// Run with: dotnet test -l "console;verbosity=detailed" to see the dump. /// public class CatalogDumpTests(ITestOutputHelper output) { @@ -21,29 +23,33 @@ public void Dump_all_objects() using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); var sb = new StringBuilder(); - sb.AppendLine($"Database: {db.DefinitionPage.FormatIdentifier} ({db.Format.Version}, page size {db.Format.PageSize})"); + sb.Append($"Database: {db.DefinitionPage.FormatIdentifier} ({db.Format.Version}, page size {db.Format.PageSize})\n"); - foreach (TableDef table in db.Catalog.Tables.OrderBy(t => t.IsSystem).ThenBy(t => t.Name)) + foreach (TableDef table in db.Catalog.Tables.OrderBy(t => t.DefinitionPage)) { // Re-read the TDEF page for per-table properties (row count, type, index count). var tdef = db.ReadTableDefinition(table.DefinitionPage); - sb.AppendLine(); - sb.AppendLine($"{(table.IsSystem ? "[SYS] " : " ")}{table.Name}"); - sb.AppendLine($" tdefPage={table.DefinitionPage} type={tdef.TableType} rows={tdef.RowCount} columns={tdef.ColumnCount} indexes={tdef.IndexCount}"); + sb.Append('\n'); + sb.Append($"{(table.IsSystem ? "[SYS] " : " ")}{table.Name}\n"); + sb.Append($" tdefPage={table.DefinitionPage} type={tdef.TableType} rows={tdef.RowCount} columns={tdef.ColumnCount} indexes={tdef.IndexCount}\n"); foreach (ColumnDef c in table.Columns) { string store = c.IsFixedLength ? $"fixed@{c.FixedOffset}" : $"var#{c.VariableIndex}"; string extra = c.IsAutoNumber ? " auto" : ""; - sb.AppendLine($" {c.Index,2}. {c.Name,-26} {c.Type,-9} len={c.Length,3} {store}{extra}"); + sb.Append($" {c.Index,2}. {c.Name,-26} {c.Type,-9} len={c.Length,3} {store}{extra}\n"); } } - _output.WriteLine(sb.ToString()); + string actual = Normalize(sb.ToString()); + _output.WriteLine(actual); - // Light sanity assertions so this still functions as a test. - Assert.True(db.Catalog.UserTables.Count() >= 12); - Assert.All(db.Catalog.Tables, t => Assert.NotEmpty(t.Columns)); + string expectedPath = Path.Combine(AppContext.BaseDirectory, "Expected", "catalog-dump.txt"); + string expected = Normalize(File.ReadAllText(expectedPath)); + + Assert.Equal(expected, actual); } + + private static string Normalize(string s) => s.Replace("\r\n", "\n").TrimEnd() + "\n"; } diff --git a/test/LibRed.Core.Tests/Expected/catalog-dump.txt b/test/LibRed.Core.Tests/Expected/catalog-dump.txt new file mode 100644 index 00000000..5eb7d31d --- /dev/null +++ b/test/LibRed.Core.Tests/Expected/catalog-dump.txt @@ -0,0 +1,286 @@ +Database: Standard ACE DB (Version12_2007, page size 4096) + +[SYS] MSysObjects + tdefPage=2 type=System rows=82 columns=17 indexes=2 + 0. Connect Memo len= 0 var#3 + 1. Database Memo len= 0 var#2 + 2. DateCreate DateTime len= 8 fixed@10 + 3. DateUpdate DateTime len= 8 fixed@18 + 4. Flags Int32 len= 4 fixed@26 + 5. ForeignName Text len=510 var#4 + 6. Id Int32 len= 4 fixed@0 + 7. Lv Ole len= 0 var#7 + 8. LvExtra Ole len= 0 var#10 + 9. LvModule Ole len= 0 var#9 + 10. LvProp Ole len= 0 var#8 + 11. Name Text len=510 var#0 + 12. Owner Binary len=510 var#1 + 13. ParentId Int32 len= 4 fixed@4 + 14. RmtInfoLong Ole len= 0 var#6 + 15. RmtInfoShort Binary len=510 var#5 + 16. Type Int16 len= 2 fixed@8 + +[SYS] MSysACEs + tdefPage=3 type=System rows=183 columns=4 indexes=1 + 0. ACM Int32 len= 4 fixed@4 + 1. FInheritable Boolean len= 1 fixed@0 + 2. ObjectId Int32 len= 4 fixed@0 + 3. SID Binary len=510 var#0 + +[SYS] MSysQueries + tdefPage=4 type=System rows=279 columns=8 indexes=1 + 0. Attribute Byte len= 1 fixed@4 + 1. Expression Memo len= 0 var#3 + 2. Flag Int16 len= 2 fixed@5 + 3. LvExtra Int32 len= 4 fixed@7 + 4. Name1 Text len=510 var#1 + 5. Name2 Text len=510 var#2 + 6. ObjectId Int32 len= 4 fixed@0 + 7. Order Binary len=510 var#0 + +[SYS] MSysRelationships + tdefPage=5 type=System rows=15 columns=8 indexes=3 + 0. ccolumn Int32 len= 4 fixed@4 + 1. grbit Int32 len= 4 fixed@0 + 2. icolumn Int32 len= 4 fixed@8 + 3. szColumn Text len=510 var#2 + 4. szObject Text len=510 var#1 + 5. szReferencedColumn Text len=510 var#4 + 6. szReferencedObject Text len=510 var#3 + 7. szRelationship Text len=510 var#0 + +[SYS] MSysComplexColumns + tdefPage=18 type=User rows=1 columns=5 indexes=3 + 0. ColumnName Text len=510 var#0 + 1. ComplexID Int32 len= 4 fixed@12 auto + 2. ComplexTypeObjectID Int32 len= 4 fixed@0 + 3. ConceptualTableID Int32 len= 4 fixed@8 + 4. FlatTableID Int32 len= 4 fixed@4 + +[SYS] MSysComplexType_UnsignedByte + tdefPage=23 type=User rows=0 columns=1 indexes=0 + 0. Value Byte len= 1 fixed@0 + +[SYS] MSysComplexType_Short + tdefPage=25 type=User rows=0 columns=1 indexes=0 + 0. Value Int16 len= 2 fixed@0 + +[SYS] MSysComplexType_Long + tdefPage=27 type=User rows=0 columns=1 indexes=0 + 0. Value Int32 len= 4 fixed@0 + +[SYS] MSysComplexType_IEEESingle + tdefPage=29 type=User rows=0 columns=1 indexes=0 + 0. Value Single len= 4 fixed@0 + +[SYS] MSysComplexType_IEEEDouble + tdefPage=31 type=User rows=0 columns=1 indexes=0 + 0. Value Double len= 8 fixed@0 + +[SYS] MSysComplexType_GUID + tdefPage=33 type=User rows=0 columns=1 indexes=0 + 0. Value Guid len= 16 fixed@0 + +[SYS] MSysComplexType_Decimal + tdefPage=35 type=User rows=0 columns=1 indexes=0 + 0. Value FixedPoint len= 9 var#0 + +[SYS] MSysComplexType_Text + tdefPage=37 type=User rows=0 columns=1 indexes=0 + 0. Value Text len=510 var#0 + +[SYS] MSysComplexType_Attachment + tdefPage=39 type=User rows=0 columns=6 indexes=0 + 0. FileData Ole len= 0 var#3 + 1. FileFlags Int32 len= 4 fixed@8 + 2. FileName Text len=510 var#1 + 3. FileTimeStamp DateTime len= 8 fixed@0 + 4. FileType Text len=510 var#2 + 5. FileURL Memo len= 0 var#0 + + Categories + tdefPage=43 type=User rows=8 columns=4 indexes=2 + 0. CategoryID Int32 len= 4 fixed@0 auto + 1. CategoryName Text len= 30 var#0 + 2. Description Memo len= 0 var#1 + 3. Picture Ole len= 0 var#2 + + CustomerCustomerDemo + tdefPage=73 type=User rows=0 columns=2 indexes=3 + 0. CustomerID Text len= 10 fixed@0 + 1. CustomerTypeID Text len= 20 fixed@10 + + CustomerDemographics + tdefPage=76 type=User rows=0 columns=2 indexes=1 + 0. CustomerTypeID Text len= 20 fixed@0 + 1. CustomerDesc Memo len= 0 var#0 + + Customers + tdefPage=79 type=User rows=91 columns=11 indexes=5 + 0. CustomerID Text len= 10 fixed@0 + 1. CompanyName Text len= 80 var#0 + 2. ContactName Text len= 60 var#1 + 3. ContactTitle Text len= 60 var#2 + 4. Address Text len=120 var#3 + 5. City Text len= 30 var#4 + 6. Region Text len= 30 var#5 + 7. PostalCode Text len= 20 var#6 + 8. Country Text len= 30 var#7 + 9. Phone Text len= 48 var#8 + 10. Fax Text len= 48 var#9 + + Employees + tdefPage=92 type=User rows=9 columns=18 indexes=4 + 0. EmployeeID Int32 len= 4 fixed@0 auto + 1. LastName Text len= 40 var#0 + 2. FirstName Text len= 20 var#1 + 3. Title Text len= 60 var#2 + 4. TitleOfCourtesy Text len= 50 var#3 + 5. BirthDate DateTime len= 8 fixed@4 + 6. HireDate DateTime len= 8 fixed@12 + 7. Address Text len=120 var#4 + 8. City Text len= 30 var#5 + 9. Region Text len= 30 var#6 + 10. PostalCode Text len= 20 var#7 + 11. Country Text len= 30 var#8 + 12. HomePhone Text len= 48 var#9 + 13. Extension Text len= 8 var#10 + 14. Photo Ole len= 0 var#11 + 15. Notes Memo len= 0 var#12 + 16. ReportsTo Int32 len= 4 fixed@20 + 17. PhotoPath Text len=510 var#13 + +[SYS] MSysAccessStorage + tdefPage=159 type=User rows=23 columns=7 indexes=3 + 0. DateCreate DateTime len= 8 fixed@0 + 1. DateUpdate DateTime len= 8 fixed@8 + 2. Id Int32 len= 4 fixed@16 auto + 3. Lv Ole len= 0 var#0 + 4. Name Text len=256 var#1 + 5. ParentId Int32 len= 4 fixed@20 + 6. Type Int32 len= 4 fixed@24 + + Order Details + tdefPage=164 type=User rows=2155 columns=5 indexes=3 + 0. OrderID Int32 len= 4 fixed@0 + 1. ProductID Int32 len= 4 fixed@4 + 2. UnitPrice Currency len= 8 fixed@8 + 3. Quantity Int16 len= 2 fixed@16 + 4. Discount Single len= 4 fixed@18 + + Orders + tdefPage=198 type=User rows=830 columns=14 indexes=7 + 0. OrderID Int32 len= 4 fixed@0 auto + 1. CustomerID Text len= 10 fixed@4 + 2. EmployeeID Int32 len= 4 fixed@14 + 3. OrderDate DateTime len= 8 fixed@18 + 4. RequiredDate DateTime len= 8 fixed@26 + 5. ShippedDate DateTime len= 8 fixed@34 + 6. ShipVia Int32 len= 4 fixed@42 + 7. Freight Currency len= 8 fixed@46 + 8. ShipName Text len= 80 var#0 + 9. ShipAddress Text len=120 var#1 + 10. ShipCity Text len= 30 var#2 + 11. ShipRegion Text len= 30 var#3 + 12. ShipPostalCode Text len= 20 var#4 + 13. ShipCountry Text len= 30 var#5 + + Products + tdefPage=266 type=User rows=77 columns=10 indexes=4 + 0. ProductID Int32 len= 4 fixed@0 auto + 1. ProductName Text len= 80 var#0 + 2. SupplierID Int32 len= 4 fixed@4 + 3. CategoryID Int32 len= 4 fixed@8 + 4. QuantityPerUnit Text len= 40 var#1 + 5. UnitPrice Currency len= 8 fixed@12 + 6. UnitsInStock Int16 len= 2 fixed@20 + 7. UnitsOnOrder Int16 len= 2 fixed@22 + 8. ReorderLevel Int16 len= 2 fixed@24 + 9. Discontinued Boolean len= 1 fixed@0 + + Region + tdefPage=275 type=User rows=4 columns=2 indexes=1 + 0. RegionID Int32 len= 4 fixed@0 + 1. RegionDescription Text len=100 fixed@4 + + Shippers + tdefPage=283 type=User rows=3 columns=3 indexes=1 + 0. ShipperID Int32 len= 4 fixed@0 auto + 1. CompanyName Text len= 80 var#0 + 2. Phone Text len= 48 var#1 + + Suppliers + tdefPage=287 type=User rows=29 columns=12 indexes=3 + 0. SupplierID Int32 len= 4 fixed@0 auto + 1. CompanyName Text len= 80 var#0 + 2. ContactName Text len= 60 var#1 + 3. ContactTitle Text len= 60 var#2 + 4. Address Text len=120 var#3 + 5. City Text len= 30 var#4 + 6. Region Text len= 30 var#5 + 7. PostalCode Text len= 20 var#6 + 8. Country Text len= 30 var#7 + 9. Phone Text len= 48 var#8 + 10. Fax Text len= 48 var#9 + 11. HomePage Memo len= 0 var#10 + + Territories + tdefPage=295 type=User rows=53 columns=3 indexes=2 + 0. TerritoryID Text len= 40 var#0 + 1. TerritoryDescription Text len=100 fixed@0 + 2. RegionID Int32 len= 4 fixed@100 + +[SYS] MSysNavPaneGroupCategories + tdefPage=312 type=User rows=3 columns=7 indexes=1 + 0. Filter Text len=510 var#4 + 1. Flags Int32 len= 4 var#6 + 2. Id Int32 len= 4 var#0 auto + 3. Name Text len=510 var#1 + 4. Position Int32 len= 4 var#5 + 5. SelectedObjectID Int32 len= 4 var#3 + 6. Type Int32 len= 4 var#2 + +[SYS] MSysNavPaneGroups + tdefPage=315 type=User rows=10 columns=7 indexes=2 + 0. Flags Int32 len= 4 var#6 + 1. GroupCategoryID Int32 len= 4 var#2 + 2. Id Int32 len= 4 var#0 auto + 3. Name Text len=510 var#1 + 4. Object Type Group Int32 len= 4 var#5 + 5. ObjectID Int32 len= 4 var#3 + 6. Position Int32 len= 4 var#4 + +[SYS] MSysNavPaneGroupToObjects + tdefPage=319 type=User rows=47 columns=7 indexes=2 + 0. Flags Int32 len= 4 var#6 + 1. GroupID Int32 len= 4 var#2 + 2. Icon Int32 len= 4 var#5 + 3. Id Int32 len= 4 var#0 auto + 4. Name Text len=510 var#3 + 5. ObjectID Int32 len= 4 var#1 + 6. Position Int32 len= 4 var#4 + +[SYS] MSysNavPaneObjectIDs + tdefPage=327 type=User rows=57 columns=3 indexes=0 + 0. Id Int32 len= 4 var#0 + 1. Name Text len=130 var#1 + 2. Type Int32 len= 4 var#2 + +[SYS] MSysResources + tdefPage=332 type=User rows=1 columns=5 indexes=2 + 0. Data Complex len= 4 fixed@0 auto + 1. Extension Text len=510 var#3 + 2. Id Int32 len= 4 var#0 auto + 3. Name Text len=510 var#1 + 4. Type Text len=510 var#2 + +[SYS] f_3B3D4E1F479C4CCCA9CB8E549B05D3B7_Data + tdefPage=334 type=User rows=1 columns=8 indexes=3 + 0. _Data Int32 len= 4 var#5 + 1. FileData Ole len= 0 var#0 + 2. FileFlags Int32 len= 4 fixed@0 + 3. FileName Text len=510 var#1 + 4. FileTimeStamp DateTime len= 8 fixed@4 + 5. FileType Text len=510 var#2 + 6. FileURL Memo len= 0 var#3 + 7. MSysResources_Data Int32 len= 4 var#4 auto diff --git a/test/LibRed.Core.Tests/LibRed.Core.Tests.csproj b/test/LibRed.Core.Tests/LibRed.Core.Tests.csproj index d7f4badd..9d079a47 100644 --- a/test/LibRed.Core.Tests/LibRed.Core.Tests.csproj +++ b/test/LibRed.Core.Tests/LibRed.Core.Tests.csproj @@ -33,6 +33,9 @@ Data\Northwind.accdb PreserveNewest + + PreserveNewest + diff --git a/test/LibRed.Core.Tests/TableScanTests.cs b/test/LibRed.Core.Tests/TableScanTests.cs index 045fef1c..a1bc3260 100644 --- a/test/LibRed.Core.Tests/TableScanTests.cs +++ b/test/LibRed.Core.Tests/TableScanTests.cs @@ -27,6 +27,21 @@ public void Scans_categories_rows_by_name() Assert.All(rows, r => Assert.IsType(r[idIdx])); } + [Fact] + public void Decodes_boolean_columns_from_the_null_bitmap() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var products = db.OpenTable("Products"); + int discontinued = products.Definition.Columns.First(c => c.Name == "Discontinued").Index; + + var rows = products.Rows().ToList(); + + Assert.All(rows, r => Assert.IsType(r[discontinued])); + // Northwind has exactly 8 discontinued products. + Assert.Equal(8, rows.Count(r => (bool)r[discontinued]!)); + } + [Fact] public void Scans_a_small_lookup_table() { From 93ed92d6fe09e2ae230c028e6e9706e0c873c482 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sat, 27 Jun 2026 22:33:01 +0800 Subject: [PATCH 011/506] LibRed: decode rows >= 256 bytes (remove incorrect jump-table guard) Jet 4 / ACE stores variable-column offsets as 2 bytes at any row size, so there is no Jet 3-style jump table and rows >= 256 bytes decode identically. The previous NotSupportedException guard (a mistaken port of Jet 3 behaviour) was silently skipping large rows -- e.g. all 9 Employees and 43 of 91 Customers. Removed it; TableCursor no longer swallows the exception. Verified: Customers=91, Employees=9, Orders=830, Order Details=2155 rows fully decoded, with correct international text. Co-Authored-By: Claude Opus 4.8 --- src/LibRed/LibRed.Core/Storage/RowDecoder.cs | 8 ++------ src/LibRed/LibRed.Core/Storage/TableCursor.cs | 13 +------------ test/LibRed.Core.Tests/TableScanTests.cs | 11 +++++++++++ 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/src/LibRed/LibRed.Core/Storage/RowDecoder.cs b/src/LibRed/LibRed.Core/Storage/RowDecoder.cs index 34e7d04e..23764cb5 100644 --- a/src/LibRed/LibRed.Core/Storage/RowDecoder.cs +++ b/src/LibRed/LibRed.Core/Storage/RowDecoder.cs @@ -13,6 +13,8 @@ namespace LibRed.Storage; /// /// The null bitmap is indexed by column id (bit set = value present). Variable columns /// are addressed via the trailing offset table in ascending column-id order. +/// Jet 4 / ACE uses 2-byte variable offsets at any row size — there is no Jet 3-style +/// jump table (1-byte offsets), so rows larger than 256 bytes decode the same way. /// public sealed class RowDecoder(IReadOnlyList columns, JetFormatBase format) { @@ -37,12 +39,6 @@ public sealed class RowDecoder(IReadOnlyList columns, JetFormatBase f int numVarCols = BinaryPrimitives.ReadUInt16LittleEndian(row.Slice(row.Length - nullBitmapSize - 2, 2)); int varTableStart = row.Length - nullBitmapSize - 2 - (numVarCols + 1) * 2; - // The 2-byte variable offset table assumed here is only valid for rows that do - // not use the >256-byte jump-table encoding. Guard so we never silently misparse. - if (numVarCols > 0 && row.Length > 256) - throw new NotSupportedException( - "Rows >= 256 bytes use the variable-offset jump table, which is not yet implemented."); - foreach (ColumnDef column in _columns) { bool present = IsPresent(nullBitmap, column.ColumnId); diff --git a/src/LibRed/LibRed.Core/Storage/TableCursor.cs b/src/LibRed/LibRed.Core/Storage/TableCursor.cs index c88c1145..062af406 100644 --- a/src/LibRed/LibRed.Core/Storage/TableCursor.cs +++ b/src/LibRed/LibRed.Core/Storage/TableCursor.cs @@ -26,18 +26,7 @@ public sealed class TableCursor(Table table) : IEnumerable RowSlot slot = page.Rows[i]; if (slot.IsDeleted || slot.HasOverflow) continue; - object?[]? values = null; - try - { - values = decoder.Decode(page.GetRow(i)); - } - catch (NotSupportedException) - { - // Rows >= 256 bytes use the variable-offset jump table (not yet - // implemented). Skip rather than fail the whole scan. TODO. - } - - if (values is not null) yield return values; + yield return decoder.Decode(page.GetRow(i)); } } } diff --git a/test/LibRed.Core.Tests/TableScanTests.cs b/test/LibRed.Core.Tests/TableScanTests.cs index a1bc3260..094a56ed 100644 --- a/test/LibRed.Core.Tests/TableScanTests.cs +++ b/test/LibRed.Core.Tests/TableScanTests.cs @@ -27,6 +27,17 @@ public void Scans_categories_rows_by_name() Assert.All(rows, r => Assert.IsType(r[idIdx])); } + [Theory] + [InlineData("Customers", 91)] // 43 rows are >= 256 bytes + [InlineData("Employees", 9)] // every row is >= 256 bytes + [InlineData("Orders", 830)] + [InlineData("Order Details", 2155)] + public void Scans_all_rows_including_large_ones(string table, int expectedRows) + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + Assert.Equal(expectedRows, db.OpenTable(table).Rows().Count()); + } + [Fact] public void Decodes_boolean_columns_from_the_null_bitmap() { From 89a806c650661517544eda1b7b98a9b04500c650 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sat, 27 Jun 2026 22:39:42 +0800 Subject: [PATCH 012/506] LibRed: read real usage maps (replace owner-scan) UsageMap now reads the table's owned-pages usage map from the TDEF pointer at 0x37 (1-byte row + 3-byte page) instead of scanning every page. Inline maps (type 0x00): a 4-byte start page followed by a bitmap where bit i marks page (startPage + i) owned. Reference maps (type 0x01, very large tables) fall back to the owner-scan for now. Verified against the brute-force scan for all 33 tables. This is also more correct: the scan over-included stale orphan pages that retain an owner stamp but are not live (e.g. MSysNavPaneObjectIDs page 329) -- the usage map excludes them, so decoded row counts now match the TDEF (57, not 112). Co-Authored-By: Claude Opus 4.8 --- .../LibRed.Core/Formats/JetFormatBase.cs | 3 + src/LibRed/LibRed.Core/Storage/UsageMap.cs | 59 ++++++++++++++++--- test/LibRed.Core.Tests/UsageMapTests.cs | 31 ++++++++++ 3 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 test/LibRed.Core.Tests/UsageMapTests.cs diff --git a/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs index 61144278..2b3494b5 100644 --- a/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs +++ b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs @@ -94,6 +94,9 @@ public abstract class JetFormatBase /// Page number of the system catalog table MSysObjects (its TDEF page). public virtual int CatalogPage => 2; + /// Offset in a TDEF of the owned-pages usage-map pointer: 1 byte row, then a 3-byte page. + public virtual int TdefOwnedPagesOffset => 0x37; + /// Page size in bytes (2048 for Jet 3, 4096 for Jet 4 and all ACE versions). public int PageSize { get; protected set; } = 4096; diff --git a/src/LibRed/LibRed.Core/Storage/UsageMap.cs b/src/LibRed/LibRed.Core/Storage/UsageMap.cs index 60b47d1a..4f215b90 100644 --- a/src/LibRed/LibRed.Core/Storage/UsageMap.cs +++ b/src/LibRed/LibRed.Core/Storage/UsageMap.cs @@ -1,27 +1,70 @@ +using System.Buffers.Binary; using LibRed.Catalog; +using LibRed.Formats; using LibRed.IO; using LibRed.Pages; namespace LibRed.Storage; /// -/// Enumerates the data pages that belong to a table. +/// Enumerates the data pages that belong to a table by reading its owned-pages usage map. /// /// -/// TEMPORARY implementation: scans every page and matches the owning-table pointer. -/// This is O(total pages) per table and ignores page order. It will be replaced by -/// parsing the table's real usage map (the inline/reference bitmap referenced from the -/// TDEF's owned-pages pointer), which lists the owned pages directly. +/// The TDEF holds a pointer (row + page) to the usage-map record. An inline map +/// (type 0x00) stores a start page and a bitmap where bit i marks page (startPage + i) +/// as owned. A reference map (type 0x01, only for very large tables) instead points at +/// dedicated bitmap pages — not yet parsed, so we fall back to a full owner-scan there. /// public sealed class UsageMap(PageChannel channel, TableDef table) { + private const byte MapTypeInline = 0x00; + private const byte MapTypeReference = 0x01; + private readonly PageChannel _channel = channel; private readonly TableDef _table = table; - /// Yields the page numbers of every data page owned by the table. + /// Yields the page numbers of every data page owned by the table, in ascending order. public IEnumerable DataPages() { - int owner = _table.DefinitionPage; + JetFormatBase format = _channel.Format; + + PageBuffer tdef = _channel.ReadPage(_table.DefinitionPage); + int mapRow = tdef.ReadByte(format.TdefOwnedPagesOffset); + int mapPage = tdef.ReadInt24(format.TdefOwnedPagesOffset + 1); + + var holder = new DataPage(); + holder.Read(_channel.ReadPage(mapPage), format); + ReadOnlySpan map = holder.GetRow(mapRow); + + return map[0] switch + { + MapTypeInline => ReadInlineMap(map), + // TODO: parse reference maps (dedicated bitmap pages). Fall back meanwhile. + MapTypeReference => OwnerScan(), + byte t => throw new NotSupportedException($"Unknown usage map type 0x{t:X2}."), + }; + } + + private static List ReadInlineMap(ReadOnlySpan map) + { + int startPage = BinaryPrimitives.ReadInt32LittleEndian(map.Slice(1, 4)); + var pages = new List(); + + for (int i = 5; i < map.Length; i++) + { + byte b = map[i]; + if (b == 0) continue; + for (int bit = 0; bit < 8; bit++) + if ((b & (1 << bit)) != 0) + pages.Add(startPage + (i - 5) * 8 + bit); + } + + return pages; + } + + /// Fallback: scan every page and match the owning-table pointer. + private IEnumerable OwnerScan() + { for (int p = 0; p < _channel.PageCount; p++) { PageBuffer buffer = _channel.ReadPage(p); @@ -29,7 +72,7 @@ public IEnumerable DataPages() var page = new DataPage(); page.Read(buffer, _channel.Format); - if (page.OwningTablePage == owner) yield return p; + if (page.OwningTablePage == _table.DefinitionPage) yield return p; } } } diff --git a/test/LibRed.Core.Tests/UsageMapTests.cs b/test/LibRed.Core.Tests/UsageMapTests.cs new file mode 100644 index 00000000..d75f1ef1 --- /dev/null +++ b/test/LibRed.Core.Tests/UsageMapTests.cs @@ -0,0 +1,31 @@ +using LibRed; +using Xunit; + +namespace LibRed.Core.Tests; + +public class UsageMapTests +{ + [Fact] + public void Inline_usage_map_lists_owned_data_pages() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var pages = db.OpenTable("MSysObjects").UsageMap.DataPages().ToList(); + + Assert.Equal([17, 274, 323], pages); + } + + [Fact] + public void Usage_map_excludes_stale_orphan_pages() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + // MSysNavPaneObjectIDs has an orphan page (stale owner stamp) that a naive + // owner-scan would double-count. The real usage map excludes it, so the decoded + // row count matches the TDEF's own count exactly. + var table = db.OpenTable("MSysNavPaneObjectIDs"); + int tdefRows = db.ReadTableDefinition(table.Definition.DefinitionPage).RowCount; + + Assert.Equal(tdefRows, table.Rows().Count()); + } +} From 9f95dcdb5b8be01ffd9ecefd310af24318e3ef73 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sat, 27 Jun 2026 22:45:02 +0800 Subject: [PATCH 013/506] LibRed: resolve memo/OLE long values (LVAL pages) + compressed Unicode LongValueReader resolves a 12-byte in-row long-value descriptor (24-bit length + flags + row/page pointer) to the full payload: inline (0x80), single LVAL page (0x40), or a chain across LVAL pages (each chunk prefixed by a 4-byte next pointer). RowDecoder now materializes Memo columns to strings and OLE columns to byte[] via the reader. Also handle Jet compressed Unicode: text values beginning with the 0xFF 0xFE marker store ASCII-range chars one byte each (not UTF-16). JetTypeCodec.DecodeText handles both forms and is used for Text and Memo. (Mid-string 1-/2-byte run toggling for mixed scripts is left as a TODO.) Verified on Northwind: all 8 Category descriptions (incl. compressed "Cheeses", "Prepared meats"), Employee Notes, and ~10 KB Picture OLE blobs chained across pages. Co-Authored-By: Claude Opus 4.8 --- .../LibRed.Core/Storage/LongValueReader.cs | 71 +++++++++++++++++++ src/LibRed/LibRed.Core/Storage/RowDecoder.cs | 13 +++- src/LibRed/LibRed.Core/Storage/TableCursor.cs | 5 +- .../LibRed.Core/Storage/Types/JetTypeCodec.cs | 19 ++++- test/LibRed.Core.Tests/TableScanTests.cs | 34 +++++++++ 5 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 src/LibRed/LibRed.Core/Storage/LongValueReader.cs diff --git a/src/LibRed/LibRed.Core/Storage/LongValueReader.cs b/src/LibRed/LibRed.Core/Storage/LongValueReader.cs new file mode 100644 index 00000000..57c639bc --- /dev/null +++ b/src/LibRed/LibRed.Core/Storage/LongValueReader.cs @@ -0,0 +1,71 @@ +using System.Buffers.Binary; +using LibRed.IO; +using LibRed.Pages; + +namespace LibRed.Storage; + +/// +/// Resolves a long value (Memo / OLE) from its 12-byte in-row descriptor to the full +/// byte payload, following LVAL pages as needed. +/// +/// +/// Descriptor layout: bytes 0-2 = length (24-bit), byte 3 = flags, bytes 4-7 = a +/// row+page pointer to the first LVAL chunk, bytes 8-11 reserved. Flags: +/// 0x80 = inline (payload follows the descriptor); 0x40 = single LVAL page (the row is +/// the whole payload); otherwise the payload is chained across LVAL pages, each row +/// beginning with a 4-byte pointer to the next chunk. +/// +public sealed class LongValueReader(PageChannel channel) +{ + private const byte FlagInline = 0x80; + private const byte FlagSinglePage = 0x40; + + private readonly PageChannel _channel = channel; + + public byte[] Resolve(ReadOnlySpan descriptor) + { + int length = descriptor[0] | (descriptor[1] << 8) | (descriptor[2] << 16); + byte flags = descriptor[3]; + + if ((flags & FlagInline) != 0) + return descriptor.Slice(12, length).ToArray(); + + int row = descriptor[4]; + int page = descriptor[5] | (descriptor[6] << 8) | (descriptor[7] << 16); + + return (flags & FlagSinglePage) != 0 + ? ReadLvalRow(page, row)[..length] + : ReadChain(page, row, length); + } + + private byte[] ReadChain(int page, int row, int length) + { + var result = new byte[length]; + int written = 0; + + while (page != 0 && written < length) + { + byte[] chunk = ReadLvalRow(page, row); + + // Each chained chunk starts with a 4-byte pointer (row + 3-byte page) to the next. + int nextRow = chunk[0]; + int nextPage = chunk[1] | (chunk[2] << 8) | (chunk[3] << 16); + + int copy = Math.Min(chunk.Length - 4, length - written); + Array.Copy(chunk, 4, result, written, copy); + written += copy; + + page = nextPage; + row = nextRow; + } + + return result; + } + + private byte[] ReadLvalRow(int page, int row) + { + var lval = new DataPage(); + lval.Read(_channel.ReadPage(page), _channel.Format); + return lval.GetRow(row).ToArray(); + } +} diff --git a/src/LibRed/LibRed.Core/Storage/RowDecoder.cs b/src/LibRed/LibRed.Core/Storage/RowDecoder.cs index 23764cb5..8209419a 100644 --- a/src/LibRed/LibRed.Core/Storage/RowDecoder.cs +++ b/src/LibRed/LibRed.Core/Storage/RowDecoder.cs @@ -16,10 +16,11 @@ namespace LibRed.Storage; /// Jet 4 / ACE uses 2-byte variable offsets at any row size — there is no Jet 3-style /// jump table (1-byte offsets), so rows larger than 256 bytes decode the same way. /// -public sealed class RowDecoder(IReadOnlyList columns, JetFormatBase format) +public sealed class RowDecoder(IReadOnlyList columns, JetFormatBase format, LongValueReader? longValues = null) { private readonly IReadOnlyList _columns = columns; private readonly JetFormatBase _format = format; + private readonly LongValueReader? _longValues = longValues; /// Decodes the row into one value per column (aligned to ). public object?[] Decode(ReadOnlySpan row) @@ -61,6 +62,16 @@ public sealed class RowDecoder(IReadOnlyList columns, JetFormatBase f ? FixedSlice(row, column) : VariableSlice(row, varTableStart, numVarCols, column.VariableIndex); + // Memo / OLE columns store a long-value descriptor, not the data itself. + if (_longValues is not null && column.Type is JetDataType.Memo or JetDataType.Ole) + { + byte[] data = _longValues.Resolve(raw); + values[column.Index] = column.Type == JetDataType.Memo + ? JetTypeCodec.DecodeText(data) + : data; + continue; + } + values[column.Index] = JetTypeCodec.Decode(column, raw); } diff --git a/src/LibRed/LibRed.Core/Storage/TableCursor.cs b/src/LibRed/LibRed.Core/Storage/TableCursor.cs index 062af406..7f76bc47 100644 --- a/src/LibRed/LibRed.Core/Storage/TableCursor.cs +++ b/src/LibRed/LibRed.Core/Storage/TableCursor.cs @@ -13,7 +13,10 @@ public sealed class TableCursor(Table table) : IEnumerable public IEnumerator GetEnumerator() { - var decoder = new RowDecoder(_table.Definition.Columns, _table.Channel.Format); + var decoder = new RowDecoder( + _table.Definition.Columns, + _table.Channel.Format, + new LongValueReader(_table.Channel)); foreach (int pageNumber in _table.UsageMap.DataPages()) { diff --git a/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs b/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs index 0fc05204..2171d057 100644 --- a/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs +++ b/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs @@ -36,7 +36,7 @@ public static class JetTypeCodec case JetDataType.Guid: return new Guid(value[..16]); case JetDataType.Text: - return Encoding.Unicode.GetString(value); + return DecodeText(value); case JetDataType.Binary: return value.ToArray(); @@ -50,6 +50,23 @@ public static class JetTypeCodec } } + /// + /// Decodes a Jet text value, honoring compressed Unicode. A value beginning with the + /// 0xFF 0xFE marker stores ASCII-range characters as one byte each; otherwise it is + /// UTF-16LE. + /// + /// + /// TODO: the full compressed format can toggle between 1-byte and 2-byte runs mid-string + /// (via embedded markers) for mixed scripts; this handles the common all-compressed case. + /// + public static string DecodeText(ReadOnlySpan value) + { + if (value.Length >= 2 && value[0] == 0xFF && value[1] == 0xFE) + return Encoding.Latin1.GetString(value[2..]); + + return Encoding.Unicode.GetString(value); + } + /// Encodes a CLR value back to its on-disk representation. public static byte[] Encode(ColumnDef column, object? value) { diff --git a/test/LibRed.Core.Tests/TableScanTests.cs b/test/LibRed.Core.Tests/TableScanTests.cs index 094a56ed..2a1c6a17 100644 --- a/test/LibRed.Core.Tests/TableScanTests.cs +++ b/test/LibRed.Core.Tests/TableScanTests.cs @@ -53,6 +53,40 @@ public void Decodes_boolean_columns_from_the_null_bitmap() Assert.Equal(8, rows.Count(r => (bool)r[discontinued]!)); } + [Fact] + public void Resolves_memo_long_values_including_compressed_unicode() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var categories = db.OpenTable("Categories"); + int nameIdx = categories.Definition.Columns.First(c => c.Name == "CategoryName").Index; + int descIdx = categories.Definition.Columns.First(c => c.Name == "Description").Index; + + var byName = categories.Rows().ToDictionary(r => (string)r[nameIdx]!, r => (string)r[descIdx]!); + + // UTF-16 stored memo: + Assert.Equal("Soft drinks, coffees, teas, beers, and ales", byName["Beverages"]); + // Compressed-Unicode (0xFF 0xFE marker, 1 byte/char) memos: + Assert.Equal("Cheeses", byName["Dairy Products"]); + Assert.Equal("Prepared meats", byName["Meat/Poultry"]); + } + + [Fact] + public void Resolves_ole_long_values_across_chained_pages() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var categories = db.OpenTable("Categories"); + int picIdx = categories.Definition.Columns.First(c => c.Name == "Picture").Index; + + // Each Picture is a ~10 KB OLE blob chained across multiple LVAL pages. + Assert.All(categories.Rows(), r => + { + var blob = Assert.IsType(r[picIdx]); + Assert.Equal(10746, blob.Length); + }); + } + [Fact] public void Scans_a_small_lookup_table() { From 0795ecaaa08b21ac423cef086709dba8c81e23cf Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 01:20:57 +0800 Subject: [PATCH 014/506] LibRed: stitch multi-page table definitions (wide tables) A table with enough columns has a TDEF that spans pages, chained via the next-page pointer at 0x04. TableDefinitionPage.Read(channel, page) now assembles the chain into one contiguous buffer before parsing: the first page whole, then each continuation page's data after its 8-byte header. Column offsets are absolute from the first page, so parsing is otherwise unchanged. Verified against a generated 200-column ACCDB (Data/WideTable.accdb, created via the included WideTable.generate.ps1 using the ACE OLE DB provider) whose definition spans pages 42->44: all 200 columns (C000..C199) parse across the boundary and the row decodes. Northwind has no multi-page TDEF, so this asset was needed to exercise it. Also mark *.accdb / *.mdb as binary in .gitattributes. Co-Authored-By: Claude Opus 4.8 --- .gitattributes | 4 ++ src/LibRed/LibRed.Core/Catalog/JetCatalog.cs | 2 +- src/LibRed/LibRed.Core/JetDatabase.cs | 2 +- .../LibRed.Core/Pages/TableDefinitionPage.cs | 36 ++++++++++++++- test/LibRed.Core.Tests/Data/WideTable.accdb | Bin 0 -> 188416 bytes .../Data/WideTable.generate.ps1 | 12 +++++ .../LibRed.Core.Tests.csproj | 5 ++ .../MultiPageTableDefinitionTests.cs | 43 ++++++++++++++++++ test/LibRed.Core.Tests/TestDatabases.cs | 4 ++ 9 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 test/LibRed.Core.Tests/Data/WideTable.accdb create mode 100644 test/LibRed.Core.Tests/Data/WideTable.generate.ps1 create mode 100644 test/LibRed.Core.Tests/MultiPageTableDefinitionTests.cs diff --git a/.gitattributes b/.gitattributes index 1ff0c423..1b136160 100644 --- a/.gitattributes +++ b/.gitattributes @@ -61,3 +61,7 @@ #*.PDF diff=astextplain #*.rtf diff=astextplain #*.RTF diff=astextplain + +# Access database files are binary. +*.accdb binary +*.mdb binary diff --git a/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs b/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs index fe509caa..9404082b 100644 --- a/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs +++ b/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs @@ -66,7 +66,7 @@ private List LoadTables() private TableDef ReadTableDefinition(int definitionPage, string name, bool isSystem) { var tdef = new TableDefinitionPage(); - tdef.Read(_channel.ReadPage(definitionPage), _channel.Format); + tdef.Read(_channel, definitionPage); return new TableDef { diff --git a/src/LibRed/LibRed.Core/JetDatabase.cs b/src/LibRed/LibRed.Core/JetDatabase.cs index 73bd544e..3f41175f 100644 --- a/src/LibRed/LibRed.Core/JetDatabase.cs +++ b/src/LibRed/LibRed.Core/JetDatabase.cs @@ -32,7 +32,7 @@ private JetDatabase(PageChannel channel) public TableDefinitionPage ReadTableDefinition(int pageNumber) { var tdef = new TableDefinitionPage(); - tdef.Read(_channel.ReadPage(pageNumber), _channel.Format); + tdef.Read(_channel, pageNumber); return tdef; } diff --git a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs index 23a8009e..3eb705b4 100644 --- a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs +++ b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs @@ -26,6 +26,39 @@ public sealed class TableDefinitionPage : Page public IReadOnlyList Columns => _columns; + /// Bytes of a continuation TDEF page that precede the resumed definition data. + private const int ContinuationHeaderSize = 8; + + /// + /// Reads a table definition starting at , transparently + /// stitching continuation pages (wide tables whose definition spans multiple pages) + /// into one contiguous buffer before parsing. + /// + public void Read(PageChannel channel, int page) + => Read(AssembleDefinition(channel, page), channel.Format); + + private static PageBuffer AssembleDefinition(PageChannel channel, int page) + { + PageBuffer first = channel.ReadPage(page); + int next = first.ReadInt32(channel.Format.TdefNextPageOffset); + if (next == 0) + return first; + + // The column offsets are absolute from the first page's start, so the first page + // is taken whole and each continuation contributes its data after the 8-byte header. + var assembled = new List(first.Span.Length * 2); + assembled.AddRange(first.Span); + + while (next != 0) + { + PageBuffer continuation = channel.ReadPage(next); + next = continuation.ReadInt32(channel.Format.TdefNextPageOffset); + assembled.AddRange(continuation.Span[ContinuationHeaderSize..]); + } + + return new PageBuffer(assembled.ToArray(), page); + } + public override void Read(PageBuffer buffer, JetFormatBase format) { PageNumber = buffer.PageNumber; @@ -41,8 +74,7 @@ public override void Read(PageBuffer buffer, JetFormatBase format) // The column descriptors follow a per-index block sized by the index count at // 0x33 (IndexCount) — NOT the index-slot count at 0x2F. The two are equal for // MSysObjects but differ for user tables (e.g. slots=2, indexes=1). - // NOTE: assumes a single-page TDEF. A multi-page TDEF (NextDefinitionPage != 0) - // must have its pages stitched into one contiguous buffer first. TODO. + // The buffer here may already be a stitched multi-page definition (see Read(channel, page)). int columnBlock = format.TdefRealIndexBlockOffset + IndexCount * format.RealIndexEntrySize; ReadColumns(buffer, format, columnBlock); } diff --git a/test/LibRed.Core.Tests/Data/WideTable.accdb b/test/LibRed.Core.Tests/Data/WideTable.accdb new file mode 100644 index 0000000000000000000000000000000000000000..1af250ee90f0e9a742e4e3e469c54077d5dd6260 GIT binary patch literal 188416 zcmeI5eUz0|y~p=+=FBj!=ga^i@}i?6Fop&*!@L+3$Vf=T%Ls_1Aq+FT2m=HIfl^>% zscBMKYGG)7yUb+QfZ-dj6i}zklU7_s;m=TlUR4H1}ZRd9UyJ zpI`iS{GZn(Zdp2^`nrbOp83;b>8V4eKQZ#Ly4HW+cI98A?dN~%wK+e(6ck{-#V=PwZB}@{^Ein=jtz%2wv%5&&GZoZZk^ibmcy09_V_+DGsHS zy1DL?{-`%%uX)&4j%^*vk)ZdR(vnD&bNt6UHZlvkLpjH6cs9kDL)Xx&8!8`xTT-`j^7O zl4U+C_BTAQ&g!?sC9!v*kC(u60oDpvjOP-pl`e(nw5+Zi&lOlxt`yH@SaD6^c@Wn4 zaKbS#K?5{E12jMbG(ZD1Km#;z3K&R452vIWbJ6dRUdPx>oqRXM)Nj6Y&ExL5N9g!Q z()q;_y3w*fx=qMByt^6+$nAVe$EVN!xbMfthag2|yTnCq8oHsg(UF_sJ9n$lVcY0d zqXQ=$R_QQSqGQxo7LnZtZ)7;aK1u9T8OzPY(VIs8g3I2&@Y@KJb1kd6w-K06 z^upeb>!Y{G`y;_O6Er{rG(ZD1Km#;D12jMbC!>KVdP8TtqpK^Q|2HXpF!f-nJvBP@ zQpJ}m=2Vna94OyWeo^_~%N{LTQ#Lkd(EnttGmn`DXn+Q2fCgxQ255i=j>bUlnAp+m zle^8%+zh^^0UCI38t~(q{jGZc3Q0zPFYB{}YUifPlzthpl@WA55L=C(Q?a^gc@iXv zoml-yzHAfumq3Z|nP-Bzyz>Tj+>V$#=FaMx)jDfU+Zy>3Ru#pWtJ{{$>RQ~l+uGXZcC1G&{ALwL z*lf+>_Jx_v%KO-CNk{uaUrt#c8?0H>u{y3*X&+m3wQY_!sO)2d_3djGFKlm{+q$JI zKIW3_h8?X}w$1K}?IyF?tJb%zUYt24McM59vpOh-n-{E400>T~?G zo_GENWkedF0UDqI8lV9hpaB}7fp^S6GTJO}{qJ>2B#SGnr|!gOp_@dnTQc)RHLzo+ zziU&KlmxnXRfUqk&77(tN#L6Tss>2{Z*!5OO?;9lM<@RqTIy4}78l z8lV9hpaB}70UDqI8lZvWWFU%0>wCD-J&fo7AE!_(mIi2m255i=Xn+Q2fCgxQ255i= zPBsG=KpmK+ud(;HReC=PNk;!t-q*;h9rn+k2w-#G*3oj~chG>0z~l0$ahf7VuU0{BF9fU zkioiTIrbNSx!R!6V^kD2Nt6Lthg99SV{- zH#0+_W{4(+IxaLo12jMbG(ZD1Km#;D12k~b8Hl3W^>g3R^~-+#-$_>y9xx5i01eOp z4bT7$&;Sk401eQvx-8c0UfsR5hJFrY`q!f1d7Xn+Q2fCgxQ255i=Xn+Q2 zfCdK20LT9a$|TF70UDqI8lV9hpaB}70UDqI8W=bO_z3ZSFFX5P0t%Qq5QUrwWyttH z-s#PS255i=Xn+Q2fCgxQ255i=Xn+Q2;1o8%{{Ja_a#=MRpaB}70UDqI8lV9hpaB}7 z0UF2{VE><&0u9gr4bT7$&;Sk401eOp4bT7$ocac$=!s(e`oJOX@pMPJEWJOqCN(Pc zY{iO-!iwGHHRT7&=9m3v>8DBumwvxwO3B}fPkk$_Aq~&~4bT7$(7-8Tph~_K*5NIr z>Bv2@XU5HE-u~yQ|M*q>Q>wVgt#%!*(=BkzT)SI~)Y#wFyZlG3t9E6s$GIx`;@M1r zyZ$)c7pOn+9lJYcyZLUL+vKLZ^{&fp@I`J#@!OE@bXU7+t{abKZn;|p-s#ri)q-9h z67ZK25MOzl6Y$1N;G+q+0714Q$jn|JAqc){w;w@T5M()2cA;i#-JD(@Ar!vmw;!QC z;ug9~T}!XEnd#4xmxgZ_?njVI-6j6fg0r|eb78#2S;7|_4?w{AkhspR^q0+B1RU9~ zfUT&R1bU0h0^jl6k6O*gMZVr`#ML*e*GIS{@a59|2(u71*#vPb@v{@n`C_!TZltd0 z^$~*NJFatrW*f5WqajPPUW-ej%ZF^ib(sGM0pvZ_@vj~&SbqY90GHuN*P`8YqXu$L zHbL5I{I=NS#{Dk8@D|ig0{CA=@*(bT9iR=z+ljNV8qZQ_lgocJWDCXDUmr!G)9@Sa z5?@@c%YRJF7AGCew?4+XXk^#m_?AP%Q5+-6tVF@-gj1vX6C{{zL7`)Rd!fsJOvoM| zs+-16>4IL1I?~CR;_@FCWeb+$>$~65^=OvTY`G=4N~NjEawtj) zNXixA1W3bot{gcYL;+}=3@FF}#kxl#2Vi?Q)c^I>W;zFk+xW4`+P+9$Y`OkYHvJa}< z*AB=pz&@w>ZJoO%>;oG|*e8j7h8?z#oD4Y?{>=Aa5t@`=EAf>fjaOD2dLEN(zhs#& zW<35pci_Jq@~iU5#ztkf?A=>whKk!bxCZ1;8lV9hpaB}70UDqI8lV9hpn;<^z|a3b zI#1k+255i=Xn+Q2fCgxQ255i=XyBbRFm)h;RS!fVl><@82~b87pZ~8T@+ZLI@DON# z255i=Xn+Q2fCgxQ255i=Xn+Qeg8}yckAqYeNdq)M12jMbG(ZD1Km#;D12jMbCzt{D z|4*CHAQ62@K1k55z?%J>zRahMr-6izgp9$xm#Ht?-!!Xn+Q2fCgxQ255i=Xn+Q2fCkKh|XLnw?|5z)td{y!Gu|5!`2>cyE7p zEFXjWIrW=UikvU@en<3h;2wkN;Qd7D)kx*e3=B9u(uXpD_@Blz`piP*f~? zBy;v~t~lN!;Xhm8@IA`22W$a8mB7t#t1@PX<{oY+}gV}n4+lJSjE)}+3=q_`ukdTCtLadkwf#(9Oocf>_1b@;14bT7$ z&;Sk401eOp4bT7$ysHMHXr}M~%Rv;mr_x)}lhOxM52o5f?z?(;3{C?yKm#;D12jMb zG(ZD1Km#;D12ixI2BPT2av8msWvqKG{U7OP(|gnRr@xTCF1;$fFnw|Q{pnHZlJq}P z|CM?n_4CwsQ+K9zr9P2blKN<>DRo|IaH^-`V8!n%o~ih8#eEgGRa{%KvSLBSw2BKW z;xhS612jMbG(ZD1Km#;D12jMbC#Qj=J0Az$b13b`{k9z6aTd8jan>Q1c4MM(`d z6fcLGA2ByePFl`Za?*0LOHNu&mYlSlEIDa8S#r{Hvf?l3 zGgcFDGXY!OjVLmm9xwf(KQk}goac~|HM6`An_~6T{NpT{g0FWHFpXaei~Jh{Vj#2o zJ&?ml7$9#9@IbaJFrZw4oSdWql?voU78)>A0V;!QC3{rE6$}CxWWXo|#Q?MA|1I8;T11K|Kf`W2@aswtRkozSS2GlE%rjjzCNdcM%uKa8Trznv3T2vY^ zRY4U%l>sdZJD8L(YJ zH9)liI~AM_aJB)v6r2NajsZ6*kl!ok8gQ$EF#ux>*sVZ*Ih|*~9SX(*j5Xjc1@aqi zoB?|j;QcIayaD$qr~#-k-~k2l*YbP=9#Vi`9qxSwJfc8;7fmqWQ3V$OTwuUn1s4Kb zXuy*S-VgA81NJGH2r$ur{R$=lOfukrf?9xD1D;b*2T*6g^9t$#>J4~7!DN8R2E3@C z0ieNvmlQMtG#c;+1x)}=2E44G8KBvKR}@?XaFGG8D)<1v2Mjoj90er}SHx+ys;KK&|Q^8b#sRp29zsk0cbIx zQo$tvml!Zq!3=;I1`Jp55rB^vFiOEpfSCqVE4UQkQUk^)m<2G)fbj}G3h+?_CMakH zXfg4qDG4X9Tz2Vjl?O$z1$%r#(&f_VV*44A5*4WP|{76q39TxP&b1@i&s8_=p? z0l)$S<|$YRu+V@73N8n@+<+?-ECN_$z*P#a0Jy?{r3w}UEH1R{~sV zz*+@Y0bFIk1_d7n__zVx3YGvYF<`5Ls{yVyV7r2)080(nsbCqvG6QxgXa{IF;3fsj z0hSwZtAY-I4g+>8=mh9A;0^^V09F`qmx7f5D-GDAU=_eB1MX9>8ep{n4=A_>;2HxS zQm_VKjRB7+SPQV$fJYT{0dyI#SHU`fbp|}CU_HQk1NJG{0Ij17Z;Glx-0NV|CUBRaSK4rig3O)_+X#?I=umfO+0smC66JVzSE|K~B zaXrBG1|$`H2H-OW6f3v^;06QA73>1oWk98Z8v$-KV5ox60({nh;R-$n@HqoUDYyyX zCIhM!+zfED0b>;00&t4~;}v`!;PVDdP;e{2tp-d~a2vpF2GlF~0>BpxXi{)H!0iT1 zQLr0ew*gZXd=cP_2DB*n62O-Xn5p2)0ADtsRlywqcNj2F!B+sjV!#3gUj_K80aqxv z6W~q*u2OIpz+DC`Rd6@J-3D|h_!_|13|OV$>i}OjV6B2Z0DBDBpx_>WdkpAS@C|@( z7_e2ry#V(buwB7@0QVWNQ^Eb{#77-zV80~%{C|DLpEN)NG(ZD1Km#;D12jMbG(ZD1 zKm!A1pn4z*>PI1oXi?H1!u{q3!7jZ)@J*m^8gP?>ZvlMEfLj$j0PuhTyA^yJ;M)e= zq2N0J-!b4W1rGu|Xuuu?4*@)6z;u?m!0QTr4)AjW-caxhfL|E!rh;Dr{L+AbD%cOO-vC#TxsUuSfL|GqRPbwn zUmH-Y;5PujF`!(*0e}MrR4RBH;AsPfDtHFq83TqZcoyJU14aQ9WAt;Pe`sn_$g2&h zx{#_5smUSL5K@gH)f7_AnbagiiDOK{n7O7j#w3i2Yo5lKgmG`p(-@O5vaNX7QNtJ)Q1@LAx3?OQ6FN|hZyxCMtz7;A7a#p7?VSc$sxw%5My$PF*(GT z9AZokF(!u?lS7QjAx1-p(GX%ZgcuDWMnj0v5Mnfh7!4stLx|B3Vl;*rjUh&3h|w5g zG=>EYQ;5+NVl;&qO(8~8h|v^cG=~_?Ax3kE z(HvqlhZxNvMstYK9AY$w7|od&(gNc)D5yc^X68x#VdKZRe7wF|?gap2pC2F4vsK&~`3)8bjN; zySF^^@b}o4uL)*FJX$)=WlBY4Wol9ObKR+7($KgT)G;o{@ zMA75r=V!`}+|%X9DGZCH0UDqI8lV9hpaB}70UDrzfidu$d$nL-N@6JkW1ux2R34bZ TSW3SP{1E3NG9^@+^v?eWs$H=1 literal 0 HcmV?d00001 diff --git a/test/LibRed.Core.Tests/Data/WideTable.generate.ps1 b/test/LibRed.Core.Tests/Data/WideTable.generate.ps1 new file mode 100644 index 00000000..b66e8d06 --- /dev/null +++ b/test/LibRed.Core.Tests/Data/WideTable.generate.ps1 @@ -0,0 +1,12 @@ +$path = "$PSScriptRoot\WideTable.accdb" +if (Test-Path $path) { Remove-Item $path -Force } +$conn = "Provider=Microsoft.ACE.OLEDB.12.0;Data Source=$path" +$cat = New-Object -ComObject ADOX.Catalog +$cat.Create($conn) | Out-Null +$cn = $cat.ActiveConnection +$cols = (0..199 | ForEach-Object { "C{0:D3} LONG" -f $_ }) -join ", " +$cn.Execute("CREATE TABLE WideTable ($cols)") | Out-Null +# insert one row: C000=1000, C199=1199, rest null +$cn.Execute("INSERT INTO WideTable (C000, C100, C199) VALUES (1000, 1100, 1199)") | Out-Null +$cn.Close() +Write-Output "created $path ($((Get-Item $path).Length) bytes)" diff --git a/test/LibRed.Core.Tests/LibRed.Core.Tests.csproj b/test/LibRed.Core.Tests/LibRed.Core.Tests.csproj index 9d079a47..88b32b7f 100644 --- a/test/LibRed.Core.Tests/LibRed.Core.Tests.csproj +++ b/test/LibRed.Core.Tests/LibRed.Core.Tests.csproj @@ -36,6 +36,11 @@ PreserveNewest + + + PreserveNewest + diff --git a/test/LibRed.Core.Tests/MultiPageTableDefinitionTests.cs b/test/LibRed.Core.Tests/MultiPageTableDefinitionTests.cs new file mode 100644 index 00000000..ffd46d18 --- /dev/null +++ b/test/LibRed.Core.Tests/MultiPageTableDefinitionTests.cs @@ -0,0 +1,43 @@ +using LibRed; +using Xunit; + +namespace LibRed.Core.Tests; + +public class MultiPageTableDefinitionTests +{ + [Fact] + public void Parses_definition_that_spans_multiple_pages() + { + using var db = JetDatabase.Open(TestDatabases.WideTableAccdb); + + var def = db.Catalog.FindTable("WideTable"); + Assert.NotNull(def); + + var tdef = db.ReadTableDefinition(def!.DefinitionPage); + + // The definition continues onto another page. + Assert.NotEqual(0, tdef.NextDefinitionPage); + Assert.Equal(200, tdef.ColumnCount); + Assert.Equal(200, tdef.Columns.Count); + + // Column names continue seamlessly across the page boundary (C000..C199). + Assert.Equal( + Enumerable.Range(0, 200).Select(i => $"C{i:D3}"), + tdef.Columns.Select(c => c.Name)); + } + + [Fact] + public void Reads_rows_from_a_wide_table() + { + using var db = JetDatabase.Open(TestDatabases.WideTableAccdb); + + var table = db.OpenTable("WideTable"); + int Idx(string n) => table.Definition.Columns.First(c => c.Name == n).Index; + + var row = Assert.Single(table.Rows()); + Assert.Equal(1000, row[Idx("C000")]); + Assert.Equal(1100, row[Idx("C100")]); + Assert.Equal(1199, row[Idx("C199")]); + Assert.Null(row[Idx("C001")]); // not inserted + } +} diff --git a/test/LibRed.Core.Tests/TestDatabases.cs b/test/LibRed.Core.Tests/TestDatabases.cs index d206c220..6104545a 100644 --- a/test/LibRed.Core.Tests/TestDatabases.cs +++ b/test/LibRed.Core.Tests/TestDatabases.cs @@ -6,4 +6,8 @@ internal static class TestDatabases /// An Access 2007 (ACE 12 / ACCDB) Northwind sample. public static string NorthwindAccdb { get; } = Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"); + + /// A 200-column ACCDB whose table definition spans multiple TDEF pages. + public static string WideTableAccdb { get; } = + Path.Combine(AppContext.BaseDirectory, "Data", "WideTable.accdb"); } From b84c020b6cc7ab9972db161ab7d8dcbf1dc4a0b8 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 01:30:02 +0800 Subject: [PATCH 015/506] LibRed: parse reference usage maps (very large tables) When a table owns more pages than an inline bitmap can cover, its usage map is a reference map (type 0x01): a list of 4-byte pointers to dedicated bitmap pages (type 0x05, bitmap at offset 4). Pointer k's bitmap covers pages starting at k * (pageSize - 4) * 8. UsageMap now parses these directly instead of the owner-scan fallback (which is removed); inline and reference share the bit-scan helper. Verified locally against a generated ~150 MB / 115k-row ACCDB whose Big table uses a reference map: UsageMap yields the same 38,534 owned pages as a brute-force owner-scan (reading 2 bitmap pages instead of 38,792). That file is too large to commit, so the generator (Data/BigTable.generate.ps1) is included and the file is gitignored; the committed Northwind tests continue to cover the inline path. Co-Authored-By: Claude Opus 4.8 --- src/LibRed/LibRed.Core/Storage/UsageMap.cs | 52 ++++++++++++------- test/LibRed.Core.Tests/Data/.gitignore | 2 + .../Data/BigTable.generate.ps1 | 21 ++++++++ 3 files changed, 56 insertions(+), 19 deletions(-) create mode 100644 test/LibRed.Core.Tests/Data/.gitignore create mode 100644 test/LibRed.Core.Tests/Data/BigTable.generate.ps1 diff --git a/src/LibRed/LibRed.Core/Storage/UsageMap.cs b/src/LibRed/LibRed.Core/Storage/UsageMap.cs index 4f215b90..dc6db710 100644 --- a/src/LibRed/LibRed.Core/Storage/UsageMap.cs +++ b/src/LibRed/LibRed.Core/Storage/UsageMap.cs @@ -12,14 +12,18 @@ namespace LibRed.Storage; /// /// The TDEF holds a pointer (row + page) to the usage-map record. An inline map /// (type 0x00) stores a start page and a bitmap where bit i marks page (startPage + i) -/// as owned. A reference map (type 0x01, only for very large tables) instead points at -/// dedicated bitmap pages — not yet parsed, so we fall back to a full owner-scan there. +/// as owned. A reference map (type 0x01, for very large tables) instead stores a list of +/// pointers to dedicated bitmap pages (type 0x05); pointer k's bitmap covers the page +/// range starting at k * (pageSize - 4) * 8. /// public sealed class UsageMap(PageChannel channel, TableDef table) { private const byte MapTypeInline = 0x00; private const byte MapTypeReference = 0x01; + /// Bytes preceding the bitmap on a dedicated usage-bitmap page (type 0x05). + private const int BitmapPageHeaderSize = 4; + private readonly PageChannel _channel = channel; private readonly TableDef _table = table; @@ -39,8 +43,7 @@ public IEnumerable DataPages() return map[0] switch { MapTypeInline => ReadInlineMap(map), - // TODO: parse reference maps (dedicated bitmap pages). Fall back meanwhile. - MapTypeReference => OwnerScan(), + MapTypeReference => ReadReferenceMap(map), byte t => throw new NotSupportedException($"Unknown usage map type 0x{t:X2}."), }; } @@ -49,30 +52,41 @@ private static List ReadInlineMap(ReadOnlySpan map) { int startPage = BinaryPrimitives.ReadInt32LittleEndian(map.Slice(1, 4)); var pages = new List(); + AppendSetBits(pages, map[5..], startPage); + return pages; + } - for (int i = 5; i < map.Length; i++) + private List ReadReferenceMap(ReadOnlySpan map) + { + int pagesPerBitmap = (_channel.PageSize - BitmapPageHeaderSize) * 8; + var pages = new List(); + + // The record is a list of 4-byte pointers to bitmap pages; pointer k's bitmap + // covers the page range starting at k * pagesPerBitmap. A zero pointer means the + // range has no owned pages. + int entryCount = (map.Length - 1) / 4; + for (int e = 0; e < entryCount; e++) { - byte b = map[i]; - if (b == 0) continue; - for (int bit = 0; bit < 8; bit++) - if ((b & (1 << bit)) != 0) - pages.Add(startPage + (i - 5) * 8 + bit); + int bitmapPage = BinaryPrimitives.ReadInt32LittleEndian(map.Slice(1 + e * 4, 4)); + if (bitmapPage == 0) continue; + + int rangeBase = e * pagesPerBitmap; + ReadOnlySpan bitmap = _channel.ReadPage(bitmapPage).Span[BitmapPageHeaderSize..]; + AppendSetBits(pages, bitmap, rangeBase); } return pages; } - /// Fallback: scan every page and match the owning-table pointer. - private IEnumerable OwnerScan() + private static void AppendSetBits(List pages, ReadOnlySpan bitmap, int basePage) { - for (int p = 0; p < _channel.PageCount; p++) + for (int i = 0; i < bitmap.Length; i++) { - PageBuffer buffer = _channel.ReadPage(p); - if (buffer.ReadByte(0) != (byte)PageType.DataPage) continue; - - var page = new DataPage(); - page.Read(buffer, _channel.Format); - if (page.OwningTablePage == _table.DefinitionPage) yield return p; + byte b = bitmap[i]; + if (b == 0) continue; + for (int bit = 0; bit < 8; bit++) + if ((b & (1 << bit)) != 0) + pages.Add(basePage + i * 8 + bit); } } } diff --git a/test/LibRed.Core.Tests/Data/.gitignore b/test/LibRed.Core.Tests/Data/.gitignore new file mode 100644 index 00000000..72486a72 --- /dev/null +++ b/test/LibRed.Core.Tests/Data/.gitignore @@ -0,0 +1,2 @@ +# Large generated DB for reference-usage-map verification — regenerate with BigTable.generate.ps1. +BigTable.accdb diff --git a/test/LibRed.Core.Tests/Data/BigTable.generate.ps1 b/test/LibRed.Core.Tests/Data/BigTable.generate.ps1 new file mode 100644 index 00000000..5b720ce5 --- /dev/null +++ b/test/LibRed.Core.Tests/Data/BigTable.generate.ps1 @@ -0,0 +1,21 @@ +# Generates BigTable.accdb: a table large enough (~115k rows, ~38k pages, ~150 MB) +# that its owned-pages usage map is a REFERENCE map (type 0x01) pointing at dedicated +# bitmap pages, rather than an inline bitmap. Northwind only exercises inline maps, and +# a file big enough for a reference map is too large to commit — so this regenerates it +# locally for verifying UsageMap's reference-map path. Requires the ACE OLE DB provider. +# +# The generated file is gitignored. Run, then point a local check at it. +$path = "$PSScriptRoot\BigTable.accdb" +if (Test-Path $path) { Remove-Item $path -Force } +$conn = "Provider=Microsoft.ACE.OLEDB.12.0;Data Source=$path" +$cat = New-Object -ComObject ADOX.Catalog +$cat.Create($conn) | Out-Null +$cn = $cat.ActiveConnection +$cn.Execute("CREATE TABLE Seed (n LONG)") | Out-Null +for ($i = 0; $i -lt 340; $i++) { $cn.Execute("INSERT INTO Seed (n) VALUES ($i)") | Out-Null } +$cn.Execute("CREATE TABLE Big (Id COUNTER PRIMARY KEY, F1 TEXT(255), F2 TEXT(255), F3 TEXT(255))") | Out-Null +$f = ('a' * 210) +$cn.Execute("INSERT INTO Big (F1,F2,F3) SELECT '$f','$f','$f' FROM Seed a, Seed b") | Out-Null +$rows = $cn.Execute("SELECT COUNT(*) AS c FROM Big").Fields.Item("c").Value +$cn.Close() +Write-Output "rows=$rows size=$((Get-Item $path).Length)" From 4a0715a2e58c24bfa4f2b3df816cb995b6ed988e Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 01:47:13 +0800 Subject: [PATCH 016/506] LibRed: parse index definitions and traverse index B-trees Using the mdbtools/Jackcess formats as reference and verifying against Northwind: Index definitions: TableDefinitionPage parses the 52-byte per-index column blocks that follow the column names -- indexed columns + sort order, the unique/required flags, and the B-tree root page -- into IndexDef, exposed via TableDef.Indexes. B-tree traversal: IndexCursor walks an index from its root and yields row pointers (RowId = page + row) in key order. It reads the entry-position bitmask at 0x1b, takes the 4-byte big-endian trailing pointer of each entry (leaf -> page<<8|row; node -> child page), recurses into node children, and follows the node header's child-tail page (0x14) for the rightmost subtree. Key bytes are not decoded, so the order-preserving key encoding is not needed to enumerate rows in order. Verified: all 36 indexes across the user tables yield exactly one entry per row (including the node-rooted Orders/Order Details indexes), and Orders' PK (a node root) traverses all 830 rows in ascending OrderID order. Known scope: index names aren't parsed yet (they sit past the logical-index entries), IsPrimaryKey is a unique+required heuristic, and index key VALUES are not decoded (only the row pointers). Co-Authored-By: Claude Opus 4.8 --- src/LibRed/LibRed.Core/Catalog/JetCatalog.cs | 1 + .../LibRed.Core/Pages/TableDefinitionPage.cs | 60 ++++++++++++- src/LibRed/LibRed.Core/Storage/IndexCursor.cs | 87 +++++++++++++++++++ src/LibRed/LibRed.Core/Storage/RowId.cs | 4 + test/LibRed.Core.Tests/IndexTests.cs | 85 ++++++++++++++++++ 5 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 src/LibRed/LibRed.Core/Storage/IndexCursor.cs create mode 100644 src/LibRed/LibRed.Core/Storage/RowId.cs create mode 100644 test/LibRed.Core.Tests/IndexTests.cs diff --git a/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs b/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs index 9404082b..f5ab5518 100644 --- a/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs +++ b/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs @@ -73,6 +73,7 @@ private TableDef ReadTableDefinition(int definitionPage, string name, bool isSys Name = name, DefinitionPage = definitionPage, Columns = tdef.Columns, + Indexes = tdef.Indexes, IsSystem = isSystem, }; } diff --git a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs index 3eb705b4..a56b6a35 100644 --- a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs +++ b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs @@ -26,9 +26,24 @@ public sealed class TableDefinitionPage : Page public IReadOnlyList Columns => _columns; + private readonly List _indexes = []; + public IReadOnlyList Indexes => _indexes; + /// Bytes of a continuation TDEF page that precede the resumed definition data. private const int ContinuationHeaderSize = 8; + // Index column block layout (Jet 4 / ACE), one per real index, following the column names. + private const int IndexBlockSize = 52; + private const int IndexMaxColumns = 10; + private const int IndexColumnSlotSize = 3; // 2-byte column id + 1-byte flags + private const int IndexColumnsOffset = 0x04; + private const int IndexRootPageOffset = 0x26; + private const int IndexFlagsOffset = 0x2E; + private const short IndexColumnUnused = -1; // 0xFFFF + private const byte IndexColumnAscending = 0x01; + private const ushort IndexFlagUnique = 0x0001; + private const ushort IndexFlagRequired = 0x0008; + /// /// Reads a table definition starting at , transparently /// stitching continuation pages (wide tables whose definition spans multiple pages) @@ -76,10 +91,49 @@ public override void Read(PageBuffer buffer, JetFormatBase format) // MSysObjects but differ for user tables (e.g. slots=2, indexes=1). // The buffer here may already be a stitched multi-page definition (see Read(channel, page)). int columnBlock = format.TdefRealIndexBlockOffset + IndexCount * format.RealIndexEntrySize; - ReadColumns(buffer, format, columnBlock); + int afterNames = ReadColumns(buffer, format, columnBlock); + ReadIndexes(buffer, afterNames); } - private void ReadColumns(PageBuffer buffer, JetFormatBase format, int columnBlock) + /// + /// Parses the per-index column blocks (52 bytes each) that follow the column names: + /// indexed columns + sort order, the unique/required flags, and the B-tree root page. + /// + private void ReadIndexes(PageBuffer buffer, int blockStart) + { + _indexes.Clear(); + var byColumnId = _columns.ToDictionary(c => c.ColumnId); + + for (int i = 0; i < IndexCount; i++) + { + int block = blockStart + i * IndexBlockSize; + + var columns = new List<(ColumnDef Column, bool Ascending)>(); + for (int slot = 0; slot < IndexMaxColumns; slot++) + { + int entry = block + IndexColumnsOffset + slot * IndexColumnSlotSize; + short columnId = buffer.ReadInt16(entry); + if (columnId == IndexColumnUnused) continue; + if (byColumnId.TryGetValue(columnId, out ColumnDef? column)) + columns.Add((column, (buffer.ReadByte(entry + 2) & IndexColumnAscending) != 0)); + } + + ushort flags = buffer.ReadUInt16(block + IndexFlagsOffset); + bool unique = (flags & IndexFlagUnique) != 0; + bool required = (flags & IndexFlagRequired) != 0; + + _indexes.Add(new IndexDef + { + Name = string.Empty, // index names live further on, after the logical-index entries (TODO) + Columns = columns, + IsUnique = unique, + IsPrimaryKey = unique && required, // heuristic; the precise flag is in the logical-index entry + RootPage = buffer.ReadInt32(block + IndexRootPageOffset), + }); + } + } + + private int ReadColumns(PageBuffer buffer, JetFormatBase format, int columnBlock) { _columns.Clear(); @@ -133,5 +187,7 @@ private void ReadColumns(PageBuffer buffer, JetFormatBase format, int columnBloc IsAutoNumber = (d.Flags & JetFormatBase.ColumnFlagAutoNumber) != 0, }); } + + return namePos; } } diff --git a/src/LibRed/LibRed.Core/Storage/IndexCursor.cs b/src/LibRed/LibRed.Core/Storage/IndexCursor.cs new file mode 100644 index 00000000..79155b6c --- /dev/null +++ b/src/LibRed/LibRed.Core/Storage/IndexCursor.cs @@ -0,0 +1,87 @@ +using LibRed.IO; +using LibRed.Pages; + +namespace LibRed.Storage; + +/// +/// Walks an index B-tree and yields the row pointers in index (key) order. +/// +/// +/// Each index page has an entry-position bitmask at whose set +/// bits give the end offsets of successive entries within the entry-data region that begins +/// at . A leaf entry ends with a 4-byte big-endian row pointer +/// (page in the high 24 bits, row in the low 8); a node entry instead ends with the 4-byte +/// child page number. Key bytes are not decoded here — only the trailing pointers are read — +/// so the order-preserving key encoding is not needed to enumerate rows in order. +/// +public sealed class IndexCursor(PageChannel channel, int rootPage) +{ + private const int EntryMaskOffset = 0x1B; + private const int EntryDataOffset = 0x1E0; + + /// On a node page, the rightmost child page — not referenced by any entry. + private const int ChildTailOffset = 0x14; + + private readonly PageChannel _channel = channel; + private readonly int _rootPage = rootPage; + + public IEnumerable RowIds() => Walk(_rootPage); + + private IEnumerable Walk(int pageNumber) + { + PageBuffer page = _channel.ReadPage(pageNumber); + var type = (PageType)page.ReadByte(0); + + foreach ((int start, int end) in EntryRanges(page)) + { + // The pointer is the last 4 bytes of the entry, stored big-endian. + int pointer = ReadInt32BigEndian(page, EntryDataOffset + end - 4); + + if (type == PageType.LeafIndexPage) + { + yield return new RowId(pointer >> 8, pointer & 0xFF); + } + else if (type == PageType.IntermediateIndexPage) + { + foreach (RowId rowId in Walk(pointer)) + yield return rowId; + } + else + { + throw new NotSupportedException($"Page {pageNumber} is not an index page (type 0x{(byte)type:X2})."); + } + + _ = start; + } + + // A node page has one more child than it has entries: the rightmost subtree, + // pointed to from the header rather than by an entry. + if (type == PageType.IntermediateIndexPage) + { + foreach (RowId rowId in Walk(page.ReadInt32(ChildTailOffset))) + yield return rowId; + } + } + + /// Yields the (start, end) byte ranges of each entry, relative to the entry-data region. + private static IEnumerable<(int Start, int End)> EntryRanges(PageBuffer page) + { + int start = 0; + for (int i = EntryMaskOffset; i < EntryDataOffset; i++) + { + byte mask = page.ReadByte(i); + if (mask == 0) continue; + for (int bit = 0; bit < 8; bit++) + { + if ((mask & (1 << bit)) == 0) continue; + int end = (i - EntryMaskOffset) * 8 + bit; + yield return (start, end); + start = end; + } + } + } + + private static int ReadInt32BigEndian(PageBuffer page, int offset) => + (page.ReadByte(offset) << 24) | (page.ReadByte(offset + 1) << 16) | + (page.ReadByte(offset + 2) << 8) | page.ReadByte(offset + 3); +} diff --git a/src/LibRed/LibRed.Core/Storage/RowId.cs b/src/LibRed/LibRed.Core/Storage/RowId.cs new file mode 100644 index 00000000..c48c52c7 --- /dev/null +++ b/src/LibRed/LibRed.Core/Storage/RowId.cs @@ -0,0 +1,4 @@ +namespace LibRed.Storage; + +/// A pointer to a row: the data page it lives on and its slot index within that page. +public readonly record struct RowId(int Page, int Row); diff --git a/test/LibRed.Core.Tests/IndexTests.cs b/test/LibRed.Core.Tests/IndexTests.cs new file mode 100644 index 00000000..ccb9262f --- /dev/null +++ b/test/LibRed.Core.Tests/IndexTests.cs @@ -0,0 +1,85 @@ +using LibRed; +using LibRed.Storage; +using Xunit; + +namespace LibRed.Core.Tests; + +public class IndexTests +{ + [Fact] + public void Parses_index_definitions() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var categories = db.Catalog.FindTable("Categories")!; + Assert.Equal(2, categories.Indexes.Count); + + var pk = Assert.Single(categories.Indexes, i => i.IsPrimaryKey); + Assert.True(pk.IsUnique); + Assert.Equal(["CategoryID"], pk.Columns.Select(c => c.Column.Name)); + Assert.True(pk.RootPage > 0); + + var byName = Assert.Single(categories.Indexes, i => !i.IsPrimaryKey); + Assert.False(byName.IsUnique); + Assert.Equal(["CategoryName"], byName.Columns.Select(c => c.Column.Name)); + } + + [Fact] + public void Traverses_leaf_index_in_key_order() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var table = db.OpenTable("Categories"); + var pk = table.Definition.Indexes.First(i => i.IsPrimaryKey); + int idIdx = table.Definition.Columns.First(c => c.Name == "CategoryID").Index; + var decoder = NewDecoder(db, table); + + var ids = new IndexCursor(table.Channel, pk.RootPage) + .RowIds() + .Select(r => (int)decoder.Decode(db.ReadDataPage(r.Page).GetRow(r.Row))[idIdx]!) + .ToList(); + + Assert.Equal(Enumerable.Range(1, 8), ids); + } + + [Fact] + public void Traverses_multilevel_index_in_key_order() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + // Orders' primary key is large enough that its B-tree root is a node page, + // exercising recursion into child entries and the child-tail page. + var table = db.OpenTable("Orders"); + var pk = table.Definition.Indexes.First(i => i.IsPrimaryKey && i.Columns.Count == 1 + && i.Columns[0].Column.Name == "OrderID"); + int idIdx = table.Definition.Columns.First(c => c.Name == "OrderID").Index; + var decoder = NewDecoder(db, table); + + var ids = new IndexCursor(table.Channel, pk.RootPage) + .RowIds() + .Select(r => (int)decoder.Decode(db.ReadDataPage(r.Page).GetRow(r.Row))[idIdx]!) + .ToList(); + + Assert.Equal(830, ids.Count); + Assert.Equal(ids.OrderBy(x => x), ids); // strictly ascending OrderID + } + + [Theory] + [InlineData("Categories")] + [InlineData("Order Details")] // node-rooted indexes + [InlineData("Orders")] + public void Every_index_has_one_entry_per_row(string tableName) + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var table = db.OpenTable(tableName); + int rowCount = table.Rows().Count(); + + Assert.NotEmpty(table.Definition.Indexes); + Assert.All(table.Definition.Indexes, ix => + Assert.Equal(rowCount, new IndexCursor(table.Channel, ix.RootPage).RowIds().Count())); + } + + private static RowDecoder NewDecoder(JetDatabase db, Table table) => + new(table.Definition.Columns, db.Format, new LongValueReader(table.Channel)); +} From 0bf54de1c9b4b0659f4f02b5beca1daab33dfb3f Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 02:02:02 +0800 Subject: [PATCH 017/506] LibRed: parse index names and accurate primary-key flag After the index-data blocks, the TDEF holds LogicalIndexCount (0x2F) logical-index info blocks (28 bytes each) followed by that many names (2-byte length + UTF-16). Each info block links a name to an index-data block (offset 0x08), carries the index type (0x17; 1 = primary) and, for relationships, a foreign-key table page (0x11). ResolveIndexNames reads these and attaches each name to its IndexDef, preferring a real index's name over a foreign-key relationship that shares the same data block, and sets IsPrimaryKey from the type byte (replacing the earlier unique+required heuristic). IndexDef is now a record. Verified against Northwind: PK_Categories/PK_Customers/PK_Orders, the composite PK_Order_Details [OrderID,ProductID], plain column indexes (City, ProductName, ...), and relationship indexes (ShippersOrders, CategoriesProducts, FK_Employees_Employees). Co-Authored-By: Claude Opus 4.8 --- src/LibRed/LibRed.Core/Catalog/IndexDef.cs | 2 +- .../LibRed.Core/Pages/TableDefinitionPage.cs | 76 ++++++++++++++++--- test/LibRed.Core.Tests/IndexTests.cs | 24 ++++++ 3 files changed, 90 insertions(+), 12 deletions(-) diff --git a/src/LibRed/LibRed.Core/Catalog/IndexDef.cs b/src/LibRed/LibRed.Core/Catalog/IndexDef.cs index be7cb962..1218c230 100644 --- a/src/LibRed/LibRed.Core/Catalog/IndexDef.cs +++ b/src/LibRed/LibRed.Core/Catalog/IndexDef.cs @@ -1,7 +1,7 @@ namespace LibRed.Catalog; /// Describes an index: its columns (with sort direction), uniqueness and root page. -public sealed class IndexDef +public sealed record IndexDef { public required string Name { get; init; } diff --git a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs index a56b6a35..151c81a9 100644 --- a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs +++ b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs @@ -32,7 +32,11 @@ public sealed class TableDefinitionPage : Page /// Bytes of a continuation TDEF page that precede the resumed definition data. private const int ContinuationHeaderSize = 8; - // Index column block layout (Jet 4 / ACE), one per real index, following the column names. + // Index structures (Jet 4 / ACE) follow the column names, in this order: + // IndexCount (0x33) data blocks : 52 bytes each — columns, flags, root page + // LogicalIndexCount (0x2F) info blocks : 28 bytes each — links a name to a data block + // LogicalIndexCount names : 2-byte length + UTF-16 + // A logical index may be a relationship (FK) sharing a data block with a real index. private const int IndexBlockSize = 52; private const int IndexMaxColumns = 10; private const int IndexColumnSlotSize = 3; // 2-byte column id + 1-byte flags @@ -42,7 +46,12 @@ public sealed class TableDefinitionPage : Page private const short IndexColumnUnused = -1; // 0xFFFF private const byte IndexColumnAscending = 0x01; private const ushort IndexFlagUnique = 0x0001; - private const ushort IndexFlagRequired = 0x0008; + + private const int IndexInfoBlockSize = 28; + private const int IndexInfoDataNumberOffset = 0x08; + private const int IndexInfoFkTablePageOffset = 0x11; + private const int IndexInfoTypeOffset = 0x17; + private const byte IndexTypePrimary = 0x01; /// /// Reads a table definition starting at , transparently @@ -96,14 +105,16 @@ public override void Read(PageBuffer buffer, JetFormatBase format) } /// - /// Parses the per-index column blocks (52 bytes each) that follow the column names: - /// indexed columns + sort order, the unique/required flags, and the B-tree root page. + /// Parses the index structures following the column names into s + /// (one per index-data block): columns + sort order, unique/primary flags, root page, + /// and the index name (resolved from the logical-index info blocks). /// private void ReadIndexes(PageBuffer buffer, int blockStart) { _indexes.Clear(); var byColumnId = _columns.ToDictionary(c => c.ColumnId); + // 1. Index-data blocks (one IndexDef each): columns, unique flag, root page. for (int i = 0; i < IndexCount; i++) { int block = blockStart + i * IndexBlockSize; @@ -118,19 +129,62 @@ private void ReadIndexes(PageBuffer buffer, int blockStart) columns.Add((column, (buffer.ReadByte(entry + 2) & IndexColumnAscending) != 0)); } - ushort flags = buffer.ReadUInt16(block + IndexFlagsOffset); - bool unique = (flags & IndexFlagUnique) != 0; - bool required = (flags & IndexFlagRequired) != 0; - _indexes.Add(new IndexDef { - Name = string.Empty, // index names live further on, after the logical-index entries (TODO) + Name = string.Empty, Columns = columns, - IsUnique = unique, - IsPrimaryKey = unique && required, // heuristic; the precise flag is in the logical-index entry + IsUnique = (buffer.ReadUInt16(block + IndexFlagsOffset) & IndexFlagUnique) != 0, + IsPrimaryKey = false, RootPage = buffer.ReadInt32(block + IndexRootPageOffset), }); } + + ResolveIndexNames(buffer, blockStart + IndexCount * IndexBlockSize); + } + + /// + /// Reads the logical-index info blocks and their names, then attaches each name (and the + /// primary-key flag) to the index-data block it references. A data block may be referenced + /// by several logical indexes (e.g. a relationship plus the real index); the real index's + /// name wins over a foreign-key relationship's. + /// + private void ResolveIndexNames(PageBuffer buffer, int infoStart) + { + int logicalCount = RealIndexCount; // 0x2F — the logical-index (slot) count + var info = new (int DataNumber, bool IsRelationship, byte Type)[logicalCount]; + for (int i = 0; i < logicalCount; i++) + { + int block = infoStart + i * IndexInfoBlockSize; + info[i] = ( + buffer.ReadInt32(block + IndexInfoDataNumberOffset), + buffer.ReadInt32(block + IndexInfoFkTablePageOffset) != 0, + buffer.ReadByte(block + IndexInfoTypeOffset)); + } + + int namePos = infoStart + logicalCount * IndexInfoBlockSize; + var priority = new int[_indexes.Count]; + for (int i = 0; i < logicalCount; i++) + { + int byteLength = buffer.ReadUInt16(namePos); + namePos += 2; + string name = Encoding.Unicode.GetString(buffer.Slice(namePos, byteLength)); + namePos += byteLength; + + (int dataNumber, bool isRelationship, byte type) = info[i]; + if (dataNumber < 0 || dataNumber >= _indexes.Count) continue; + + // Prefer a real index name over a relationship's; prefer the primary among real ones. + int p = isRelationship ? 1 : type == IndexTypePrimary ? 3 : 2; + if (p > priority[dataNumber]) + { + priority[dataNumber] = p; + _indexes[dataNumber] = _indexes[dataNumber] with + { + Name = name, + IsPrimaryKey = !isRelationship && type == IndexTypePrimary, + }; + } + } } private int ReadColumns(PageBuffer buffer, JetFormatBase format, int columnBlock) diff --git a/test/LibRed.Core.Tests/IndexTests.cs b/test/LibRed.Core.Tests/IndexTests.cs index ccb9262f..8b4f565b 100644 --- a/test/LibRed.Core.Tests/IndexTests.cs +++ b/test/LibRed.Core.Tests/IndexTests.cs @@ -15,15 +15,39 @@ public void Parses_index_definitions() Assert.Equal(2, categories.Indexes.Count); var pk = Assert.Single(categories.Indexes, i => i.IsPrimaryKey); + Assert.Equal("PK_Categories", pk.Name); Assert.True(pk.IsUnique); Assert.Equal(["CategoryID"], pk.Columns.Select(c => c.Column.Name)); Assert.True(pk.RootPage > 0); var byName = Assert.Single(categories.Indexes, i => !i.IsPrimaryKey); + Assert.Equal("CategoryName", byName.Name); Assert.False(byName.IsUnique); Assert.Equal(["CategoryName"], byName.Columns.Select(c => c.Column.Name)); } + [Fact] + public void Resolves_index_names_including_relationships() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + string[] Names(string table) => + db.Catalog.FindTable(table)!.Indexes.Select(i => i.Name).OrderBy(n => n).ToArray(); + + // Composite primary key. + var orderDetails = db.Catalog.FindTable("Order Details")!; + var pk = Assert.Single(orderDetails.Indexes, i => i.IsPrimaryKey); + Assert.Equal("PK_Order_Details", pk.Name); + Assert.Equal(["OrderID", "ProductID"], pk.Columns.Select(c => c.Column.Name)); + + // Real-index names win over the relationship sharing the same data block, and + // relationship/FK indexes keep their own names. + Assert.Contains("PK_Orders", Names("Orders")); + Assert.Contains("ShippersOrders", Names("Orders")); // relationship to Shippers + Assert.Contains("FK_Employees_Employees", Names("Employees")); // self-referencing FK + Assert.All(db.Catalog.FindTable("Customers")!.Indexes, i => Assert.NotEqual("", i.Name)); + } + [Fact] public void Traverses_leaf_index_in_key_order() { From 97f783f335b468488cec19d13e3e2d0fa81af905 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 02:10:54 +0800 Subject: [PATCH 018/506] LibRed: decode index key values (reversible fixed types) IndexKeyDecoder reverses Jet's order-preserving key encoding for the fixed/numeric types: a per-column flag byte (0x7F start / 0x00 null ascending; 0x80 / 0xFF descending), then integers via sign-bit flip + big-endian, floating point / DateTime via the IEEE transform, currency as a scaled int64, and booleans as a single constant byte. Text/Binary/GUID keys use Jet's lossy collation encoding and are not reversible, so decoding stops at the first such column (returns null for it onward) -- matching Jackcess, which also does not decode text keys back to values. IndexCursor.Entries(columns) yields each entry's decoded key plus its RowId, in index order, reconstructing full keys across per-page prefix compression (count at 0x18). Verified against Northwind: across 18 numeric-lead indexes every decoded key equals the row value it points at (Int32, Int16, DateTime, and the composite [OrderID,ProductID] PK), including node-rooted, prefix-compressed pages. Constants/transforms confirmed from Jackcess IndexData.java + IndexCodes.java. Co-Authored-By: Claude Opus 4.8 --- src/LibRed/LibRed.Core/Storage/IndexCursor.cs | 63 ++++++++ .../LibRed.Core/Storage/IndexKeyDecoder.cs | 136 ++++++++++++++++++ test/LibRed.Core.Tests/IndexTests.cs | 57 ++++++++ 3 files changed, 256 insertions(+) create mode 100644 src/LibRed/LibRed.Core/Storage/IndexKeyDecoder.cs diff --git a/src/LibRed/LibRed.Core/Storage/IndexCursor.cs b/src/LibRed/LibRed.Core/Storage/IndexCursor.cs index 79155b6c..f07840fe 100644 --- a/src/LibRed/LibRed.Core/Storage/IndexCursor.cs +++ b/src/LibRed/LibRed.Core/Storage/IndexCursor.cs @@ -1,8 +1,12 @@ +using LibRed.Catalog; using LibRed.IO; using LibRed.Pages; namespace LibRed.Storage; +/// An index entry: the decoded key values (in index column order) and the row they point at. +public readonly record struct IndexEntry(object?[] Key, RowId Row); + /// /// Walks an index B-tree and yields the row pointers in index (key) order. /// @@ -19,6 +23,9 @@ public sealed class IndexCursor(PageChannel channel, int rootPage) private const int EntryMaskOffset = 0x1B; private const int EntryDataOffset = 0x1E0; + /// Per-page count of leading key bytes shared by every entry (prefix compression). + private const int CompressedByteCountOffset = 0x18; + /// On a node page, the rightmost child page — not referenced by any entry. private const int ChildTailOffset = 0x14; @@ -27,6 +34,62 @@ public sealed class IndexCursor(PageChannel channel, int rootPage) public IEnumerable RowIds() => Walk(_rootPage); + /// + /// Yields each entry with its decoded key (per ) in index order. + /// Key columns that use Jet's lossy text/binary collation decode as null. + /// + public IEnumerable Entries(IReadOnlyList<(ColumnDef Column, bool Ascending)> columns) => + WalkEntries(_rootPage, columns); + + private IEnumerable WalkEntries(int pageNumber, IReadOnlyList<(ColumnDef, bool)> columns) + { + PageBuffer page = _channel.ReadPage(pageNumber); + var type = (PageType)page.ReadByte(0); + + if (type == PageType.LeafIndexPage) + { + int compress = page.ReadUInt16(CompressedByteCountOffset); + byte[] prefix = []; + bool first = true; + + foreach ((int start, int end) in EntryRanges(page)) + { + int entryStart = EntryDataOffset + start; + int pointer = ReadInt32BigEndian(page, EntryDataOffset + end - 4); + + ReadOnlySpan storedKey = page.Slice(entryStart, end - start - 4); + byte[] key = first ? storedKey.ToArray() : Concat(prefix, storedKey); + if (first) + { + prefix = key[..compress]; + first = false; + } + + yield return new IndexEntry(IndexKeyDecoder.Decode(columns, key), new RowId(pointer >> 8, pointer & 0xFF)); + } + } + else if (type == PageType.IntermediateIndexPage) + { + foreach ((int _, int end) in EntryRanges(page)) + foreach (IndexEntry e in WalkEntries(ReadInt32BigEndian(page, EntryDataOffset + end - 4), columns)) + yield return e; + foreach (IndexEntry e in WalkEntries(page.ReadInt32(ChildTailOffset), columns)) + yield return e; + } + else + { + throw new NotSupportedException($"Page {pageNumber} is not an index page (type 0x{(byte)type:X2})."); + } + } + + private static byte[] Concat(ReadOnlySpan a, ReadOnlySpan b) + { + var result = new byte[a.Length + b.Length]; + a.CopyTo(result); + b.CopyTo(result.AsSpan(a.Length)); + return result; + } + private IEnumerable Walk(int pageNumber) { PageBuffer page = _channel.ReadPage(pageNumber); diff --git a/src/LibRed/LibRed.Core/Storage/IndexKeyDecoder.cs b/src/LibRed/LibRed.Core/Storage/IndexKeyDecoder.cs new file mode 100644 index 00000000..c2e75544 --- /dev/null +++ b/src/LibRed/LibRed.Core/Storage/IndexKeyDecoder.cs @@ -0,0 +1,136 @@ +using System.Buffers.Binary; +using LibRed.Catalog; + +namespace LibRed.Storage; + +/// +/// Decodes the order-preserving key bytes of an index entry back into column values. +/// +/// +/// Each non-boolean column is prefixed by a flag byte (0x7F start / 0x00 null for ascending; +/// 0x80 / 0xFF for descending). Fixed/numeric types use a reversible transform (sign-bit flip +/// + big-endian for integers; an IEEE transform for floating point). TEXT/Binary/GUID keys use +/// Jet's collation encoding, which is lossy and not reversible — decoding stops at the first +/// such column (its value and any following columns are returned as null). +/// +public static class IndexKeyDecoder +{ + private const byte AscStartFlag = 0x7F; + private const byte AscNullFlag = 0x00; + private const byte DescStartFlag = 0x80; + private const byte DescNullFlag = 0xFF; + private const byte AscBooleanTrue = 0x00; // ascending: true sorts before false + + public static object?[] Decode(IReadOnlyList<(ColumnDef Column, bool Ascending)> columns, ReadOnlySpan key) + { + var values = new object?[columns.Count]; + int pos = 0; + + for (int i = 0; i < columns.Count; i++) + { + (ColumnDef column, bool ascending) = columns[i]; + if (pos >= key.Length) break; + + // Booleans carry no flag byte — the value IS the byte. + if (column.Type == JetDataType.Boolean) + { + byte b = key[pos++]; + values[i] = ascending ? b == AscBooleanTrue : b != AscBooleanTrue; + continue; + } + + byte flag = key[pos++]; + if (flag == (ascending ? AscNullFlag : DescNullFlag)) + { + values[i] = null; + continue; + } + // Otherwise flag is the start flag (0x7F / 0x80). + + int size = FixedKeySize(column.Type); + if (size <= 0 || pos + size > key.Length) + break; // text/binary/unsupported (lossy) — cannot reliably continue + + Span raw = key.Slice(pos, size).ToArray(); + pos += size; + values[i] = DecodeFixed(column.Type, raw, ascending); + } + + return values; + } + + private static int FixedKeySize(JetDataType type) => type switch + { + JetDataType.Byte => 1, + JetDataType.Int16 => 2, + JetDataType.Int32 => 4, + JetDataType.Single => 4, + JetDataType.Double or JetDataType.DateTime => 8, + JetDataType.Currency => 8, + _ => -1, + }; + + private static object DecodeFixed(JetDataType type, Span raw, bool ascending) + { + switch (type) + { + case JetDataType.Byte: + if (!ascending) raw[0] = (byte)~raw[0]; + return raw[0]; + + case JetDataType.Int16: + return (short)DecodeInteger(raw, ascending); + case JetDataType.Int32: + return (int)DecodeInteger(raw, ascending); + case JetDataType.Currency: + return DecodeInteger(raw, ascending) / 10000m; + + case JetDataType.Single: + return BitConverter.Int32BitsToSingle((int)DecodeFloatBits(raw, ascending)); + case JetDataType.Double: + return BitConverter.Int64BitsToDouble(DecodeFloatBits(raw, ascending)); + case JetDataType.DateTime: + return DateTime.FromOADate(BitConverter.Int64BitsToDouble(DecodeFloatBits(raw, ascending))); + + default: + throw new NotSupportedException($"Index key type {type} is not decodable."); + } + } + + /// Reverses the integer key transform (descending = bytes inverted; sign bit flipped; big-endian). + private static long DecodeInteger(Span raw, bool ascending) + { + if (!ascending) + for (int i = 0; i < raw.Length; i++) raw[i] = (byte)~raw[i]; + raw[0] ^= 0x80; + + long value = 0; + bool negative = (raw[0] & 0x80) != 0; + if (negative) value = -1; // sign-extend + foreach (byte b in raw) value = (value << 8) | b; + return value; + } + + /// Reverses the floating-point key transform, returning the raw IEEE bits big-endian. + private static long DecodeFloatBits(Span raw, bool ascending) + { + if (ascending) + { + if ((raw[0] & 0x80) != 0) raw[0] ^= 0x80; // was positive: undo first-bit flip + else for (int i = 0; i < raw.Length; i++) raw[i] = (byte)~raw[i]; // was negative: undo full invert + } + else + { + if ((raw[0] & 0x80) == 0) // was positive + { + for (int i = 0; i < raw.Length; i++) raw[i] = (byte)~raw[i]; + raw[0] ^= 0x80; + } + // was negative: stored as-is + } + + long bits = 0; + foreach (byte b in raw) bits = (bits << 8) | b; + return bits; + } +} diff --git a/test/LibRed.Core.Tests/IndexTests.cs b/test/LibRed.Core.Tests/IndexTests.cs index 8b4f565b..dd20d9c6 100644 --- a/test/LibRed.Core.Tests/IndexTests.cs +++ b/test/LibRed.Core.Tests/IndexTests.cs @@ -104,6 +104,63 @@ public void Every_index_has_one_entry_per_row(string tableName) Assert.Equal(rowCount, new IndexCursor(table.Channel, ix.RootPage).RowIds().Count())); } + [Fact] + public void Decodes_numeric_index_keys() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var table = db.OpenTable("Categories"); + var pk = table.Definition.Indexes.First(i => i.IsPrimaryKey); + + var keys = new IndexCursor(table.Channel, pk.RootPage) + .Entries(pk.Columns) + .Select(e => (int)e.Key[0]!) + .ToList(); + + Assert.Equal(Enumerable.Range(1, 8), keys); + } + + [Fact] + public void Decodes_composite_index_keys() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var table = db.OpenTable("Order Details"); + var pk = table.Definition.Indexes.First(i => i.IsPrimaryKey); // [OrderID, ProductID] + + var first = new IndexCursor(table.Channel, pk.RootPage).Entries(pk.Columns).First(); + + Assert.Equal(2, first.Key.Length); + Assert.Equal(10248, first.Key[0]); + Assert.Equal(11, first.Key[1]); + } + + [Theory] + [InlineData("Categories")] + [InlineData("Orders")] // includes a DateTime index and node-rooted (prefix-compressed) pages + [InlineData("Order Details")] + public void Decoded_keys_match_the_row_they_point_at(string tableName) + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var table = db.OpenTable(tableName); + var decoder = NewDecoder(db, table); + + foreach (var index in table.Definition.Indexes) + { + foreach (var entry in new IndexCursor(table.Channel, index.RootPage).Entries(index.Columns)) + { + var row = decoder.Decode(db.ReadDataPage(entry.Row.Page).GetRow(entry.Row.Row)); + for (int c = 0; c < index.Columns.Count; c++) + { + object? keyValue = entry.Key[c]; + if (keyValue is null) break; // lossy (text) key column — not decoded + Assert.Equal(row[index.Columns[c].Column.Index], keyValue); + } + } + } + } + private static RowDecoder NewDecoder(JetDatabase db, Table table) => new(table.Definition.Columns, db.Format, new LongValueReader(table.Channel)); } From 017a3ed2f5dbc6fef36e906237b8fba450cb323d Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 02:15:04 +0800 Subject: [PATCH 019/506] LibRed: include index details in the catalog dump; add WideTable dump The dump test now renders each table's indexes (PK/unique marker, key columns with sort direction, root page) in addition to columns, and covers two databases via golden files: Northwind (catalog-dump.txt, regenerated) and the multi-page-TDEF WideTable (widetable-dump.txt). Confirms the full decode -- columns and indexes -- for both, e.g. the composite PK_Order_Details [OrderID, ProductID] and relationship indexes. Co-Authored-By: Claude Opus 4.8 --- test/LibRed.Core.Tests/CatalogDumpTests.cs | 48 ++- .../Expected/catalog-dump.txt | 82 +++++ .../Expected/widetable-dump.txt | 317 ++++++++++++++++++ 3 files changed, 432 insertions(+), 15 deletions(-) create mode 100644 test/LibRed.Core.Tests/Expected/widetable-dump.txt diff --git a/test/LibRed.Core.Tests/CatalogDumpTests.cs b/test/LibRed.Core.Tests/CatalogDumpTests.cs index 1b46d0c9..34489f37 100644 --- a/test/LibRed.Core.Tests/CatalogDumpTests.cs +++ b/test/LibRed.Core.Tests/CatalogDumpTests.cs @@ -7,10 +7,10 @@ namespace LibRed.Core.Tests; /// -/// Dumps every object in the database (user and system tables, in page order) with its -/// properties and columns/data types, then asserts the whole rendering against a golden -/// file (Expected/catalog-dump.txt). This pins the schema decode end to end; regenerate -/// the golden file if the decode intentionally changes. +/// Dumps every object in a database (user and system, in page order) with its properties, +/// columns/data types, and indexes (name, key columns, unique/primary, root page), then +/// asserts the whole rendering against a golden file. Pins the schema decode end to end; +/// regenerate the golden files if the decode intentionally changes. /// Run with: dotnet test -l "console;verbosity=detailed" to see the dump. /// public class CatalogDumpTests(ITestOutputHelper output) @@ -18,21 +18,34 @@ public class CatalogDumpTests(ITestOutputHelper output) private readonly ITestOutputHelper _output = output; [Fact] - public void Dump_all_objects() + public void Dump_northwind() => VerifyDump(TestDatabases.NorthwindAccdb, "catalog-dump.txt"); + + [Fact] + public void Dump_widetable() => VerifyDump(TestDatabases.WideTableAccdb, "widetable-dump.txt"); + + private void VerifyDump(string databasePath, string goldenFile) + { + string actual = BuildDump(databasePath); + _output.WriteLine(actual); + + string expected = Normalize(File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "Expected", goldenFile))); + Assert.Equal(expected, actual); + } + + private static string BuildDump(string path) { - using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + using var db = JetDatabase.Open(path); var sb = new StringBuilder(); sb.Append($"Database: {db.DefinitionPage.FormatIdentifier} ({db.Format.Version}, page size {db.Format.PageSize})\n"); foreach (TableDef table in db.Catalog.Tables.OrderBy(t => t.DefinitionPage)) { - // Re-read the TDEF page for per-table properties (row count, type, index count). var tdef = db.ReadTableDefinition(table.DefinitionPage); sb.Append('\n'); sb.Append($"{(table.IsSystem ? "[SYS] " : " ")}{table.Name}\n"); - sb.Append($" tdefPage={table.DefinitionPage} type={tdef.TableType} rows={tdef.RowCount} columns={tdef.ColumnCount} indexes={tdef.IndexCount}\n"); + sb.Append($" tdefPage={table.DefinitionPage} type={tdef.TableType} rows={tdef.RowCount} columns={tdef.ColumnCount} indexes={table.Indexes.Count}\n"); foreach (ColumnDef c in table.Columns) { @@ -40,15 +53,20 @@ public void Dump_all_objects() string extra = c.IsAutoNumber ? " auto" : ""; sb.Append($" {c.Index,2}. {c.Name,-26} {c.Type,-9} len={c.Length,3} {store}{extra}\n"); } - } - - string actual = Normalize(sb.ToString()); - _output.WriteLine(actual); - string expectedPath = Path.Combine(AppContext.BaseDirectory, "Expected", "catalog-dump.txt"); - string expected = Normalize(File.ReadAllText(expectedPath)); + if (table.Indexes.Count > 0) + { + sb.Append(" --- indexes ---\n"); + foreach (IndexDef ix in table.Indexes) + { + string kind = ix.IsPrimaryKey ? "PK " : ix.IsUnique ? "U " : " "; + string cols = string.Join(", ", ix.Columns.Select(c => c.Column.Name + (c.Ascending ? "" : " DESC"))); + sb.Append($" {kind}{("\"" + ix.Name + "\""),-26} [{cols}] root={ix.RootPage}\n"); + } + } + } - Assert.Equal(expected, actual); + return Normalize(sb.ToString()); } private static string Normalize(string s) => s.Replace("\r\n", "\n").TrimEnd() + "\n"; diff --git a/test/LibRed.Core.Tests/Expected/catalog-dump.txt b/test/LibRed.Core.Tests/Expected/catalog-dump.txt index 5eb7d31d..74b9e987 100644 --- a/test/LibRed.Core.Tests/Expected/catalog-dump.txt +++ b/test/LibRed.Core.Tests/Expected/catalog-dump.txt @@ -19,6 +19,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 14. RmtInfoLong Ole len= 0 var#6 15. RmtInfoShort Binary len=510 var#5 16. Type Int16 len= 2 fixed@8 + --- indexes --- + U "ParentIdName" [ParentId, Name] root=7 + PK "Id" [Id] root=8 [SYS] MSysACEs tdefPage=3 type=System rows=183 columns=4 indexes=1 @@ -26,6 +29,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 1. FInheritable Boolean len= 1 fixed@0 2. ObjectId Int32 len= 4 fixed@0 3. SID Binary len=510 var#0 + --- indexes --- + "ObjectId" [ObjectId] root=10 [SYS] MSysQueries tdefPage=4 type=System rows=279 columns=8 indexes=1 @@ -37,6 +42,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 5. Name2 Text len=510 var#2 6. ObjectId Int32 len= 4 fixed@0 7. Order Binary len=510 var#0 + --- indexes --- + PK "ObjectIdAttribute" [ObjectId, Attribute, Order] root=12 [SYS] MSysRelationships tdefPage=5 type=System rows=15 columns=8 indexes=3 @@ -48,6 +55,10 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 5. szReferencedColumn Text len=510 var#4 6. szReferencedObject Text len=510 var#3 7. szRelationship Text len=510 var#0 + --- indexes --- + "szRelationship" [szRelationship] root=14 + "szObject" [szObject] root=15 + "szReferencedObject" [szReferencedObject] root=16 [SYS] MSysComplexColumns tdefPage=18 type=User rows=1 columns=5 indexes=3 @@ -56,6 +67,10 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 2. ComplexTypeObjectID Int32 len= 4 fixed@0 3. ConceptualTableID Int32 len= 4 fixed@8 4. FlatTableID Int32 len= 4 fixed@4 + --- indexes --- + PK "IdxID" [ComplexID] root=20 + "IdxConceptualTableID" [ConceptualTableID] root=21 + "IdxFlatTableID" [FlatTableID] root=22 [SYS] MSysComplexType_UnsignedByte tdefPage=23 type=User rows=0 columns=1 indexes=0 @@ -104,16 +119,25 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 1. CategoryName Text len= 30 var#0 2. Description Memo len= 0 var#1 3. Picture Ole len= 0 var#2 + --- indexes --- + "CategoryName" [CategoryName] root=71 + PK "PK_Categories" [CategoryID] root=72 CustomerCustomerDemo tdefPage=73 type=User rows=0 columns=2 indexes=3 0. CustomerID Text len= 10 fixed@0 1. CustomerTypeID Text len= 20 fixed@10 + --- indexes --- + PK "PK_CustomerCustomerDemo" [CustomerID, CustomerTypeID] root=75 + "FK_CustomerCustomerDemo" [CustomerTypeID] root=303 + "FK_CustomerCustomerDemo_Customers" [CustomerID] root=304 CustomerDemographics tdefPage=76 type=User rows=0 columns=2 indexes=1 0. CustomerTypeID Text len= 20 fixed@0 1. CustomerDesc Memo len= 0 var#0 + --- indexes --- + PK "PK_CustomerDemographics" [CustomerTypeID] root=78 Customers tdefPage=79 type=User rows=91 columns=11 indexes=5 @@ -128,6 +152,12 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 8. Country Text len= 30 var#7 9. Phone Text len= 48 var#8 10. Fax Text len= 48 var#9 + --- indexes --- + "City" [City] root=87 + "CompanyName" [CompanyName] root=88 + PK "PK_Customers" [CustomerID] root=89 + "PostalCode" [PostalCode] root=90 + "Region" [Region] root=91 Employees tdefPage=92 type=User rows=9 columns=18 indexes=4 @@ -149,6 +179,11 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 15. Notes Memo len= 0 var#12 16. ReportsTo Int32 len= 4 fixed@20 17. PhotoPath Text len=510 var#13 + --- indexes --- + "LastName" [LastName] root=151 + PK "PK_Employees" [EmployeeID] root=152 + "PostalCode" [PostalCode] root=153 + "FK_Employees_Employees" [ReportsTo] root=305 [SYS] MSysAccessStorage tdefPage=159 type=User rows=23 columns=7 indexes=3 @@ -159,6 +194,10 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 4. Name Text len=256 var#1 5. ParentId Int32 len= 4 fixed@20 6. Type Int32 len= 4 fixed@24 + --- indexes --- + PK "Id" [Id] root=161 + U "ParentIdId" [ParentId, Id] root=162 + U "ParentIdName" [ParentId, Name] root=163 Order Details tdefPage=164 type=User rows=2155 columns=5 indexes=3 @@ -167,6 +206,10 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 2. UnitPrice Currency len= 8 fixed@8 3. Quantity Int16 len= 2 fixed@16 4. Discount Single len= 4 fixed@18 + --- indexes --- + "OrderID" [OrderID] root=181 + PK "PK_Order_Details" [OrderID, ProductID] root=186 + "ProductID" [ProductID] root=194 Orders tdefPage=198 type=User rows=830 columns=14 indexes=7 @@ -184,6 +227,14 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 11. ShipRegion Text len= 30 var#3 12. ShipPostalCode Text len= 20 var#4 13. ShipCountry Text len= 30 var#5 + --- indexes --- + "CustomerID" [CustomerID] root=240 + "EmployeeID" [EmployeeID] root=244 + "OrderDate" [OrderDate] root=247 + PK "PK_Orders" [OrderID] root=251 + "ShippedDate" [ShippedDate] root=254 + "ShippersOrders" [ShipVia] root=258 + "ShipPostalCode" [ShipPostalCode] root=261 Products tdefPage=266 type=User rows=77 columns=10 indexes=4 @@ -197,17 +248,26 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 7. UnitsOnOrder Int16 len= 2 fixed@22 8. ReorderLevel Int16 len= 2 fixed@24 9. Discontinued Boolean len= 1 fixed@0 + --- indexes --- + "CategoriesProducts" [CategoryID] root=270 + PK "PK_Products" [ProductID] root=271 + "ProductName" [ProductName] root=272 + "SupplierID" [SupplierID] root=273 Region tdefPage=275 type=User rows=4 columns=2 indexes=1 0. RegionID Int32 len= 4 fixed@0 1. RegionDescription Text len=100 fixed@4 + --- indexes --- + PK "PK_Region" [RegionID] root=278 Shippers tdefPage=283 type=User rows=3 columns=3 indexes=1 0. ShipperID Int32 len= 4 fixed@0 auto 1. CompanyName Text len= 80 var#0 2. Phone Text len= 48 var#1 + --- indexes --- + PK "PK_Shippers" [ShipperID] root=286 Suppliers tdefPage=287 type=User rows=29 columns=12 indexes=3 @@ -223,12 +283,19 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 9. Phone Text len= 48 var#8 10. Fax Text len= 48 var#9 11. HomePage Memo len= 0 var#10 + --- indexes --- + "CompanyName" [CompanyName] root=292 + PK "PK_Suppliers" [SupplierID] root=293 + "PostalCode" [PostalCode] root=294 Territories tdefPage=295 type=User rows=53 columns=3 indexes=2 0. TerritoryID Text len= 40 var#0 1. TerritoryDescription Text len=100 fixed@0 2. RegionID Int32 len= 4 fixed@100 + --- indexes --- + PK "PK_Territories" [TerritoryID] root=299 + "FK_Territories_Region" [RegionID] root=308 [SYS] MSysNavPaneGroupCategories tdefPage=312 type=User rows=3 columns=7 indexes=1 @@ -239,6 +306,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 4. Position Int32 len= 4 var#5 5. SelectedObjectID Int32 len= 4 var#3 6. Type Int32 len= 4 var#2 + --- indexes --- + PK "Id" [Id] root=314 [SYS] MSysNavPaneGroups tdefPage=315 type=User rows=10 columns=7 indexes=2 @@ -249,6 +318,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 4. Object Type Group Int32 len= 4 var#5 5. ObjectID Int32 len= 4 var#3 6. Position Int32 len= 4 var#4 + --- indexes --- + PK "Id" [Id] root=317 + "GroupCategoryID" [GroupCategoryID] root=318 [SYS] MSysNavPaneGroupToObjects tdefPage=319 type=User rows=47 columns=7 indexes=2 @@ -259,6 +331,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 4. Name Text len=510 var#3 5. ObjectID Int32 len= 4 var#1 6. Position Int32 len= 4 var#4 + --- indexes --- + PK "Id" [Id] root=321 + "GroupID" [GroupID] root=322 [SYS] MSysNavPaneObjectIDs tdefPage=327 type=User rows=57 columns=3 indexes=0 @@ -273,6 +348,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 2. Id Int32 len= 4 var#0 auto 3. Name Text len=510 var#1 4. Type Text len=510 var#2 + --- indexes --- + U "Data_BF92175BD03C433695464F95CDF97589" [Data] root=340 + PK "Id" [Id] root=341 [SYS] f_3B3D4E1F479C4CCCA9CB8E549B05D3B7_Data tdefPage=334 type=User rows=1 columns=8 indexes=3 @@ -284,3 +362,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 5. FileType Text len=510 var#2 6. FileURL Memo len= 0 var#3 7. MSysResources_Data Int32 len= 4 var#4 auto + --- indexes --- + PK "MSysComplexPKIndex" [MSysResources_Data] root=336 + "_Data" [_Data] root=337 + U "IdxFKPrimaryScalar" [_Data, FileName] root=338 diff --git a/test/LibRed.Core.Tests/Expected/widetable-dump.txt b/test/LibRed.Core.Tests/Expected/widetable-dump.txt new file mode 100644 index 00000000..d90c5e0c --- /dev/null +++ b/test/LibRed.Core.Tests/Expected/widetable-dump.txt @@ -0,0 +1,317 @@ +Database: Standard ACE DB (Version12_2007, page size 4096) + +[SYS] MSysObjects + tdefPage=2 type=System rows=19 columns=17 indexes=2 + 0. Connect Memo len= 0 var#3 + 1. Database Memo len= 0 var#2 + 2. DateCreate DateTime len= 8 fixed@10 + 3. DateUpdate DateTime len= 8 fixed@18 + 4. Flags Int32 len= 4 fixed@26 + 5. ForeignName Text len=510 var#4 + 6. Id Int32 len= 4 fixed@0 + 7. Lv Ole len= 0 var#7 + 8. LvExtra Ole len= 0 var#10 + 9. LvModule Ole len= 0 var#9 + 10. LvProp Ole len= 0 var#8 + 11. Name Text len=510 var#0 + 12. Owner Binary len=510 var#1 + 13. ParentId Int32 len= 4 fixed@4 + 14. RmtInfoLong Ole len= 0 var#6 + 15. RmtInfoShort Binary len=510 var#5 + 16. Type Int16 len= 2 fixed@8 + --- indexes --- + U "ParentIdName" [ParentId, Name] root=7 + PK "Id" [Id] root=8 + +[SYS] MSysACEs + tdefPage=3 type=System rows=38 columns=4 indexes=1 + 0. ACM Int32 len= 4 fixed@4 + 1. FInheritable Boolean len= 1 fixed@0 + 2. ObjectId Int32 len= 4 fixed@0 + 3. SID Binary len=510 var#0 + --- indexes --- + "ObjectId" [ObjectId] root=10 + +[SYS] MSysQueries + tdefPage=4 type=System rows=0 columns=8 indexes=1 + 0. Attribute Byte len= 1 fixed@4 + 1. Expression Memo len= 0 var#3 + 2. Flag Int16 len= 2 fixed@5 + 3. LvExtra Int32 len= 4 fixed@7 + 4. Name1 Text len=510 var#1 + 5. Name2 Text len=510 var#2 + 6. ObjectId Int32 len= 4 fixed@0 + 7. Order Binary len=510 var#0 + --- indexes --- + PK "ObjectIdAttribute" [ObjectId, Attribute, Order] root=12 + +[SYS] MSysRelationships + tdefPage=5 type=System rows=0 columns=8 indexes=3 + 0. ccolumn Int32 len= 4 fixed@4 + 1. grbit Int32 len= 4 fixed@0 + 2. icolumn Int32 len= 4 fixed@8 + 3. szColumn Text len=510 var#2 + 4. szObject Text len=510 var#1 + 5. szReferencedColumn Text len=510 var#4 + 6. szReferencedObject Text len=510 var#3 + 7. szRelationship Text len=510 var#0 + --- indexes --- + "szRelationship" [szRelationship] root=14 + "szObject" [szObject] root=15 + "szReferencedObject" [szReferencedObject] root=16 + +[SYS] MSysComplexColumns + tdefPage=18 type=User rows=0 columns=5 indexes=3 + 0. ColumnName Text len=510 var#0 + 1. ComplexID Int32 len= 4 fixed@12 auto + 2. ComplexTypeObjectID Int32 len= 4 fixed@0 + 3. ConceptualTableID Int32 len= 4 fixed@8 + 4. FlatTableID Int32 len= 4 fixed@4 + --- indexes --- + PK "IdxID" [ComplexID] root=20 + "IdxConceptualTableID" [ConceptualTableID] root=21 + "IdxFlatTableID" [FlatTableID] root=22 + +[SYS] MSysComplexType_UnsignedByte + tdefPage=23 type=User rows=0 columns=1 indexes=0 + 0. Value Byte len= 1 fixed@0 + +[SYS] MSysComplexType_Short + tdefPage=25 type=User rows=0 columns=1 indexes=0 + 0. Value Int16 len= 2 fixed@0 + +[SYS] MSysComplexType_Long + tdefPage=27 type=User rows=0 columns=1 indexes=0 + 0. Value Int32 len= 4 fixed@0 + +[SYS] MSysComplexType_IEEESingle + tdefPage=29 type=User rows=0 columns=1 indexes=0 + 0. Value Single len= 4 fixed@0 + +[SYS] MSysComplexType_IEEEDouble + tdefPage=31 type=User rows=0 columns=1 indexes=0 + 0. Value Double len= 8 fixed@0 + +[SYS] MSysComplexType_GUID + tdefPage=33 type=User rows=0 columns=1 indexes=0 + 0. Value Guid len= 16 fixed@0 + +[SYS] MSysComplexType_Decimal + tdefPage=35 type=User rows=0 columns=1 indexes=0 + 0. Value FixedPoint len= 9 var#0 + +[SYS] MSysComplexType_Text + tdefPage=37 type=User rows=0 columns=1 indexes=0 + 0. Value Text len=510 var#0 + +[SYS] MSysComplexType_Attachment + tdefPage=39 type=User rows=0 columns=6 indexes=0 + 0. FileData Ole len= 0 var#3 + 1. FileFlags Int32 len= 4 fixed@8 + 2. FileName Text len=510 var#1 + 3. FileTimeStamp DateTime len= 8 fixed@0 + 4. FileType Text len=510 var#2 + 5. FileURL Memo len= 0 var#0 + + WideTable + tdefPage=42 type=User rows=1 columns=200 indexes=0 + 0. C000 Int32 len= 4 fixed@0 + 1. C001 Int32 len= 4 fixed@4 + 2. C002 Int32 len= 4 fixed@8 + 3. C003 Int32 len= 4 fixed@12 + 4. C004 Int32 len= 4 fixed@16 + 5. C005 Int32 len= 4 fixed@20 + 6. C006 Int32 len= 4 fixed@24 + 7. C007 Int32 len= 4 fixed@28 + 8. C008 Int32 len= 4 fixed@32 + 9. C009 Int32 len= 4 fixed@36 + 10. C010 Int32 len= 4 fixed@40 + 11. C011 Int32 len= 4 fixed@44 + 12. C012 Int32 len= 4 fixed@48 + 13. C013 Int32 len= 4 fixed@52 + 14. C014 Int32 len= 4 fixed@56 + 15. C015 Int32 len= 4 fixed@60 + 16. C016 Int32 len= 4 fixed@64 + 17. C017 Int32 len= 4 fixed@68 + 18. C018 Int32 len= 4 fixed@72 + 19. C019 Int32 len= 4 fixed@76 + 20. C020 Int32 len= 4 fixed@80 + 21. C021 Int32 len= 4 fixed@84 + 22. C022 Int32 len= 4 fixed@88 + 23. C023 Int32 len= 4 fixed@92 + 24. C024 Int32 len= 4 fixed@96 + 25. C025 Int32 len= 4 fixed@100 + 26. C026 Int32 len= 4 fixed@104 + 27. C027 Int32 len= 4 fixed@108 + 28. C028 Int32 len= 4 fixed@112 + 29. C029 Int32 len= 4 fixed@116 + 30. C030 Int32 len= 4 fixed@120 + 31. C031 Int32 len= 4 fixed@124 + 32. C032 Int32 len= 4 fixed@128 + 33. C033 Int32 len= 4 fixed@132 + 34. C034 Int32 len= 4 fixed@136 + 35. C035 Int32 len= 4 fixed@140 + 36. C036 Int32 len= 4 fixed@144 + 37. C037 Int32 len= 4 fixed@148 + 38. C038 Int32 len= 4 fixed@152 + 39. C039 Int32 len= 4 fixed@156 + 40. C040 Int32 len= 4 fixed@160 + 41. C041 Int32 len= 4 fixed@164 + 42. C042 Int32 len= 4 fixed@168 + 43. C043 Int32 len= 4 fixed@172 + 44. C044 Int32 len= 4 fixed@176 + 45. C045 Int32 len= 4 fixed@180 + 46. C046 Int32 len= 4 fixed@184 + 47. C047 Int32 len= 4 fixed@188 + 48. C048 Int32 len= 4 fixed@192 + 49. C049 Int32 len= 4 fixed@196 + 50. C050 Int32 len= 4 fixed@200 + 51. C051 Int32 len= 4 fixed@204 + 52. C052 Int32 len= 4 fixed@208 + 53. C053 Int32 len= 4 fixed@212 + 54. C054 Int32 len= 4 fixed@216 + 55. C055 Int32 len= 4 fixed@220 + 56. C056 Int32 len= 4 fixed@224 + 57. C057 Int32 len= 4 fixed@228 + 58. C058 Int32 len= 4 fixed@232 + 59. C059 Int32 len= 4 fixed@236 + 60. C060 Int32 len= 4 fixed@240 + 61. C061 Int32 len= 4 fixed@244 + 62. C062 Int32 len= 4 fixed@248 + 63. C063 Int32 len= 4 fixed@252 + 64. C064 Int32 len= 4 fixed@256 + 65. C065 Int32 len= 4 fixed@260 + 66. C066 Int32 len= 4 fixed@264 + 67. C067 Int32 len= 4 fixed@268 + 68. C068 Int32 len= 4 fixed@272 + 69. C069 Int32 len= 4 fixed@276 + 70. C070 Int32 len= 4 fixed@280 + 71. C071 Int32 len= 4 fixed@284 + 72. C072 Int32 len= 4 fixed@288 + 73. C073 Int32 len= 4 fixed@292 + 74. C074 Int32 len= 4 fixed@296 + 75. C075 Int32 len= 4 fixed@300 + 76. C076 Int32 len= 4 fixed@304 + 77. C077 Int32 len= 4 fixed@308 + 78. C078 Int32 len= 4 fixed@312 + 79. C079 Int32 len= 4 fixed@316 + 80. C080 Int32 len= 4 fixed@320 + 81. C081 Int32 len= 4 fixed@324 + 82. C082 Int32 len= 4 fixed@328 + 83. C083 Int32 len= 4 fixed@332 + 84. C084 Int32 len= 4 fixed@336 + 85. C085 Int32 len= 4 fixed@340 + 86. C086 Int32 len= 4 fixed@344 + 87. C087 Int32 len= 4 fixed@348 + 88. C088 Int32 len= 4 fixed@352 + 89. C089 Int32 len= 4 fixed@356 + 90. C090 Int32 len= 4 fixed@360 + 91. C091 Int32 len= 4 fixed@364 + 92. C092 Int32 len= 4 fixed@368 + 93. C093 Int32 len= 4 fixed@372 + 94. C094 Int32 len= 4 fixed@376 + 95. C095 Int32 len= 4 fixed@380 + 96. C096 Int32 len= 4 fixed@384 + 97. C097 Int32 len= 4 fixed@388 + 98. C098 Int32 len= 4 fixed@392 + 99. C099 Int32 len= 4 fixed@396 + 100. C100 Int32 len= 4 fixed@400 + 101. C101 Int32 len= 4 fixed@404 + 102. C102 Int32 len= 4 fixed@408 + 103. C103 Int32 len= 4 fixed@412 + 104. C104 Int32 len= 4 fixed@416 + 105. C105 Int32 len= 4 fixed@420 + 106. C106 Int32 len= 4 fixed@424 + 107. C107 Int32 len= 4 fixed@428 + 108. C108 Int32 len= 4 fixed@432 + 109. C109 Int32 len= 4 fixed@436 + 110. C110 Int32 len= 4 fixed@440 + 111. C111 Int32 len= 4 fixed@444 + 112. C112 Int32 len= 4 fixed@448 + 113. C113 Int32 len= 4 fixed@452 + 114. C114 Int32 len= 4 fixed@456 + 115. C115 Int32 len= 4 fixed@460 + 116. C116 Int32 len= 4 fixed@464 + 117. C117 Int32 len= 4 fixed@468 + 118. C118 Int32 len= 4 fixed@472 + 119. C119 Int32 len= 4 fixed@476 + 120. C120 Int32 len= 4 fixed@480 + 121. C121 Int32 len= 4 fixed@484 + 122. C122 Int32 len= 4 fixed@488 + 123. C123 Int32 len= 4 fixed@492 + 124. C124 Int32 len= 4 fixed@496 + 125. C125 Int32 len= 4 fixed@500 + 126. C126 Int32 len= 4 fixed@504 + 127. C127 Int32 len= 4 fixed@508 + 128. C128 Int32 len= 4 fixed@512 + 129. C129 Int32 len= 4 fixed@516 + 130. C130 Int32 len= 4 fixed@520 + 131. C131 Int32 len= 4 fixed@524 + 132. C132 Int32 len= 4 fixed@528 + 133. C133 Int32 len= 4 fixed@532 + 134. C134 Int32 len= 4 fixed@536 + 135. C135 Int32 len= 4 fixed@540 + 136. C136 Int32 len= 4 fixed@544 + 137. C137 Int32 len= 4 fixed@548 + 138. C138 Int32 len= 4 fixed@552 + 139. C139 Int32 len= 4 fixed@556 + 140. C140 Int32 len= 4 fixed@560 + 141. C141 Int32 len= 4 fixed@564 + 142. C142 Int32 len= 4 fixed@568 + 143. C143 Int32 len= 4 fixed@572 + 144. C144 Int32 len= 4 fixed@576 + 145. C145 Int32 len= 4 fixed@580 + 146. C146 Int32 len= 4 fixed@584 + 147. C147 Int32 len= 4 fixed@588 + 148. C148 Int32 len= 4 fixed@592 + 149. C149 Int32 len= 4 fixed@596 + 150. C150 Int32 len= 4 fixed@600 + 151. C151 Int32 len= 4 fixed@604 + 152. C152 Int32 len= 4 fixed@608 + 153. C153 Int32 len= 4 fixed@612 + 154. C154 Int32 len= 4 fixed@616 + 155. C155 Int32 len= 4 fixed@620 + 156. C156 Int32 len= 4 fixed@624 + 157. C157 Int32 len= 4 fixed@628 + 158. C158 Int32 len= 4 fixed@632 + 159. C159 Int32 len= 4 fixed@636 + 160. C160 Int32 len= 4 fixed@640 + 161. C161 Int32 len= 4 fixed@644 + 162. C162 Int32 len= 4 fixed@648 + 163. C163 Int32 len= 4 fixed@652 + 164. C164 Int32 len= 4 fixed@656 + 165. C165 Int32 len= 4 fixed@660 + 166. C166 Int32 len= 4 fixed@664 + 167. C167 Int32 len= 4 fixed@668 + 168. C168 Int32 len= 4 fixed@672 + 169. C169 Int32 len= 4 fixed@676 + 170. C170 Int32 len= 4 fixed@680 + 171. C171 Int32 len= 4 fixed@684 + 172. C172 Int32 len= 4 fixed@688 + 173. C173 Int32 len= 4 fixed@692 + 174. C174 Int32 len= 4 fixed@696 + 175. C175 Int32 len= 4 fixed@700 + 176. C176 Int32 len= 4 fixed@704 + 177. C177 Int32 len= 4 fixed@708 + 178. C178 Int32 len= 4 fixed@712 + 179. C179 Int32 len= 4 fixed@716 + 180. C180 Int32 len= 4 fixed@720 + 181. C181 Int32 len= 4 fixed@724 + 182. C182 Int32 len= 4 fixed@728 + 183. C183 Int32 len= 4 fixed@732 + 184. C184 Int32 len= 4 fixed@736 + 185. C185 Int32 len= 4 fixed@740 + 186. C186 Int32 len= 4 fixed@744 + 187. C187 Int32 len= 4 fixed@748 + 188. C188 Int32 len= 4 fixed@752 + 189. C189 Int32 len= 4 fixed@756 + 190. C190 Int32 len= 4 fixed@760 + 191. C191 Int32 len= 4 fixed@764 + 192. C192 Int32 len= 4 fixed@768 + 193. C193 Int32 len= 4 fixed@772 + 194. C194 Int32 len= 4 fixed@776 + 195. C195 Int32 len= 4 fixed@780 + 196. C196 Int32 len= 4 fixed@784 + 197. C197 Int32 len= 4 fixed@788 + 198. C198 Int32 len= 4 fixed@792 + 199. C199 Int32 len= 4 fixed@796 From aa9be66aefd142766d09d59f307419caaad2ee3c Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 02:21:07 +0800 Subject: [PATCH 020/506] LibRed: parse foreign keys (relationships) from MSysRelationships JetCatalog now reads the MSysRelationships system table into ForeignKey records: relationship name, referencing (child) table, referenced (parent) table, the ordered child->parent column pairs (icolumn), and the enforce / cascade-update / cascade-delete flags from grbit (DAO RelationAttributeEnum: 0x02 don't-enforce, 0x100 update cascade, 0x1000 delete cascade). Catalog.Relationships lists all; ForeignKeysOf(table) filters to a table's outgoing keys. The catalog dump now renders each table's foreign keys, and the goldens (Northwind + WideTable) are regenerated to include them. Verified against Northwind: all 13 user relationships (incl. FK_Orders_Shippers where ShipVia -> Shippers.ShipperID, and the self-referencing FK_Employees_Employees), plus the cascade-update/delete MSysNavPane relationships. Co-Authored-By: Claude Opus 4.8 --- src/LibRed/LibRed.Core/Catalog/ForeignKey.cs | 20 +++++++ src/LibRed/LibRed.Core/Catalog/JetCatalog.cs | 55 +++++++++++++++++++ test/LibRed.Core.Tests/CatalogDumpTests.cs | 15 +++++ .../Expected/catalog-dump.txt | 21 +++++++ test/LibRed.Core.Tests/ForeignKeyTests.cs | 52 ++++++++++++++++++ 5 files changed, 163 insertions(+) create mode 100644 src/LibRed/LibRed.Core/Catalog/ForeignKey.cs create mode 100644 test/LibRed.Core.Tests/ForeignKeyTests.cs diff --git a/src/LibRed/LibRed.Core/Catalog/ForeignKey.cs b/src/LibRed/LibRed.Core/Catalog/ForeignKey.cs new file mode 100644 index 00000000..9ef3886c --- /dev/null +++ b/src/LibRed/LibRed.Core/Catalog/ForeignKey.cs @@ -0,0 +1,20 @@ +namespace LibRed.Catalog; + +/// +/// A relationship (foreign key) between two tables, as recorded in MSysRelationships. +/// +/// The relationship name. +/// The referencing (child / foreign-key) table. +/// The referenced (parent) table. +/// The column pairs (child column → referenced column), in key order. +/// Whether referential integrity is enforced. +/// Whether updates to the parent key cascade. +/// Whether deletes of the parent row cascade. +public sealed record ForeignKey( + string Name, + string Table, + string ReferencedTable, + IReadOnlyList<(string Column, string ReferencedColumn)> Columns, + bool IsEnforced, + bool CascadeUpdate, + bool CascadeDelete); diff --git a/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs b/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs index f5ab5518..c2ea0bb9 100644 --- a/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs +++ b/src/LibRed/LibRed.Core/Catalog/JetCatalog.cs @@ -20,12 +20,25 @@ public sealed class JetCatalog(PageChannel channel) /// MSysObjects.Flags bits marking a system object. private const uint SystemObjectFlags = 0x80000002; + // MSysRelationships.grbit flags (DAO RelationAttributeEnum). + private const int RelationshipDontEnforce = 0x00000002; + private const int RelationshipUpdateCascade = 0x00000100; + private const int RelationshipDeleteCascade = 0x00001000; + private readonly PageChannel _channel = channel; private List? _tables; + private List? _relationships; /// All tables in the database (user and system). public IReadOnlyList Tables => _tables ??= LoadTables(); + /// All relationships (foreign keys) defined in the database. + public IReadOnlyList Relationships => _relationships ??= LoadRelationships(); + + /// Relationships for which is the referencing (child) table. + public IEnumerable ForeignKeysOf(string table) => + Relationships.Where(r => string.Equals(r.Table, table, StringComparison.OrdinalIgnoreCase)); + /// User (non-system) tables only. public IEnumerable UserTables => Tables.Where(t => !t.IsSystem); @@ -63,6 +76,48 @@ private List LoadTables() return tables; } + private List LoadRelationships() + { + TableDef? def = FindTable("MSysRelationships"); + if (def is null) return []; + + var c = def.Columns; + int nameIdx = ColumnIndex(c, "szRelationship"); + int childTableIdx = ColumnIndex(c, "szObject"); + int childColumnIdx = ColumnIndex(c, "szColumn"); + int parentTableIdx = ColumnIndex(c, "szReferencedObject"); + int parentColumnIdx = ColumnIndex(c, "szReferencedColumn"); + int orderIdx = ColumnIndex(c, "icolumn"); + int flagsIdx = ColumnIndex(c, "grbit"); + + // One row per column; group by relationship name and order columns by icolumn. + var groups = new Dictionary Columns)>(); + + foreach (object?[] row in new Table(_channel, def).Rows()) + { + string name = (string)row[nameIdx]!; + if (!groups.TryGetValue(name, out var g)) + { + g = ((string)row[childTableIdx]!, (string)row[parentTableIdx]!, + (int)row[flagsIdx]!, []); + groups[name] = g; + } + g.Columns.Add(((int)row[orderIdx]!, (string)row[childColumnIdx]!, (string)row[parentColumnIdx]!)); + } + + return groups + .Select(kvp => new ForeignKey( + kvp.Key, + kvp.Value.Child, + kvp.Value.Parent, + kvp.Value.Columns.OrderBy(x => x.Order).Select(x => (x.Column, x.ReferencedColumn)).ToList(), + (kvp.Value.Flags & RelationshipDontEnforce) == 0, + (kvp.Value.Flags & RelationshipUpdateCascade) != 0, + (kvp.Value.Flags & RelationshipDeleteCascade) != 0)) + .ToList(); + } + private TableDef ReadTableDefinition(int definitionPage, string name, bool isSystem) { var tdef = new TableDefinitionPage(); diff --git a/test/LibRed.Core.Tests/CatalogDumpTests.cs b/test/LibRed.Core.Tests/CatalogDumpTests.cs index 34489f37..99b574bd 100644 --- a/test/LibRed.Core.Tests/CatalogDumpTests.cs +++ b/test/LibRed.Core.Tests/CatalogDumpTests.cs @@ -64,6 +64,21 @@ private static string BuildDump(string path) sb.Append($" {kind}{("\"" + ix.Name + "\""),-26} [{cols}] root={ix.RootPage}\n"); } } + + var fks = db.Catalog.ForeignKeysOf(table.Name).OrderBy(f => f.Name).ToList(); + if (fks.Count > 0) + { + sb.Append(" --- foreign keys ---\n"); + foreach (ForeignKey fk in fks) + { + string cols = string.Join(", ", fk.Columns.Select(c => c.Column)); + string refcols = string.Join(", ", fk.Columns.Select(c => c.ReferencedColumn)); + string extra = (fk.IsEnforced ? "" : " (not enforced)") + + (fk.CascadeUpdate ? " cascadeUpdate" : "") + + (fk.CascadeDelete ? " cascadeDelete" : ""); + sb.Append($" {("\"" + fk.Name + "\""),-44} [{cols}] -> {fk.ReferencedTable} [{refcols}]{extra}\n"); + } + } } return Normalize(sb.ToString()); diff --git a/test/LibRed.Core.Tests/Expected/catalog-dump.txt b/test/LibRed.Core.Tests/Expected/catalog-dump.txt index 74b9e987..0064b1e4 100644 --- a/test/LibRed.Core.Tests/Expected/catalog-dump.txt +++ b/test/LibRed.Core.Tests/Expected/catalog-dump.txt @@ -131,6 +131,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) PK "PK_CustomerCustomerDemo" [CustomerID, CustomerTypeID] root=75 "FK_CustomerCustomerDemo" [CustomerTypeID] root=303 "FK_CustomerCustomerDemo_Customers" [CustomerID] root=304 + --- foreign keys --- + "FK_CustomerCustomerDemo" [CustomerTypeID] -> CustomerDemographics [CustomerTypeID] + "FK_CustomerCustomerDemo_Customers" [CustomerID] -> Customers [CustomerID] CustomerDemographics tdefPage=76 type=User rows=0 columns=2 indexes=1 @@ -184,6 +187,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) PK "PK_Employees" [EmployeeID] root=152 "PostalCode" [PostalCode] root=153 "FK_Employees_Employees" [ReportsTo] root=305 + --- foreign keys --- + "FK_Employees_Employees" [ReportsTo] -> Employees [EmployeeID] [SYS] MSysAccessStorage tdefPage=159 type=User rows=23 columns=7 indexes=3 @@ -210,6 +215,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) "OrderID" [OrderID] root=181 PK "PK_Order_Details" [OrderID, ProductID] root=186 "ProductID" [ProductID] root=194 + --- foreign keys --- + "FK_Order_Details_Orders" [OrderID] -> Orders [OrderID] + "FK_Order_Details_Products" [ProductID] -> Products [ProductID] Orders tdefPage=198 type=User rows=830 columns=14 indexes=7 @@ -235,6 +243,10 @@ Database: Standard ACE DB (Version12_2007, page size 4096) "ShippedDate" [ShippedDate] root=254 "ShippersOrders" [ShipVia] root=258 "ShipPostalCode" [ShipPostalCode] root=261 + --- foreign keys --- + "FK_Orders_Customers" [CustomerID] -> Customers [CustomerID] + "FK_Orders_Employees" [EmployeeID] -> Employees [EmployeeID] + "FK_Orders_Shippers" [ShipVia] -> Shippers [ShipperID] Products tdefPage=266 type=User rows=77 columns=10 indexes=4 @@ -253,6 +265,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) PK "PK_Products" [ProductID] root=271 "ProductName" [ProductName] root=272 "SupplierID" [SupplierID] root=273 + --- foreign keys --- + "FK_Products_Categories" [CategoryID] -> Categories [CategoryID] + "FK_Products_Suppliers" [SupplierID] -> Suppliers [SupplierID] Region tdefPage=275 type=User rows=4 columns=2 indexes=1 @@ -296,6 +311,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) --- indexes --- PK "PK_Territories" [TerritoryID] root=299 "FK_Territories_Region" [RegionID] root=308 + --- foreign keys --- + "FK_Territories_Region" [RegionID] -> Region [RegionID] [SYS] MSysNavPaneGroupCategories tdefPage=312 type=User rows=3 columns=7 indexes=1 @@ -321,6 +338,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) --- indexes --- PK "Id" [Id] root=317 "GroupCategoryID" [GroupCategoryID] root=318 + --- foreign keys --- + "MSysNavPaneGroupCategoriesMSysNavPaneGroups" [GroupCategoryID] -> MSysNavPaneGroupCategories [Id] cascadeUpdate cascadeDelete [SYS] MSysNavPaneGroupToObjects tdefPage=319 type=User rows=47 columns=7 indexes=2 @@ -334,6 +353,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) --- indexes --- PK "Id" [Id] root=321 "GroupID" [GroupID] root=322 + --- foreign keys --- + "MSysNavPaneGroupsMSysNavPaneGroupToObjects" [GroupID] -> MSysNavPaneGroups [Id] cascadeUpdate cascadeDelete [SYS] MSysNavPaneObjectIDs tdefPage=327 type=User rows=57 columns=3 indexes=0 diff --git a/test/LibRed.Core.Tests/ForeignKeyTests.cs b/test/LibRed.Core.Tests/ForeignKeyTests.cs new file mode 100644 index 00000000..97282c42 --- /dev/null +++ b/test/LibRed.Core.Tests/ForeignKeyTests.cs @@ -0,0 +1,52 @@ +using LibRed; +using Xunit; + +namespace LibRed.Core.Tests; + +public class ForeignKeyTests +{ + [Fact] + public void Reads_relationships_from_the_catalog() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var fk = Assert.Single(db.Catalog.Relationships, r => r.Name == "FK_Orders_Customers"); + Assert.Equal("Orders", fk.Table); + Assert.Equal("Customers", fk.ReferencedTable); + Assert.Equal([("CustomerID", "CustomerID")], fk.Columns); + Assert.True(fk.IsEnforced); + Assert.False(fk.CascadeUpdate); + Assert.False(fk.CascadeDelete); + + // A foreign key whose column name differs from the referenced column. + var shippers = Assert.Single(db.Catalog.Relationships, r => r.Name == "FK_Orders_Shippers"); + Assert.Equal([("ShipVia", "ShipperID")], shippers.Columns); + + // A self-referencing relationship. + var selfRef = Assert.Single(db.Catalog.Relationships, r => r.Name == "FK_Employees_Employees"); + Assert.Equal("Employees", selfRef.Table); + Assert.Equal("Employees", selfRef.ReferencedTable); + Assert.Equal([("ReportsTo", "EmployeeID")], selfRef.Columns); + } + + [Fact] + public void Lists_a_tables_outgoing_foreign_keys() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var names = db.Catalog.ForeignKeysOf("Orders").Select(f => f.Name).OrderBy(n => n).ToList(); + + Assert.Equal(["FK_Orders_Customers", "FK_Orders_Employees", "FK_Orders_Shippers"], names); + } + + [Fact] + public void Reads_cascade_flags() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + // The MSysNavPane relationships cascade on update and delete (grbit 0x1100). + var cascading = db.Catalog.Relationships.First(r => r.CascadeDelete); + Assert.True(cascading.CascadeUpdate); + Assert.True(cascading.CascadeDelete); + } +} From 0310af4bfbc06e20046a4df2e71b7d0ddcc2ad61 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 02:27:09 +0800 Subject: [PATCH 021/506] LibRed: add verified on-disk format spec (our HACKING-style reference) docs/jet-ace-file-format.md documents the Jet 4 / ACE format LibRed implements: page types; the TDEF header, body order, column descriptors, index-data and index-info blocks, multi-page stitching; the data-page header and row format (fixed/var/null bitmap, boolean-in-bitmap); data types; compressed Unicode; long values (LVAL); inline and reference usage maps; index B-tree pages (entry mask, prefix compression, row pointers, order-preserving key encoding); and the system catalog (MSysObjects, MSysRelationships). Every offset is verified against real files; Jet 3 differences and unimplemented areas are flagged. Linked as the source of truth from README.md and CLAUDE.md. Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 8 +- src/LibRed/README.md | 5 + src/LibRed/docs/jet-ace-file-format.md | 364 +++++++++++++++++++++++++ 3 files changed, 374 insertions(+), 3 deletions(-) create mode 100644 src/LibRed/docs/jet-ace-file-format.md diff --git a/CLAUDE.md b/CLAUDE.md index 78463ab7..82ab3362 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -134,9 +134,11 @@ are **not** stamped `[SupportedOSPlatform("windows")]`. Strong-naming is preserv node types rather than rewrites): `text → ISqlParser → AST → Binder(ISchemaProvider) → BoundStatement → QueryPlanner → PlanNode → QueryExecutor → ResultSet` -**Status:** structural scaffold — almost every method body is a documented `TODO`. -Reference implementations for the binary layouts: **mdbtools** (`src/libmdb/`) and -**Jackcess**. The 10-step build order and full detail live in `src/LibRed/README.md`. +**Format spec:** `src/LibRed/docs/jet-ace-file-format.md` is LibRed's own verified +reference for the on-disk Jet 4 / ACE format (page types, TDEF/row/index/usage-map/long-value +layouts, key encodings). It is the source of truth — keep it updated as the format +understanding grows. Reference implementations for the binary layouts: **mdbtools** +(`src/libmdb/`) and **Jackcess**. The build order and project detail live in `src/LibRed/README.md`. ANTLR is present but **not** wired into the build yet (commented `Antlr4BuildTasks` block in `LibRed.Sql.csproj`). diff --git a/src/LibRed/README.md b/src/LibRed/README.md index 1b87c0f3..cac1768e 100644 --- a/src/LibRed/README.md +++ b/src/LibRed/README.md @@ -5,6 +5,11 @@ Jet/ACE database engine — the format behind Access `.mdb` and `.accdb` files. Unlike the `EntityFrameworkCore.Jet` projects (which rely on Windows-only ODBC/OleDb), LibRed reads and writes the file format directly. +> **Format spec:** [`docs/jet-ace-file-format.md`](docs/jet-ace-file-format.md) is LibRed's +> authoritative, verified reference for the on-disk Jet 4 / ACE format — every page type, +> structure, and encoding we implement. Treat it as the source of truth (it supersedes +> ad-hoc reads of mdbtools/Jackcess) and update it whenever the format understanding changes. + ## Projects | Project | Responsibility | Depends on | diff --git a/src/LibRed/docs/jet-ace-file-format.md b/src/LibRed/docs/jet-ace-file-format.md new file mode 100644 index 00000000..7b65cce1 --- /dev/null +++ b/src/LibRed/docs/jet-ace-file-format.md @@ -0,0 +1,364 @@ +# Jet / ACE file format — LibRed reference + +This is LibRed's own specification of the Microsoft Jet 4 / ACE (Access `.mdb` / `.accdb`) +on-disk format. **Every offset and structure here has been verified byte-for-byte against +real database files** (the Northwind ACE-2007 sample, a generated 200-column wide table, +and a generated ~150 MB large table) and cross-checked against mdbtools (`src/libmdb/`) +and Jackcess. Where something is assumed or unverified, it says so explicitly. + +Unless noted, everything below describes **Jet 4 and ACE (12/14/16/17)**, which share one +structural layout. **Jet 3** (Access 97) differs in many of these and is *not yet +implemented* — see [Version differences](#version-differences). + +Implemented by `src/LibRed/LibRed.Core/`. The canonical offsets live in +`Formats/JetFormatBase.cs`. + +--- + +## 1. Conventions + +- **Endianness:** little-endian for all integers, **except** index-page key/pointer values, + which are big-endian (noted in §10). +- **Page size:** 4096 bytes (Jet 4 / ACE). Jet 3 is 2048. +- **Pages** are numbered from 0; a page's byte offset in the file is `pageNumber * pageSize`. +- **Page type** is the first byte of every page: + + | Byte | Page type | LibRed | + | --- | --- | --- | + | `0x00` | Database definition (page 0 only) | `DatabaseDefinitionPage` | + | `0x01` | Data page (also long-value/LVAL pages) | `DataPage` | + | `0x02` | Table definition (TDEF) | `TableDefinitionPage` | + | `0x03` | Index B-tree node (intermediate) | `IndexCursor` | + | `0x04` | Index B-tree leaf | `IndexCursor` | + | `0x05` | Page-usage bitmap | `UsageMap` | + +--- + +## 2. Page 0 — database definition + +| Offset | Size | Meaning | +| --- | --- | --- | +| `0x00` | 1 | Page type, `0x00` | +| `0x04` | 15 | Format identifier ASCII: `Standard Jet DB` or `Standard ACE DB` | +| `0x14` | 1 | Version byte (see below) | +| `0x18`+ | … | Obfuscated/encrypted (code page, collation, creation date, password) — **not decoded** | + +Version byte → format: + +| `0x14` | Version | Page size | Family | +| --- | --- | --- | --- | +| `0x00` | Jet 3 (Access 97) | 2048 | MDB | +| `0x01` | Jet 4 (Access 2000–2003) | 4096 | MDB | +| `0x02` | ACE 12 (Access 2007) | 4096 | ACCDB | +| `0x03` | ACE 14 (Access 2010) | 4096 | ACCDB | +| `0x05` | ACE 16 (Access 2016) | 4096 | ACCDB | +| `0x06` | ACE 17 (Access 2019+) | 4096 | ACCDB | + +Everything from `0x18` onward on page 0 is obfuscated; LibRed reads only the identifier and +version. (Page-level encryption for password-protected files is not implemented.) + +--- + +## 3. Table definition (TDEF) page — type `0x02` + +### 3.1 Header + +| Offset | Size | Meaning | +| --- | --- | --- | +| `0x00` | 1 | Page type `0x02` | +| `0x04` | 4 | Next TDEF page (0 if the definition fits one page) | +| `0x08` | 4 | TDEF length (total logical bytes) | +| `0x10` | 4 | Row count | +| `0x14` | 4 | Next auto-number value | +| `0x28` | 1 | Table type: `0x4E` 'N' user, `0x53` 'S' system | +| `0x2B` | 2 | Variable-length column count | +| `0x2D` | 2 | Column count | +| `0x2F` | 4 | **Logical** index count (a.k.a. index slots) | +| `0x33` | 4 | **Real** index count (number of index-data blocks) | +| `0x37` | 4 | Owned-pages usage-map pointer: 1-byte row + 3-byte page | +| `0x3B` | 4 | Free-space-pages usage-map pointer | +| `0x3F` | — | Start of the real-index block (precedes column descriptors) | + +> ⚠️ `0x2F` vs `0x33`: these are equal for MSysObjects (which hid the distinction during +> reverse-engineering) but differ for user tables. `0x33` (real index count) sizes the +> index-data blocks **and** the `0x3F` pre-column block; `0x2F` (logical count) is the number +> of logical-index info blocks and index names. A relationship adds a *logical* index that +> shares a real index's data, so logical ≥ real. + +### 3.2 Multi-page TDEFs + +If a table has enough columns, the definition spans pages chained by the `0x04` pointer. +Reassemble before parsing: take the **first page whole**, then append each continuation +page's bytes **from offset 8** (continuation pages have an 8-byte header). Column offsets are +absolute from the first page, so parsing is otherwise unchanged. + +### 3.3 Body layout (in order, after the header) + +``` +0x3F : real-index block RealIndexCount(0x33) × 12 bytes (skipped to find columns) + column descriptors ColumnCount(0x2D) × 25 bytes + column names ColumnCount × (2-byte length + UTF-16LE) + index-data blocks RealIndexCount(0x33) × 52 bytes + index-info blocks LogicalIndexCount(0x2F) × 28 bytes + index names LogicalIndexCount × (2-byte length + UTF-16LE) +``` + +### 3.4 Column descriptor (25 bytes) + +| Offset | Size | Meaning | +| --- | --- | --- | +| `0x00` | 1 | Data type (see §6) | +| `0x05` | 2 | Column id (a.k.a. column number) | +| `0x0F` | 1 | Flags: `0x01` fixed-length, `0x04` auto-number | +| `0x15` | 2 | Fixed-data offset within the row's fixed region | +| `0x17` | 2 | Length (bytes) | + +Variable-length columns are assigned a *variable index* = their rank among variable columns +ordered by ascending column id (used by the row's variable-offset table, §5). + +### 3.5 Index-data block (52 bytes) + +| Offset | Size | Meaning | +| --- | --- | --- | +| `0x00` | 4 | Marker (`0x00000783` = 1923, or 0) | +| `0x04` | 30 | 10 column slots × (2-byte column id + 1-byte flags); column id `0xFFFF` = unused; flag `0x01` = ascending | +| `0x22` | 1 | Usage-map row | +| `0x23` | 3 | Usage-map page | +| `0x26` | 4 | **B-tree root page** | +| `0x2E` | 2 | Flags: `0x01` unique, `0x08` required, `0x80` always-set (Access 2000+) | + +### 3.6 Index-info block (28 bytes) — one per *logical* index + +| Offset | Size | Meaning | +| --- | --- | --- | +| `0x00` | 4 | Marker (`0x00000659` = 1625, or 0) | +| `0x04` | 4 | Logical index number | +| `0x08` | 4 | Index-data block number this logical index uses | +| `0x0C` | 1 | Foreign-key index type | +| `0x0D` | 4 | Foreign-key index number | +| `0x11` | 4 | Foreign-key table page (non-zero ⇒ a relationship index) | +| `0x15` | 1 | Update action | +| `0x16` | 1 | Delete action | +| `0x17` | 1 | Index type (`1` = primary) | + +The index **name** read at the same ordinal applies to this logical index. To name the +physical (data-block) index, prefer a real index's name over a foreign-key relationship's +(distinguished by `0x11` ≠ 0), and take `IsPrimaryKey` from the type byte `0x17`. + +--- + +## 4. Data page — type `0x01` + +| Offset | Size | Meaning | +| --- | --- | --- | +| `0x00` | 1 | Page type `0x01` | +| `0x02` | 2 | Free space | +| `0x04` | 4 | Owning table's TDEF page — **or** the ASCII marker `LVAL` (`0x4C41564C`) for long-value pages | +| `0x0C` | 2 | Row count on this page | +| `0x0E` | 2×N | Row slot directory: one 2-byte entry per row | + +Row slot entry: lower 13 bits (`& 0x1FFF`) = the row's byte offset in the page; `0x8000` = +deleted, `0x4000` = overflow/lookup pointer (not an inline row). Rows are packed from the end +of the page backward, so a slot runs from its offset up to where the previous slot's row began. + +--- + +## 5. Row record format + +``` +[ colCount : 2 ] +[ fixed-length column data ... ] +[ variable-length column data ... ] +[ variable-offset table : (numVarCols + 1) × 2, stored end-first ] +[ numVarCols : 2 ] +[ null bitmap : ceil(colCount / 8) bytes ] ← the very end of the row +``` + +- **Null bitmap** is indexed by **column id**; a **set bit = the value is present** (non-null). +- **Fixed** column value is at `rowStart + 2 + fixedOffset`, `length` bytes. +- **Variable** column value: with `varTableStart = rowEnd − nullBitmapSize − 2 − (numVarCols+1)×2`, + variable column `j` spans `[offset(numVarCols − j), offset(numVarCols − j − 1))`, where + `offset(k)` is the little-endian 16-bit value at `varTableStart + k×2`. (The table is stored + end-first, i.e. ascending column-id order maps to descending table index.) +- **Booleans** carry **no data** — the value *is* the null-bitmap bit (set = true). Boolean + columns are never null. +- Variable offsets are **always 2 bytes** in Jet 4 / ACE, at any row size. There is **no + jump table** (that is a Jet 3 construct for its 1-byte offsets). + +--- + +## 6. Data types + +| Code | Type | Storage / decode | +| --- | --- | --- | +| `0x01` | Boolean | A bit in the null bitmap (no data) | +| `0x02` | Byte | 1 byte | +| `0x03` | Int16 | 2 bytes LE | +| `0x04` | Int32 | 4 bytes LE | +| `0x05` | Currency | int64 LE, scaled: value / 10000 | +| `0x06` | Single | 4-byte IEEE | +| `0x07` | Double | 8-byte IEEE | +| `0x08` | DateTime | 8-byte IEEE double, OLE-automation epoch (1899-12-30) | +| `0x09` | Binary | raw bytes | +| `0x0A` | Text | UTF-16LE, or compressed Unicode (§7); inline ≤ 255 chars | +| `0x0B` | OLE | long value (§8) | +| `0x0C` | Memo | long value (§8); text once resolved | +| `0x0F` | GUID | 16 raw bytes | +| `0x10` | FixedPoint (Numeric/Decimal) | not yet decoded | +| `0x12` | Complex (multi-value / attachment) | descriptor parsed; contents not materialized (out of scope for SQL/EF) | + +--- + +## 7. Compressed Unicode + +A text value that begins with the 2-byte marker `FF FE` is **compressed**: the following bytes +are one per character (ASCII range), not UTF-16. Otherwise the value is UTF-16LE. Applies to +both `Text` and resolved `Memo`. + +> Not yet handled: the full format can toggle between 1-byte and 2-byte runs mid-string for +> mixed scripts. LibRed handles the common all-compressed case. + +--- + +## 8. Long values (Memo / OLE) + +The in-row value for a Memo/OLE column is a **12-byte descriptor**, not the data: + +| Offset | Size | Meaning | +| --- | --- | --- | +| `0x00` | 3 | Length (24-bit) | +| `0x03` | 1 | Flags | +| `0x04` | 1 | Row | +| `0x05` | 3 | Page | +| `0x08` | 4 | reserved | + +Flags: +- `0x80` **inline** — the payload follows the descriptor in the row. +- `0x40` **single LVAL page** — the row at (page, row) *is* the whole payload. +- otherwise **multi-page** — the payload is chained across LVAL pages; each chunk's row begins + with a 4-byte pointer (row + 3-byte page) to the next chunk, followed by chunk data. + +LVAL pages are data pages (type `0x01`) whose owner field (`0x04`) is the ASCII marker `LVAL`. + +--- + +## 9. Usage maps + +Each table has an *owned-pages* usage map, referenced from TDEF `0x37` (1-byte row + 3-byte +page) pointing at a row on a data page. The first byte of that row is the map type. + +**Inline map (type `0x00`):** + +| Offset | Size | Meaning | +| --- | --- | --- | +| `0x00` | 1 | Type `0x00` | +| `0x01` | 4 | Start page | +| `0x05` | … | Bitmap; bit `i` ⇒ page `startPage + i` is owned | + +**Reference map (type `0x01`, for very large tables):** the row is a list of 4-byte pointers +to dedicated **bitmap pages** (type `0x05`). Pointer `k` (zero ⇒ none) points at a bitmap page +covering the page range starting at `k × (pageSize − 4) × 8`; on a bitmap page the bitmap data +begins at **offset 4**. + +> The usage map is authoritative: a brute-force owner-scan can over-count, because deleted/ +> orphaned pages can retain a stale owner stamp that the map correctly omits. + +--- + +## 10. Index B-tree pages — types `0x03` (node) and `0x04` (leaf) + +### 10.1 Header + +| Offset | Size | Meaning | +| --- | --- | --- | +| `0x00` | 1 | Page type (`0x03` node / `0x04` leaf) | +| `0x04` | 4 | Owning table TDEF page | +| `0x14` | 4 | **Child-tail** page (node pages: the rightmost child, referenced by no entry) | +| `0x18` | 2 | Compressed-byte count (shared key prefix length, §10.3) | +| `0x1B` | … | Entry-position bitmask | +| `0x1E0` | — | Start of entry data | + +### 10.2 Entries + +The entry bitmask (`0x1B` up to `0x1E0`) is a bitmap whose set bits, read in order, give the +**end offsets** of successive entries within the entry-data region. Entry `n` spans +`[prevEnd, end_n)` relative to `0x1E0` (first entry starts at 0). + +Each entry ends with a **4-byte big-endian** trailing pointer: +- **Leaf:** `pointer` is a row id — page = `pointer >> 8`, row = `pointer & 0xFF`. +- **Node:** the trailing 4 bytes are the **child page**; recurse into it. After all entries, + also recurse into the header's child-tail page (`0x14`). + +### 10.3 Prefix compression + +Entries on a page share a leading key prefix of `compressedByteCount` (`0x18`) bytes. The +**first** entry is stored in full; its first `compressedByteCount` bytes are the shared prefix, +which every subsequent entry omits. Reconstruct: `fullKey = prefix ++ storedKey`. (The trailing +pointer is never compressed, so reading row pointers needs none of this.) + +### 10.4 Key encoding (order-preserving) + +Each key column is encoded so that raw byte comparison equals value comparison. Non-boolean +columns are prefixed by a **flag byte**: + +| | Ascending | Descending | +| --- | --- | --- | +| value present | `0x7F` | `0x80` | +| null | `0x00` | `0xFF` | + +Then the value, transformed: + +- **Integers** (Byte/Int16/Int32) and **Currency** (int64): big-endian, with the **sign bit of + the first byte flipped**. Descending additionally inverts all bytes. (Decode reverses this.) +- **Single / Double / DateTime** (IEEE): if non-negative, flip the first bit; if negative, + invert all bytes (ascending). Decode: first byte's top bit set ⇒ was positive (un-flip); + else ⇒ was negative (invert all). DateTime is the resulting double via the OLE epoch. +- **Boolean:** no flag byte — a single constant: ascending `0x00` = true, `0xFF` = false + (true sorts first). +- **Text / Binary / GUID:** Jet's collation encoding, which is **lossy** (case/diacritics + folded) and **not reversible**. LibRed extracts row pointers from such indexes and decodes + the leading reversible columns, but cannot recover text key *values*. (A future keyed *seek* + needs the *encoder* + collation tables, not a decoder.) + +--- + +## 11. System catalog + +- **MSysObjects** (TDEF at page **2**) lists every object. Columns include `Id`, `Name`, + `Type`, `Flags`, `ParentId`. For a **table** object (`Type == 1`), **`Id` is the table's TDEF + page number**. An object is a system object if `Flags & 0x80000002` is set, or its name + begins with `MSys`/`~`. Bootstrap: build a TableDef for MSysObjects from page 2 and read its + rows like any table. +- **MSysRelationships** defines foreign keys (one row per relationship column): `szRelationship` + (name), `szObject` (child/referencing table), `szColumn` (child column), `szReferencedObject` + (parent table), `szReferencedColumn`, `icolumn` (order), `grbit` (flags: `0x02` don't-enforce, + `0x100` cascade-update, `0x1000` cascade-delete). + +--- + +## 12. Version differences + +**Jet 3 (Access 97) is the odd one out** and is *not yet implemented*. Known differences from +the Jet 4 / ACE layout documented above: + +- 2048-byte pages. +- 18-byte column descriptors (vs 25). +- 1-byte ASCII name lengths (vs 2-byte UTF-16). +- 1-byte row column-count (vs 2-byte), and 1-byte variable-offset entries **with a jump table** + for rows > 256 bytes. +- Different data-page and TDEF header offsets. + +Jet 4 and all later ACE versions (12/14/16/17) share the structural layout above; differences +between *those* are additive at the type/feature level (new data types, encryption schemes), +not the page offsets. In LibRed this is reflected by `JetFormatBase` virtual members with +Jet 4/ACE defaults; a future `Jet3Format` overrides the ones that differ. + +--- + +## Provenance + +Verified against: `Northwind.accdb` (ACE 2007), a generated 200-column ACCDB (multi-page TDEF), +and a generated ~150 MB ACCDB (reference usage map). Cross-referenced with mdbtools +`src/libmdb/` (`table.c`, `data.c`, `index.c`) and Jackcess (`TableImpl`, `ColumnImpl`, +`IndexData`, `IndexCodes`). The LibRed test suite (`test/LibRed.Core.Tests/`) pins these +structures, including whole-database golden dumps. From c7dd08c03fe91bfed796e3863463937aee574733 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 13:38:14 +0800 Subject: [PATCH 022/506] docs(LibRed): fill TDEF header gaps in the format spec The TDEF header table jumped from 0x14 to 0x28. Document the intervening and omitted fields, all verified across MSysObjects/Customers/Orders: page flags (0x01), free space (0x02), the 0x0C constant (0x659), the 0x18 constant (0x01) and 0x1C-0x27 reserved zeros, and the maximum-column count (0x29). The 0x14 next-auto-number is confirmed (Orders reads 11077, its next OrderID). The header is now gap-free from 0x00 to 0x3F. Co-Authored-By: Claude Opus 4.8 --- src/LibRed/docs/jet-ace-file-format.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/LibRed/docs/jet-ace-file-format.md b/src/LibRed/docs/jet-ace-file-format.md index 7b65cce1..83913dcd 100644 --- a/src/LibRed/docs/jet-ace-file-format.md +++ b/src/LibRed/docs/jet-ace-file-format.md @@ -66,11 +66,17 @@ version. (Page-level encryption for password-protected files is not implemented. | Offset | Size | Meaning | | --- | --- | --- | | `0x00` | 1 | Page type `0x02` | +| `0x01` | 1 | Flags (observed `0x01`) | +| `0x02` | 2 | Free space remaining in this page | | `0x04` | 4 | Next TDEF page (0 if the definition fits one page) | | `0x08` | 4 | TDEF length (total logical bytes) | +| `0x0C` | 4 | Unknown — a constant `0x00000659` (1625) observed in every file | | `0x10` | 4 | Row count | -| `0x14` | 4 | Next auto-number value | +| `0x14` | 4 | Next auto-number value (e.g. next AutoNumber id) | +| `0x18` | 4 | Unknown — observed constant `0x01` (possibly the ACE complex-type auto-number) | +| `0x1C` | 12 | Unknown / reserved (zero observed) | | `0x28` | 1 | Table type: `0x4E` 'N' user, `0x53` 'S' system | +| `0x29` | 2 | Maximum column count | | `0x2B` | 2 | Variable-length column count | | `0x2D` | 2 | Column count | | `0x2F` | 4 | **Logical** index count (a.k.a. index slots) | @@ -79,6 +85,10 @@ version. (Page-level encryption for password-protected files is not implemented. | `0x3B` | 4 | Free-space-pages usage-map pointer | | `0x3F` | — | Start of the real-index block (precedes column descriptors) | +> The `0x0C` and `0x18` entries are constants across every file inspected; their exact +> meaning is unconfirmed, so they are recorded as observed rather than named. LibRed does not +> read them. + > ⚠️ `0x2F` vs `0x33`: these are equal for MSysObjects (which hid the distinction during > reverse-engineering) but differ for user tables. `0x33` (real index count) sizes the > index-data blocks **and** the `0x3F` pre-column block; `0x2F` (logical count) is the number From 89e91feba4763e3f5e283dd60db302a1b93609f9 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 13:56:41 +0800 Subject: [PATCH 023/506] docs(LibRed): clarify TDEF 0x0C is not the code page Note that 0x0C (constant 0x659/1625) is not a code page (not a valid one, and the code page is a database-wide page-0 value, not per-table); 1625 is the same magic as the index-info block marker, so 0x0C is most likely a format sentinel. Co-Authored-By: Claude Opus 4.8 --- src/LibRed/docs/jet-ace-file-format.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/LibRed/docs/jet-ace-file-format.md b/src/LibRed/docs/jet-ace-file-format.md index 83913dcd..925e0b6a 100644 --- a/src/LibRed/docs/jet-ace-file-format.md +++ b/src/LibRed/docs/jet-ace-file-format.md @@ -87,7 +87,10 @@ version. (Page-level encryption for password-protected files is not implemented. > The `0x0C` and `0x18` entries are constants across every file inspected; their exact > meaning is unconfirmed, so they are recorded as observed rather than named. LibRed does not -> read them. +> read them. Note `0x0C` is **not** the code page — `0x659` (1625) is not a valid code page, +> and the code page is a database-wide value on page 0, not per-table. 1625 is in fact the +> same magic constant used as the index-info block marker (and `1923`/`0x783` is the +> index-data block marker), so `0x0C` is most likely a format sentinel. > ⚠️ `0x2F` vs `0x33`: these are equal for MSysObjects (which hid the distinction during > reverse-engineering) but differ for user tables. `0x33` (real index count) sizes the From 4d93c9a1053c64259f4a0e3c5a33bbf84f84019a Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 14:07:13 +0800 Subject: [PATCH 024/506] docs(LibRed): document 0x659/0x783 as recurring TDEF record markers Scanning the TDEF showed 0x659 (1625) is not a one-off header constant: it recurs as a fixed 4-byte field at column-descriptor +1, index-info block +0, and header 0x0C, while 0x783 (1923) marks each index-data block +0. They are constant within and across files (mdbtools: "usually 1625/1923 or 0"), i.e. reserved record markers the engine ignores. Add the column-descriptor +1 field and generalize the note accordingly. Co-Authored-By: Claude Opus 4.8 --- src/LibRed/docs/jet-ace-file-format.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/LibRed/docs/jet-ace-file-format.md b/src/LibRed/docs/jet-ace-file-format.md index 925e0b6a..85d8bdf2 100644 --- a/src/LibRed/docs/jet-ace-file-format.md +++ b/src/LibRed/docs/jet-ace-file-format.md @@ -85,12 +85,14 @@ version. (Page-level encryption for password-protected files is not implemented. | `0x3B` | 4 | Free-space-pages usage-map pointer | | `0x3F` | — | Start of the real-index block (precedes column descriptors) | -> The `0x0C` and `0x18` entries are constants across every file inspected; their exact -> meaning is unconfirmed, so they are recorded as observed rather than named. LibRed does not -> read them. Note `0x0C` is **not** the code page — `0x659` (1625) is not a valid code page, -> and the code page is a database-wide value on page 0, not per-table. 1625 is in fact the -> same magic constant used as the index-info block marker (and `1923`/`0x783` is the -> index-data block marker), so `0x0C` is most likely a format sentinel. +> **The `0x659` / `0x783` record markers.** `0x0C` holds `0x659` (1625) in every file. This is +> not isolated: `0x659` recurs as a fixed 4-byte field at the start of each repeating TDEF +> record — column descriptor `+1` (§3.4), index-info block `+0` (§3.6), and this header slot — +> while `0x783` (1923) marks each index-data block `+0` (§3.5). They are constant within and +> across files; mdbtools describes them as "usually 1625 / 1923 *or 0*", so they appear to be +> reserved record markers/tags the engine does not depend on (LibRed ignores them). `0x0C` is +> therefore **not** the code page — `1625` is not a valid code page, and the code page is a +> database-wide value on page 0, not per-table. > ⚠️ `0x2F` vs `0x33`: these are equal for MSysObjects (which hid the distinction during > reverse-engineering) but differ for user tables. `0x33` (real index count) sizes the @@ -121,11 +123,15 @@ absolute from the first page, so parsing is otherwise unchanged. | Offset | Size | Meaning | | --- | --- | --- | | `0x00` | 1 | Data type (see §6) | +| `0x01` | 4 | Record marker constant `0x659` (see §3.1 note); ignored | | `0x05` | 2 | Column id (a.k.a. column number) | | `0x0F` | 1 | Flags: `0x01` fixed-length, `0x04` auto-number | | `0x15` | 2 | Fixed-data offset within the row's fixed region | | `0x17` | 2 | Length (bytes) | +Bytes `0x07`–`0x0E` and `0x10`–`0x14` carry additional per-column flags/metadata LibRed does +not currently use. + Variable-length columns are assigned a *variable index* = their rank among variable columns ordered by ascending column id (used by the row's variable-offset table, §5). From a494ebe6eea7ef245cdd240ad07f0cd390e7fb58 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 14:22:52 +0800 Subject: [PATCH 025/506] LibRed: parse the TDEF index-statistics block (index cardinality) The 12-byte-per-real-index block at 0x3F (previously skipped to reach the columns) is the per-index statistics block: [+0] total entry count (= row count), [+4] distinct value count (cardinality), [+8] reserved. IndexDef now exposes it as UniqueValueCount; the catalog dump shows it as distinct=N (goldens regenerated). Verified against Northwind: cardinality matches the actual distinct values (e.g. Orders EmployeeID=9, ShipVia=3, PK=830). A unique index always has distinct==rowCount; the converse does not hold (a non-unique index can have all-distinct data, e.g. Customers.CompanyName). Format spec section 3.3.1 documents the block. Co-Authored-By: Claude Opus 4.8 --- src/LibRed/LibRed.Core/Catalog/IndexDef.cs | 3 + .../LibRed.Core/Pages/TableDefinitionPage.cs | 10 +- src/LibRed/LibRed.Sql/LibRed.Sql.csproj | 16 +-- src/LibRed/docs/jet-ace-file-format.md | 17 ++- test/LibRed.Core.Tests/CatalogDumpTests.cs | 2 +- .../Expected/catalog-dump.txt | 118 +++++++++--------- .../Expected/widetable-dump.txt | 20 +-- .../LibRed.Core.Tests/IndexStatisticsTests.cs | 46 +++++++ 8 files changed, 148 insertions(+), 84 deletions(-) create mode 100644 test/LibRed.Core.Tests/IndexStatisticsTests.cs diff --git a/src/LibRed/LibRed.Core/Catalog/IndexDef.cs b/src/LibRed/LibRed.Core/Catalog/IndexDef.cs index 1218c230..b261a749 100644 --- a/src/LibRed/LibRed.Core/Catalog/IndexDef.cs +++ b/src/LibRed/LibRed.Core/Catalog/IndexDef.cs @@ -11,6 +11,9 @@ public sealed record IndexDef public bool IsUnique { get; init; } public bool IsPrimaryKey { get; init; } + /// Number of distinct key values in the index (cardinality), from the TDEF statistics block. + public int UniqueValueCount { get; init; } + /// Page number of the index B-tree root. public int RootPage { get; init; } } diff --git a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs index 151c81a9..586a1628 100644 --- a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs +++ b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs @@ -101,7 +101,7 @@ public override void Read(PageBuffer buffer, JetFormatBase format) // The buffer here may already be a stitched multi-page definition (see Read(channel, page)). int columnBlock = format.TdefRealIndexBlockOffset + IndexCount * format.RealIndexEntrySize; int afterNames = ReadColumns(buffer, format, columnBlock); - ReadIndexes(buffer, afterNames); + ReadIndexes(buffer, format, afterNames); } /// @@ -109,7 +109,7 @@ public override void Read(PageBuffer buffer, JetFormatBase format) /// (one per index-data block): columns + sort order, unique/primary flags, root page, /// and the index name (resolved from the logical-index info blocks). /// - private void ReadIndexes(PageBuffer buffer, int blockStart) + private void ReadIndexes(PageBuffer buffer, JetFormatBase format, int blockStart) { _indexes.Clear(); var byColumnId = _columns.ToDictionary(c => c.ColumnId); @@ -119,6 +119,11 @@ private void ReadIndexes(PageBuffer buffer, int blockStart) { int block = blockStart + i * IndexBlockSize; + // Per-index statistics live in the 12-byte block at TdefRealIndexBlockOffset: + // [+0] total entries (= row count), [+4] distinct value count, [+8] reserved. + int statsBlock = format.TdefRealIndexBlockOffset + i * format.RealIndexEntrySize; + int uniqueValueCount = buffer.ReadInt32(statsBlock + 4); + var columns = new List<(ColumnDef Column, bool Ascending)>(); for (int slot = 0; slot < IndexMaxColumns; slot++) { @@ -135,6 +140,7 @@ private void ReadIndexes(PageBuffer buffer, int blockStart) Columns = columns, IsUnique = (buffer.ReadUInt16(block + IndexFlagsOffset) & IndexFlagUnique) != 0, IsPrimaryKey = false, + UniqueValueCount = uniqueValueCount, RootPage = buffer.ReadInt32(block + IndexRootPageOffset), }); } diff --git a/src/LibRed/LibRed.Sql/LibRed.Sql.csproj b/src/LibRed/LibRed.Sql/LibRed.Sql.csproj index 1ab06195..70fdae2e 100644 --- a/src/LibRed/LibRed.Sql/LibRed.Sql.csproj +++ b/src/LibRed/LibRed.Sql/LibRed.Sql.csproj @@ -7,16 +7,10 @@ Jet/ACE SQL front end: lexer, parser, AST and binder. Has no dependency on the storage engine; binding resolves names through an injected schema abstraction. - + + + + + diff --git a/src/LibRed/docs/jet-ace-file-format.md b/src/LibRed/docs/jet-ace-file-format.md index 85d8bdf2..55cbca27 100644 --- a/src/LibRed/docs/jet-ace-file-format.md +++ b/src/LibRed/docs/jet-ace-file-format.md @@ -110,7 +110,7 @@ absolute from the first page, so parsing is otherwise unchanged. ### 3.3 Body layout (in order, after the header) ``` -0x3F : real-index block RealIndexCount(0x33) × 12 bytes (skipped to find columns) +0x3F : index statistics RealIndexCount(0x33) × 12 bytes (per-index, §3.3.1) column descriptors ColumnCount(0x2D) × 25 bytes column names ColumnCount × (2-byte length + UTF-16LE) index-data blocks RealIndexCount(0x33) × 52 bytes @@ -118,6 +118,21 @@ absolute from the first page, so parsing is otherwise unchanged. index names LogicalIndexCount × (2-byte length + UTF-16LE) ``` +### 3.3.1 Index statistics block (12 bytes, one per real index) + +The block at `0x3F`, in the same order as the index-data blocks (§3.5), holds per-index +statistics: + +| Offset | Size | Meaning | +| --- | --- | --- | +| `0x00` | 4 | Total entry count (= the table's row count; every row is indexed) | +| `0x04` | 4 | **Distinct value count** (index cardinality) | +| `0x08` | 4 | Reserved (zero observed) | + +A unique index necessarily has `distinct == rowCount`; the converse does not hold (a +non-unique index can have all-distinct data). LibRed exposes the cardinality as +`IndexDef.UniqueValueCount`. + ### 3.4 Column descriptor (25 bytes) | Offset | Size | Meaning | diff --git a/test/LibRed.Core.Tests/CatalogDumpTests.cs b/test/LibRed.Core.Tests/CatalogDumpTests.cs index 99b574bd..6183817c 100644 --- a/test/LibRed.Core.Tests/CatalogDumpTests.cs +++ b/test/LibRed.Core.Tests/CatalogDumpTests.cs @@ -61,7 +61,7 @@ private static string BuildDump(string path) { string kind = ix.IsPrimaryKey ? "PK " : ix.IsUnique ? "U " : " "; string cols = string.Join(", ", ix.Columns.Select(c => c.Column.Name + (c.Ascending ? "" : " DESC"))); - sb.Append($" {kind}{("\"" + ix.Name + "\""),-26} [{cols}] root={ix.RootPage}\n"); + sb.Append($" {kind}{("\"" + ix.Name + "\""),-26} [{cols}] root={ix.RootPage} distinct={ix.UniqueValueCount}\n"); } } diff --git a/test/LibRed.Core.Tests/Expected/catalog-dump.txt b/test/LibRed.Core.Tests/Expected/catalog-dump.txt index 0064b1e4..3ebe5c22 100644 --- a/test/LibRed.Core.Tests/Expected/catalog-dump.txt +++ b/test/LibRed.Core.Tests/Expected/catalog-dump.txt @@ -20,8 +20,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 15. RmtInfoShort Binary len=510 var#5 16. Type Int16 len= 2 fixed@8 --- indexes --- - U "ParentIdName" [ParentId, Name] root=7 - PK "Id" [Id] root=8 + U "ParentIdName" [ParentId, Name] root=7 distinct=82 + PK "Id" [Id] root=8 distinct=82 [SYS] MSysACEs tdefPage=3 type=System rows=183 columns=4 indexes=1 @@ -30,7 +30,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 2. ObjectId Int32 len= 4 fixed@0 3. SID Binary len=510 var#0 --- indexes --- - "ObjectId" [ObjectId] root=10 + "ObjectId" [ObjectId] root=10 distinct=82 [SYS] MSysQueries tdefPage=4 type=System rows=279 columns=8 indexes=1 @@ -43,7 +43,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 6. ObjectId Int32 len= 4 fixed@0 7. Order Binary len=510 var#0 --- indexes --- - PK "ObjectIdAttribute" [ObjectId, Attribute, Order] root=12 + PK "ObjectIdAttribute" [ObjectId, Attribute, Order] root=12 distinct=279 [SYS] MSysRelationships tdefPage=5 type=System rows=15 columns=8 indexes=3 @@ -56,9 +56,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 6. szReferencedObject Text len=510 var#3 7. szRelationship Text len=510 var#0 --- indexes --- - "szRelationship" [szRelationship] root=14 - "szObject" [szObject] root=15 - "szReferencedObject" [szReferencedObject] root=16 + "szRelationship" [szRelationship] root=14 distinct=15 + "szObject" [szObject] root=15 distinct=9 + "szReferencedObject" [szReferencedObject] root=16 distinct=12 [SYS] MSysComplexColumns tdefPage=18 type=User rows=1 columns=5 indexes=3 @@ -68,9 +68,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 3. ConceptualTableID Int32 len= 4 fixed@8 4. FlatTableID Int32 len= 4 fixed@4 --- indexes --- - PK "IdxID" [ComplexID] root=20 - "IdxConceptualTableID" [ConceptualTableID] root=21 - "IdxFlatTableID" [FlatTableID] root=22 + PK "IdxID" [ComplexID] root=20 distinct=1 + "IdxConceptualTableID" [ConceptualTableID] root=21 distinct=1 + "IdxFlatTableID" [FlatTableID] root=22 distinct=1 [SYS] MSysComplexType_UnsignedByte tdefPage=23 type=User rows=0 columns=1 indexes=0 @@ -120,17 +120,17 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 2. Description Memo len= 0 var#1 3. Picture Ole len= 0 var#2 --- indexes --- - "CategoryName" [CategoryName] root=71 - PK "PK_Categories" [CategoryID] root=72 + "CategoryName" [CategoryName] root=71 distinct=8 + PK "PK_Categories" [CategoryID] root=72 distinct=8 CustomerCustomerDemo tdefPage=73 type=User rows=0 columns=2 indexes=3 0. CustomerID Text len= 10 fixed@0 1. CustomerTypeID Text len= 20 fixed@10 --- indexes --- - PK "PK_CustomerCustomerDemo" [CustomerID, CustomerTypeID] root=75 - "FK_CustomerCustomerDemo" [CustomerTypeID] root=303 - "FK_CustomerCustomerDemo_Customers" [CustomerID] root=304 + PK "PK_CustomerCustomerDemo" [CustomerID, CustomerTypeID] root=75 distinct=0 + "FK_CustomerCustomerDemo" [CustomerTypeID] root=303 distinct=0 + "FK_CustomerCustomerDemo_Customers" [CustomerID] root=304 distinct=0 --- foreign keys --- "FK_CustomerCustomerDemo" [CustomerTypeID] -> CustomerDemographics [CustomerTypeID] "FK_CustomerCustomerDemo_Customers" [CustomerID] -> Customers [CustomerID] @@ -140,7 +140,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 0. CustomerTypeID Text len= 20 fixed@0 1. CustomerDesc Memo len= 0 var#0 --- indexes --- - PK "PK_CustomerDemographics" [CustomerTypeID] root=78 + PK "PK_CustomerDemographics" [CustomerTypeID] root=78 distinct=0 Customers tdefPage=79 type=User rows=91 columns=11 indexes=5 @@ -156,11 +156,11 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 9. Phone Text len= 48 var#8 10. Fax Text len= 48 var#9 --- indexes --- - "City" [City] root=87 - "CompanyName" [CompanyName] root=88 - PK "PK_Customers" [CustomerID] root=89 - "PostalCode" [PostalCode] root=90 - "Region" [Region] root=91 + "City" [City] root=87 distinct=69 + "CompanyName" [CompanyName] root=88 distinct=91 + PK "PK_Customers" [CustomerID] root=89 distinct=91 + "PostalCode" [PostalCode] root=90 distinct=87 + "Region" [Region] root=91 distinct=19 Employees tdefPage=92 type=User rows=9 columns=18 indexes=4 @@ -183,10 +183,10 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 16. ReportsTo Int32 len= 4 fixed@20 17. PhotoPath Text len=510 var#13 --- indexes --- - "LastName" [LastName] root=151 - PK "PK_Employees" [EmployeeID] root=152 - "PostalCode" [PostalCode] root=153 - "FK_Employees_Employees" [ReportsTo] root=305 + "LastName" [LastName] root=151 distinct=9 + PK "PK_Employees" [EmployeeID] root=152 distinct=9 + "PostalCode" [PostalCode] root=153 distinct=9 + "FK_Employees_Employees" [ReportsTo] root=305 distinct=3 --- foreign keys --- "FK_Employees_Employees" [ReportsTo] -> Employees [EmployeeID] @@ -200,9 +200,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 5. ParentId Int32 len= 4 fixed@20 6. Type Int32 len= 4 fixed@24 --- indexes --- - PK "Id" [Id] root=161 - U "ParentIdId" [ParentId, Id] root=162 - U "ParentIdName" [ParentId, Name] root=163 + PK "Id" [Id] root=161 distinct=24 + U "ParentIdId" [ParentId, Id] root=162 distinct=24 + U "ParentIdName" [ParentId, Name] root=163 distinct=24 Order Details tdefPage=164 type=User rows=2155 columns=5 indexes=3 @@ -212,9 +212,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 3. Quantity Int16 len= 2 fixed@16 4. Discount Single len= 4 fixed@18 --- indexes --- - "OrderID" [OrderID] root=181 - PK "PK_Order_Details" [OrderID, ProductID] root=186 - "ProductID" [ProductID] root=194 + "OrderID" [OrderID] root=181 distinct=830 + PK "PK_Order_Details" [OrderID, ProductID] root=186 distinct=2155 + "ProductID" [ProductID] root=194 distinct=77 --- foreign keys --- "FK_Order_Details_Orders" [OrderID] -> Orders [OrderID] "FK_Order_Details_Products" [ProductID] -> Products [ProductID] @@ -236,13 +236,13 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 12. ShipPostalCode Text len= 20 var#4 13. ShipCountry Text len= 30 var#5 --- indexes --- - "CustomerID" [CustomerID] root=240 - "EmployeeID" [EmployeeID] root=244 - "OrderDate" [OrderDate] root=247 - PK "PK_Orders" [OrderID] root=251 - "ShippedDate" [ShippedDate] root=254 - "ShippersOrders" [ShipVia] root=258 - "ShipPostalCode" [ShipPostalCode] root=261 + "CustomerID" [CustomerID] root=240 distinct=89 + "EmployeeID" [EmployeeID] root=244 distinct=9 + "OrderDate" [OrderDate] root=247 distinct=480 + PK "PK_Orders" [OrderID] root=251 distinct=830 + "ShippedDate" [ShippedDate] root=254 distinct=388 + "ShippersOrders" [ShipVia] root=258 distinct=3 + "ShipPostalCode" [ShipPostalCode] root=261 distinct=85 --- foreign keys --- "FK_Orders_Customers" [CustomerID] -> Customers [CustomerID] "FK_Orders_Employees" [EmployeeID] -> Employees [EmployeeID] @@ -261,10 +261,10 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 8. ReorderLevel Int16 len= 2 fixed@24 9. Discontinued Boolean len= 1 fixed@0 --- indexes --- - "CategoriesProducts" [CategoryID] root=270 - PK "PK_Products" [ProductID] root=271 - "ProductName" [ProductName] root=272 - "SupplierID" [SupplierID] root=273 + "CategoriesProducts" [CategoryID] root=270 distinct=8 + PK "PK_Products" [ProductID] root=271 distinct=77 + "ProductName" [ProductName] root=272 distinct=77 + "SupplierID" [SupplierID] root=273 distinct=29 --- foreign keys --- "FK_Products_Categories" [CategoryID] -> Categories [CategoryID] "FK_Products_Suppliers" [SupplierID] -> Suppliers [SupplierID] @@ -274,7 +274,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 0. RegionID Int32 len= 4 fixed@0 1. RegionDescription Text len=100 fixed@4 --- indexes --- - PK "PK_Region" [RegionID] root=278 + PK "PK_Region" [RegionID] root=278 distinct=4 Shippers tdefPage=283 type=User rows=3 columns=3 indexes=1 @@ -282,7 +282,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 1. CompanyName Text len= 80 var#0 2. Phone Text len= 48 var#1 --- indexes --- - PK "PK_Shippers" [ShipperID] root=286 + PK "PK_Shippers" [ShipperID] root=286 distinct=3 Suppliers tdefPage=287 type=User rows=29 columns=12 indexes=3 @@ -299,9 +299,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 10. Fax Text len= 48 var#9 11. HomePage Memo len= 0 var#10 --- indexes --- - "CompanyName" [CompanyName] root=292 - PK "PK_Suppliers" [SupplierID] root=293 - "PostalCode" [PostalCode] root=294 + "CompanyName" [CompanyName] root=292 distinct=29 + PK "PK_Suppliers" [SupplierID] root=293 distinct=29 + "PostalCode" [PostalCode] root=294 distinct=29 Territories tdefPage=295 type=User rows=53 columns=3 indexes=2 @@ -309,8 +309,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 1. TerritoryDescription Text len=100 fixed@0 2. RegionID Int32 len= 4 fixed@100 --- indexes --- - PK "PK_Territories" [TerritoryID] root=299 - "FK_Territories_Region" [RegionID] root=308 + PK "PK_Territories" [TerritoryID] root=299 distinct=53 + "FK_Territories_Region" [RegionID] root=308 distinct=4 --- foreign keys --- "FK_Territories_Region" [RegionID] -> Region [RegionID] @@ -324,7 +324,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 5. SelectedObjectID Int32 len= 4 var#3 6. Type Int32 len= 4 var#2 --- indexes --- - PK "Id" [Id] root=314 + PK "Id" [Id] root=314 distinct=3 [SYS] MSysNavPaneGroups tdefPage=315 type=User rows=10 columns=7 indexes=2 @@ -336,8 +336,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 5. ObjectID Int32 len= 4 var#3 6. Position Int32 len= 4 var#4 --- indexes --- - PK "Id" [Id] root=317 - "GroupCategoryID" [GroupCategoryID] root=318 + PK "Id" [Id] root=317 distinct=10 + "GroupCategoryID" [GroupCategoryID] root=318 distinct=3 --- foreign keys --- "MSysNavPaneGroupCategoriesMSysNavPaneGroups" [GroupCategoryID] -> MSysNavPaneGroupCategories [Id] cascadeUpdate cascadeDelete @@ -351,8 +351,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 5. ObjectID Int32 len= 4 var#1 6. Position Int32 len= 4 var#4 --- indexes --- - PK "Id" [Id] root=321 - "GroupID" [GroupID] root=322 + PK "Id" [Id] root=321 distinct=47 + "GroupID" [GroupID] root=322 distinct=2 --- foreign keys --- "MSysNavPaneGroupsMSysNavPaneGroupToObjects" [GroupID] -> MSysNavPaneGroups [Id] cascadeUpdate cascadeDelete @@ -370,8 +370,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 3. Name Text len=510 var#1 4. Type Text len=510 var#2 --- indexes --- - U "Data_BF92175BD03C433695464F95CDF97589" [Data] root=340 - PK "Id" [Id] root=341 + U "Data_BF92175BD03C433695464F95CDF97589" [Data] root=340 distinct=1 + PK "Id" [Id] root=341 distinct=1 [SYS] f_3B3D4E1F479C4CCCA9CB8E549B05D3B7_Data tdefPage=334 type=User rows=1 columns=8 indexes=3 @@ -384,6 +384,6 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 6. FileURL Memo len= 0 var#3 7. MSysResources_Data Int32 len= 4 var#4 auto --- indexes --- - PK "MSysComplexPKIndex" [MSysResources_Data] root=336 - "_Data" [_Data] root=337 - U "IdxFKPrimaryScalar" [_Data, FileName] root=338 + PK "MSysComplexPKIndex" [MSysResources_Data] root=336 distinct=1 + "_Data" [_Data] root=337 distinct=1 + U "IdxFKPrimaryScalar" [_Data, FileName] root=338 distinct=1 diff --git a/test/LibRed.Core.Tests/Expected/widetable-dump.txt b/test/LibRed.Core.Tests/Expected/widetable-dump.txt index d90c5e0c..661c1222 100644 --- a/test/LibRed.Core.Tests/Expected/widetable-dump.txt +++ b/test/LibRed.Core.Tests/Expected/widetable-dump.txt @@ -20,8 +20,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 15. RmtInfoShort Binary len=510 var#5 16. Type Int16 len= 2 fixed@8 --- indexes --- - U "ParentIdName" [ParentId, Name] root=7 - PK "Id" [Id] root=8 + U "ParentIdName" [ParentId, Name] root=7 distinct=19 + PK "Id" [Id] root=8 distinct=19 [SYS] MSysACEs tdefPage=3 type=System rows=38 columns=4 indexes=1 @@ -30,7 +30,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 2. ObjectId Int32 len= 4 fixed@0 3. SID Binary len=510 var#0 --- indexes --- - "ObjectId" [ObjectId] root=10 + "ObjectId" [ObjectId] root=10 distinct=19 [SYS] MSysQueries tdefPage=4 type=System rows=0 columns=8 indexes=1 @@ -43,7 +43,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 6. ObjectId Int32 len= 4 fixed@0 7. Order Binary len=510 var#0 --- indexes --- - PK "ObjectIdAttribute" [ObjectId, Attribute, Order] root=12 + PK "ObjectIdAttribute" [ObjectId, Attribute, Order] root=12 distinct=0 [SYS] MSysRelationships tdefPage=5 type=System rows=0 columns=8 indexes=3 @@ -56,9 +56,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 6. szReferencedObject Text len=510 var#3 7. szRelationship Text len=510 var#0 --- indexes --- - "szRelationship" [szRelationship] root=14 - "szObject" [szObject] root=15 - "szReferencedObject" [szReferencedObject] root=16 + "szRelationship" [szRelationship] root=14 distinct=0 + "szObject" [szObject] root=15 distinct=0 + "szReferencedObject" [szReferencedObject] root=16 distinct=0 [SYS] MSysComplexColumns tdefPage=18 type=User rows=0 columns=5 indexes=3 @@ -68,9 +68,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 3. ConceptualTableID Int32 len= 4 fixed@8 4. FlatTableID Int32 len= 4 fixed@4 --- indexes --- - PK "IdxID" [ComplexID] root=20 - "IdxConceptualTableID" [ConceptualTableID] root=21 - "IdxFlatTableID" [FlatTableID] root=22 + PK "IdxID" [ComplexID] root=20 distinct=0 + "IdxConceptualTableID" [ConceptualTableID] root=21 distinct=0 + "IdxFlatTableID" [FlatTableID] root=22 distinct=0 [SYS] MSysComplexType_UnsignedByte tdefPage=23 type=User rows=0 columns=1 indexes=0 diff --git a/test/LibRed.Core.Tests/IndexStatisticsTests.cs b/test/LibRed.Core.Tests/IndexStatisticsTests.cs new file mode 100644 index 00000000..7b75be6c --- /dev/null +++ b/test/LibRed.Core.Tests/IndexStatisticsTests.cs @@ -0,0 +1,46 @@ +using LibRed; +using Xunit; + +namespace LibRed.Core.Tests; + +public class IndexStatisticsTests +{ + [Fact] + public void Unique_indexes_have_cardinality_equal_to_row_count() + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + foreach (var table in db.Catalog.UserTables) + { + int rows = db.ReadTableDefinition(table.DefinitionPage).RowCount; + foreach (var ix in table.Indexes) + { + Assert.InRange(ix.UniqueValueCount, 0, rows); + if (ix.IsUnique) + Assert.Equal(rows, ix.UniqueValueCount); // a unique index cannot have duplicates + } + } + } + + [Theory] + [InlineData("EmployeeID", 9)] // 9 employees + [InlineData("ShippersOrders", 3)] // 3 shippers (ShipVia) + [InlineData("PK_Orders", 830)] // unique = row count + public void Cardinality_matches_actual_distinct_values(string indexName, int expected) + { + using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); + + var table = db.OpenTable("Orders"); + var index = table.Definition.Indexes.First(i => i.Name == indexName); + int columnIndex = index.Columns[0].Column.Index; + + int actualDistinct = table.Rows() + .Select(r => r[columnIndex]) + .Where(v => v is not null) // index cardinality excludes nulls + .Distinct() + .Count(); + + Assert.Equal(expected, index.UniqueValueCount); + Assert.Equal(expected, actualDistinct); + } +} From 1d8202e1574a6b737fe741436264c957ed748e64 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 14:33:54 +0800 Subject: [PATCH 026/506] LibRed: correct index stat to UniqueEntryCount (cumulative, not live cardinality) Jackcess reads the same +4 field as uniqueEntryCount and documents that Access only ever increments it, never decrements it. So it is NOT a guaranteed current distinct-value count -- it equals the cardinality only when no rows have been deleted (which is why it matched fresh Northwind exactly). Rename IndexDef.UniqueValueCount -> UniqueEntryCount with accurate semantics, fix the spec (section 3.3.1) and the dump label (distinct -> uniqueEntries, goldens regenerated), and note the no-deletions caveat in the tests. Co-Authored-By: Claude Opus 4.8 --- src/LibRed/LibRed.Core/Catalog/IndexDef.cs | 9 +- .../LibRed.Core/Pages/TableDefinitionPage.cs | 7 +- src/LibRed/docs/jet-ace-file-format.md | 11 +- test/LibRed.Core.Tests/CatalogDumpTests.cs | 2 +- .../Expected/catalog-dump.txt | 118 +++++++++--------- .../Expected/widetable-dump.txt | 20 +-- .../LibRed.Core.Tests/IndexStatisticsTests.cs | 16 ++- 7 files changed, 98 insertions(+), 85 deletions(-) diff --git a/src/LibRed/LibRed.Core/Catalog/IndexDef.cs b/src/LibRed/LibRed.Core/Catalog/IndexDef.cs index b261a749..86026e58 100644 --- a/src/LibRed/LibRed.Core/Catalog/IndexDef.cs +++ b/src/LibRed/LibRed.Core/Catalog/IndexDef.cs @@ -11,8 +11,13 @@ public sealed record IndexDef public bool IsUnique { get; init; } public bool IsPrimaryKey { get; init; } - /// Number of distinct key values in the index (cardinality), from the TDEF statistics block. - public int UniqueValueCount { get; init; } + /// + /// The index's unique-entry count from the TDEF statistics block. This is a cumulative + /// count of distinct entries ever added that Access increments but never decrements, + /// so it equals the current distinct-value count (cardinality) only when no rows have been + /// deleted. (Same semantics as Jackcess's uniqueEntryCount.) + /// + public int UniqueEntryCount { get; init; } /// Page number of the index B-tree root. public int RootPage { get; init; } diff --git a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs index 586a1628..13b6d90f 100644 --- a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs +++ b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs @@ -120,9 +120,10 @@ private void ReadIndexes(PageBuffer buffer, JetFormatBase format, int blockStart int block = blockStart + i * IndexBlockSize; // Per-index statistics live in the 12-byte block at TdefRealIndexBlockOffset: - // [+0] total entries (= row count), [+4] distinct value count, [+8] reserved. + // [+0] total entries (= row count), [+4] unique entry count (cumulative, never + // decremented by Access), [+8] reserved. int statsBlock = format.TdefRealIndexBlockOffset + i * format.RealIndexEntrySize; - int uniqueValueCount = buffer.ReadInt32(statsBlock + 4); + int uniqueEntryCount = buffer.ReadInt32(statsBlock + 4); var columns = new List<(ColumnDef Column, bool Ascending)>(); for (int slot = 0; slot < IndexMaxColumns; slot++) @@ -140,7 +141,7 @@ private void ReadIndexes(PageBuffer buffer, JetFormatBase format, int blockStart Columns = columns, IsUnique = (buffer.ReadUInt16(block + IndexFlagsOffset) & IndexFlagUnique) != 0, IsPrimaryKey = false, - UniqueValueCount = uniqueValueCount, + UniqueEntryCount = uniqueEntryCount, RootPage = buffer.ReadInt32(block + IndexRootPageOffset), }); } diff --git a/src/LibRed/docs/jet-ace-file-format.md b/src/LibRed/docs/jet-ace-file-format.md index 55cbca27..f674d74f 100644 --- a/src/LibRed/docs/jet-ace-file-format.md +++ b/src/LibRed/docs/jet-ace-file-format.md @@ -126,12 +126,15 @@ statistics: | Offset | Size | Meaning | | --- | --- | --- | | `0x00` | 4 | Total entry count (= the table's row count; every row is indexed) | -| `0x04` | 4 | **Distinct value count** (index cardinality) | +| `0x04` | 4 | **Unique entry count** — distinct entries ever added (see note) | | `0x08` | 4 | Reserved (zero observed) | -A unique index necessarily has `distinct == rowCount`; the converse does not hold (a -non-unique index can have all-distinct data). LibRed exposes the cardinality as -`IndexDef.UniqueValueCount`. +The unique entry count is **cumulative**: Access increments it but never decrements it (the +same behaviour Jackcess documents for `uniqueEntryCount`). It therefore equals the *current* +distinct-value count only when no rows have been deleted; after deletions it drifts higher. +On a database with no deletions a unique index has `uniqueEntryCount == rowCount`; the converse +does not hold (a non-unique index can have all-distinct data). LibRed exposes it as +`IndexDef.UniqueEntryCount`. ### 3.4 Column descriptor (25 bytes) diff --git a/test/LibRed.Core.Tests/CatalogDumpTests.cs b/test/LibRed.Core.Tests/CatalogDumpTests.cs index 6183817c..304f0ca7 100644 --- a/test/LibRed.Core.Tests/CatalogDumpTests.cs +++ b/test/LibRed.Core.Tests/CatalogDumpTests.cs @@ -61,7 +61,7 @@ private static string BuildDump(string path) { string kind = ix.IsPrimaryKey ? "PK " : ix.IsUnique ? "U " : " "; string cols = string.Join(", ", ix.Columns.Select(c => c.Column.Name + (c.Ascending ? "" : " DESC"))); - sb.Append($" {kind}{("\"" + ix.Name + "\""),-26} [{cols}] root={ix.RootPage} distinct={ix.UniqueValueCount}\n"); + sb.Append($" {kind}{("\"" + ix.Name + "\""),-26} [{cols}] root={ix.RootPage} uniqueEntries={ix.UniqueEntryCount}\n"); } } diff --git a/test/LibRed.Core.Tests/Expected/catalog-dump.txt b/test/LibRed.Core.Tests/Expected/catalog-dump.txt index 3ebe5c22..8005e512 100644 --- a/test/LibRed.Core.Tests/Expected/catalog-dump.txt +++ b/test/LibRed.Core.Tests/Expected/catalog-dump.txt @@ -20,8 +20,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 15. RmtInfoShort Binary len=510 var#5 16. Type Int16 len= 2 fixed@8 --- indexes --- - U "ParentIdName" [ParentId, Name] root=7 distinct=82 - PK "Id" [Id] root=8 distinct=82 + U "ParentIdName" [ParentId, Name] root=7 uniqueEntries=82 + PK "Id" [Id] root=8 uniqueEntries=82 [SYS] MSysACEs tdefPage=3 type=System rows=183 columns=4 indexes=1 @@ -30,7 +30,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 2. ObjectId Int32 len= 4 fixed@0 3. SID Binary len=510 var#0 --- indexes --- - "ObjectId" [ObjectId] root=10 distinct=82 + "ObjectId" [ObjectId] root=10 uniqueEntries=82 [SYS] MSysQueries tdefPage=4 type=System rows=279 columns=8 indexes=1 @@ -43,7 +43,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 6. ObjectId Int32 len= 4 fixed@0 7. Order Binary len=510 var#0 --- indexes --- - PK "ObjectIdAttribute" [ObjectId, Attribute, Order] root=12 distinct=279 + PK "ObjectIdAttribute" [ObjectId, Attribute, Order] root=12 uniqueEntries=279 [SYS] MSysRelationships tdefPage=5 type=System rows=15 columns=8 indexes=3 @@ -56,9 +56,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 6. szReferencedObject Text len=510 var#3 7. szRelationship Text len=510 var#0 --- indexes --- - "szRelationship" [szRelationship] root=14 distinct=15 - "szObject" [szObject] root=15 distinct=9 - "szReferencedObject" [szReferencedObject] root=16 distinct=12 + "szRelationship" [szRelationship] root=14 uniqueEntries=15 + "szObject" [szObject] root=15 uniqueEntries=9 + "szReferencedObject" [szReferencedObject] root=16 uniqueEntries=12 [SYS] MSysComplexColumns tdefPage=18 type=User rows=1 columns=5 indexes=3 @@ -68,9 +68,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 3. ConceptualTableID Int32 len= 4 fixed@8 4. FlatTableID Int32 len= 4 fixed@4 --- indexes --- - PK "IdxID" [ComplexID] root=20 distinct=1 - "IdxConceptualTableID" [ConceptualTableID] root=21 distinct=1 - "IdxFlatTableID" [FlatTableID] root=22 distinct=1 + PK "IdxID" [ComplexID] root=20 uniqueEntries=1 + "IdxConceptualTableID" [ConceptualTableID] root=21 uniqueEntries=1 + "IdxFlatTableID" [FlatTableID] root=22 uniqueEntries=1 [SYS] MSysComplexType_UnsignedByte tdefPage=23 type=User rows=0 columns=1 indexes=0 @@ -120,17 +120,17 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 2. Description Memo len= 0 var#1 3. Picture Ole len= 0 var#2 --- indexes --- - "CategoryName" [CategoryName] root=71 distinct=8 - PK "PK_Categories" [CategoryID] root=72 distinct=8 + "CategoryName" [CategoryName] root=71 uniqueEntries=8 + PK "PK_Categories" [CategoryID] root=72 uniqueEntries=8 CustomerCustomerDemo tdefPage=73 type=User rows=0 columns=2 indexes=3 0. CustomerID Text len= 10 fixed@0 1. CustomerTypeID Text len= 20 fixed@10 --- indexes --- - PK "PK_CustomerCustomerDemo" [CustomerID, CustomerTypeID] root=75 distinct=0 - "FK_CustomerCustomerDemo" [CustomerTypeID] root=303 distinct=0 - "FK_CustomerCustomerDemo_Customers" [CustomerID] root=304 distinct=0 + PK "PK_CustomerCustomerDemo" [CustomerID, CustomerTypeID] root=75 uniqueEntries=0 + "FK_CustomerCustomerDemo" [CustomerTypeID] root=303 uniqueEntries=0 + "FK_CustomerCustomerDemo_Customers" [CustomerID] root=304 uniqueEntries=0 --- foreign keys --- "FK_CustomerCustomerDemo" [CustomerTypeID] -> CustomerDemographics [CustomerTypeID] "FK_CustomerCustomerDemo_Customers" [CustomerID] -> Customers [CustomerID] @@ -140,7 +140,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 0. CustomerTypeID Text len= 20 fixed@0 1. CustomerDesc Memo len= 0 var#0 --- indexes --- - PK "PK_CustomerDemographics" [CustomerTypeID] root=78 distinct=0 + PK "PK_CustomerDemographics" [CustomerTypeID] root=78 uniqueEntries=0 Customers tdefPage=79 type=User rows=91 columns=11 indexes=5 @@ -156,11 +156,11 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 9. Phone Text len= 48 var#8 10. Fax Text len= 48 var#9 --- indexes --- - "City" [City] root=87 distinct=69 - "CompanyName" [CompanyName] root=88 distinct=91 - PK "PK_Customers" [CustomerID] root=89 distinct=91 - "PostalCode" [PostalCode] root=90 distinct=87 - "Region" [Region] root=91 distinct=19 + "City" [City] root=87 uniqueEntries=69 + "CompanyName" [CompanyName] root=88 uniqueEntries=91 + PK "PK_Customers" [CustomerID] root=89 uniqueEntries=91 + "PostalCode" [PostalCode] root=90 uniqueEntries=87 + "Region" [Region] root=91 uniqueEntries=19 Employees tdefPage=92 type=User rows=9 columns=18 indexes=4 @@ -183,10 +183,10 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 16. ReportsTo Int32 len= 4 fixed@20 17. PhotoPath Text len=510 var#13 --- indexes --- - "LastName" [LastName] root=151 distinct=9 - PK "PK_Employees" [EmployeeID] root=152 distinct=9 - "PostalCode" [PostalCode] root=153 distinct=9 - "FK_Employees_Employees" [ReportsTo] root=305 distinct=3 + "LastName" [LastName] root=151 uniqueEntries=9 + PK "PK_Employees" [EmployeeID] root=152 uniqueEntries=9 + "PostalCode" [PostalCode] root=153 uniqueEntries=9 + "FK_Employees_Employees" [ReportsTo] root=305 uniqueEntries=3 --- foreign keys --- "FK_Employees_Employees" [ReportsTo] -> Employees [EmployeeID] @@ -200,9 +200,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 5. ParentId Int32 len= 4 fixed@20 6. Type Int32 len= 4 fixed@24 --- indexes --- - PK "Id" [Id] root=161 distinct=24 - U "ParentIdId" [ParentId, Id] root=162 distinct=24 - U "ParentIdName" [ParentId, Name] root=163 distinct=24 + PK "Id" [Id] root=161 uniqueEntries=24 + U "ParentIdId" [ParentId, Id] root=162 uniqueEntries=24 + U "ParentIdName" [ParentId, Name] root=163 uniqueEntries=24 Order Details tdefPage=164 type=User rows=2155 columns=5 indexes=3 @@ -212,9 +212,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 3. Quantity Int16 len= 2 fixed@16 4. Discount Single len= 4 fixed@18 --- indexes --- - "OrderID" [OrderID] root=181 distinct=830 - PK "PK_Order_Details" [OrderID, ProductID] root=186 distinct=2155 - "ProductID" [ProductID] root=194 distinct=77 + "OrderID" [OrderID] root=181 uniqueEntries=830 + PK "PK_Order_Details" [OrderID, ProductID] root=186 uniqueEntries=2155 + "ProductID" [ProductID] root=194 uniqueEntries=77 --- foreign keys --- "FK_Order_Details_Orders" [OrderID] -> Orders [OrderID] "FK_Order_Details_Products" [ProductID] -> Products [ProductID] @@ -236,13 +236,13 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 12. ShipPostalCode Text len= 20 var#4 13. ShipCountry Text len= 30 var#5 --- indexes --- - "CustomerID" [CustomerID] root=240 distinct=89 - "EmployeeID" [EmployeeID] root=244 distinct=9 - "OrderDate" [OrderDate] root=247 distinct=480 - PK "PK_Orders" [OrderID] root=251 distinct=830 - "ShippedDate" [ShippedDate] root=254 distinct=388 - "ShippersOrders" [ShipVia] root=258 distinct=3 - "ShipPostalCode" [ShipPostalCode] root=261 distinct=85 + "CustomerID" [CustomerID] root=240 uniqueEntries=89 + "EmployeeID" [EmployeeID] root=244 uniqueEntries=9 + "OrderDate" [OrderDate] root=247 uniqueEntries=480 + PK "PK_Orders" [OrderID] root=251 uniqueEntries=830 + "ShippedDate" [ShippedDate] root=254 uniqueEntries=388 + "ShippersOrders" [ShipVia] root=258 uniqueEntries=3 + "ShipPostalCode" [ShipPostalCode] root=261 uniqueEntries=85 --- foreign keys --- "FK_Orders_Customers" [CustomerID] -> Customers [CustomerID] "FK_Orders_Employees" [EmployeeID] -> Employees [EmployeeID] @@ -261,10 +261,10 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 8. ReorderLevel Int16 len= 2 fixed@24 9. Discontinued Boolean len= 1 fixed@0 --- indexes --- - "CategoriesProducts" [CategoryID] root=270 distinct=8 - PK "PK_Products" [ProductID] root=271 distinct=77 - "ProductName" [ProductName] root=272 distinct=77 - "SupplierID" [SupplierID] root=273 distinct=29 + "CategoriesProducts" [CategoryID] root=270 uniqueEntries=8 + PK "PK_Products" [ProductID] root=271 uniqueEntries=77 + "ProductName" [ProductName] root=272 uniqueEntries=77 + "SupplierID" [SupplierID] root=273 uniqueEntries=29 --- foreign keys --- "FK_Products_Categories" [CategoryID] -> Categories [CategoryID] "FK_Products_Suppliers" [SupplierID] -> Suppliers [SupplierID] @@ -274,7 +274,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 0. RegionID Int32 len= 4 fixed@0 1. RegionDescription Text len=100 fixed@4 --- indexes --- - PK "PK_Region" [RegionID] root=278 distinct=4 + PK "PK_Region" [RegionID] root=278 uniqueEntries=4 Shippers tdefPage=283 type=User rows=3 columns=3 indexes=1 @@ -282,7 +282,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 1. CompanyName Text len= 80 var#0 2. Phone Text len= 48 var#1 --- indexes --- - PK "PK_Shippers" [ShipperID] root=286 distinct=3 + PK "PK_Shippers" [ShipperID] root=286 uniqueEntries=3 Suppliers tdefPage=287 type=User rows=29 columns=12 indexes=3 @@ -299,9 +299,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 10. Fax Text len= 48 var#9 11. HomePage Memo len= 0 var#10 --- indexes --- - "CompanyName" [CompanyName] root=292 distinct=29 - PK "PK_Suppliers" [SupplierID] root=293 distinct=29 - "PostalCode" [PostalCode] root=294 distinct=29 + "CompanyName" [CompanyName] root=292 uniqueEntries=29 + PK "PK_Suppliers" [SupplierID] root=293 uniqueEntries=29 + "PostalCode" [PostalCode] root=294 uniqueEntries=29 Territories tdefPage=295 type=User rows=53 columns=3 indexes=2 @@ -309,8 +309,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 1. TerritoryDescription Text len=100 fixed@0 2. RegionID Int32 len= 4 fixed@100 --- indexes --- - PK "PK_Territories" [TerritoryID] root=299 distinct=53 - "FK_Territories_Region" [RegionID] root=308 distinct=4 + PK "PK_Territories" [TerritoryID] root=299 uniqueEntries=53 + "FK_Territories_Region" [RegionID] root=308 uniqueEntries=4 --- foreign keys --- "FK_Territories_Region" [RegionID] -> Region [RegionID] @@ -324,7 +324,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 5. SelectedObjectID Int32 len= 4 var#3 6. Type Int32 len= 4 var#2 --- indexes --- - PK "Id" [Id] root=314 distinct=3 + PK "Id" [Id] root=314 uniqueEntries=3 [SYS] MSysNavPaneGroups tdefPage=315 type=User rows=10 columns=7 indexes=2 @@ -336,8 +336,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 5. ObjectID Int32 len= 4 var#3 6. Position Int32 len= 4 var#4 --- indexes --- - PK "Id" [Id] root=317 distinct=10 - "GroupCategoryID" [GroupCategoryID] root=318 distinct=3 + PK "Id" [Id] root=317 uniqueEntries=10 + "GroupCategoryID" [GroupCategoryID] root=318 uniqueEntries=3 --- foreign keys --- "MSysNavPaneGroupCategoriesMSysNavPaneGroups" [GroupCategoryID] -> MSysNavPaneGroupCategories [Id] cascadeUpdate cascadeDelete @@ -351,8 +351,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 5. ObjectID Int32 len= 4 var#1 6. Position Int32 len= 4 var#4 --- indexes --- - PK "Id" [Id] root=321 distinct=47 - "GroupID" [GroupID] root=322 distinct=2 + PK "Id" [Id] root=321 uniqueEntries=47 + "GroupID" [GroupID] root=322 uniqueEntries=2 --- foreign keys --- "MSysNavPaneGroupsMSysNavPaneGroupToObjects" [GroupID] -> MSysNavPaneGroups [Id] cascadeUpdate cascadeDelete @@ -370,8 +370,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 3. Name Text len=510 var#1 4. Type Text len=510 var#2 --- indexes --- - U "Data_BF92175BD03C433695464F95CDF97589" [Data] root=340 distinct=1 - PK "Id" [Id] root=341 distinct=1 + U "Data_BF92175BD03C433695464F95CDF97589" [Data] root=340 uniqueEntries=1 + PK "Id" [Id] root=341 uniqueEntries=1 [SYS] f_3B3D4E1F479C4CCCA9CB8E549B05D3B7_Data tdefPage=334 type=User rows=1 columns=8 indexes=3 @@ -384,6 +384,6 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 6. FileURL Memo len= 0 var#3 7. MSysResources_Data Int32 len= 4 var#4 auto --- indexes --- - PK "MSysComplexPKIndex" [MSysResources_Data] root=336 distinct=1 - "_Data" [_Data] root=337 distinct=1 - U "IdxFKPrimaryScalar" [_Data, FileName] root=338 distinct=1 + PK "MSysComplexPKIndex" [MSysResources_Data] root=336 uniqueEntries=1 + "_Data" [_Data] root=337 uniqueEntries=1 + U "IdxFKPrimaryScalar" [_Data, FileName] root=338 uniqueEntries=1 diff --git a/test/LibRed.Core.Tests/Expected/widetable-dump.txt b/test/LibRed.Core.Tests/Expected/widetable-dump.txt index 661c1222..26600f05 100644 --- a/test/LibRed.Core.Tests/Expected/widetable-dump.txt +++ b/test/LibRed.Core.Tests/Expected/widetable-dump.txt @@ -20,8 +20,8 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 15. RmtInfoShort Binary len=510 var#5 16. Type Int16 len= 2 fixed@8 --- indexes --- - U "ParentIdName" [ParentId, Name] root=7 distinct=19 - PK "Id" [Id] root=8 distinct=19 + U "ParentIdName" [ParentId, Name] root=7 uniqueEntries=19 + PK "Id" [Id] root=8 uniqueEntries=19 [SYS] MSysACEs tdefPage=3 type=System rows=38 columns=4 indexes=1 @@ -30,7 +30,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 2. ObjectId Int32 len= 4 fixed@0 3. SID Binary len=510 var#0 --- indexes --- - "ObjectId" [ObjectId] root=10 distinct=19 + "ObjectId" [ObjectId] root=10 uniqueEntries=19 [SYS] MSysQueries tdefPage=4 type=System rows=0 columns=8 indexes=1 @@ -43,7 +43,7 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 6. ObjectId Int32 len= 4 fixed@0 7. Order Binary len=510 var#0 --- indexes --- - PK "ObjectIdAttribute" [ObjectId, Attribute, Order] root=12 distinct=0 + PK "ObjectIdAttribute" [ObjectId, Attribute, Order] root=12 uniqueEntries=0 [SYS] MSysRelationships tdefPage=5 type=System rows=0 columns=8 indexes=3 @@ -56,9 +56,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 6. szReferencedObject Text len=510 var#3 7. szRelationship Text len=510 var#0 --- indexes --- - "szRelationship" [szRelationship] root=14 distinct=0 - "szObject" [szObject] root=15 distinct=0 - "szReferencedObject" [szReferencedObject] root=16 distinct=0 + "szRelationship" [szRelationship] root=14 uniqueEntries=0 + "szObject" [szObject] root=15 uniqueEntries=0 + "szReferencedObject" [szReferencedObject] root=16 uniqueEntries=0 [SYS] MSysComplexColumns tdefPage=18 type=User rows=0 columns=5 indexes=3 @@ -68,9 +68,9 @@ Database: Standard ACE DB (Version12_2007, page size 4096) 3. ConceptualTableID Int32 len= 4 fixed@8 4. FlatTableID Int32 len= 4 fixed@4 --- indexes --- - PK "IdxID" [ComplexID] root=20 distinct=0 - "IdxConceptualTableID" [ConceptualTableID] root=21 distinct=0 - "IdxFlatTableID" [FlatTableID] root=22 distinct=0 + PK "IdxID" [ComplexID] root=20 uniqueEntries=0 + "IdxConceptualTableID" [ConceptualTableID] root=21 uniqueEntries=0 + "IdxFlatTableID" [FlatTableID] root=22 uniqueEntries=0 [SYS] MSysComplexType_UnsignedByte tdefPage=23 type=User rows=0 columns=1 indexes=0 diff --git a/test/LibRed.Core.Tests/IndexStatisticsTests.cs b/test/LibRed.Core.Tests/IndexStatisticsTests.cs index 7b75be6c..4114aa7c 100644 --- a/test/LibRed.Core.Tests/IndexStatisticsTests.cs +++ b/test/LibRed.Core.Tests/IndexStatisticsTests.cs @@ -5,8 +5,12 @@ namespace LibRed.Core.Tests; public class IndexStatisticsTests { + // Note: UniqueEntryCount is cumulative and never decremented by Access, so these + // assertions rely on Northwind having no deleted rows (uniqueEntryCount == current + // distinct count). They would not hold on a database that has had deletions. + [Fact] - public void Unique_indexes_have_cardinality_equal_to_row_count() + public void Unique_indexes_have_entry_count_equal_to_row_count() { using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); @@ -15,9 +19,9 @@ public void Unique_indexes_have_cardinality_equal_to_row_count() int rows = db.ReadTableDefinition(table.DefinitionPage).RowCount; foreach (var ix in table.Indexes) { - Assert.InRange(ix.UniqueValueCount, 0, rows); + Assert.InRange(ix.UniqueEntryCount, 0, rows); if (ix.IsUnique) - Assert.Equal(rows, ix.UniqueValueCount); // a unique index cannot have duplicates + Assert.Equal(rows, ix.UniqueEntryCount); // a unique index cannot have duplicates } } } @@ -26,7 +30,7 @@ public void Unique_indexes_have_cardinality_equal_to_row_count() [InlineData("EmployeeID", 9)] // 9 employees [InlineData("ShippersOrders", 3)] // 3 shippers (ShipVia) [InlineData("PK_Orders", 830)] // unique = row count - public void Cardinality_matches_actual_distinct_values(string indexName, int expected) + public void Unique_entry_count_matches_actual_distinct_values(string indexName, int expected) { using var db = JetDatabase.Open(TestDatabases.NorthwindAccdb); @@ -36,11 +40,11 @@ public void Cardinality_matches_actual_distinct_values(string indexName, int exp int actualDistinct = table.Rows() .Select(r => r[columnIndex]) - .Where(v => v is not null) // index cardinality excludes nulls + .Where(v => v is not null) // index entries exclude nulls .Distinct() .Count(); - Assert.Equal(expected, index.UniqueValueCount); + Assert.Equal(expected, index.UniqueEntryCount); Assert.Equal(expected, actualDistinct); } } From a63f86a24197c5b651c41166482f7f0222c0f5f4 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 15:02:46 +0800 Subject: [PATCH 027/506] docs(LibRed): complete the column-descriptor layout Map the full 25-byte column descriptor, verified against Northwind: the 0x0659 marker is a 2-byte field at +1 (not 4); the variable-table index is at +7 (its low half matches the var-column ranking LibRed computes); column id at +5 (repeated at +9); the flags byte at +0x0F (0x01 fixed, 0x02 updatable, 0x04 auto-number, 0x40 auto-number GUID, 0x80 hyperlink); ext flags at +0x10; fixed-data offset at +0x15; length at +0x17. Note 0x0B/0x0C read as the 0x0409 en-US locale on non-numeric columns and hold precision/scale for Decimal/Numeric per Jackcess (no Decimal column in Northwind to verify). Fix the section 3.1 marker note (2 bytes in column descriptors vs 4 in the index blocks). Co-Authored-By: Claude Opus 4.8 --- src/LibRed/docs/jet-ace-file-format.md | 32 +++++++++++++++++++------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/LibRed/docs/jet-ace-file-format.md b/src/LibRed/docs/jet-ace-file-format.md index f674d74f..5bb43a54 100644 --- a/src/LibRed/docs/jet-ace-file-format.md +++ b/src/LibRed/docs/jet-ace-file-format.md @@ -86,9 +86,9 @@ version. (Page-level encryption for password-protected files is not implemented. | `0x3F` | — | Start of the real-index block (precedes column descriptors) | > **The `0x659` / `0x783` record markers.** `0x0C` holds `0x659` (1625) in every file. This is -> not isolated: `0x659` recurs as a fixed 4-byte field at the start of each repeating TDEF -> record — column descriptor `+1` (§3.4), index-info block `+0` (§3.6), and this header slot — -> while `0x783` (1923) marks each index-data block `+0` (§3.5). They are constant within and +> not isolated: `0x659` recurs as a fixed marker at the start of each repeating TDEF record — +> column descriptor `+1` (2 bytes, §3.4), index-info block `+0` (4 bytes, §3.6), and this header +> slot (4 bytes) — while `0x783` (1923) marks each index-data block `+0` (§3.5). They are constant within and > across files; mdbtools describes them as "usually 1625 / 1923 *or 0*", so they appear to be > reserved record markers/tags the engine does not depend on (LibRed ignores them). `0x0C` is > therefore **not** the code page — `1625` is not a valid code page, and the code page is a @@ -141,14 +141,30 @@ does not hold (a non-unique index can have all-distinct data). LibRed exposes it | Offset | Size | Meaning | | --- | --- | --- | | `0x00` | 1 | Data type (see §6) | -| `0x01` | 4 | Record marker constant `0x659` (see §3.1 note); ignored | -| `0x05` | 2 | Column id (a.k.a. column number) | -| `0x0F` | 1 | Flags: `0x01` fixed-length, `0x04` auto-number | +| `0x01` | 2 | Record marker `0x0659` (see §3.1 note); ignored | +| `0x03` | 2 | Unknown (zero observed) | +| `0x05` | 2 | Column id | +| `0x07` | 2 | Variable-length table index — this column's position among the variable columns (0 for fixed columns) | +| `0x09` | 2 | Column number (equals the column id `0x05` in every file observed) | +| `0x0B` | 1 | Numeric **precision** (Decimal/Numeric columns); otherwise the low byte of the locale id, `0x09` | +| `0x0C` | 1 | Numeric **scale** (Decimal/Numeric columns); otherwise the high byte of the locale id, `0x04` | +| `0x0D` | 2 | Unknown (zero observed) | +| `0x0F` | 1 | Flags (see below) | +| `0x10` | 1 | Extended flags: `0x01` compressed-Unicode capable, `0xC0` calculated column | +| `0x11` | 4 | Unknown (zero observed) | | `0x15` | 2 | Fixed-data offset within the row's fixed region | | `0x17` | 2 | Length (bytes) | -Bytes `0x07`–`0x0E` and `0x10`–`0x14` carry additional per-column flags/metadata LibRed does -not currently use. +**Flags (`0x0F`):** `0x01` fixed-length, `0x02` updatable, `0x04` auto-number, +`0x40` auto-number GUID, `0x80` hyperlink (on a Memo column). + +> `0x0B`–`0x0C` reads as the constant `0x0409` (the en-US LCID / text collation) on every +> non-numeric column in the files inspected; per Jackcess these two bytes instead hold the +> precision and scale for Decimal/Numeric columns. Northwind has no Decimal column, so the +> numeric interpretation is taken from Jackcess and not independently verified here. +> +> LibRed currently *derives* the variable-table index (`0x07`) by ranking variable columns by +> column id rather than reading it; the stored value matches that ranking in every file tested. Variable-length columns are assigned a *variable index* = their rank among variable columns ordered by ascending column id (used by the row's variable-offset table, §5). From cf622db510ffda8adaf241a1e72151a42fd2e161 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 15:18:48 +0800 Subject: [PATCH 028/506] docs(LibRed): verify column precision/scale (0x0B/0x0C) A DECIMAL(12,3) column (type 0x10 FixedPoint) reads precision=12 at +0x0B and scale=3 at +0x0C, while non-numeric columns read the 0x0409 en-US locale there -- confirming the 0x0B/0x0C union directly rather than via Jackcess. (My earlier "DECIMAL fails via OLE DB" was a misdiagnosis: the failing statement used the reserved column name "Note", not the DECIMAL type.) Co-Authored-By: Claude Opus 4.8 --- src/LibRed/docs/jet-ace-file-format.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/LibRed/docs/jet-ace-file-format.md b/src/LibRed/docs/jet-ace-file-format.md index 5bb43a54..6f912d64 100644 --- a/src/LibRed/docs/jet-ace-file-format.md +++ b/src/LibRed/docs/jet-ace-file-format.md @@ -158,10 +158,10 @@ does not hold (a non-unique index can have all-distinct data). LibRed exposes it **Flags (`0x0F`):** `0x01` fixed-length, `0x02` updatable, `0x04` auto-number, `0x40` auto-number GUID, `0x80` hyperlink (on a Memo column). -> `0x0B`–`0x0C` reads as the constant `0x0409` (the en-US LCID / text collation) on every -> non-numeric column in the files inspected; per Jackcess these two bytes instead hold the -> precision and scale for Decimal/Numeric columns. Northwind has no Decimal column, so the -> numeric interpretation is taken from Jackcess and not independently verified here. +> `0x0B`–`0x0C` is a union keyed by type: for a Decimal/Numeric column (type `0x10`) it holds +> the **precision** (`0x0B`) and **scale** (`0x0C`) — verified with a `DECIMAL(12,3)` column, +> which reads precision = 12, scale = 3; for every other type it reads the constant `0x0409` +> (the en-US LCID / text collation). > > LibRed currently *derives* the variable-table index (`0x07`) by ranking variable columns by > column id rather than reading it; the stored value matches that ranking in every file tested. From ee7f36698b6b230a2d3f8e8d32c48fe9b60d525b Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 15:31:00 +0800 Subject: [PATCH 029/506] LibRed: decode Decimal/Numeric (FixedPoint) values JetTypeCodec now decodes type 0x10 (Decimal/Numeric) to System.Decimal. The 17-byte storage is a sign byte (0x80 = negative) plus a 128-bit magnitude as four 32-bit little-endian words in big-endian word order (low word last); value = magnitude / 10^scale. The column descriptor's precision/scale (0x0B/0x0C) are parsed into ColumnDef.Precision/Scale (FixedPoint columns only). Verified with a new committed asset (Data/Decimals.accdb, + generator): DECIMAL(12,3) and DECIMAL(28,4) values decode exactly, including a negative 20-digit value spanning the high 32-bit word. Spec section 6 updated; values beyond System.Decimal's range throw OverflowException. Co-Authored-By: Claude Opus 4.8 --- .../LibRed.Core/Formats/JetFormatBase.cs | 2 + .../LibRed.Core/Pages/TableDefinitionPage.cs | 15 ++++-- .../LibRed.Core/Storage/Types/JetTypeCodec.cs | 22 ++++++++- src/LibRed/docs/jet-ace-file-format.md | 2 +- test/LibRed.Core.Tests/Data/Decimals.accdb | Bin 0 -> 188416 bytes .../Data/Decimals.generate.ps1 | 14 ++++++ test/LibRed.Core.Tests/DecimalTests.cs | 44 ++++++++++++++++++ .../LibRed.Core.Tests.csproj | 3 ++ test/LibRed.Core.Tests/TestDatabases.cs | 4 ++ 9 files changed, 101 insertions(+), 5 deletions(-) create mode 100644 test/LibRed.Core.Tests/Data/Decimals.accdb create mode 100644 test/LibRed.Core.Tests/Data/Decimals.generate.ps1 create mode 100644 test/LibRed.Core.Tests/DecimalTests.cs diff --git a/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs index 2b3494b5..81b8edd7 100644 --- a/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs +++ b/src/LibRed/LibRed.Core/Formats/JetFormatBase.cs @@ -64,6 +64,8 @@ public abstract class JetFormatBase // --- Column descriptor layout (offsets within a single descriptor) --- public virtual int ColumnTypeOffset => 0x00; public virtual int ColumnNumberOffset => 0x05; + public virtual int ColumnPrecisionOffset => 0x0B; // Decimal/Numeric columns only + public virtual int ColumnScaleOffset => 0x0C; // Decimal/Numeric columns only public virtual int ColumnFlagsOffset => 0x0F; public virtual int ColumnFixedOffsetOffset => 0x15; public virtual int ColumnLengthOffset => 0x17; diff --git a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs index 13b6d90f..97617687 100644 --- a/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs +++ b/src/LibRed/LibRed.Core/Pages/TableDefinitionPage.cs @@ -199,16 +199,23 @@ private int ReadColumns(PageBuffer buffer, JetFormatBase format, int columnBlock _columns.Clear(); // Pass 1: fixed-size column descriptors. - var descriptors = new (JetDataType Type, int ColumnId, byte Flags, int FixedOffset, int Length)[ColumnCount]; + var descriptors = new (JetDataType Type, int ColumnId, byte Flags, int FixedOffset, int Length, byte Precision, byte Scale)[ColumnCount]; for (int i = 0; i < ColumnCount; i++) { int entry = columnBlock + i * format.ColumnDescriptorSize; + var type = (JetDataType)buffer.ReadByte(entry + format.ColumnTypeOffset); + + // Precision/scale share these bytes with a locale id; they are only meaningful + // for Decimal/Numeric columns. + bool numeric = type == JetDataType.FixedPoint; descriptors[i] = ( - (JetDataType)buffer.ReadByte(entry + format.ColumnTypeOffset), + type, buffer.ReadUInt16(entry + format.ColumnNumberOffset), buffer.ReadByte(entry + format.ColumnFlagsOffset), buffer.ReadUInt16(entry + format.ColumnFixedOffsetOffset), - buffer.ReadUInt16(entry + format.ColumnLengthOffset)); + buffer.ReadUInt16(entry + format.ColumnLengthOffset), + numeric ? buffer.ReadByte(entry + format.ColumnPrecisionOffset) : (byte)0, + numeric ? buffer.ReadByte(entry + format.ColumnScaleOffset) : (byte)0); } // Variable columns are addressed (in the row's var-offset table) in ascending @@ -246,6 +253,8 @@ private int ReadColumns(PageBuffer buffer, JetFormatBase format, int columnBlock VariableIndex = isFixed ? -1 : variableIndex[d.ColumnId], IsFixedLength = isFixed, IsAutoNumber = (d.Flags & JetFormatBase.ColumnFlagAutoNumber) != 0, + Precision = d.Precision, + Scale = d.Scale, }); } diff --git a/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs b/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs index 2171d057..ba3a12b0 100644 --- a/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs +++ b/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs @@ -39,17 +39,37 @@ public static class JetTypeCodec return DecodeText(value); case JetDataType.Binary: return value.ToArray(); + case JetDataType.FixedPoint: + return DecodeNumeric(value, column.Scale); // Long values stored on LVAL pages — needs the long-value reader. TODO. case JetDataType.Memo: case JetDataType.Ole: case JetDataType.Complex: - case JetDataType.FixedPoint: default: return value.ToArray(); } } + /// + /// Decodes a Jet Decimal/Numeric value (17 bytes): a sign byte (0x80 = negative) followed + /// by a 128-bit magnitude stored as four 32-bit little-endian words in big-endian word + /// order (the low word last). The value is the magnitude divided by 10^scale. + /// + private static decimal DecodeNumeric(ReadOnlySpan value, byte scale) + { + bool negative = (value[0] & 0x80) != 0; + uint lo = BinaryPrimitives.ReadUInt32LittleEndian(value.Slice(13, 4)); + uint mid = BinaryPrimitives.ReadUInt32LittleEndian(value.Slice(9, 4)); + uint hi = BinaryPrimitives.ReadUInt32LittleEndian(value.Slice(5, 4)); + uint top = BinaryPrimitives.ReadUInt32LittleEndian(value.Slice(1, 4)); + + if (top != 0) + throw new OverflowException("Numeric value exceeds the range of System.Decimal."); + + return new decimal((int)lo, (int)mid, (int)hi, negative, scale); + } + /// /// Decodes a Jet text value, honoring compressed Unicode. A value beginning with the /// 0xFF 0xFE marker stores ASCII-range characters as one byte each; otherwise it is diff --git a/src/LibRed/docs/jet-ace-file-format.md b/src/LibRed/docs/jet-ace-file-format.md index 6f912d64..2ccf0248 100644 --- a/src/LibRed/docs/jet-ace-file-format.md +++ b/src/LibRed/docs/jet-ace-file-format.md @@ -257,7 +257,7 @@ of the page backward, so a slot runs from its offset up to where the previous sl | `0x0B` | OLE | long value (§8) | | `0x0C` | Memo | long value (§8); text once resolved | | `0x0F` | GUID | 16 raw bytes | -| `0x10` | FixedPoint (Numeric/Decimal) | not yet decoded | +| `0x10` | FixedPoint (Numeric/Decimal) | 17 bytes: sign byte (`0x80` = negative) + 128-bit magnitude (four 32-bit little-endian words, low word last); value = magnitude / 10^scale. Precision/scale from the column descriptor (§3.4) | | `0x12` | Complex (multi-value / attachment) | descriptor parsed; contents not materialized (out of scope for SQL/EF) | --- diff --git a/test/LibRed.Core.Tests/Data/Decimals.accdb b/test/LibRed.Core.Tests/Data/Decimals.accdb new file mode 100644 index 0000000000000000000000000000000000000000..bfb80737fc77f57d6af036715837896026334b6d GIT binary patch literal 188416 zcmeI*Yj9lWUB~hNSsh8M%kGh!#JSk6m75bKb#N}zhE8U19K|HYvZKgu>SV^TWXXZHR9$}nZH z`2C-=XU}SPr6`shS@!pA&hE3%{d~@@oaa2x?pkCve0*eTY-DDv@80|F?;F@=1vC5V zRLkgrj=$LZ)|U>?Z2bxQ{HMS0;_P$3b^BjE^R;)jeEmCr``ZV<{qKMIh3)_Q{MUEQ z?tc5jH~i=4fAo#--~7EN3(p_e-1o^3{nG2-dAYpxvitsY-OG0lzWK~!XQEHt@_YaK zsoyMa{Eh#6TWw8S;4qtg>Q6`0%j*(j7q`{9je9gnTcOO#aE+!B^1^r*6?35gD|X+l}u zt4|2M3U+fy&PE;Hcw6-V?@&qKD&0*_MBd$4#}&HU?dM=lReI)Cby;g&!r7?B6I!Zm zq(|;*$LG&Zrvf~A*D{M zonp0?PTOri9eO>ZQV*%D_iak`cdl=k3p#j^B zO=tk;16Cg}_Gn-@YELTU!&x5;w%M>A2g1Nktu8P~E9i)PsPEe^g!_dCVvQ{^XbNcD zA%Fk^2q1s}0tg_000IbHcmjp!Ov!sy(#&-2V7FoXulaUOV0v^&(xQPVEF8!Wso>@F zL3?^xz%LnuIseusivhx{bek%!y<*U+YY3I8{kaM4$`~fm^GgZYx`MM&NQkS80rG@} zuQ4>6k6)u}499iz>k6ZqhEAuueUkdAFv)yOliL$vLb+I-?yjH6?yOE!->xeP+0^VE zO|DM(MD~oXQuuX@$~5u}RhQsb4w{x|Yq%&; zxss3`i?2P%t14?n!9k7tkLsGhq?VI+YC-vot`PX@gR<^x(|UA`;1gkKeU>|osa>0J zL&)>6M*fGw=--#GA5>^;PPzGW)z-DXT9+@sKVG>$aZJnGCv?@~-Iu2qbS}d>5FPl> zhJ*Jqx_Gm|YKaz=$-XTjFQX4KEKnv^nVuB4(oS~KD7@hE>lY!9r}??|z2@3RH7x|1 zq`epG#Vc}sr0Puo0R#|0009ILKmY**R--@^{a3c)T|-^}{=eIkx65BFkCr!-|EctA zrCp_t(rewPx&K>I?}bNZqk1>9y5JL009ILKmY**5I|rt0`+rZiz!o|=X8Ap z&k;c2(i8~unoB+OrBxC~KkAydhCZ{czNFu0Y<&hjOvLu-ajSOg^E*%Sz-jGaCg1Z! z;c1T&w|X?&$_bo$L2aMfy<>96&K*aFj`%-ey-^xDF|>cjmI><@X8dGhR?-sfLeVZG&fGVB`~8rnTJt)mFpN(*E=vTt;6C0qAA z+4hf(?hW;H&6DBCBV!ZkQFYFfV{+(3nxSW&4AY}W_U#=V+P(ASWZLJBJmc8T$A$(c zQ{6a^eqwrPVqfKywB^x_XSF>|9GyOp*0IK)7s5H+y)ztGSkbd`uTfe?009ILKmY** z5I_Kd_eUU(PWVUvn{I`;*weT5w7v_yCeSn|kr%4Ksng-AO>gW0jq`e2J)ny@y=@-Q zF9r0rdq5v^=q-BSN(DMRaJ2%R9$2S9mj^Z|(CvYZ3Y0u>tpa5aT(3alfg2U*@xVT0R#|0009ILKmY**5I9c)QM4iZ z47Y}d>H7ck)D+bsfB*srAbMM1M59T>j?V8G6WB7 z@E9#9=nv}=Jn#XJxn6;=Fu?;GJw{6m`ojtZ5A=DA)*AGOr3xOn)?>8jp#O#}aAQ~i zubA}n0t%#`EKuOOuohl{utLFE!U}y4gik1V;QGo6h5pO3z~#n<00IagfB*srAb{w-LG{GcYU|>XFAt(e!1hGjvp2;eie=o0R#|0009IpI)PsQ zeOS|nNt4KKee-vJ|E?Flw>5k8gWY8(Y|@U|qi1eW&idQU)@NNdYu4*OJX@)z>#dhU zHQA$nWB2r+4cjxS=02NNbx+w*8@H!zkK&$E=yAK(j_WpJ2ki-^kK2@L+jO8tkq#yqE^}gbp0H2b$ihjp&zlD~-JQ-Snvbq4KzxMh4W#L7UJKjp&GXHN91v z($D-Zp{bv=z4o9DG(DOJ!dVJv`gP$YH1eR`A8rFWizjR}oW*xJOZtK1WoU3%h*NfW zF%7O;QiD5nWZuxbye#w^&r3L}VZF$wZALHg9Zhf5mxO*?dI`JE6!ix9#xe7 zp>n^*L%z+b2VIB;N~*}dqAu=h^H92}fjQf*R-L(4|Ka$s+hsbF+848R=Jj^yRqAgw z-_Ay{C&d1WFi-F|zFD_oLPAtc$l9qSwA3XOvl4~61m(BO!@s`S+jN&;{9oENb^j*& zKO&pz;a1ct!=NokLSie+a3;hcGl+mub_j? zeqR0;1BjQ=Av zb-f|Yqr;Y`g=%irKhI{3mZOrM<*4KWs3X?*|7SDu7eIID z0s;sifB*srAb>KopII@xQlUuve0k z)n_Z!Bicm(0R#|0009ILKmY**5I_Kdz^+5I_I{1Q0*~0R#|0V3i5Hhxz}=qWs`)(EPvlo_zj)jS>r$3?0n> z=Tw9cKmY**5I_I{1Q0*~0R#|OaRKK4SNs&v0R#|0009ILKmY**5I_I{1kSs_xz7J@ z()_>n&V2sABhKgl&qkg0Q5&(NcEZMOcJ{m<1`Qy900IagfB*srAb`k^MKbEXdI+Fh`|6BQ+G9d+n!`?`+G+_^ed1Hnb*fQg zl-ob{gYHdl%KS^C_ia+ zQGU|uqWq-QMfpjqi}I6J7nS}%UyW)ZdM(WK-zB4TEx)hlZ?gX1BU{490+iG2R%Oxd zo2U2|mi(tdUyk3N2DP9+{79geip;Myr1@j(O%uFJ1zWEYX~Gpsi0wfe)BEcoo6$o2 zgIa9gX4@1xY|n;jBdx2S@4te&^}$NppAC=s#{~3rR-4DhaX5}GYH3PsjoZ`qxZP>D z>mvgnw!3xjp~tnve_MF=xZP!U=yoSd`hzKV2q1s}0tg_000IagfB*ukOCX9i=rT*X zwN%#sXU};=009ILKmY**5I_I{1Q0*~0evIfGHiXzP=_s{k~nHD` z$K;Qa=c?-gmQXE45kLR|1Q0*~0R#|0009ILK%ibA{QU+`a*&b o;ooR*_uwBq`u$ID`VVCcpKh c.Name == "Price"); + var big = cols.First(c => c.Name == "Big"); + + Assert.Equal(JetDataType.FixedPoint, price.Type); + Assert.Equal((12, 3), (price.Precision, price.Scale)); + Assert.Equal((28, 4), (big.Precision, big.Scale)); + } + + [Fact] + public void Decodes_decimal_values() + { + using var db = JetDatabase.Open(TestDatabases.DecimalsAccdb); + + var table = db.OpenTable("Nums"); + int price = table.Definition.Columns.First(c => c.Name == "Price").Index; + int big = table.Definition.Columns.First(c => c.Name == "Big").Index; + + var rows = table.Rows().ToList(); + + // Every value is a System.Decimal. + Assert.All(rows, r => { Assert.IsType(r[price]); Assert.IsType(r[big]); }); + + var prices = rows.Select(r => (decimal)r[price]!).ToList(); + var bigs = rows.Select(r => (decimal)r[big]!).ToList(); + + Assert.Equal([12.345m, -9.999m, 0.000m], prices); + // Includes a negative 20-digit value that spans the high 32-bit word. + Assert.Equal([123456789012.3456m, -9876543210987654.3210m, 0.0000m], bigs); + } +} diff --git a/test/LibRed.Core.Tests/LibRed.Core.Tests.csproj b/test/LibRed.Core.Tests/LibRed.Core.Tests.csproj index 88b32b7f..38e56214 100644 --- a/test/LibRed.Core.Tests/LibRed.Core.Tests.csproj +++ b/test/LibRed.Core.Tests/LibRed.Core.Tests.csproj @@ -41,6 +41,9 @@ PreserveNewest + + PreserveNewest + diff --git a/test/LibRed.Core.Tests/TestDatabases.cs b/test/LibRed.Core.Tests/TestDatabases.cs index 6104545a..f7dcf7ad 100644 --- a/test/LibRed.Core.Tests/TestDatabases.cs +++ b/test/LibRed.Core.Tests/TestDatabases.cs @@ -10,4 +10,8 @@ internal static class TestDatabases /// A 200-column ACCDB whose table definition spans multiple TDEF pages. public static string WideTableAccdb { get; } = Path.Combine(AppContext.BaseDirectory, "Data", "WideTable.accdb"); + + /// An ACCDB with Decimal/Numeric columns and known values. + public static string DecimalsAccdb { get; } = + Path.Combine(AppContext.BaseDirectory, "Data", "Decimals.accdb"); } From 5fdbe704e2790949edc1fac082d56b26f9239506 Mon Sep 17 00:00:00 2001 From: Christopher Jolly Date: Sun, 28 Jun 2026 15:48:11 +0800 Subject: [PATCH 030/506] LibRed: support ACE 16 BIGINT and DATETIME2 types Office 2016 added two types, exposed by the Microsoft.ACE.OLEDB.16.0 engine (which creates version-byte 0x06 databases): - BIGINT (Large Number), type code 0x13: an 8-byte little-endian signed int64, stored as a variable-length column. Decoded to System.Int64. - DATETIME2, type code 0x14: a fixed 42-byte ASCII string ":