diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj index 8f624eb24d..7680564e42 100644 --- a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj +++ b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj @@ -212,6 +212,7 @@ + @@ -329,6 +330,8 @@ + + diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters index 5fc2e79a90..d187f219e1 100644 --- a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters +++ b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters @@ -99,6 +99,9 @@ Header Files + + Header Files + @@ -419,6 +422,15 @@ Source Files\Repository + + Source Files + + + Source Files\Repository + + + Source Files\Repository + diff --git a/src/AppInstallerCLITests/SQLiteIndex.cpp b/src/AppInstallerCLITests/SQLiteIndex.cpp index 99b6dd8786..713a993018 100644 --- a/src/AppInstallerCLITests/SQLiteIndex.cpp +++ b/src/AppInstallerCLITests/SQLiteIndex.cpp @@ -2,13 +2,14 @@ // Licensed under the MIT License. #include "pch.h" #include "TestCommon.h" +#include "SQLiteIndexTestCommon.h" #include #include #include #include #include #include -#include +#include #include #include @@ -35,22 +36,6 @@ using namespace AppInstaller::SQLite; using namespace AppInstaller::Utility; using UtilityVersion = AppInstaller::Utility::Version; -using SQLiteVersion = AppInstaller::SQLite::Version; - -SQLiteIndex CreateTestIndex(const std::string& filePath, std::optional version = {}) -{ - // If no specific version requested, then use generator to run against the last 3 versions. - if (!version) - { - SQLiteVersion latestVersion{ 2, 0 }; - SQLiteVersion versionMinus1 = SQLiteVersion{ 1, 7 }; - SQLiteVersion versionMinus2 = SQLiteVersion{ 1, 6 }; - - version = GENERATE_COPY(SQLiteVersion{ versionMinus2 }, SQLiteVersion{ versionMinus1 }, SQLiteVersion{ latestVersion }); - } - - return SQLiteIndex::CreateNew(filePath, version.value()); -} SQLiteVersion TestPrepareForRead(SQLiteIndex& index) { @@ -82,26 +67,6 @@ SQLiteVersion TestPrepareForRead(SQLiteIndex& index) return index.GetVersion(); } -std::string GetPathFromManifest(Manifest& manifest) -{ - auto publisher = manifest.Id; - AppInstaller::Utility::FindAndReplace(publisher, ".", "/"); - - return AppInstaller::Utility::ToLower(publisher).append("/").append(manifest.Version); -} - -void CreateFakeManifest(Manifest& manifest, string_t publisher, string_t version = "1.0.0") -{ - manifest.Installers.push_back({}); - manifest.Id = publisher.append(".").append("Id"); - manifest.DefaultLocalization.Add(publisher.append(" Name")); - manifest.Moniker = "testmoniker"; - manifest.Version = version; - manifest.Channel = "test"; - manifest.DefaultLocalization.Add({ "t1", "t2" }); - manifest.Installers[0].Commands = { "test1", "test2" }; -} - SQLiteIndex SimpleTestSetup(const std::string& filePath, Manifest& manifest, std::optional version = {}) { SQLiteIndex index = CreateTestIndex(filePath, version); @@ -116,176 +81,6 @@ SQLiteIndex SimpleTestSetup(const std::string& filePath, Manifest& manifest, std return index; } -struct IndexFields -{ - IndexFields( - std::string id, - std::string name, - std::string moniker, - std::string version, - std::string channel, - std::vector tags, - std::vector commands, - std::string path - ) : - Id(std::move(id)), - Name(std::move(name)), - Moniker(std::move(moniker)), - Version(std::move(version)), - Channel(std::move(channel)), - Tags(std::move(tags)), - Commands(std::move(commands)), - Path(std::move(path)) - {} - - IndexFields( - std::string id, - std::string name, - std::string moniker, - std::string version, - std::string channel, - std::vector tags, - std::vector commands, - std::string path, - std::vector packageFamilyNames, - std::vector productCodes - ) : - Id(std::move(id)), - Name(std::move(name)), - Moniker(std::move(moniker)), - Version(std::move(version)), - Channel(std::move(channel)), - Tags(std::move(tags)), - Commands(std::move(commands)), - Path(std::move(path)), - PackageFamilyNames(std::move(packageFamilyNames)), - ProductCodes(std::move(productCodes)) - {} - - IndexFields( - std::string id, - std::string name, - std::string publisher, - std::string moniker, - std::string version, - std::string channel, - std::vector tags, - std::vector commands, - std::string path, - std::vector packageFamilyNames, - std::vector productCodes - ) : - Id(std::move(id)), - Name(std::move(name)), - Publisher(std::move(publisher)), - Moniker(std::move(moniker)), - Version(std::move(version)), - Channel(std::move(channel)), - Tags(std::move(tags)), - Commands(std::move(commands)), - Path(std::move(path)), - PackageFamilyNames(std::move(packageFamilyNames)), - ProductCodes(std::move(productCodes)) - {} - - IndexFields( - std::string id, - std::string name, - std::string publisher, - std::string moniker, - std::string version, - std::string channel, - std::vector tags, - std::vector commands, - std::string path, - std::vector packageFamilyNames, - std::vector productCodes, - std::string arpName, - std::string arpPublisher - ) : - Id(std::move(id)), - Name(std::move(name)), - Publisher(std::move(publisher)), - Moniker(std::move(moniker)), - Version(std::move(version)), - Channel(std::move(channel)), - Tags(std::move(tags)), - Commands(std::move(commands)), - Path(std::move(path)), - PackageFamilyNames(std::move(packageFamilyNames)), - ProductCodes(std::move(productCodes)), - ArpName(std::move(arpName)), - ArpPublisher(std::move(arpPublisher)) - {} - - std::string Id; - std::string Name; - std::string Publisher; - std::string Moniker; - std::string Version; - std::string Channel; - std::vector Tags; - std::vector Commands; - std::string Path; - std::vector PackageFamilyNames; - std::vector ProductCodes; - std::string ArpName; - std::string ArpPublisher; -}; - -SQLiteIndex SearchTestSetup(const std::string& filePath, std::initializer_list data = {}, std::optional version = {}) -{ - SQLiteIndex index = CreateTestIndex(filePath, version); - - Manifest manifest; - - auto addFunc = [&](const IndexFields& d) - { - manifest.Id = d.Id; - manifest.DefaultLocalization.Add(d.Name); - manifest.DefaultLocalization.Add(d.Publisher); - manifest.Moniker = d.Moniker; - manifest.Version = d.Version; - manifest.DefaultLocalization.Add(d.Tags); - - manifest.Installers.resize(std::max(d.PackageFamilyNames.size(), d.ProductCodes.size())); - - if (manifest.Installers.size() == 0) - { - manifest.Installers.push_back({}); - } - - manifest.Channel = d.Channel; - manifest.Installers[0].Commands = d.Commands; - - for (size_t i = 0; i < d.PackageFamilyNames.size(); ++i) - { - manifest.Installers[i].PackageFamilyName = d.PackageFamilyNames[i]; - } - - for (size_t i = 0; i < d.ProductCodes.size(); ++i) - { - manifest.Installers[i].ProductCode = d.ProductCodes[i]; - } - - if (!d.ArpName.empty() || !d.ArpPublisher.empty()) - { - manifest.Installers[0].AppsAndFeaturesEntries.push_back({}); - manifest.Installers[0].AppsAndFeaturesEntries[0].DisplayName = d.ArpName; - manifest.Installers[0].AppsAndFeaturesEntries[0].Publisher = d.ArpPublisher; - } - - index.AddManifest(manifest, d.Path); - }; - - for (const auto& d : data) - { - addFunc(d); - } - - return index; -} - bool ArePackageFamilyNameAndProductCodeSupported(const SQLiteIndex& index, const SQLiteVersion& testVersion) { UNSCOPED_INFO("Index " << index.GetVersion() << " | Test " << testVersion); @@ -3582,7 +3377,7 @@ TEST_CASE("SQLiteIndex_MigrateTo_Data", "[sqliteindex][V2_0]") REQUIRE(index.GetVersion() == SQLiteVersion{ 2, 0 }); Connection connection = Connection::Create(tempFile, Connection::OpenDisposition::ReadWrite); - auto updateData = Schema::V2_0::PackageUpdateTrackingTable::GetUpdatesSince(connection, 0); + auto updateData = Schema::V2_0::PackageUpdateTrackingTable::GetUpdatesSince(connection, 0, Schema::V2_0::PackageUpdateTrackingTable::RemovalBehavior::Delete); REQUIRE(updateData.size() == 3); REQUIRE(std::count_if(updateData.begin(), updateData.end(), [&](const auto& x) { return x.PackageIdentifier == packageId1; }) == 1); @@ -3607,38 +3402,6 @@ TEST_CASE("SQLiteIndex_Property_IntermediateFilePath", "[sqliteindex]") REQUIRE(contextData.Get() == intermediateFilePath); } -struct ManifestAndPath -{ - Manifest Manifest; - std::string Path; -}; - -void CreateFakeManifestAndPath( - ManifestAndPath& manifestAndPath, - const string_t& publisher, - std::string_view version = "1.0.0", - std::optional arpMinVersion = {}, - std::optional arpMaxVersion = {}) -{ - CreateFakeManifest(manifestAndPath.Manifest, publisher, version); - manifestAndPath.Path = ConvertToUTF8(CreateNewGuidNameWString()); - manifestAndPath.Manifest.StreamSha256 = SHA256::ComputeHash(manifestAndPath.Path); - - if (arpMinVersion) - { - manifestAndPath.Manifest.Installers[0].BaseInstallerType = InstallerTypeEnum::Exe; - manifestAndPath.Manifest.Installers[0].AppsAndFeaturesEntries.push_back({}); - manifestAndPath.Manifest.Installers[0].AppsAndFeaturesEntries.back().DisplayVersion = arpMinVersion.value(); - } - - if (arpMaxVersion) - { - manifestAndPath.Manifest.Installers[0].BaseInstallerType = InstallerTypeEnum::Exe; - manifestAndPath.Manifest.Installers[0].AppsAndFeaturesEntries.push_back({}); - manifestAndPath.Manifest.Installers[0].AppsAndFeaturesEntries.back().DisplayVersion = arpMaxVersion.value(); - } -} - std::filesystem::path GetOnlyChild(const std::filesystem::path& parent) { auto parentDirectoryIterator = std::filesystem::directory_iterator{ parent }; @@ -3963,3 +3726,4 @@ TEST_CASE("SQLiteIndex_VersionStringPreserved", "[sqliteindex]") REQUIRE(extractedVersion == version); } + diff --git a/src/AppInstallerCLITests/SQLiteIndexDelta.cpp b/src/AppInstallerCLITests/SQLiteIndexDelta.cpp new file mode 100644 index 0000000000..bfeb528136 --- /dev/null +++ b/src/AppInstallerCLITests/SQLiteIndexDelta.cpp @@ -0,0 +1,1857 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "TestCommon.h" +#include "SQLiteIndexTestCommon.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace std::string_literals; +using namespace std::string_view_literals; +using namespace TestCommon; +using namespace AppInstaller::Manifest; +using namespace AppInstaller::Repository; +using namespace AppInstaller::Repository::Microsoft; +using namespace AppInstaller::SQLite; +using namespace AppInstaller::Utility; + +using Tracking = Schema::V2_0::PackageUpdateTrackingTable; +namespace Delta = AppInstaller::Repository::Microsoft::Schema::V2_1::Delta; + +namespace +{ + // The first version that can produce or consume a delta. + const SQLiteVersion s_DeltaVersion{ 2, 1 }; + + // Runs a query that produces a single integer. + int64_t GetScalar(const Connection& connection, const std::string& sql) + { + Statement statement = Statement::Create(connection, sql); + REQUIRE(statement.Step()); + return statement.GetColumn(0); + } + + // Counts the rows in a table. + int64_t GetRowCount(const Connection& connection, std::string_view tableName) + { + return GetScalar(connection, "SELECT COUNT(*) FROM [" + std::string{ tableName } + "]"); + } + + // Reads every value in a single column of a query as a set. + std::set GetStrings(const Connection& connection, const std::string& sql) + { + std::set result; + Statement statement = Statement::Create(connection, sql); + + while (statement.Step()) + { + result.insert(statement.GetColumn(0)); + } + + return result; + } + + std::set ToStringSet(const std::vector& values) + { + return std::set(values.begin(), values.end()); + } + + // Reads the rowid that a prepared index gave a package identifier. + std::optional GetPreparedPackageRowId(const std::filesystem::path& indexPath, std::string_view packageIdentifier) + { + Connection connection = Connection::Create(indexPath.u8string(), Connection::OpenDisposition::ReadOnly); + Statement statement = Statement::Create(connection, "SELECT rowid FROM packages WHERE id = ?"); + statement.Bind(1, std::string{ packageIdentifier }); + + if (statement.Step()) + { + return statement.GetColumn(0); + } + + return {}; + } + + // Reads a package's associated 1:N values through the given connection. + std::set GetOneToManyValues( + const Connection& connection, + std::string_view tableName, + std::string_view valueName, + std::string_view packageId) + { + std::string sql = + "SELECT [v].[" + std::string{ valueName } + "] FROM [" + std::string{ tableName } + "] AS [v] " + "JOIN [" + std::string{ tableName } + "_map] AS [m] ON [m].[" + std::string{ valueName } + "] = [v].[rowid] " + "JOIN [packages] AS [p] ON [p].[rowid] = [m].[package] " + "WHERE [p].[id] = '" + std::string{ packageId } + "'"; + + return GetStrings(connection, sql); + } + + // Reads a package's system reference values, which are stored directly against the package. + std::set GetSystemReferenceValues( + const Connection& connection, + std::string_view tableName, + std::string_view valueName, + std::string_view packageId) + { + std::string sql = + "SELECT [s].[" + std::string{ valueName } + "] FROM [" + std::string{ tableName } + "] AS [s] " + "JOIN [packages] AS [p] ON [p].[rowid] = [s].[package] " + "WHERE [p].[id] = '" + std::string{ packageId } + "'"; + + return GetStrings(connection, sql); + } + + struct DeltaTestContext + { + TempFile WorkingFile{ "delta_working"s, ".db"s }; + TempFile BaselineFile{ "delta_baseline"s, ".db"s }; + TempFile DeltaFile{ "delta_output"s, ".db"s }; + + DeltaTestContext() = default; + + // Creates the working index and fills it with the data the baseline will hold. + DeltaTestContext(std::initializer_list baselineData) + { + CreateWorking(baselineData); + CaptureBaseline(); + } + + void CreateWorking(std::initializer_list baselineData) + { + SQLiteIndex index = SQLiteIndex::CreateNew(WorkingFile, s_DeltaVersion); + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, "0"); + + for (const auto& fields : baselineData) + { + Manifest manifest = CreateManifest(fields); + index.AddManifest(manifest, fields.Path); + } + } + + // Copies the working index out, prepares it, and designates it as a baseline. + // Everything the working index does afterwards is what the delta will describe. + void CaptureBaseline(bool markAsBaseline = true) + { + std::filesystem::copy_file(WorkingFile.GetPath(), BaselineFile.GetPath(), std::filesystem::copy_options::overwrite_existing); + + SQLiteIndex prepared = SQLiteIndex::Open(BaselineFile.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); + prepared.PrepareForPackaging(); + + if (markAsBaseline) + { + prepared.MarkAsBaseline(); + } + + m_baselineCaptured = true; + } + + // Opens the working index for the changes that the delta will carry. + SQLiteIndex OpenWorkingForChanges(bool resetBaseTimeIfNeeded = true) + { + SQLiteIndex index = SQLiteIndex::Open(WorkingFile, SQLiteStorageBase::OpenDisposition::ReadWrite); + + if (resetBaseTimeIfNeeded && !m_baseTimeReset) + { + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, ""); + m_baseTimeReset = true; + } + + return index; + } + + void Add(const IndexFields& fields, bool resetBaseTimeIfNeeded = true) + { + SQLiteIndex index = OpenWorkingForChanges(resetBaseTimeIfNeeded); + Manifest manifest = CreateManifest(fields); + index.AddManifest(manifest, fields.Path); + } + + void Update(const IndexFields& fields) + { + SQLiteIndex index = OpenWorkingForChanges(); + Manifest manifest = CreateManifest(fields); + REQUIRE(index.UpdateManifest(manifest, fields.Path)); + } + + void Remove(const IndexFields& fields) + { + SQLiteIndex index = OpenWorkingForChanges(); + Manifest manifest = CreateManifest(fields); + index.RemoveManifest(manifest, fields.Path); + } + + // Prepares the working index, producing the delta as a side effect. + void GenerateDelta() + { + REQUIRE(m_baselineCaptured); + + SQLiteIndex index = SQLiteIndex::Open(WorkingFile, SQLiteStorageBase::OpenDisposition::ReadWrite); + index.SetProperty(SQLiteIndex::Property::DeltaBaselineIndexPath, BaselineFile.GetPath().u8string()); + index.SetProperty(SQLiteIndex::Property::DeltaOutputPath, DeltaFile.GetPath().u8string()); + index.PrepareForPackaging(); + + m_deltaGenerated = true; + } + + // Opens the delta on its own, to inspect what generation actually wrote. + Connection OpenDeltaConnection() + { + REQUIRE(std::filesystem::exists(DeltaFile.GetPath())); + return Connection::Create(DeltaFile.GetPath().u8string(), Connection::OpenDisposition::ReadOnly); + } + + // Opens the delta with its baseline attached and the merged views in place, without going + // through SQLiteIndex, so that a test can read the merged tables directly. + Connection OpenMergedConnection() + { + Connection connection = OpenDeltaConnection(); + Delta::SetupReadMode(connection, DatabaseSpecifier{ BaselineFile.GetPath().u8string(), DatabaseDisposition::Read }); + return connection; + } + + SQLiteIndex OpenCombined(SQLiteStorageBase::OpenDisposition disposition = SQLiteStorageBase::OpenDisposition::Read) + { + REQUIRE(m_deltaGenerated); + return SQLiteIndex::OpenWithBaseline(DeltaFile.GetPath().u8string(), BaselineFile.GetPath().u8string(), disposition); + } + + // The working index, once prepared, is an ordinary full index built from the same data that + // the delta describes. That makes it the reference for equivalence. + SQLiteIndex OpenFullIndex() + { + REQUIRE(m_deltaGenerated); + return SQLiteIndex::Open(WorkingFile.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::Read); + } + + std::string DeltaTable(std::string_view baseTableName) const + { + return Delta::GetTableName(baseTableName); + } + + std::string DeltaMapTable(std::string_view baseTableName) const + { + return Delta::GetMapTableName(baseTableName); + } + + private: + bool m_baselineCaptured = false; + bool m_baseTimeReset = false; + bool m_deltaGenerated = false; + }; + + IndexFields MakePackage( + std::string id, + std::string name, + std::vector tags = { "t1", "t2" }, + std::vector commands = { "c1" }, + std::vector packageFamilyNames = {}, + std::vector productCodes = {}, + std::string version = "1.0"s) + { + std::string path = id + "/" + version; + + return IndexFields{ + id, + std::move(name), + "Publisher"s, + "moniker"s, + std::move(version), + ""s, + std::move(tags), + std::move(commands), + std::move(path), + std::move(packageFamilyNames), + std::move(productCodes) }; + } + + // Collects the identifiers that a search returns. + std::set GetSearchedIds(const SQLiteIndex& index, const SearchRequest& request = {}) + { + std::set result; + + for (const auto& match : index.Search(request).Matches) + { + auto id = index.GetPropertyByPrimaryId(match.first, PackageVersionProperty::Id); + REQUIRE(id.has_value()); + result.insert(id.value()); + } + + return result; + } +} + +// --------------------------------------------------------------------------------------------- +// Group B - package rowid identity +// +// The merged package's view suppresses a baseline row when the delta names the same rowid, and +// every association refers to a package by that rowid. These cases cover the ways that identity +// can be broken. +// --------------------------------------------------------------------------------------------- + +// B1. A package removed and re-added lands on a new rowid, so the delta has to both introduce it +// at the new rowid and vacate the old one. Recording only the add leaves the baseline row in +// place, and the package appears twice. +TEST_CASE("SQLiteIndex_Delta_PackageRemovedThenReAdded", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + auto p3 = MakePackage("Publisher3.Id", "Package 3"); + + DeltaTestContext context{ { p1, p2, p3 } }; + + rowid_t originalRowId = GetPreparedPackageRowId(context.BaselineFile.GetPath(), p2.Id).value(); + + // Removing the middle package frees its rowid, but the next insert takes one above the highest + // in use, so the re-add cannot land back on it. + context.Remove(p2); + context.Add(p2); + + context.GenerateDelta(); + + rowid_t newRowId = GetPreparedPackageRowId(context.WorkingFile.GetPath(), p2.Id).value(); + REQUIRE(newRowId != originalRowId); + + { + Connection delta = context.OpenDeltaConnection(); + + // The old rowid must be vacated, or nothing suppresses the baseline row. + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [delta_packages] WHERE [rowid] = " + std::to_string(originalRowId) + " AND [is_removed] = 1") == 1); + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [delta_packages] WHERE [rowid] = " + std::to_string(newRowId) + " AND [is_removed] = 0") == 1); + } + + SQLiteIndex combined = context.OpenCombined(); + + SearchRequest request; + request.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::Id, MatchType::Exact, p2.Id)); + + // Exactly one, and from the delta rather than the stale baseline row. + auto results = combined.Search(request); + REQUIRE(results.Matches.size() == 1); + + REQUIRE(GetSearchedIds(combined) == std::set{ p1.Id, p2.Id, p3.Id }); +} + +// B2. Removing the package that holds the highest rowid frees it for the next package added, so +// a removal and a change can name the same rowid. Writing both violates the delta's primary key. +TEST_CASE("SQLiteIndex_Delta_RemovedPackageRowIdReused", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + auto p3 = MakePackage("Publisher3.Id", "Package 3"); + + DeltaTestContext context{ { p1, p2, p3 } }; + + rowid_t reusedRowId = GetPreparedPackageRowId(context.BaselineFile.GetPath(), p3.Id).value(); + + auto p4 = MakePackage("Publisher4.Id", "Package 4"); + + context.Remove(p3); + context.Add(p4); + + REQUIRE_NOTHROW(context.GenerateDelta()); + + // The new package took the rowid that the removed one gave up. + REQUIRE(GetPreparedPackageRowId(context.WorkingFile.GetPath(), p4.Id).value() == reusedRowId); + + { + Connection delta = context.OpenDeltaConnection(); + + // Only the change is recorded. The removal would be redundant, since a delta row at that + // rowid already displaces the baseline row, and it cannot be written in any case. + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [delta_packages] WHERE [rowid] = " + std::to_string(reusedRowId)) == 1); + REQUIRE(GetStrings(delta, "SELECT [id] FROM [delta_packages] WHERE [rowid] = " + std::to_string(reusedRowId)) == + std::set{ p4.Id }); + } + + SQLiteIndex combined = context.OpenCombined(); + REQUIRE(GetSearchedIds(combined) == std::set{ p1.Id, p2.Id, p4.Id }); +} + +// B2 continued. The displaced package's associations must go with it. They are diffed against the +// baseline at the shared rowid, so the values belonging to the old occupant are removed. +TEST_CASE("SQLiteIndex_Delta_ReusedRowIdReplacesAssociations", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1", { "keep" }); + auto p2 = MakePackage("Publisher2.Id", "Package 2", { "old1", "old2" }, { "oldcmd" }, {}, { "OLDPC" }); + + DeltaTestContext context{ { p1, p2 } }; + + auto p3 = MakePackage("Publisher3.Id", "Package 3", { "new1" }, { "newcmd" }, {}, { "NEWPC" }); + + context.Remove(p2); + context.Add(p3); + context.GenerateDelta(); + + Connection merged = context.OpenMergedConnection(); + + // Nothing of the old occupant survives at the shared rowid. + REQUIRE(GetOneToManyValues(merged, "tags2", "tag", p3.Id) == ToStringSet(p3.Tags)); + REQUIRE(GetOneToManyValues(merged, "commands2", "command", p3.Id) == ToStringSet(p3.Commands)); + REQUIRE(GetSystemReferenceValues(merged, "productcodes2", "productcode", p3.Id) == std::set{ "newpc" }); + + // The untouched package is unaffected. + REQUIRE(GetOneToManyValues(merged, "tags2", "tag", p1.Id) == ToStringSet(p1.Tags)); +} + +// B6. Remove, re-add, and remove again leaves a tombstone for each rowid the package vacated. Both +// are reported, but only one of them names a rowid the baseline holds, so only one removal is +// written — a delta that recorded the transient rowid too would claim to suppress a row that has +// never existed. +TEST_CASE("SQLiteIndex_Delta_RemoveAddRemove", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + auto p3 = MakePackage("Publisher3.Id", "Package 3"); + + DeltaTestContext context{ { p1, p2, p3 } }; + + rowid_t originalRowId = GetPreparedPackageRowId(context.BaselineFile.GetPath(), p2.Id).value(); + + context.Remove(p2); + context.Add(p2); + context.Remove(p2); + + { + Connection working = Connection::Create(context.WorkingFile.GetPath().u8string(), Connection::OpenDisposition::ReadOnly); + + // One tombstone per vacated rowid, and generation is told about both of them. + REQUIRE(GetScalar(working, "SELECT COUNT(*) FROM [update_tracking] WHERE [package] = '" + p2.Id + "' AND [is_removed] = 1") == 2); + + auto removals = Tracking::GetRemovalsSince(working, 0, Tracking::RemovalBehavior::Record); + REQUIRE(removals.size() == 2); + REQUIRE(removals.count(originalRowId) == 1); + } + + REQUIRE_NOTHROW(context.GenerateDelta()); + + { + Connection delta = context.OpenDeltaConnection(); + // Only the rowid the baseline actually holds is written; the one the re-add briefly + // occupied is above the baseline's range, so there is nothing there to suppress. + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [delta_packages] WHERE [id] = '" + p2.Id + "'") == 1); + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [delta_packages] WHERE [rowid] = " + std::to_string(originalRowId) + " AND [is_removed] = 1") == 1); + } + + SQLiteIndex combined = context.OpenCombined(); + REQUIRE(GetSearchedIds(combined) == std::set{ p1.Id, p3.Id }); +} + +// B14. The same sequence, but the re-add changes the casing of the identifier. The tracking table +// freezes the casing of each row at insert, so the two tombstones carry byte different names. With +// identity settled on the rowid this is uneventful, which is exactly what it is asserting: no part +// of the removal path compares the two spellings, so nothing can get the comparison wrong. +TEST_CASE("SQLiteIndex_Delta_RemoveAddRemove_CasingChanged", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + auto p3 = MakePackage("Publisher3.Id", "Package 3"); + auto p2Recased = MakePackage("publisher2.id", "Package 2"); + + DeltaTestContext context{ { p1, p2, p3 } }; + + rowid_t originalRowId = GetPreparedPackageRowId(context.BaselineFile.GetPath(), p2.Id).value(); + + // Publisher3 holds the highest rowid, so the re-add lands on a new one and leaves the first + // tombstone in place rather than reviving it. + context.Remove(p2); + context.Add(p2Recased); + context.Remove(p2Recased); + + { + Connection working = Connection::Create(context.WorkingFile.GetPath().u8string(), Connection::OpenDisposition::ReadOnly); + + // Two tombstones at two rowids. The differing casing does not participate in identity at + // all, which is the point: nothing here has to reconcile the two spellings. + auto removals = Tracking::GetRemovalsSince(working, 0, Tracking::RemovalBehavior::Record); + REQUIRE(removals.size() == 2); + REQUIRE(removals.count(originalRowId) == 1); + } + + REQUIRE_NOTHROW(context.GenerateDelta()); + + { + Connection delta = context.OpenDeltaConnection(); + + // Only the baseline rowid was written, and the identifier stored with it is the one the + // baseline holds rather than either spelling the tracking table recorded. + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [delta_packages] WHERE [is_removed] = 1") == 1); + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [delta_packages] WHERE [rowid] = " + std::to_string(originalRowId) + " AND [is_removed] = 1") == 1); + } + + SQLiteIndex combined = context.OpenCombined(); + REQUIRE(GetSearchedIds(combined) == std::set{ p1.Id, p3.Id }); +} + +// B7. When the re-add happens to land on the rowid the package just gave up, the tracking row is +// updated in place. Leaving a tombstone next to the live row would contradict it. +TEST_CASE("SQLiteIndex_Delta_ReAddOnSameRowIdUpdatesInPlace", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + + DeltaTestContext context{ { p1, p2 } }; + + rowid_t originalRowId = GetPreparedPackageRowId(context.BaselineFile.GetPath(), p2.Id).value(); + + // Publisher2 holds the highest rowid, so removing it frees the value that the re-add takes. + context.Remove(p2); + context.Add(p2); + + context.GenerateDelta(); + + REQUIRE(GetPreparedPackageRowId(context.WorkingFile.GetPath(), p2.Id).value() == originalRowId); + + Connection delta = context.OpenDeltaConnection(); + + // A single live row, and no tombstone left behind to contradict it. + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [delta_packages] WHERE [id] = '" + p2.Id + "'") == 1); + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [delta_packages] WHERE [id] = '" + p2.Id + "' AND [is_removed] = 0") == 1); + + SQLiteIndex combined = context.OpenCombined(); + REQUIRE(GetSearchedIds(combined) == std::set{ p1.Id, p2.Id }); +} + +// B8. Two identifiers legitimately share a rowid in the tracking table when one vacates it and the +// other takes it. A unique index on the rowid alone would reject exactly the case that the +// tombstone exists to record; only the live rows are constrained. +TEST_CASE("SQLiteIndex_Delta_TrackingAllowsSharedRowIdAcrossPackages", "[sqliteindex][V2_1][updatetracking]") +{ + TempFile indexFile{ "updatetracking"s, ".db"s }; + + ManifestAndPath m1; + CreateFakeManifestAndPath(m1, "Publisher1", "1.0"); + ManifestAndPath m2; + CreateFakeManifestAndPath(m2, "Publisher2", "1.0"); + + { + SQLiteIndex index = SQLiteIndex::CreateNew(indexFile, s_DeltaVersion); + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, "0"); + index.AddManifest(m1.Manifest, m1.Path); + index.RemoveManifest(m1.Manifest, m1.Path); + + // Publisher1 gave up the only rowid in use, so Publisher2 takes it. + REQUIRE_NOTHROW(index.AddManifest(m2.Manifest, m2.Path)); + REQUIRE(index.CheckConsistency(true)); + } + + Connection connection = Connection::Create(indexFile, Connection::OpenDisposition::ReadOnly); + + int64_t sharedRowId = GetScalar(connection, "SELECT [package_rowid] FROM [update_tracking] WHERE [package] = '" + m2.Manifest.Id + "'"); + REQUIRE(GetScalar(connection, "SELECT [package_rowid] FROM [update_tracking] WHERE [package] = '" + m1.Manifest.Id + "'") == sharedRowId); + + REQUIRE(GetScalar(connection, "SELECT COUNT(*) FROM [update_tracking] WHERE [package_rowid] = " + std::to_string(sharedRowId)) == 2); + REQUIRE(GetScalar(connection, "SELECT COUNT(*) FROM [update_tracking] WHERE [package_rowid] = " + std::to_string(sharedRowId) + " AND [is_removed] = 0") == 1); +} + +// B9. Identifiers that differ only by case are the same package. The index decides that with the +// ICU LIKE implementation, which folds beyond ASCII, so a NOCASE index would disagree here. +TEST_CASE("SQLiteIndex_Delta_TrackingIdentityIsIcuCaseInsensitive", "[sqliteindex][V2_1][updatetracking]") +{ + TempFile indexFile{ "updatetracking"s, ".db"s }; + + // Cyrillic, where the case mapping is well defined but outside the ASCII range that the + // built in NOCASE collation folds. + ManifestAndPath lower; + CreateFakeManifestAndPath(lower, "\xd0\xbf\xd1\x80\xd0\xb8\xd0\xbc\xd0\xb5\xd1\x80", "1.0"); + ManifestAndPath upper; + CreateFakeManifestAndPath(upper, "\xd0\x9f\xd0\xa0\xd0\x98\xd0\x9c\xd0\x95\xd0\xa0", "2.0"); + + { + SQLiteIndex index = SQLiteIndex::CreateNew(indexFile, s_DeltaVersion); + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, "0"); + index.AddManifest(lower.Manifest, lower.Path); + index.AddManifest(upper.Manifest, upper.Path); + REQUIRE(index.CheckConsistency(true)); + } + + Connection connection = Connection::Create(indexFile, Connection::OpenDisposition::ReadOnly); + + // One package, so one live tracking row. Two would mean the two spellings were treated as + // different packages, and the live row constraint failed to see them as one. + REQUIRE(GetScalar(connection, "SELECT COUNT(*) FROM [update_tracking] WHERE [is_removed] = 0") == 1); + REQUIRE(GetScalar(connection, "SELECT COUNT(DISTINCT [package_rowid]) FROM [update_tracking]") == 1); +} + +// B10. The rowid the tracking table stores has to be the one the prepared index assigns, since +// that is what generation and the merged views agree on. This also fails if pinning regresses. +TEST_CASE("SQLiteIndex_Delta_TrackingRowIdMatchesPreparedIndex", "[sqliteindex][V2_1][updatetracking]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + auto p3 = MakePackage("Publisher3.Id", "Package 3"); + + DeltaTestContext context{ { p1, p2, p3 } }; + + std::vector> tracked; + + { + Connection working = Connection::Create(context.WorkingFile.GetPath().u8string(), Connection::OpenDisposition::ReadOnly); + Statement statement = Statement::Create(working, "SELECT [package], [package_rowid] FROM [update_tracking] WHERE [is_removed] = 0"); + + while (statement.Step()) + { + tracked.emplace_back(statement.GetColumn(0), statement.GetColumn(1)); + } + } + + REQUIRE(tracked.size() == 3); + + for (const auto& [packageId, trackedRowId] : tracked) + { + INFO(packageId); + auto preparedRowId = GetPreparedPackageRowId(context.BaselineFile.GetPath(), packageId); + REQUIRE(preparedRowId.has_value()); + REQUIRE(preparedRowId.value() == trackedRowId); + } +} + +// B12. Package identity is case insensitive everywhere in the index: the ids table collapses +// LIKE equal identifiers onto one rowid and overwrites the stored string with the most recent +// casing, while the tracking table freezes the casing it first saw. Generation therefore has to +// resolve a package across a casing difference between the two. +// +// On the changed path, byte equality fails to resolve the package and generation throws for the +// entire index. +TEST_CASE("SQLiteIndex_Delta_IdentifierCasingChange_Changed", "[sqliteindex][V2_1][delta]") +{ + auto original = MakePackage("Publisher1.Id", "Package 1"); + auto unchanged = MakePackage("Publisher2.Id", "Package 2"); + auto recased = MakePackage("publisher1.id", "Package 1 V2", { "t1", "t2" }, { "c1" }, {}, {}, "2.0"s); + + DeltaTestContext context{ { original, unchanged } }; + + // Adding a version under a different casing rewrites the ids table entry, and with it the + // identifier that packaging will put in the packages table. The tracking row keeps the + // original casing. + context.Add(recased); + + REQUIRE_NOTHROW(context.GenerateDelta()); + + SQLiteIndex combined = context.OpenCombined(); + + // Exactly one row for the package. Resolving to no rowid would have left the baseline row + // unsuppressed alongside the delta's, showing it twice. + REQUIRE(GetSearchedIds(combined) == std::set{ recased.Id, unchanged.Id }); +} + +// B13. The silent half of the same defect. Here the casing changed before the baseline was taken, +// so the baseline holds the new casing while the tracking table still holds the old. A removal +// that cannot be resolved against the baseline is treated as "never existed there" and skipped, +// leaving the baseline row visible forever. +TEST_CASE("SQLiteIndex_Delta_IdentifierCasingChange_Removed", "[sqliteindex][V2_1][delta]") +{ + auto original = MakePackage("Publisher1.Id", "Package 1"); + auto recased = MakePackage("publisher1.id", "Package 1 V2", { "t1", "t2" }, { "c1" }, {}, {}, "2.0"s); + auto unchanged = MakePackage("Publisher2.Id", "Package 2"); + + DeltaTestContext context; + context.CreateWorking({ original, unchanged }); + context.Add(recased, false); + context.CaptureBaseline(); + + REQUIRE(GetPreparedPackageRowId(context.BaselineFile.GetPath(), recased.Id).has_value()); + + context.Remove(original); + context.Remove(recased); + + context.GenerateDelta(); + + SQLiteIndex combined = context.OpenCombined(); + + REQUIRE(GetSearchedIds(combined) == std::set{ unchanged.Id }); +} + +// B11. The migration adds the rowid column to a table whose rows predate it. A backfill that left +// nulls behind would break the first delta generated after an upgrade. +TEST_CASE("SQLiteIndex_Delta_TrackingMigrationBackfillsRowIds", "[sqliteindex][V2_1][updatetracking]") +{ + TempFile indexFile{ "updatetracking"s, ".db"s }; + + ManifestAndPath m1; + CreateFakeManifestAndPath(m1, "Publisher1", "1.0"); + ManifestAndPath m2; + CreateFakeManifestAndPath(m2, "Publisher2", "1.0"); + + { + SQLiteIndex index = SQLiteIndex::CreateNew(indexFile, SQLiteVersion{ 2, 0 }); + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, "0"); + index.AddManifest(m1.Manifest, m1.Path); + index.AddManifest(m2.Manifest, m2.Path); + } + + { + SQLiteIndex index = SQLiteIndex::Open(indexFile, SQLiteStorageBase::OpenDisposition::ReadWrite); + REQUIRE(index.MigrateTo(s_DeltaVersion)); + REQUIRE(index.CheckConsistency(true)); + } + + Connection connection = Connection::Create(indexFile, Connection::OpenDisposition::ReadOnly); + + REQUIRE(GetRowCount(connection, "update_tracking") == 2); + + // The column arrives with a default of 0, so an unbackfilled row is 0 rather than null. + REQUIRE(GetScalar(connection, "SELECT COUNT(*) FROM [update_tracking] WHERE [package_rowid] = 0") == 0); + + // The backfilled value has to be the one the index itself uses, not just any non null. + for (const auto& packageId : { m1.Manifest.Id, m2.Manifest.Id }) + { + INFO(packageId); + int64_t tracked = GetScalar(connection, "SELECT [package_rowid] FROM [update_tracking] WHERE [package] = '" + std::string{ packageId } + "'"); + auto idRowId = Schema::V1_0::IdTable::SelectIdByValue(connection, std::string{ packageId }); + REQUIRE(idRowId.has_value()); + REQUIRE(idRowId.value() == tracked); + } +} + +// B3/B4. Rowids have to survive repeated preparation, not just one round. Two rounds cannot +// distinguish a stable assignment from one that happens to repeat. +TEST_CASE("SQLiteIndex_Delta_RowIdsAreStableAcrossPrepares", "[sqliteindex][V2_0]") +{ + TempFile workingFile{ "rowid_working"s, ".db"s }; + + std::vector manifests(5); + CreateFakeManifestAndPath(manifests[0], "Publisher1", "1.0"); + CreateFakeManifestAndPath(manifests[1], "Publisher2", "1.0"); + CreateFakeManifestAndPath(manifests[2], "Publisher3", "1.0"); + CreateFakeManifestAndPath(manifests[3], "Publisher4", "1.0"); + CreateFakeManifestAndPath(manifests[4], "Publisher5", "1.0"); + + { + SQLiteIndex index = SQLiteIndex::CreateNew(workingFile, SQLiteVersion{ 2, 0 }); + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, "0"); + index.AddManifest(manifests[0].Manifest, manifests[0].Path); + index.AddManifest(manifests[1].Manifest, manifests[1].Path); + index.AddManifest(manifests[2].Manifest, manifests[2].Path); + } + + auto prepareCopy = [&](const TempFile& target) + { + std::filesystem::copy_file(workingFile.GetPath(), target.GetPath(), std::filesystem::copy_options::overwrite_existing); + SQLiteIndex prepared = SQLiteIndex::Open(target.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); + prepared.PrepareForPackaging(); + }; + + TempFile first{ "rowid_first"s, ".db"s }; + prepareCopy(first); + + // Round two: drop one package and add another. + { + SQLiteIndex index = SQLiteIndex::Open(workingFile, SQLiteStorageBase::OpenDisposition::ReadWrite); + index.RemoveManifest(manifests[0].Manifest, manifests[0].Path); + index.AddManifest(manifests[3].Manifest, manifests[3].Path); + } + + TempFile second{ "rowid_second"s, ".db"s }; + prepareCopy(second); + + // Round three, to catch an assignment that only appears stable over a single step. + { + SQLiteIndex index = SQLiteIndex::Open(workingFile, SQLiteStorageBase::OpenDisposition::ReadWrite); + index.RemoveManifest(manifests[1].Manifest, manifests[1].Path); + index.AddManifest(manifests[4].Manifest, manifests[4].Path); + } + + TempFile third{ "rowid_third"s, ".db"s }; + prepareCopy(third); + + // Publisher3 is present throughout and must never move. + rowid_t p3First = GetPreparedPackageRowId(first.GetPath(), manifests[2].Manifest.Id).value(); + REQUIRE(GetPreparedPackageRowId(second.GetPath(), manifests[2].Manifest.Id).value() == p3First); + REQUIRE(GetPreparedPackageRowId(third.GetPath(), manifests[2].Manifest.Id).value() == p3First); + + // Publisher4 survives from round two to round three. + REQUIRE(GetPreparedPackageRowId(third.GetPath(), manifests[3].Manifest.Id).value() == GetPreparedPackageRowId(second.GetPath(), manifests[3].Manifest.Id).value()); + + // B5. A package added after the baseline must land above everything the baseline holds, or it + // would collide with an untouched package when the two are merged. + rowid_t maxInFirst = 0; + for (const auto& id : { manifests[0].Manifest.Id, manifests[1].Manifest.Id, manifests[2].Manifest.Id }) + { + maxInFirst = std::max(maxInFirst, GetPreparedPackageRowId(first.GetPath(), id).value()); + } + + REQUIRE(GetPreparedPackageRowId(second.GetPath(), manifests[3].Manifest.Id).value() > maxInFirst); +} + +// --------------------------------------------------------------------------------------------- +// Group C - generation of the packages table +// --------------------------------------------------------------------------------------------- + +TEST_CASE("SQLiteIndex_Delta_AddedPackage", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + + DeltaTestContext context{ { p1 } }; + + context.Add(p2); + context.GenerateDelta(); + + Connection delta = context.OpenDeltaConnection(); + + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [delta_packages] WHERE [is_removed] = 0") == 1); + REQUIRE(GetStrings(delta, "SELECT [id] FROM [delta_packages] WHERE [is_removed] = 0") == std::set{ p2.Id }); +} + +TEST_CASE("SQLiteIndex_Delta_RemovedPackage", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + + DeltaTestContext context{ { p1, p2 } }; + + context.Remove(p2); + context.GenerateDelta(); + + Connection delta = context.OpenDeltaConnection(); + + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [delta_packages] WHERE [is_removed] = 1") == 1); + REQUIRE(GetStrings(delta, "SELECT [id] FROM [delta_packages] WHERE [is_removed] = 1") == std::set{ p2.Id }); + + // A removal carries no data beyond identity, so the rest of the row stays null. + REQUIRE(GetScalar(delta, + "SELECT COUNT(*) FROM [delta_packages] WHERE [is_removed] = 1 AND [name] IS NULL AND [latest_version] IS NULL AND [hash] IS NULL") == 1); +} + +// C4. Only the identifier and name have ever been asserted, so a column dropped from the copy +// would go unnoticed. +TEST_CASE("SQLiteIndex_Delta_ChangedPackageCopiesEveryColumn", "[sqliteindex][V2_1][delta]") +{ + DeltaTestContext context{ { MakePackage("Publisher1.Id", "Package 1") } }; + ManifestAndPath added; + CreateFakeManifestAndPath(added, "Publisher2", "3.4.5", "1.2"sv, "6.7"sv); + + { + SQLiteIndex index = context.OpenWorkingForChanges(); + index.AddManifest(added.Manifest, added.Path); + } + + context.GenerateDelta(); + + Connection delta = context.OpenDeltaConnection(); + Connection source = Connection::Create(context.WorkingFile.GetPath().u8string(), Connection::OpenDisposition::ReadOnly); + + for (std::string_view column : { "id"sv, "name"sv, "moniker"sv, "latest_version"sv, "arp_min_version"sv, "arp_max_version"sv }) + { + INFO(column); + + auto fromDelta = GetStrings(delta, "SELECT [" + std::string{ column } + "] FROM [delta_packages] WHERE [id] = '" + added.Manifest.Id + "'"); + auto fromSource = GetStrings(source, "SELECT [" + std::string{ column } + "] FROM [packages] WHERE [id] = '" + added.Manifest.Id + "'"); + + REQUIRE(fromDelta == fromSource); + REQUIRE(fromDelta.size() == 1); + REQUIRE(!fromDelta.begin()->empty()); + } + + // The hash is a blob, so compare it as one rather than through the string accessor. + Statement deltaHash = Statement::Create(delta, "SELECT [hash] FROM [delta_packages] WHERE [id] = '" + added.Manifest.Id + "'"); + REQUIRE(deltaHash.Step()); + Statement sourceHash = Statement::Create(source, "SELECT [hash] FROM [packages] WHERE [id] = '" + added.Manifest.Id + "'"); + REQUIRE(sourceHash.Step()); + + auto hashValue = deltaHash.GetColumn(0); + REQUIRE(!hashValue.empty()); + REQUIRE(hashValue == sourceHash.GetColumn(0)); +} + +TEST_CASE("SQLiteIndex_Delta_MultipleChangesAndRemovals", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + auto p3 = MakePackage("Publisher3.Id", "Package 3"); + auto p4 = MakePackage("Publisher4.Id", "Package 4"); + auto p3Updated = MakePackage(p3.Id, "Renamed 3"); + auto p5 = MakePackage("Publisher5.Id", "Package 5"); + auto p6 = MakePackage("Publisher6.Id", "Package 6"); + + DeltaTestContext context{ { p1, p2, p3, p4 } }; + + context.Remove(p1); + context.Remove(p2); + context.Update(p3Updated); + context.Add(p5); + context.Add(p6); + + context.GenerateDelta(); + + Connection delta = context.OpenDeltaConnection(); + + REQUIRE(GetStrings(delta, "SELECT [id] FROM [delta_packages] WHERE [is_removed] = 1") == + std::set{ p1.Id, p2.Id }); + REQUIRE(GetStrings(delta, "SELECT [id] FROM [delta_packages] WHERE [is_removed] = 0") == + std::set{ p3Updated.Id, p5.Id, p6.Id }); + + SQLiteIndex combined = context.OpenCombined(); + REQUIRE(GetSearchedIds(combined) == std::set{ p3Updated.Id, p4.Id, p5.Id, p6.Id }); +} + +// C6. A package that came and went within the window was never in the baseline, so there is +// nothing to suppress and the delta must not claim to remove it. +TEST_CASE("SQLiteIndex_Delta_PackageAddedAndRemovedWithinWindow", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto transient = MakePackage("Transient.Id", "Transient"); + + DeltaTestContext context{ { p1 } }; + + context.Add(transient); + context.Remove(transient); + + REQUIRE_NOTHROW(context.GenerateDelta()); + + Connection delta = context.OpenDeltaConnection(); + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [delta_packages] WHERE [id] LIKE '" + transient.Id + "'") == 0); + + SQLiteIndex combined = context.OpenCombined(); + REQUIRE(GetSearchedIds(combined) == std::set{ p1.Id }); +} + +// C8. The identifier lookup used to be a LIKE, which treats these characters as wildcards. A +// package whose identifier contains them would have matched the wrong row. +TEST_CASE("SQLiteIndex_Delta_IdentifierWithLikeWildcards", "[sqliteindex][V2_1][delta]") +{ + auto wild = MakePackage("Publisher_A.Id", "Wildcard Underscore"); + auto decoy = MakePackage("PublisherXA.Id", "Decoy"); + auto percent = MakePackage("Pub%cent.Id", "Wildcard Percent"); + + DeltaTestContext context{ { wild, decoy, percent } }; + + context.Remove(wild); + context.GenerateDelta(); + + Connection delta = context.OpenDeltaConnection(); + + // Only the package actually removed; a LIKE would also have matched the decoy. + REQUIRE(GetStrings(delta, "SELECT [id] FROM [delta_packages] WHERE [is_removed] = 1") == std::set{ wild.Id }); + + SQLiteIndex combined = context.OpenCombined(); + REQUIRE(GetSearchedIds(combined) == std::set{ decoy.Id, percent.Id }); +} + +TEST_CASE("SQLiteIndex_Delta_NoChanges_EmptyDelta", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + + DeltaTestContext context{ { p1 } }; + + // Reset the base time without making any change, so nothing is reported. + context.OpenWorkingForChanges(); + context.GenerateDelta(); + + Connection delta = context.OpenDeltaConnection(); + + // The delta is a database in its own right, carrying the metadata that any index has. Without + // it the delta could not be opened, since opening reads the schema version to pick an interface. + REQUIRE(MetadataTable::GetNamedValue(delta, s_MetadataValueName_MajorVersion) == 2); + REQUIRE(MetadataTable::GetNamedValue(delta, s_MetadataValueName_MinorVersion) == 1); + REQUIRE(!MetadataTable::TryGetNamedValue(delta, s_MetadataValueName_DatabaseIdentifier).value_or(std::string{}).empty()); + REQUIRE(MetadataTable::TryGetNamedValue(delta, s_MetadataValueName_LastWriteTime).has_value()); + + REQUIRE(GetRowCount(delta, "delta_packages") == 0); + + for (const auto& table : Delta::SystemReferenceTables()) + { + INFO(table.TableName); + REQUIRE(GetRowCount(delta, Delta::GetTableName(table.TableName)) == 0); + } + + for (const auto& table : Delta::OneToManyTables()) + { + INFO(table.TableName); + REQUIRE(GetRowCount(delta, Delta::GetTableName(table.TableName)) == 0); + REQUIRE(GetRowCount(delta, Delta::GetMapTableName(table.TableName)) == 0); + } + + // J4. The delta ships as plain tables, just as a prepared 2.0 index does. Its indexes serve + // only generation, and the merged views probe nothing but primary keys. + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [sqlite_master] WHERE [type] = 'index'") == 0); + + // An empty delta still has to merge cleanly. + SQLiteIndex combined = context.OpenCombined(); + REQUIRE(GetSearchedIds(combined) == std::set{ p1.Id }); +} + +// --------------------------------------------------------------------------------------------- +// Group D - generation of the system reference tables +// +// These decide whether ARP correlation still works after an update, and nothing covered them. +// --------------------------------------------------------------------------------------------- + +TEST_CASE("SQLiteIndex_Delta_SystemReference_AddAndRemove", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1", { "t1" }, { "c1" }, { "Family1_8wekyb3d8bbwe" }, { "PC-KEEP", "PC-DROP" }); + auto p2 = MakePackage("Publisher2.Id", "Package 2", { "t1" }, { "c1" }, { "Family2_8wekyb3d8bbwe" }, { "PC-OTHER" }); + auto p1Updated = MakePackage(p1.Id, p1.Name, { "t1" }, { "c1" }, { "Family3_8wekyb3d8bbwe" }, { "PC-KEEP", "PC-NEW" }); + + DeltaTestContext context{ { p1, p2 } }; + + // Trade one product code for another while keeping a third, and swap the family name. + context.Update(p1Updated); + + context.GenerateDelta(); + + { + Connection delta = context.OpenDeltaConnection(); + + // D6. Only the difference is recorded. Writing the whole current set would destroy the size + // benefit that justifies the feature. + auto productCodeTable = Delta::GetTableName("productcodes2"); + REQUIRE(GetStrings(delta, "SELECT [productcode] FROM [" + productCodeTable + "] WHERE [is_removed] = 0") == + std::set{ "pc-new" }); + REQUIRE(GetStrings(delta, "SELECT [productcode] FROM [" + productCodeTable + "] WHERE [is_removed] = 1") == + std::set{ "pc-drop" }); + + // The kept value is not mentioned at all. + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [" + productCodeTable + "] WHERE [productcode] = 'pc-keep'") == 0); + } + + Connection merged = context.OpenMergedConnection(); + + // D3. Suppression is per row: the kept code survives even though the package changed. + REQUIRE(GetSystemReferenceValues(merged, "productcodes2", "productcode", p1Updated.Id) == + std::set{ "pc-keep", "pc-new" }); + REQUIRE(GetSystemReferenceValues(merged, "pfns2", "pfn", p1Updated.Id) == + std::set{ "family3_8wekyb3d8bbwe" }); + + // The untouched package keeps everything. + REQUIRE(GetSystemReferenceValues(merged, "productcodes2", "productcode", p2.Id) == + std::set{ "pc-other" }); +} + +// D7. The user visible consequence: correlation by product code has to find the updated package. +TEST_CASE("SQLiteIndex_Delta_SystemReference_CorrelationThroughCombined", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1", { "t1" }, { "c1" }, { "Family1_8wekyb3d8bbwe" }, { "PC-KEEP", "PC-DROP" }); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + auto p1Updated = MakePackage(p1.Id, p1.Name, { "t1" }, { "c1" }, p1.PackageFamilyNames, { "PC-KEEP", "PC-NEW" }); + + DeltaTestContext context{ { p1, p2 } }; + + context.Update(p1Updated); + context.GenerateDelta(); + + SQLiteIndex combined = context.OpenCombined(); + + for (const auto& productCode : p1Updated.ProductCodes) + { + INFO(productCode); + + SearchRequest request; + request.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, std::string{ productCode })); + + REQUIRE(GetSearchedIds(combined, request) == std::set{ p1Updated.Id }); + } + + // The dropped code must no longer correlate to anything. + SearchRequest dropped; + dropped.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, p1.ProductCodes[1])); + REQUIRE(combined.Search(dropped).Matches.empty()); + + // And the family name still resolves through the merged view. + SearchRequest family; + family.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::PackageFamilyName, MatchType::Exact, p1Updated.PackageFamilyNames[0])); + REQUIRE(GetSearchedIds(combined, family) == std::set{ p1Updated.Id }); +} + +// D5. The normalized name and publisher tables are populated as a side effect of the package name, +// so a rename has to move them. Nothing asserted on them before. +TEST_CASE("SQLiteIndex_Delta_SystemReference_NormalizedNameFollowsRename", "[sqliteindex][V2_1][delta]") +{ + auto original = MakePackage("Publisher1.Id", "Original Name"); + auto renamed = MakePackage(original.Id, "Replacement Name"); + + DeltaTestContext context{ { original } }; + + context.Update(renamed); + context.GenerateDelta(); + + Connection merged = context.OpenMergedConnection(); + + auto names = GetSystemReferenceValues(merged, "norm_names2", "norm_name", renamed.Id); + REQUIRE(names.size() == 1); + + // The old name must be gone rather than merely joined by the new one. + for (const auto& name : names) + { + REQUIRE(name.find("original") == std::string::npos); + } + + // Rather than predicting what normalization produces, require that the merged form holds + // exactly what a full index built from the same data holds. + Connection reference = Connection::Create(context.WorkingFile.GetPath().u8string(), Connection::OpenDisposition::ReadOnly); + + REQUIRE(GetSystemReferenceValues(merged, "norm_names2", "norm_name", renamed.Id) == + GetSystemReferenceValues(reference, "norm_names2", "norm_name", renamed.Id)); + + REQUIRE(GetSystemReferenceValues(merged, "norm_publishers2", "norm_publisher", renamed.Id) == + GetSystemReferenceValues(reference, "norm_publishers2", "norm_publisher", renamed.Id)); +} + +TEST_CASE("SQLiteIndex_Delta_SystemReference_RemovedPackageValuesAreInvisible", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1", { "t1" }, { "c1" }, { "Family1_8wekyb3d8bbwe" }, { "PC-1" }); + auto p2 = MakePackage("Publisher2.Id", "Package 2", { "t1" }, { "c1" }, { "Family2_8wekyb3d8bbwe" }, { "PC-2" }); + + DeltaTestContext context{ { p1, p2 } }; + + context.Remove(p2); + context.GenerateDelta(); + + { + Connection delta = context.OpenDeltaConnection(); + + // F3. A removal is recorded once, against the package. There are no per value tombstones, + // so the package row is the only thing that can suppress these. + REQUIRE(GetRowCount(delta, Delta::GetTableName("productcodes2")) == 0); + REQUIRE(GetRowCount(delta, Delta::GetTableName("pfns2")) == 0); + } + + Connection merged = context.OpenMergedConnection(); + + REQUIRE(GetSystemReferenceValues(merged, "productcodes2", "productcode", p2.Id).empty()); + REQUIRE(GetSystemReferenceValues(merged, "pfns2", "pfn", p2.Id).empty()); + REQUIRE(GetSystemReferenceValues(merged, "productcodes2", "productcode", p1.Id) == std::set{ "pc-1" }); + + SQLiteIndex combined = context.OpenCombined(); + + SearchRequest request; + request.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, p2.ProductCodes[0])); + REQUIRE(combined.Search(request).Matches.empty()); +} + +// --------------------------------------------------------------------------------------------- +// Group E - generation of the one to many tables +// --------------------------------------------------------------------------------------------- + +// E1. A package that trades one tag for another while keeping a third. Suppressing the baseline +// at the level of the package would lose the kept tag, because the delta never mentions it. +TEST_CASE("SQLiteIndex_Delta_OneToMany_AssociationsAreSuppressedPerRow", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1", { "keep", "drop", "alsokeep" }, { "cmdkeep", "cmddrop" }); + auto p2 = MakePackage("Publisher2.Id", "Package 2", { "other" }, { "othercmd" }); + auto p1Updated = MakePackage(p1.Id, p1.Name, { "keep", "added", "alsokeep" }, { "cmdkeep", "cmdadded" }); + + DeltaTestContext context{ { p1, p2 } }; + + context.Update(p1Updated); + context.Remove(p2); + + context.GenerateDelta(); + + Connection merged = context.OpenMergedConnection(); + + REQUIRE(GetOneToManyValues(merged, "tags2", "tag", p1Updated.Id) == ToStringSet(p1Updated.Tags)); + + // E2. The same defect on the other one to many table. + REQUIRE(GetOneToManyValues(merged, "commands2", "command", p1Updated.Id) == ToStringSet(p1Updated.Commands)); + + // F3 again, for the map tables. + REQUIRE(GetOneToManyValues(merged, "tags2", "tag", p2.Id).empty()); + REQUIRE(GetOneToManyValues(merged, "commands2", "command", p2.Id).empty()); +} + +// E3/E4. A value the baseline already knows is referenced at its existing rowid rather than copied, +// and a genuinely new value is numbered above everything the baseline holds. +TEST_CASE("SQLiteIndex_Delta_OneToMany_ValueRowIdAllocation", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1", { "shared", "only1" }); + auto p2 = MakePackage("Publisher2.Id", "Package 2", { "other" }); + auto p2Updated = MakePackage(p2.Id, p2.Name, { "other", "shared", "brandnew" }); + + DeltaTestContext context{ { p1, p2 } }; + + // Publisher2 gains a tag the baseline already has, plus one it does not. + context.Update(p2Updated); + context.GenerateDelta(); + + rowid_t baselineMaxTagRowId = 0; + rowid_t sharedRowIdInBaseline = 0; + + { + Connection baseline = Connection::Create(context.BaselineFile.GetPath().u8string(), Connection::OpenDisposition::ReadOnly); + baselineMaxTagRowId = static_cast(GetScalar(baseline, "SELECT MAX([rowid]) FROM [tags2]")); + sharedRowIdInBaseline = static_cast(GetScalar(baseline, "SELECT [rowid] FROM [tags2] WHERE [tag] = '" + p1.Tags[0] + "'")); + } + + Connection delta = context.OpenDeltaConnection(); + + // E4. The shared value is not copied into the delta; the map points at the baseline rowid. + REQUIRE(GetStrings(delta, "SELECT [tag] FROM [delta_tags2]") == std::set{ p2Updated.Tags[2] }); + + auto mapTable = Delta::GetMapTableName("tags2"); + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [" + mapTable + "] WHERE [tag] = " + std::to_string(sharedRowIdInBaseline) + " AND [is_removed] = 0") == 1); + + // E3. The new value is numbered above the baseline, so the union cannot collide. + REQUIRE(GetScalar(delta, "SELECT [rowid] FROM [delta_tags2] WHERE [tag] = '" + p2Updated.Tags[2] + "'") > baselineMaxTagRowId); + + // F4. Both sides resolve through the unioned value view. + Connection merged = context.OpenMergedConnection(); + REQUIRE(GetOneToManyValues(merged, "tags2", "tag", p2Updated.Id) == ToStringSet(p2Updated.Tags)); +} + +// E5. One new value shared by two packages is stored once and mapped twice. +TEST_CASE("SQLiteIndex_Delta_OneToMany_NewValueSharedByPackages", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1", { "t1" }); + auto p2 = MakePackage("Publisher2.Id", "Package 2", { "t2" }); + auto p1Updated = MakePackage(p1.Id, p1.Name, { "t1", "commontag" }); + auto p2Updated = MakePackage(p2.Id, p2.Name, { "t2", "commontag" }); + + DeltaTestContext context{ { p1, p2 } }; + + context.Update(p1Updated); + context.Update(p2Updated); + + context.GenerateDelta(); + + Connection delta = context.OpenDeltaConnection(); + + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [delta_tags2] WHERE [tag] = '" + p1Updated.Tags[1] + "'") == 1); + + rowid_t valueRowId = static_cast(GetScalar(delta, "SELECT [rowid] FROM [delta_tags2] WHERE [tag] = '" + p1Updated.Tags[1] + "'")); + auto mapTable = Delta::GetMapTableName("tags2"); + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [" + mapTable + "] WHERE [tag] = " + std::to_string(valueRowId) + " AND [is_removed] = 0") == 2); + + Connection merged = context.OpenMergedConnection(); + REQUIRE(GetOneToManyValues(merged, "tags2", "tag", p1Updated.Id) == ToStringSet(p1Updated.Tags)); + REQUIRE(GetOneToManyValues(merged, "tags2", "tag", p2Updated.Id) == ToStringSet(p2Updated.Tags)); +} + +// E6. Every value removed from a package. An empty current set must not read as "nothing changed". +TEST_CASE("SQLiteIndex_Delta_OneToMany_AllValuesRemoved", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1", { "t1", "t2" }, { "c1" }); + auto p2 = MakePackage("Publisher2.Id", "Package 2", { "t3" }, { "c2" }); + auto p1Updated = MakePackage(p1.Id, p1.Name, {}, {}); + + DeltaTestContext context{ { p1, p2 } }; + + context.Update(p1Updated); + context.GenerateDelta(); + + { + Connection delta = context.OpenDeltaConnection(); + auto mapTable = Delta::GetMapTableName("tags2"); + REQUIRE(GetScalar(delta, "SELECT COUNT(*) FROM [" + mapTable + "] WHERE [is_removed] = 1") == 2); + } + + Connection merged = context.OpenMergedConnection(); + REQUIRE(GetOneToManyValues(merged, "tags2", "tag", p1Updated.Id).empty()); + REQUIRE(GetOneToManyValues(merged, "commands2", "command", p1Updated.Id).empty()); + + // The other package is untouched. + REQUIRE(GetOneToManyValues(merged, "tags2", "tag", p2.Id) == ToStringSet(p2.Tags)); +} + +// E7. A baseline with no values at all, so the maximum rowid query has nothing to report. +TEST_CASE("SQLiteIndex_Delta_OneToMany_EmptyBaselineValueTable", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1", {}, {}); + auto p1Updated = MakePackage(p1.Id, p1.Name, { "first" }, { "firstcmd" }); + + DeltaTestContext context{ { p1 } }; + + { + Connection baseline = Connection::Create(context.BaselineFile.GetPath().u8string(), Connection::OpenDisposition::ReadOnly); + REQUIRE(GetRowCount(baseline, "tags2") == 0); + } + + context.Update(p1Updated); + REQUIRE_NOTHROW(context.GenerateDelta()); + + Connection delta = context.OpenDeltaConnection(); + + // Numbering has to start somewhere valid; rowid 0 is not. + REQUIRE(GetScalar(delta, "SELECT [rowid] FROM [delta_tags2] WHERE [tag] = '" + p1Updated.Tags[0] + "'") > 0); + + Connection merged = context.OpenMergedConnection(); + REQUIRE(GetOneToManyValues(merged, "tags2", "tag", p1Updated.Id) == ToStringSet(p1Updated.Tags)); +} + +// --------------------------------------------------------------------------------------------- +// Group F - merged views +// --------------------------------------------------------------------------------------------- + +// F5. In delta read mode the packages table is a view, so the check that decides whether an index +// has been prepared cannot rely on the table existing. Without this the 1.7 internal interface +// would be built over the views. +TEST_CASE("SQLiteIndex_Delta_CombinedIndexIsInPreparedState", "[sqliteindex][V2_1][delta]") +{ + DeltaTestContext context{ { MakePackage("Publisher1.Id", "Package 1") } }; + + context.Add(MakePackage("Publisher2.Id", "Package 2")); + context.GenerateDelta(); + + SQLiteIndex combined = context.OpenCombined(); + + // A prepared index reports no manifest paths; reaching the 1.7 interface would change that, + // and would more likely fail outright looking for tables that are not there. + auto results = combined.Search({}); + REQUIRE(results.Matches.size() == 2); + + REQUIRE(combined.GetPropertyByPrimaryId(results.Matches[0].first, PackageVersionProperty::Id).has_value()); +} + +// --------------------------------------------------------------------------------------------- +// Group G - combined open, affinity, and the negative paths +// --------------------------------------------------------------------------------------------- + +TEST_CASE("SQLiteIndex_Delta_OpenWithBaseline_Search", "[sqliteindex][V2_1][delta]") +{ + // Immutable is the disposition the shipped path uses, and it is the one that reaches SQLite as + // a URI rather than a plain path, so both files have to be named that way for the attach to + // resolve at all. + auto disposition = GENERATE(SQLiteStorageBase::OpenDisposition::Read, SQLiteStorageBase::OpenDisposition::Immutable); + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + + DeltaTestContext context{ { p1 } }; + + context.Add(p2); + context.GenerateDelta(); + + SQLiteIndex combined = context.OpenCombined(disposition); + REQUIRE(GetSearchedIds(combined) == std::set{ p1.Id, p2.Id }); +} + +TEST_CASE("SQLiteIndex_Delta_OpenWithBaseline_RemovedPackageExcluded", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + + DeltaTestContext context{ { p1, p2 } }; + + context.Remove(p2); + context.GenerateDelta(); + + SQLiteIndex combined = context.OpenCombined(); + REQUIRE(GetSearchedIds(combined) == std::set{ p1.Id }); +} + +TEST_CASE("SQLiteIndex_Delta_UnmarkedBaselineRejected", "[sqliteindex][V2_1][delta]") +{ + DeltaTestContext context; + context.CreateWorking({ MakePackage("Publisher1.Id", "Package 1") }); + context.CaptureBaseline(false); + + context.Add(MakePackage("Publisher2.Id", "Package 2")); + + // Without a designation the baseline has no identity, so nothing could tie the delta to it. + REQUIRE_THROWS_HR(context.GenerateDelta(), APPINSTALLER_CLI_ERROR_INDEX_INTEGRITY_COMPROMISED); +} + +TEST_CASE("SQLiteIndex_Delta_MismatchedBaselineRejected", "[sqliteindex][V2_1][delta]") +{ + DeltaTestContext context{ { MakePackage("Publisher1.Id", "Package 1") } }; + + // A second baseline with byte identical contents. Designating each gives it its own identity, + // which is the point: a delta is tied to the baseline it was computed from, not to data that + // happens to look like it. + TempFile otherBaselineFile{ "delta_baseline_other"s, ".db"s }; + std::filesystem::copy_file(context.WorkingFile.GetPath(), otherBaselineFile.GetPath(), std::filesystem::copy_options::overwrite_existing); + + { + SQLiteIndex other = SQLiteIndex::Open(otherBaselineFile.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); + other.PrepareForPackaging(); + other.MarkAsBaseline(); + } + + context.Add(MakePackage("Publisher2.Id", "Package 2")); + context.GenerateDelta(); + + REQUIRE_NOTHROW(context.OpenCombined()); + + REQUIRE_THROWS_HR( + SQLiteIndex::OpenWithBaseline(context.DeltaFile.GetPath().u8string(), otherBaselineFile.GetPath().u8string()), + APPINSTALLER_CLI_ERROR_INDEX_INTEGRITY_COMPROMISED); +} + +// G9. Designating an index a second time mints a new identity, which invalidates any delta made +// against the first. Designation is deliberately not idempotent. +TEST_CASE("SQLiteIndex_Delta_ReMarkingBaselineInvalidatesDelta", "[sqliteindex][V2_1][delta]") +{ + DeltaTestContext context{ { MakePackage("Publisher1.Id", "Package 1") } }; + + context.Add(MakePackage("Publisher2.Id", "Package 2")); + context.GenerateDelta(); + + REQUIRE_NOTHROW(context.OpenCombined()); + + { + SQLiteIndex baseline = SQLiteIndex::Open(context.BaselineFile.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); + baseline.MarkAsBaseline(); + } + + REQUIRE_THROWS_HR(context.OpenCombined(), APPINSTALLER_CLI_ERROR_INDEX_INTEGRITY_COMPROMISED); +} + +// G5. The combined form is a set of views over a union, so there is nothing to write back to. +TEST_CASE("SQLiteIndex_Delta_OpenWithBaseline_ReadWriteRejected", "[sqliteindex][V2_1][delta]") +{ + DeltaTestContext context{ { MakePackage("Publisher1.Id", "Package 1") } }; + + context.Add(MakePackage("Publisher2.Id", "Package 2")); + context.GenerateDelta(); + + REQUIRE_THROWS_HR(context.OpenCombined(SQLiteStorageBase::OpenDisposition::ReadWrite), E_INVALIDARG); +} + +// G6. A baseline that cannot be read at all has to fail rather than silently produce a delta only +// view of the world. +TEST_CASE("SQLiteIndex_Delta_OpenWithBaseline_MissingBaseline", "[sqliteindex][V2_1][delta]") +{ + DeltaTestContext context{ { MakePackage("Publisher1.Id", "Package 1") } }; + + context.Add(MakePackage("Publisher2.Id", "Package 2")); + context.GenerateDelta(); + + TempFile missing{ "delta_missing_baseline"s, ".db"s }; + std::filesystem::remove(missing.GetPath()); + + REQUIRE_THROWS(SQLiteIndex::OpenWithBaseline(context.DeltaFile.GetPath().u8string(), missing.GetPath().u8string())); +} + +// G7/G8. The delta entry points are 2.1 only, and the base implementations say so rather than +// doing something undefined. +TEST_CASE("SQLiteIndex_Delta_NotSupportedBefore_2_1", "[sqliteindex][V2_0][delta]") +{ + TempFile indexFile{ "delta_unsupported"s, ".db"s }; + + ManifestAndPath m1; + CreateFakeManifestAndPath(m1, "Publisher1", "1.0"); + + SQLiteIndex index = SQLiteIndex::CreateNew(indexFile, SQLiteVersion{ 2, 0 }); + index.AddManifest(m1.Manifest, m1.Path); + index.PrepareForPackaging(); + + REQUIRE_THROWS_HR(index.MarkAsBaseline(), HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED)); + + // G7. A 2.0 file in the delta position is nonsense, and the interface is what says so. The + // baseline is never touched, so it does not need to exist. + TempFile baselineFile{ "delta_unsupported_baseline"s, ".db"s }; + + REQUIRE_THROWS_HR( + SQLiteIndex::OpenWithBaseline(indexFile.GetPath().u8string(), baselineFile.GetPath().u8string()), + HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED)); +} + +// --------------------------------------------------------------------------------------------- +// Group I - consistency +// --------------------------------------------------------------------------------------------- + +// I2/I3. A package recorded as removed while still present in the index is a real inconsistency, +// but a package that was removed and re-added is not: it legitimately has both a tombstone and a +// live row. Confusing the two turns a silent bug into a loud but wrong integrity failure. +TEST_CASE("SQLiteIndex_Delta_CheckConsistency_ReAddedPackageIsNotCorruption", "[sqliteindex][V2_1][updatetracking]") +{ + TempFile indexFile{ "updatetracking"s, ".db"s }; + + ManifestAndPath m1; + CreateFakeManifestAndPath(m1, "Publisher1", "1.0"); + ManifestAndPath m2; + CreateFakeManifestAndPath(m2, "Publisher2", "1.0"); + ManifestAndPath m3; + CreateFakeManifestAndPath(m3, "Publisher3", "1.0"); + + SQLiteIndex index = SQLiteIndex::CreateNew(indexFile, s_DeltaVersion); + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, "0"); + index.AddManifest(m1.Manifest, m1.Path); + index.AddManifest(m2.Manifest, m2.Path); + index.AddManifest(m3.Manifest, m3.Path); + + // Removing the middle package and re-adding it leaves a tombstone beside a live row. + index.RemoveManifest(m2.Manifest, m2.Path); + index.AddManifest(m2.Manifest, m2.Path); + + REQUIRE(index.CheckConsistency(true)); + + Connection connection = Connection::Create(indexFile, Connection::OpenDisposition::ReadOnly); + REQUIRE(GetScalar(connection, "SELECT COUNT(*) FROM [update_tracking] WHERE [package] = '" + m2.Manifest.Id + "'") == 2); + + // The removal is still reported, since the old rowid genuinely was vacated. + auto removals = Tracking::GetRemovalsSince(connection, 0, Tracking::RemovalBehavior::Record); + REQUIRE(removals.size() == 1); + + // And it is not also reported as an update under that identity being gone. + auto updates = Tracking::GetUpdatesSince(connection, 0, Tracking::RemovalBehavior::Record); + REQUIRE(std::count_if(updates.begin(), updates.end(), [&](const auto& u) { return u.PackageIdentifier == m2.Manifest.Id; }) == 1); +} + +TEST_CASE("SQLiteIndex_Delta_CheckConsistency_OnCombinedIndex", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1", { "t1" }, { "c1" }, {}, { "PC-1" }); + auto p2 = MakePackage("Publisher2.Id", "Package 2", { "t2" }, { "c2" }, {}, { "PC-2" }); + auto p1Updated = MakePackage(p1.Id, p1.Name, { "t1", "t3" }, p1.Commands, p1.PackageFamilyNames, p1.ProductCodes); + + DeltaTestContext context{ { p1, p2 } }; + + context.Update(p1Updated); + context.Remove(p2); + context.GenerateDelta(); + + SQLiteIndex combined = context.OpenCombined(); + REQUIRE(combined.CheckConsistency(true)); +} + +// --------------------------------------------------------------------------------------------- +// Group H - equivalence +// +// Enumerating what could go wrong in a merge is unbounded; comparing against a full index built +// from the same data is not. The targeted cases above exist to localize what this detects. +// --------------------------------------------------------------------------------------------- + +namespace +{ + // Compares a combined index against a full index built from the same data. + void RequireEquivalent(const SQLiteIndex& combined, const SQLiteIndex& full) + { + // H1. A battery covering the fields that the merged views are responsible for. + std::vector requests; + + requests.emplace_back(); + + auto addFieldRequest = [&](PackageMatchField field, MatchType type, std::string value, std::string second = {}) + { + SearchRequest request; + if (second.empty()) + { + request.Inclusions.emplace_back(PackageMatchFilter(field, type, std::move(value))); + } + else + { + request.Inclusions.emplace_back(PackageMatchFilter(field, type, std::move(value), std::move(second))); + } + requests.emplace_back(std::move(request)); + }; + + for (MatchType type : { MatchType::Exact, MatchType::Substring, MatchType::StartsWith }) + { + addFieldRequest(PackageMatchField::Id, type, "Equivalence"); + addFieldRequest(PackageMatchField::Name, type, "Package"); + addFieldRequest(PackageMatchField::Moniker, type, "moniker"); + addFieldRequest(PackageMatchField::Tag, type, "shared"); + addFieldRequest(PackageMatchField::Tag, type, "changed"); + addFieldRequest(PackageMatchField::Command, type, "cmd"); + addFieldRequest(PackageMatchField::ProductCode, type, "PC"); + addFieldRequest(PackageMatchField::PackageFamilyName, type, "Family"); + } + + { + SearchRequest query; + query.Query = RequestMatch(MatchType::Substring, "Package"); + requests.emplace_back(std::move(query)); + } + + addFieldRequest(PackageMatchField::NormalizedNameAndPublisher, MatchType::Exact, "Package Untouched", "Publisher"); + addFieldRequest(PackageMatchField::NormalizedNameAndPublisher, MatchType::Exact, "Package Replacement", "Publisher"); + addFieldRequest(PackageMatchField::NormalizedNameAndPublisher, MatchType::Exact, "Package Original", "Publisher"); + + size_t matchedRequests = 0; + + for (size_t i = 0; i < requests.size(); ++i) + { + INFO("request index " << i); + + // A field and match type combination that the index does not support is not a delta + // concern, but the two forms still have to agree about it. + std::optional> combinedIds; + std::optional> fullIds; + + try + { + combinedIds = GetSearchedIds(combined, requests[i]); + } + catch (...) {} + + try + { + fullIds = GetSearchedIds(full, requests[i]); + } + catch (...) {} + + REQUIRE(combinedIds.has_value() == fullIds.has_value()); + REQUIRE(combinedIds == fullIds); + + if (fullIds && !fullIds->empty()) + { + ++matchedRequests; + } + } + + // Agreement is only meaningful if the battery actually matched something. Without this the + // group passes when every request throws or returns nothing, which is exactly what a defect + // in the merged views could cause. + REQUIRE(matchedRequests != 0); + + // H3. The identifiers agreeing is not enough; the rowids that name them have to agree too, + // since that is what the merge is built on and what a caller carries around. + std::map combinedPrimaryIds; + for (const auto& match : combined.Search({}).Matches) + { + combinedPrimaryIds[combined.GetPropertyByPrimaryId(match.first, PackageVersionProperty::Id).value()] = match.first; + } + + std::map fullPrimaryIds; + for (const auto& match : full.Search({}).Matches) + { + fullPrimaryIds[full.GetPropertyByPrimaryId(match.first, PackageVersionProperty::Id).value()] = match.first; + } + + REQUIRE(combinedPrimaryIds.size() == fullPrimaryIds.size()); + + // H2 and H3 iterate this map, so an empty one would make both pass vacuously. + REQUIRE(!combinedPrimaryIds.empty()); + + for (const auto& [packageId, primaryId] : combinedPrimaryIds) + { + INFO(packageId); + + auto full_itr = fullPrimaryIds.find(packageId); + REQUIRE(full_itr != fullPrimaryIds.end()); + REQUIRE(primaryId == full_itr->second); + + // H2. Found is not the same as correct. + for (PackageVersionProperty property : { + PackageVersionProperty::Id, + PackageVersionProperty::Name, + PackageVersionProperty::Moniker, + PackageVersionProperty::Version }) + { + REQUIRE(combined.GetPropertyByPrimaryId(primaryId, property) == full.GetPropertyByPrimaryId(full_itr->second, property)); + } + + for (PackageVersionMultiProperty property : { + PackageVersionMultiProperty::Tag, + PackageVersionMultiProperty::Command, + PackageVersionMultiProperty::PackageFamilyName, + PackageVersionMultiProperty::ProductCode, + PackageVersionMultiProperty::UpgradeCode, + PackageVersionMultiProperty::Name, + PackageVersionMultiProperty::Publisher }) + { + auto combinedValues = combined.GetMultiPropertyByPrimaryId(primaryId, property); + auto fullValues = full.GetMultiPropertyByPrimaryId(full_itr->second, property); + + std::sort(combinedValues.begin(), combinedValues.end()); + std::sort(fullValues.begin(), fullValues.end()); + + REQUIRE(combinedValues == fullValues); + } + } + } +} + +TEST_CASE("SQLiteIndex_Delta_EquivalenceWithFullIndex", "[sqliteindex][V2_1][delta]") +{ + // The mutation set covers every kind of change the delta has to describe: a package added, one + // removed, one whose values change, one renamed, and one left entirely alone. + auto untouched = MakePackage("Equivalence.Untouched", "Package Untouched", { "shared", "keep" }, { "cmdkeep" }, { "Family0_8wekyb3d8bbwe" }, { "PC-UNTOUCHED" }); + auto removed = MakePackage("Equivalence.Removed", "Package Removed", { "shared", "gone" }, { "cmdgone" }, { "Family1_8wekyb3d8bbwe" }, { "PC-REMOVED" }); + auto retagged = MakePackage("Equivalence.Retagged", "Package Retagged", { "shared", "changed" }, { "cmdold" }, { "Family2_8wekyb3d8bbwe" }, { "PC-OLD", "PC-BOTH" }); + auto renamed = MakePackage("Equivalence.Renamed", "Package Original", { "shared" }, { "cmdkeep" }, { "Family3_8wekyb3d8bbwe" }, { "PC-RENAMED" }); + auto roundTrip = MakePackage("Equivalence.RoundTrip", "Package RoundTrip", { "shared" }, { "cmdkeep" }, {}, { "PC-ROUND" }); + auto retaggedUpdated = MakePackage(retagged.Id, retagged.Name, { "shared", "changednew" }, { "cmdnew" }, retagged.PackageFamilyNames, { "PC-NEW", "PC-BOTH" }); + auto renamedUpdated = MakePackage(renamed.Id, "Package Replacement", renamed.Tags, renamed.Commands, renamed.PackageFamilyNames, renamed.ProductCodes); + auto added = MakePackage("Equivalence.Added", "Package Added", { "shared", "brand" }, { "cmdadded" }, { "Family4_8wekyb3d8bbwe" }, { "PC-ADDED" }); + + DeltaTestContext context{ { untouched, removed, retagged, renamed, roundTrip } }; + + context.Remove(removed); + context.Update(retaggedUpdated); + context.Update(renamedUpdated); + context.Add(added); + + // A removal and re-add in the same window, which is where rowid identity is hardest. + context.Remove(roundTrip); + context.Add(roundTrip); + + context.GenerateDelta(); + + SQLiteIndex combined = context.OpenCombined(); + SQLiteIndex full = context.OpenFullIndex(); + + REQUIRE(GetSearchedIds(full) == std::set{ + untouched.Id, retaggedUpdated.Id, renamedUpdated.Id, roundTrip.Id, added.Id }); + + RequireEquivalent(combined, full); +} + +// H5. The degenerate case, which a consumer must not have to special case. +TEST_CASE("SQLiteIndex_Delta_EquivalenceWithEmptyDelta", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Equivalence.One", "Package One", { "shared" }, { "cmdkeep" }, { "Family0_8wekyb3d8bbwe" }, { "PC-1" }); + auto p2 = MakePackage("Equivalence.Two", "Package Two", { "shared", "changed" }, { "cmd2" }, {}, { "PC-2" }); + + DeltaTestContext context{ { p1, p2 } }; + + context.OpenWorkingForChanges(); + context.GenerateDelta(); + + SQLiteIndex combined = context.OpenCombined(); + SQLiteIndex full = context.OpenFullIndex(); + + RequireEquivalent(combined, full); +} + + +// --------------------------------------------------------------------------------------------- +// Group K - the change sequence +// +// The window a delta describes is defined by a monotonic sequence rather than by the write time. +// These cases cover the boundary that a whole second time cannot express, and the one failure +// mode a sequence has that a time does not. +// --------------------------------------------------------------------------------------------- + +// K1. Every write takes a new sequence, and a package written twice keeps only the later one. +// A sequence that did not advance on update would leave the second change outside any window +// opened after the first. +TEST_CASE("SQLiteIndex_Delta_ChangeSequenceAdvancesOnEveryWrite", "[sqliteindex][V2_1][updatetracking]") +{ + TempFile indexFile{ "changeseq"s, ".db"s }; + + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + auto p1Updated = MakePackage(p1.Id, "Package 1 Renamed"); + + { + SQLiteIndex index = SQLiteIndex::CreateNew(indexFile, s_DeltaVersion); + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, "0"); + + Manifest m1 = CreateManifest(p1); + index.AddManifest(m1, p1.Path); + + Manifest m2 = CreateManifest(p2); + index.AddManifest(m2, p2.Path); + + Manifest m1Updated = CreateManifest(p1Updated); + REQUIRE(index.UpdateManifest(m1Updated, p1.Path)); + } + + Connection connection = Connection::Create(indexFile, Connection::OpenDisposition::ReadOnly); + + // Three writes, but only two rows: the update replaced the first package's sequence rather + // than adding a row. + REQUIRE(GetRowCount(connection, "update_tracking") == 2); + REQUIRE(GetScalar(connection, "SELECT MAX([change_seq]) FROM [update_tracking]") == 3); + + int64_t first = GetScalar(connection, "SELECT [change_seq] FROM [update_tracking] WHERE [package] = '" + p1Updated.Id + "'"); + int64_t second = GetScalar(connection, "SELECT [change_seq] FROM [update_tracking] WHERE [package] = '" + p2.Id + "'"); + + // The updated package is now the more recent of the two, having started as the older. + REQUIRE(first == 3); + REQUIRE(second == 2); +} + +// K2. The boundary case that the time based window could not express at all: a baseline captured +// in the same instant as the data it holds. Under whole second times compared inclusively, the +// entire baseline fell inside its own delta window. +TEST_CASE("SQLiteIndex_Delta_BaselineCapturedImmediatelyExcludesItsOwnData", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + + // No delay anywhere: creation, capture and generation all happen as fast as they can. + DeltaTestContext context{ { p1, p2 } }; + context.GenerateDelta(); + + { + Connection baseline = Connection::Create(context.BaselineFile.GetPath().u8string(), Connection::OpenDisposition::ReadOnly); + + // Both packages were written before the baseline was taken, so the baseline's own sequence + // must already account for them. + REQUIRE(MetadataTable::GetNamedValue(baseline, "deltaBaselineSequence") == 2); + } + + Connection delta = context.OpenDeltaConnection(); + REQUIRE(GetRowCount(delta, "delta_packages") == 0); + + SQLiteIndex combined = context.OpenCombined(); + REQUIRE(GetSearchedIds(combined) == std::set{ p1.Id, p2.Id }); +} + +// K3. A migrated table has no sequences, so every row backfills to the same value and the first +// one issued afterwards is above it. An index designated as a baseline at that moment records 0, +// and the exclusive window correctly reports nothing that preceded the migration. +TEST_CASE("SQLiteIndex_Delta_TrackingMigrationBackfillsChangeSequence", "[sqliteindex][V2_1][updatetracking]") +{ + TempFile indexFile{ "changeseq_migrate"s, ".db"s }; + + ManifestAndPath m1; + CreateFakeManifestAndPath(m1, "Publisher1", "1.0"); + ManifestAndPath m2; + CreateFakeManifestAndPath(m2, "Publisher2", "1.0"); + ManifestAndPath m3; + CreateFakeManifestAndPath(m3, "Publisher3", "1.0"); + + { + SQLiteIndex index = SQLiteIndex::CreateNew(indexFile, SQLiteVersion{ 2, 0 }); + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, "0"); + index.AddManifest(m1.Manifest, m1.Path); + index.AddManifest(m2.Manifest, m2.Path); + } + + { + SQLiteIndex index = SQLiteIndex::Open(indexFile, SQLiteStorageBase::OpenDisposition::ReadWrite); + REQUIRE(index.MigrateTo(s_DeltaVersion)); + REQUIRE(index.CheckConsistency(true)); + } + + { + Connection connection = Connection::Create(indexFile, Connection::OpenDisposition::ReadOnly); + REQUIRE(GetScalar(connection, "SELECT COUNT(*) FROM [update_tracking] WHERE [change_seq] = 0") == 2); + } + + // A write after the migration has to be distinguishable from everything that preceded it. + { + SQLiteIndex index = SQLiteIndex::Open(indexFile, SQLiteStorageBase::OpenDisposition::ReadWrite); + index.AddManifest(m3.Manifest, m3.Path); + } + + Connection connection = Connection::Create(indexFile, Connection::OpenDisposition::ReadOnly); + REQUIRE(GetScalar(connection, "SELECT [change_seq] FROM [update_tracking] WHERE [package] = '" + m3.Manifest.Id + "'") == 1); +} + +// K4. The one direction a sequence fails in that a time does not. If the working index is rebuilt, +// its counter restarts below the baseline's and the window is empty, which would produce a +// silently empty delta. Generation has to refuse instead. +TEST_CASE("SQLiteIndex_Delta_SequenceBelowBaselineIsRejected", "[sqliteindex][V2_1][delta]") +{ + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + auto p3 = MakePackage("Publisher3.Id", "Package 3"); + + DeltaTestContext context{ { p1, p2, p3 } }; + + { + Connection baseline = Connection::Create(context.BaselineFile.GetPath().u8string(), Connection::OpenDisposition::ReadOnly); + REQUIRE(MetadataTable::GetNamedValue(baseline, "deltaBaselineSequence") == 3); + } + + // A rebuilt working index, holding the same data but having recorded far fewer changes. + TempFile rebuiltFile{ "changeseq_rebuilt"s, ".db"s }; + TempFile rebuiltDeltaFile{ "changeseq_rebuilt_delta"s, ".db"s }; + + { + SQLiteIndex index = SQLiteIndex::CreateNew(rebuiltFile, s_DeltaVersion); + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, "0"); + Manifest manifest = CreateManifest(p1); + index.AddManifest(manifest, p1.Path); + } + + SQLiteIndex rebuilt = SQLiteIndex::Open(rebuiltFile, SQLiteStorageBase::OpenDisposition::ReadWrite); + rebuilt.SetProperty(SQLiteIndex::Property::DeltaBaselineIndexPath, context.BaselineFile.GetPath().u8string()); + rebuilt.SetProperty(SQLiteIndex::Property::DeltaOutputPath, rebuiltDeltaFile.GetPath().u8string()); + + REQUIRE_THROWS_HR(rebuilt.PrepareForPackaging(), APPINSTALLER_CLI_ERROR_INDEX_INTEGRITY_COMPROMISED); +} diff --git a/src/AppInstallerCLITests/SQLiteIndexTestCommon.cpp b/src/AppInstallerCLITests/SQLiteIndexTestCommon.cpp new file mode 100644 index 0000000000..83b44015c0 --- /dev/null +++ b/src/AppInstallerCLITests/SQLiteIndexTestCommon.cpp @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "SQLiteIndexTestCommon.h" +#include "TestCommon.h" + +#include +#include + +#include + +using namespace std::string_literals; +using namespace AppInstaller::Manifest; +using namespace AppInstaller::Repository::Microsoft; +using namespace AppInstaller::Utility; + +namespace TestCommon +{ + SQLiteIndex CreateTestIndex(const std::string& filePath, std::optional version) + { + // If no specific version requested, then use generator to run against the last 3 versions. + if (!version) + { + SQLiteVersion latestVersion{ 2, 1 }; + SQLiteVersion versionMinus1 = SQLiteVersion{ 2, 0 }; + SQLiteVersion versionMinus2 = SQLiteVersion{ 1, 7 }; + + version = GENERATE_COPY(SQLiteVersion{ versionMinus2 }, SQLiteVersion{ versionMinus1 }, SQLiteVersion{ latestVersion }); + } + + return SQLiteIndex::CreateNew(filePath, version.value()); + } + + std::string GetPathFromManifest(Manifest& manifest) + { + auto publisher = manifest.Id; + AppInstaller::Utility::FindAndReplace(publisher, ".", "/"); + + return AppInstaller::Utility::ToLower(publisher).append("/").append(manifest.Version); + } + + void CreateFakeManifest(Manifest& manifest, string_t publisher, string_t version) + { + manifest.Installers.push_back({}); + manifest.Id = publisher.append(".").append("Id"); + manifest.DefaultLocalization.Add(publisher.append(" Name")); + manifest.Moniker = "testmoniker"; + manifest.Version = version; + manifest.Channel = "test"; + manifest.DefaultLocalization.Add({ "t1", "t2" }); + manifest.Installers[0].Commands = { "test1", "test2" }; + } + + void CreateFakeManifestAndPath( + ManifestAndPath& manifestAndPath, + const string_t& publisher, + std::string_view version, + std::optional arpMinVersion, + std::optional arpMaxVersion) + { + CreateFakeManifest(manifestAndPath.Manifest, publisher, string_t{ version }); + manifestAndPath.Path = ConvertToUTF8(CreateNewGuidNameWString()); + manifestAndPath.Manifest.StreamSha256 = AppInstaller::Utility::SHA256::ComputeHash(manifestAndPath.Path); + + if (arpMinVersion) + { + manifestAndPath.Manifest.Installers[0].BaseInstallerType = InstallerTypeEnum::Exe; + manifestAndPath.Manifest.Installers[0].AppsAndFeaturesEntries.push_back({}); + manifestAndPath.Manifest.Installers[0].AppsAndFeaturesEntries.back().DisplayVersion = arpMinVersion.value(); + } + + if (arpMaxVersion) + { + manifestAndPath.Manifest.Installers[0].BaseInstallerType = InstallerTypeEnum::Exe; + manifestAndPath.Manifest.Installers[0].AppsAndFeaturesEntries.push_back({}); + manifestAndPath.Manifest.Installers[0].AppsAndFeaturesEntries.back().DisplayVersion = arpMaxVersion.value(); + } + } + + void ApplyIndexFields(Manifest& manifest, const IndexFields& fields) + { + manifest.Id = fields.Id; + manifest.DefaultLocalization.Add(fields.Name); + manifest.DefaultLocalization.Add(fields.Publisher); + manifest.Moniker = fields.Moniker; + manifest.Version = fields.Version; + manifest.DefaultLocalization.Add(fields.Tags); + + manifest.Installers.resize(std::max(fields.PackageFamilyNames.size(), fields.ProductCodes.size())); + + if (manifest.Installers.size() == 0) + { + manifest.Installers.push_back({}); + } + + manifest.Channel = fields.Channel; + manifest.Installers[0].Commands = fields.Commands; + + for (size_t i = 0; i < fields.PackageFamilyNames.size(); ++i) + { + manifest.Installers[i].PackageFamilyName = fields.PackageFamilyNames[i]; + } + + for (size_t i = 0; i < fields.ProductCodes.size(); ++i) + { + manifest.Installers[i].ProductCode = fields.ProductCodes[i]; + } + + if (!fields.ArpName.empty() || !fields.ArpPublisher.empty()) + { + manifest.Installers[0].AppsAndFeaturesEntries.push_back({}); + manifest.Installers[0].AppsAndFeaturesEntries[0].DisplayName = fields.ArpName; + manifest.Installers[0].AppsAndFeaturesEntries[0].Publisher = fields.ArpPublisher; + } + } + + Manifest CreateManifest(const IndexFields& fields) + { + Manifest manifest; + ApplyIndexFields(manifest, fields); + return manifest; + } + + SQLiteIndex SearchTestSetup(const std::string& filePath, std::initializer_list data, std::optional version) + { + SQLiteIndex index = CreateTestIndex(filePath, version); + + // A single manifest is carried across the whole set, matching what this helper has always + // done. Anything the fields do not overwrite therefore persists from the previous entry. + Manifest manifest; + + for (const auto& d : data) + { + ApplyIndexFields(manifest, d); + index.AddManifest(manifest, d.Path); + } + + return index; + } +} diff --git a/src/AppInstallerCLITests/SQLiteIndexTestCommon.h b/src/AppInstallerCLITests/SQLiteIndexTestCommon.h new file mode 100644 index 0000000000..56d25cd702 --- /dev/null +++ b/src/AppInstallerCLITests/SQLiteIndexTestCommon.h @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include +#include +#include + + +// Fixture helpers shared by the index test files. +namespace TestCommon +{ + using SQLiteVersion = AppInstaller::SQLite::Version; + + // Creates an index to test against. + // When no version is given, the test is generated once for each of the last few schema + // versions, so callers that care about a specific version must name it. + AppInstaller::Repository::Microsoft::SQLiteIndex CreateTestIndex(const std::string& filePath, std::optional version = {}); + + // Gets the relative path that the given manifest would be stored at. + std::string GetPathFromManifest(AppInstaller::Manifest::Manifest& manifest); + + // Fills in a manifest with values derived from the given publisher. + void CreateFakeManifest( + AppInstaller::Manifest::Manifest& manifest, + AppInstaller::Manifest::string_t publisher, + AppInstaller::Manifest::string_t version = "1.0.0"); + + // A manifest along with the path that it is stored at. + struct ManifestAndPath + { + AppInstaller::Manifest::Manifest Manifest; + std::string Path; + }; + + // Fills in a manifest and gives it a unique path, along with the hash of that path. + void CreateFakeManifestAndPath( + ManifestAndPath& manifestAndPath, + const AppInstaller::Manifest::string_t& publisher, + std::string_view version = "1.0.0", + std::optional arpMinVersion = {}, + std::optional arpMaxVersion = {}); + + // The values that a test places into an index for a single manifest. + struct IndexFields + { + IndexFields( + std::string id, + std::string name, + std::string moniker, + std::string version, + std::string channel, + std::vector tags, + std::vector commands, + std::string path + ) : + Id(std::move(id)), + Name(std::move(name)), + Moniker(std::move(moniker)), + Version(std::move(version)), + Channel(std::move(channel)), + Tags(std::move(tags)), + Commands(std::move(commands)), + Path(std::move(path)) + {} + + IndexFields( + std::string id, + std::string name, + std::string moniker, + std::string version, + std::string channel, + std::vector tags, + std::vector commands, + std::string path, + std::vector packageFamilyNames, + std::vector productCodes + ) : + Id(std::move(id)), + Name(std::move(name)), + Moniker(std::move(moniker)), + Version(std::move(version)), + Channel(std::move(channel)), + Tags(std::move(tags)), + Commands(std::move(commands)), + Path(std::move(path)), + PackageFamilyNames(std::move(packageFamilyNames)), + ProductCodes(std::move(productCodes)) + {} + + IndexFields( + std::string id, + std::string name, + std::string publisher, + std::string moniker, + std::string version, + std::string channel, + std::vector tags, + std::vector commands, + std::string path, + std::vector packageFamilyNames, + std::vector productCodes + ) : + Id(std::move(id)), + Name(std::move(name)), + Publisher(std::move(publisher)), + Moniker(std::move(moniker)), + Version(std::move(version)), + Channel(std::move(channel)), + Tags(std::move(tags)), + Commands(std::move(commands)), + Path(std::move(path)), + PackageFamilyNames(std::move(packageFamilyNames)), + ProductCodes(std::move(productCodes)) + {} + + IndexFields( + std::string id, + std::string name, + std::string publisher, + std::string moniker, + std::string version, + std::string channel, + std::vector tags, + std::vector commands, + std::string path, + std::vector packageFamilyNames, + std::vector productCodes, + std::string arpName, + std::string arpPublisher + ) : + Id(std::move(id)), + Name(std::move(name)), + Publisher(std::move(publisher)), + Moniker(std::move(moniker)), + Version(std::move(version)), + Channel(std::move(channel)), + Tags(std::move(tags)), + Commands(std::move(commands)), + Path(std::move(path)), + PackageFamilyNames(std::move(packageFamilyNames)), + ProductCodes(std::move(productCodes)), + ArpName(std::move(arpName)), + ArpPublisher(std::move(arpPublisher)) + {} + + std::string Id; + std::string Name; + std::string Publisher; + std::string Moniker; + std::string Version; + std::string Channel; + std::vector Tags; + std::vector Commands; + std::string Path; + std::vector PackageFamilyNames; + std::vector ProductCodes; + std::string ArpName; + std::string ArpPublisher; + }; + + // Applies the given fields to a manifest, overwriting what it names and leaving the rest. + void ApplyIndexFields(AppInstaller::Manifest::Manifest& manifest, const IndexFields& fields); + + // Produces the manifest described by the given fields. + AppInstaller::Manifest::Manifest CreateManifest(const IndexFields& fields); + + // Creates an index containing the given data. + AppInstaller::Repository::Microsoft::SQLiteIndex SearchTestSetup( + const std::string& filePath, + std::initializer_list data = {}, + std::optional version = {}); +} diff --git a/src/AppInstallerCLITests/SQLiteWrapper.cpp b/src/AppInstallerCLITests/SQLiteWrapper.cpp index 10fd960372..50cf907909 100644 --- a/src/AppInstallerCLITests/SQLiteWrapper.cpp +++ b/src/AppInstallerCLITests/SQLiteWrapper.cpp @@ -730,6 +730,310 @@ TEST_CASE("SQLBuilder_InsertValueBinding", "[sqlbuilder]") } } +TEST_CASE("SQLBuilder_AssignValueNull", "[sqlbuilder]") +{ + Connection connection = Connection::Create(SQLITE_MEMORY_DB_CONNECTION_TARGET, Connection::OpenDisposition::Create); + + CreateSimpleTestTable(connection); + InsertIntoSimpleTestTable(connection, 1, "value"); + + { + INFO("Equals(nullptr) remains blocked as a filter"); + Builder::StatementBuilder builder; + REQUIRE_THROWS_HR(builder.Select(s_firstColumn).From(s_tableName).Where(s_secondColumn).Equals(nullptr), E_NOTIMPL); + } + + { + INFO("AssignValue(nullptr) assigns NULL in an update"); + Builder::StatementBuilder update; + update.Update(s_tableName).Set().Column(s_secondColumn).AssignValue(nullptr).Where(s_firstColumn).Equals(1); + update.Execute(connection); + } + + { + INFO("The value is now NULL"); + Builder::StatementBuilder select; + select.Select({ s_firstColumn, s_secondColumn }).From(s_tableName); + + Statement statement = select.Prepare(connection); + REQUIRE(statement.Step()); + REQUIRE(statement.GetColumn(0) == 1); + REQUIRE(statement.GetColumnIsNull(1)); + REQUIRE(!statement.Step()); + } +} + +TEST_CASE("SQLBuilder_AddColumnWithConstraints", "[sqlbuilder]") +{ + Connection connection = Connection::Create(SQLITE_MEMORY_DB_CONNECTION_TARGET, Connection::OpenDisposition::Create); + + CreateSimpleTestTable(connection); + InsertIntoSimpleTestTable(connection, 1, "one"); + + constexpr std::string_view addedColumn = "added"; + + { + // SQLite requires a non-null default when adding a column declared as not null, + // so the plain Add(column, type) form cannot express this. + INFO("Add a not null column with a default"); + Builder::StatementBuilder alter; + alter.AlterTable(s_tableName).Add(Builder::ColumnBuilder(addedColumn, Builder::Type::Int64).NotNull().Default(0)); + alter.Execute(connection); + } + + { + INFO("The existing row receives the default rather than null"); + Builder::StatementBuilder select; + select.Select(addedColumn).From(s_tableName); + + Statement statement = select.Prepare(connection); + REQUIRE(statement.Step()); + REQUIRE(!statement.GetColumnIsNull(0)); + REQUIRE(statement.GetColumn(0) == 0); + REQUIRE(!statement.Step()); + } +} + +TEST_CASE("SQLBuilder_CreateTempView", "[sqlbuilder]") +{ + Connection connection = Connection::Create(SQLITE_MEMORY_DB_CONNECTION_TARGET, Connection::OpenDisposition::Create); + + CreateSimpleTestTable(connection); + InsertIntoSimpleTestTable(connection, 1, "one"); + InsertIntoSimpleTestTable(connection, 2, "two"); + + constexpr std::string_view viewName = "simpleview"; + + { + // Note that SQLite prohibits bound parameters in a view definition, so the + // statement that defines a view must be structural only. + INFO("Create a view over the table"); + Builder::StatementBuilder createView; + createView.CreateTempView(viewName).Select({ s_firstColumn, s_secondColumn }).From(s_tableName).OrderBy(s_firstColumn); + createView.Execute(connection); + } + + { + INFO("The view returns the underlying rows"); + Builder::StatementBuilder select; + select.Select({ s_firstColumn, s_secondColumn }).From(viewName); + + Statement statement = select.Prepare(connection); + REQUIRE(statement.Step()); + REQUIRE(statement.GetColumn(0) == 1); + REQUIRE(statement.GetColumn(1) == "one"); + REQUIRE(statement.Step()); + REQUIRE(statement.GetColumn(0) == 2); + REQUIRE(statement.GetColumn(1) == "two"); + REQUIRE(!statement.Step()); + } + + { + INFO("A filter can still be applied when reading the view"); + Builder::StatementBuilder select; + select.Select(s_secondColumn).From(viewName).Where(s_firstColumn).Equals(2); + + Statement statement = select.Prepare(connection); + REQUIRE(statement.Step()); + REQUIRE(statement.GetColumn(0) == "two"); + REQUIRE(!statement.Step()); + } +} + +TEST_CASE("SQLBuilder_UnionAll", "[sqlbuilder]") +{ + Connection connection = Connection::Create(SQLITE_MEMORY_DB_CONNECTION_TARGET, Connection::OpenDisposition::Create); + + CreateSimpleTestTable(connection); + InsertIntoSimpleTestTable(connection, 1, "one"); + InsertIntoSimpleTestTable(connection, 2, "two"); + + Builder::StatementBuilder select; + select.Select(s_firstColumn).From(s_tableName).Where(s_firstColumn).Equals(1). + UnionAll(). + Select(s_firstColumn).From(s_tableName).Where(s_firstColumn).Equals(2); + + Statement statement = select.Prepare(connection); + + REQUIRE(statement.Step()); + REQUIRE(statement.GetColumn(0) == 1); + REQUIRE(statement.Step()); + REQUIRE(statement.GetColumn(0) == 2); + REQUIRE(!statement.Step()); +} + +TEST_CASE("SQLBuilder_NotExists", "[sqlbuilder]") +{ + Connection connection = Connection::Create(SQLITE_MEMORY_DB_CONNECTION_TARGET, Connection::OpenDisposition::Create); + + constexpr std::string_view otherTable = "othertest"; + + CreateSimpleTestTable(connection); + InsertIntoSimpleTestTable(connection, 1, "one"); + InsertIntoSimpleTestTable(connection, 2, "two"); + + { + Builder::StatementBuilder createTable; + createTable.CreateTable(otherTable).Columns({ Builder::ColumnBuilder(s_firstColumn, Builder::Type::Int) }); + createTable.Execute(connection); + + Builder::StatementBuilder insert; + insert.InsertInto(otherTable).Columns(s_firstColumn).Values(2); + insert.Execute(connection); + } + + // Select rows from the simple table that have no matching row in the other table. + Builder::StatementBuilder select; + select.Select(Builder::QualifiedColumn{ s_tableName, s_firstColumn }).From(s_tableName). + Where().NotExists().BeginParenthetical(). + Select(Builder::QualifiedColumn{ otherTable, s_firstColumn }).From(otherTable). + Where(Builder::QualifiedColumn{ otherTable, s_firstColumn }).Equals(Builder::QualifiedColumn{ s_tableName, s_firstColumn }). + EndParenthetical(); + + Statement statement = select.Prepare(connection); + + REQUIRE(statement.Step()); + REQUIRE(statement.GetColumn(0) == 1); + REQUIRE(!statement.Step()); +} + +TEST_CASE("SQLBuilder_AttachAndTempView", "[sqlbuilder]") +{ + TestCommon::TempFile baselineFile{ "repolibtest_baseline"s, ".db"s }; + INFO("Using temporary file named: " << baselineFile.GetPath()); + + { + INFO("Create the database that will be attached"); + Connection baseline = Connection::Create(baselineFile, Connection::OpenDisposition::Create); + CreateSimpleTestTable(baseline); + InsertIntoSimpleTestTable(baseline, 1, "baseline"); + } + + Connection connection = Connection::Create(SQLITE_MEMORY_DB_CONNECTION_TARGET, Connection::OpenDisposition::Create); + + constexpr std::string_view baselineAlias = "baseline"; + constexpr std::string_view deltaTable = "deltatest"; + + { + INFO("Create a local table with a distinct row"); + Builder::StatementBuilder createTable; + createTable.CreateTable(deltaTable).Columns({ + Builder::ColumnBuilder(s_firstColumn, Builder::Type::Int), + Builder::ColumnBuilder(s_secondColumn, Builder::Type::Text), + }); + createTable.Execute(connection); + + Builder::StatementBuilder insert; + insert.InsertInto(deltaTable).Columns({ s_firstColumn, s_secondColumn }).Values(2, "delta"sv); + insert.Execute(connection); + } + + { + INFO("Attach the baseline database"); + Builder::StatementBuilder attach; + attach.Attach(DatabaseSpecifier{ baselineFile.GetPath().u8string(), DatabaseDisposition::Read }, baselineAlias); + attach.Execute(connection); + } + + { + INFO("A temp view can span the local and attached databases"); + Builder::StatementBuilder createView; + createView.CreateTempView(s_tableName). + Select({ s_firstColumn, s_secondColumn }).From(deltaTable). + UnionAll(). + Select({ s_firstColumn, s_secondColumn }).From(Builder::QualifiedTable{ baselineAlias, s_tableName }); + createView.Execute(connection); + } + + { + INFO("Reading the view returns the merged rows"); + Builder::StatementBuilder select; + select.Select({ s_firstColumn, s_secondColumn }).From(s_tableName).OrderBy(s_firstColumn); + + Statement statement = select.Prepare(connection); + + REQUIRE(statement.Step()); + REQUIRE(statement.GetColumn(0) == 1); + REQUIRE(statement.GetColumn(1) == "baseline"); + + REQUIRE(statement.Step()); + REQUIRE(statement.GetColumn(0) == 2); + REQUIRE(statement.GetColumn(1) == "delta"); + + REQUIRE(!statement.Step()); + } +} + +TEST_CASE("SQLBuilder_ViewWithTombstoneSuppression", "[sqlbuilder]") +{ + Connection connection = Connection::Create(SQLITE_MEMORY_DB_CONNECTION_TARGET, Connection::OpenDisposition::Create); + + constexpr std::string_view valueTombstoneTable = "valuetombstone"; + constexpr std::string_view ownerTombstoneTable = "ownertombstone"; + constexpr std::string_view removedColumn = "is_removed"; + constexpr std::string_view valueAlias = "v"; + constexpr std::string_view ownerAlias = "o"; + constexpr std::string_view viewName = "survivors"; + + CreateSimpleTestTable(connection); + InsertIntoSimpleTestTable(connection, 1, "kept"); + InsertIntoSimpleTestTable(connection, 2, "value removed"); + InsertIntoSimpleTestTable(connection, 3, "owner removed"); + + auto createTombstoneTable = [&](std::string_view tableName, int suppressedValue) + { + Builder::StatementBuilder createTable; + createTable.CreateTable(tableName).Columns({ + Builder::ColumnBuilder(s_firstColumn, Builder::Type::Int), + Builder::ColumnBuilder(removedColumn, Builder::Type::Int), + }); + createTable.Execute(connection); + + Builder::StatementBuilder insertSuppressed; + insertSuppressed.InsertInto(tableName).Columns({ s_firstColumn, removedColumn }).Values(suppressedValue, 1); + insertSuppressed.Execute(connection); + + // Row 1 is named by both tables without being removed by either, which is what + // distinguishes a test of the removal flag from a test of mere presence. + Builder::StatementBuilder insertMentioned; + insertMentioned.InsertInto(tableName).Columns({ s_firstColumn, removedColumn }).Values(1, 0); + insertMentioned.Execute(connection); + }; + + createTombstoneTable(valueTombstoneTable, 2); + createTombstoneTable(ownerTombstoneTable, 3); + + { + INFO("A view cannot contain bound parameters, so its comparisons have to be literals"); + Builder::StatementBuilder createView; + createView.CreateTempView(viewName). + Select({ s_firstColumn, s_secondColumn }).From(s_tableName). + Where().NotExists().BeginParenthetical(). + Select(s_firstColumn).From(valueTombstoneTable).As(valueAlias). + Where(Builder::QualifiedColumn{ valueAlias, s_firstColumn }).Equals(Builder::QualifiedColumn{ s_tableName, s_firstColumn }). + And(Builder::QualifiedColumn{ valueAlias, removedColumn }).EqualsLiteral(1). + EndParenthetical(). + And().NotExists().BeginParenthetical(). + Select(s_firstColumn).From(ownerTombstoneTable).As(ownerAlias). + Where(Builder::QualifiedColumn{ ownerAlias, s_firstColumn }).Equals(Builder::QualifiedColumn{ s_tableName, s_firstColumn }). + And(Builder::QualifiedColumn{ ownerAlias, removedColumn }).EqualsLiteral(1). + EndParenthetical(); + createView.Execute(connection); + } + + INFO("Only the row that neither tombstone removes survives"); + Builder::StatementBuilder select; + select.Select({ s_firstColumn, s_secondColumn }).From(viewName).OrderBy(s_firstColumn); + + Statement statement = select.Prepare(connection); + + REQUIRE(statement.Step()); + REQUIRE(statement.GetColumn(0) == 1); + REQUIRE(statement.GetColumn(1) == "kept"); + + REQUIRE(!statement.Step()); +} + TEST_CASE("SQLiteWrapperTransactionRollback", "[sqlitewrapper]") { Connection connection = Connection::Create(SQLITE_MEMORY_DB_CONNECTION_TARGET, Connection::OpenDisposition::Create); @@ -854,3 +1158,47 @@ TEST_CASE("SQLiteWrapperTransactionWriteConflict", "[sqlitewrapper]") SelectFromSimpleTestTableOnlyOneRow(connection, firstVal, secondVal); } + +TEST_CASE("SQLiteDatabaseSpecifierTargets", "[sqlitewrapper]") +{ + // A disposition that SQLite can express with flags alone hands over the path untouched. + DatabaseSpecifier plain{ "D:\\test\\index.db"s, DatabaseDisposition::Read }; + REQUIRE(plain.Target() == "D:\\test\\index.db"); + REQUIRE(plain.ConnectionDisposition() == Connection::OpenDisposition::ReadOnly); + + // Immutability can only be asked for through a URI query parameter. + DatabaseSpecifier immutable{ "D:\\test\\index.db"s, DatabaseDisposition::Immutable }; + REQUIRE(immutable.Target() == "file:/D:/test/index.db?immutable=1"); + REQUIRE(immutable.ConnectionDisposition() == Connection::OpenDisposition::ReadOnly); + + // Characters that would otherwise start the query or fragment are escaped, and repeated + // separators collapse, per the conversion the URI documentation prescribes. + DatabaseSpecifier escaped{ "D:\\a#b\\\\c?d\\index.db"s, DatabaseDisposition::Immutable }; + REQUIRE(escaped.Target() == "file:/D:/a%23b/c%3fd/index.db?immutable=1"); + + // Every disposition enables URI handling, because a connection that did not ask for it cannot + // attach one later regardless of how the attached database is named. + REQUIRE(plain.ConnectionFlags() == Connection::OpenFlags::Uri); + REQUIRE(immutable.ConnectionFlags() == Connection::OpenFlags::Uri); +} + +TEST_CASE("SQLiteDatabaseSpecifierImmutableOpen", "[sqlitewrapper]") +{ + TestCommon::TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + int firstVal = 1; + std::string secondVal = "test"; + + { + Connection connection = Connection::Create(tempFile, Connection::OpenDisposition::Create); + + CreateSimpleTestTable(connection); + + InsertIntoSimpleTestTable(connection, firstVal, secondVal); + } + + Connection connection = Connection::Create(DatabaseSpecifier{ tempFile.GetPath().u8string(), DatabaseDisposition::Immutable }); + + SelectFromSimpleTestTableOnlyOneRow(connection, firstVal, secondVal); +} \ No newline at end of file diff --git a/src/AppInstallerCLITests/main.cpp b/src/AppInstallerCLITests/main.cpp index a91ca2ab11..77aaef8671 100644 --- a/src/AppInstallerCLITests/main.cpp +++ b/src/AppInstallerCLITests/main.cpp @@ -33,12 +33,22 @@ struct LoggingBreakListener : public Catch::EventListenerBase if (!testCaseStats.totals.delta(lastTotals).testCases.allOk()) { TestCommon::TempFile::SetTestFailed(true); + if (testCaseStats.testInfo) + { + GetFailedTests().push_back(testCaseStats.testInfo->name); + } } lastTotals = testCaseStats.totals; Catch::EventListenerBase::testCaseEnded(testCaseStats); } Catch::Totals lastTotals{}; + + static std::vector& GetFailedTests() + { + static std::vector failedTests; + return failedTests; + } }; CATCH_REGISTER_LISTENER(LoggingBreakListener); @@ -78,6 +88,7 @@ int main(int argc, char** argv) bool hasSetTestDataBasePath = false; bool waitBeforeReturn = false; bool keepSQLLogging = false; + bool printSummary = false; std::vector args; for (int i = 0; i < argc; ++i) @@ -132,6 +143,10 @@ int main(int argc, char** argv) Debugging::EnableSelfInitiatedMinidump(std::filesystem::path{ argv[i] }); } } + else if ("-summary"s == argv[i]) + { + printSummary = true; + } else { args.push_back(argv[i]); @@ -175,6 +190,15 @@ int main(int argc, char** argv) int result = Catch::Session().run(static_cast(args.size()), args.data()); + if (printSummary) + { + std::cout << "\nFailed tests:\n"; + for (const auto& test : LoggingBreakListener::GetFailedTests()) + { + std::cout << " " << test << "\n"; + } + } + if (waitBeforeReturn) { // Wait for some input before returning diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj index 3a5a39da6a..3f69f65c86 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -331,6 +331,10 @@ + + + + @@ -443,6 +447,10 @@ + + + + diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters index 64f1b186c2..2071579398 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -127,6 +127,9 @@ {b4c8d9e2-f3a5-6b7c-0d1e-2f3a4b5c6d7e} + + {b4b0b84d-5dea-42bb-817b-7cf269008f52} + @@ -519,6 +522,18 @@ Header Files + + Microsoft\Schema\2_1 + + + Microsoft\Schema\2_1 + + + Microsoft\Schema\2_1 + + + Microsoft\Schema\2_1 + @@ -812,6 +827,18 @@ Source Files + + Microsoft\Schema\2_1 + + + Microsoft\Schema\2_1 + + + Microsoft\Schema\2_1 + + + Microsoft\Schema\2_1 + diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp index 61247ebf5f..10d7a5699c 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp @@ -1,379 +1,431 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -#include "pch.h" -#include "SQLiteIndex.h" -#include -#include "ArpVersionValidation.h" -#include - -namespace AppInstaller::Repository::Microsoft -{ - namespace - { - size_t GetPageSizeFromOptions(SQLiteIndex::CreateOptions options) - { - return WI_IsFlagSet(options, SQLiteIndex::CreateOptions::LargePageSize) ? 65536 : 0; - } - } - - SQLiteIndex SQLiteIndex::CreateNew(const std::string& filePath, SQLite::Version version, CreateOptions options) - { - AICLI_LOG(Repo, Info, << "Creating new SQLite Index with version [" << version << "] at '" << filePath << "'"); - SQLiteIndex result{ filePath, version, options }; - - SQLite::Savepoint savepoint = SQLite::Savepoint::Create(result.m_dbconn, "sqliteindex_createnew"); - - // Use calculated version, as incoming version could be 'latest' - result.m_version.SetSchemaVersion(result.m_dbconn); - - result.m_interface->CreateTables(result.m_dbconn, options); - - result.SetLastWriteTime(); - - savepoint.Commit(); - - return result; - } - - SQLiteIndex SQLiteIndex::Open(const std::string& filePath, OpenDisposition disposition, Utility::ManagedFile&& indexFile) - { - return { filePath, disposition, std::move(indexFile) }; - } - - SQLiteIndex SQLiteIndex::CopyFrom(const std::string& filePath, SQLiteIndex& source) - { - return { filePath, source }; - } - - SQLiteIndex::SQLiteIndex(const std::string& target, const SQLite::Version& version, CreateOptions options) : SQLiteStorageBase(target, version, GetPageSizeFromOptions(options)) - { - m_dbconn.EnableICU(); - m_interface = Schema::CreateISQLiteIndex(version); - m_version = m_interface->GetVersion(); - SetDatabaseFilePath(target); - } - - SQLiteIndex::SQLiteIndex(const std::string& target, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile) : - SQLiteStorageBase(target, disposition, std::move(indexFile)) - { - m_dbconn.EnableICU(); - AICLI_LOG(Repo, Info, << "Opened SQLite Index with version [" << m_version << "], last write [" << GetLastWriteTime() << "]"); - m_interface = Schema::CreateISQLiteIndex(m_version); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_CANNOT_WRITE_TO_UPLEVEL_INDEX, disposition == SQLiteStorageBase::OpenDisposition::ReadWrite && m_version != m_interface->GetVersion()); - SetDatabaseFilePath(target); - } - - SQLiteIndex::SQLiteIndex(const std::string& target, SQLiteIndex& source) : - SQLiteStorageBase(target, source) - { - m_dbconn.EnableICU(); - m_interface = Schema::CreateISQLiteIndex(m_version); - SetDatabaseFilePath(target); - } - - void SQLiteIndex::SetDatabaseFilePath(const std::string& target) - { - if (target != SQLITE_MEMORY_DB_CONNECTION_TARGET) - { - m_contextData.Add(Utility::ConvertToUTF16(target)); - } - } - -#ifndef AICLI_DISABLE_TEST_HOOKS - void SQLiteIndex::ForceVersion(const SQLite::Version& version) - { - m_interface = Schema::CreateISQLiteIndex(version); - } - - SQLite::Version SQLiteIndex::GetLatestVersion() - { - return Schema::CreateISQLiteIndex(SQLite::Version::Latest())->GetVersion(); - } - - const Schema::SQLiteIndexContextData& SQLiteIndex::GetContextData() const - { - return m_contextData; - } -#endif - - SQLiteIndex::IdType SQLiteIndex::AddManifest(const std::filesystem::path& manifestPath, const std::filesystem::path& relativePath) - { - AICLI_LOG(Repo, Verbose, << "Adding manifest from file [" << manifestPath << "]"); - - Manifest::Manifest manifest = Manifest::YamlParser::CreateFromPath(manifestPath); - return AddManifestInternal(manifest, relativePath); - } - - SQLiteIndex::IdType SQLiteIndex::AddManifest(const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) - { - return AddManifestInternal(manifest, relativePath); - } - - SQLiteIndex::IdType SQLiteIndex::AddManifest(const Manifest::Manifest& manifest) - { - return AddManifestInternal(manifest, {}); - } - - SQLiteIndex::IdType SQLiteIndex::AddManifestInternal(const Manifest::Manifest& manifest, const std::optional& relativePath) - { - std::lock_guard lockInterface{ *m_interfaceLock }; - return AddManifestInternalHoldingLock(manifest, relativePath); - } - - SQLiteIndex::IdType SQLiteIndex::AddManifestInternalHoldingLock(const Manifest::Manifest& manifest, const std::optional& relativePath) - { - AICLI_LOG(Repo, Verbose, << "Adding manifest for [" << manifest.Id << ", " << manifest.Version << "] at relative path [" << relativePath.value_or("") << "]"); - - SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "sqliteindex_addmanifest"); - - IdType result = m_interface->AddManifest(m_dbconn, manifest, relativePath); - - SetLastWriteTime(); - - savepoint.Commit(); - - return result; - } - - bool SQLiteIndex::UpdateManifest(const std::filesystem::path& manifestPath, const std::filesystem::path& relativePath) - { - AICLI_LOG(Repo, Verbose, << "Updating manifest from file [" << manifestPath << "]"); - - Manifest::Manifest manifest = Manifest::YamlParser::CreateFromPath(manifestPath); - return UpdateManifestInternal(manifest, relativePath); - } - - bool SQLiteIndex::UpdateManifest(const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) - { - return UpdateManifestInternal(manifest, relativePath); - } - - bool SQLiteIndex::UpdateManifest(const Manifest::Manifest& manifest) - { - return UpdateManifestInternal(manifest, {}); - } - - bool SQLiteIndex::UpdateManifestInternal(const Manifest::Manifest& manifest, const std::optional& relativePath) - { - std::lock_guard lockInterface{ *m_interfaceLock }; - return UpdateManifestInternalHoldingLock(manifest, relativePath); - } - - bool SQLiteIndex::UpdateManifestInternalHoldingLock(const Manifest::Manifest& manifest, const std::optional& relativePath) - { - AICLI_LOG(Repo, Verbose, << "Updating manifest for [" << manifest.Id << ", " << manifest.Version << "] at relative path [" << relativePath.value_or("") << "]"); - - SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "sqliteindex_updatemanifest"); - - bool result = m_interface->UpdateManifest(m_dbconn, manifest, relativePath).first; - - if (result) - { - SetLastWriteTime(); - - savepoint.Commit(); - } - - return result; - } - - bool SQLiteIndex::AddOrUpdateManifest(const std::filesystem::path& manifestPath, const std::filesystem::path& relativePath) - { - AICLI_LOG(Repo, Verbose, << "Adding or Updating manifest from file [" << manifestPath << "]"); - - Manifest::Manifest manifest = Manifest::YamlParser::CreateFromPath(manifestPath); - return AddOrUpdateManifestInternal(manifest, relativePath); - } - - bool SQLiteIndex::AddOrUpdateManifest(const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) - { - return AddOrUpdateManifestInternal(manifest, relativePath); - } - - bool SQLiteIndex::AddOrUpdateManifest(const Manifest::Manifest& manifest) - { - return AddOrUpdateManifestInternal(manifest, {}); - } - - bool SQLiteIndex::AddOrUpdateManifestInternal(const Manifest::Manifest& manifest, const std::optional& relativePath) - { - std::lock_guard lockInterface{ *m_interfaceLock }; - AICLI_LOG(Repo, Verbose, << "Adding or Updating manifest for [" << manifest.Id << ", " << manifest.Version << "] at relative path [" << relativePath.value_or("") << "]"); - - if (m_interface->GetManifestIdByManifest(m_dbconn, manifest)) - { - UpdateManifestInternalHoldingLock(manifest, relativePath); - return false; - } - else - { - AddManifestInternalHoldingLock(manifest, relativePath); - return true; - } - } - - void SQLiteIndex::RemoveManifest(const std::filesystem::path& manifestPath, const std::filesystem::path& relativePath) - { - AICLI_LOG(Repo, Verbose, << "Removing manifest from file [" << manifestPath << "]"); - - Manifest::Manifest manifest = Manifest::YamlParser::CreateFromPath(manifestPath); - RemoveManifest(manifest, relativePath); - } - - void SQLiteIndex::RemoveManifest(const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) - { - AICLI_LOG(Repo, Verbose, << "Removing manifest for [" << manifest.Id << ", " << manifest.Version << "] at relative path [" << relativePath << "]"); - RemoveManifest(manifest); - } - - void SQLiteIndex::RemoveManifest(const Manifest::Manifest& manifest) - { - std::lock_guard lockInterface{ *m_interfaceLock }; - SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "sqliteindex_removemanifest"); - - m_interface->RemoveManifest(m_dbconn, manifest); - - SetLastWriteTime(); - - savepoint.Commit(); - } - - void SQLiteIndex::RemoveManifestById(IdType manifestId) - { - SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "SQLiteIndex_RemoveManifestById"); - - m_interface->RemoveManifestById(m_dbconn, manifestId); - - SetLastWriteTime(); - - savepoint.Commit(); - } - - void SQLiteIndex::PrepareForPackaging() - { - std::lock_guard lockInterface{ *m_interfaceLock }; - AICLI_LOG(Repo, Info, << "Preparing index for packaging"); - - m_interface->PrepareForPackaging(Schema::SQLiteIndexContext{ m_dbconn, m_contextData }); - } - - bool SQLiteIndex::CheckConsistency(bool log) const - { - std::lock_guard lockInterface{ *m_interfaceLock }; - AICLI_LOG(Repo, Info, << "Checking index consistency..."); - - bool result = m_interface->CheckConsistency(m_dbconn, log); - - AICLI_LOG(Repo, Info, << "...index *WAS" << (result ? "*" : " NOT*") << " consistent."); - - return result; - } - - Schema::ISQLiteIndex::SearchResult SQLiteIndex::Search(const SearchRequest& request) const - { - std::lock_guard lockInterface{ *m_interfaceLock }; - AICLI_LOG(Repo, Verbose, << "Performing search: " << request.ToString()); - - return m_interface->Search(m_dbconn, request); - } - - std::optional SQLiteIndex::GetPropertyByPrimaryId(IdType primaryId, PackageVersionProperty property) const - { - std::lock_guard lockInterface{ *m_interfaceLock }; - return m_interface->GetPropertyByPrimaryId(m_dbconn, primaryId, property); - } - - std::vector SQLiteIndex::GetMultiPropertyByPrimaryId(IdType primaryId, PackageVersionMultiProperty property) const - { - std::lock_guard lockInterface{ *m_interfaceLock }; - return m_interface->GetMultiPropertyByPrimaryId(m_dbconn, primaryId, property); - } - - std::optional SQLiteIndex::GetManifestIdByKey(IdType id, std::string_view version, std::string_view channel) const - { - std::lock_guard lockInterface{ *m_interfaceLock }; - return m_interface->GetManifestIdByKey(m_dbconn, id, version, channel); - } - - std::optional SQLiteIndex::GetManifestIdByManifest(const Manifest::Manifest& manifest) const - { - return m_interface->GetManifestIdByManifest(m_dbconn, manifest); - } - - std::vector SQLiteIndex::GetVersionKeysById(IdType id) const - { - std::lock_guard lockInterface{ *m_interfaceLock }; - return m_interface->GetVersionKeysById(m_dbconn, id); - } - - SQLiteIndex::MetadataResult SQLiteIndex::GetMetadataByManifestId(SQLite::rowid_t manifestId) const - { - std::lock_guard lockInterface{ *m_interfaceLock }; - return m_interface->GetMetadataByManifestId(m_dbconn, manifestId); - } - - void SQLiteIndex::SetMetadataByManifestId(IdType manifestId, PackageVersionMetadata metadata, std::string_view value) - { - std::lock_guard lockInterface{ *m_interfaceLock }; - m_interface->SetMetadataByManifestId(m_dbconn, manifestId, metadata, value); - } - - Utility::NormalizedName SQLiteIndex::NormalizeName(std::string_view name, std::string_view publisher) const - { - std::lock_guard lockInterface{ *m_interfaceLock }; - return m_interface->NormalizeName(name, publisher); - } - - std::set> SQLiteIndex::GetDependenciesByManifestRowId(SQLite::rowid_t manifestRowId) const - { - return m_interface->GetDependenciesByManifestRowId(m_dbconn, manifestRowId); - } - - std::vector> SQLiteIndex::GetDependentsById(AppInstaller::Manifest::string_t packageId) const - { - return m_interface->GetDependentsById(m_dbconn, packageId); - } - - bool SQLiteIndex::MigrateTo(SQLite::Version version) - { - std::lock_guard lockInterface{ *m_interfaceLock }; - SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "sqliteindex_migrate_to"); - - AICLI_LOG(Repo, Info, << "Attempting to migrate index from [" << m_interface->GetVersion() << "] to [" << version << "]..."); - std::unique_ptr newInterface = Schema::CreateISQLiteIndex(version); - - bool result = newInterface->MigrateFrom(m_dbconn, m_interface.get()); - - AICLI_LOG(Repo, Info, << "...migration was " << (result ? "" : "NOT ") << "successful"); - if (result) - { - version.SetSchemaVersion(m_dbconn); - SetLastWriteTime(); - savepoint.Commit(); - - m_version = version; - m_interface = std::move(newInterface); - } - - return result; - } - - void SQLiteIndex::SetProperty(Property property, const std::string& value) - { - std::lock_guard lockInterface{ *m_interfaceLock }; - - switch (property) - { - case Property::PackageUpdateTrackingBaseTime: - m_interface->SetProperty(m_dbconn, Schema::Property::PackageUpdateTrackingBaseTime, value); - break; - case Property::IntermediateFileOutputPath: - { - std::filesystem::path pathValue{ Utility::ConvertToUTF16(value) }; - THROW_HR_IF(E_INVALIDARG, pathValue.empty() || pathValue.is_relative()); - m_contextData.Add(std::move(pathValue)); - } - break; - } - } -} +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "SQLiteIndex.h" +#include +#include "ArpVersionValidation.h" +#include + +namespace AppInstaller::Repository::Microsoft +{ + namespace + { + size_t GetPageSizeFromOptions(SQLiteIndex::CreateOptions options) + { + return WI_IsFlagSet(options, SQLiteIndex::CreateOptions::LargePageSize) ? 65536 : 0; + } + } + + SQLiteIndex SQLiteIndex::CreateNew(const std::string& filePath, SQLite::Version version, CreateOptions options) + { + AICLI_LOG(Repo, Info, << "Creating new SQLite Index with version [" << version << "] at '" << filePath << "'"); + SQLiteIndex result{ filePath, version, options }; + + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(result.m_dbconn, "sqliteindex_createnew"); + + // Use calculated version, as incoming version could be 'latest' + result.m_version.SetSchemaVersion(result.m_dbconn); + + result.m_interface->CreateTables(result.m_dbconn, options); + + result.SetLastWriteTime(); + + savepoint.Commit(); + + return result; + } + + SQLiteIndex SQLiteIndex::Open(const std::string& filePath, OpenDisposition disposition, Utility::ManagedFile&& indexFile) + { + return { filePath, disposition, std::move(indexFile) }; + } + + SQLiteIndex SQLiteIndex::CopyFrom(const std::string& filePath, SQLiteIndex& source) + { + return { filePath, source }; + } + + SQLiteIndex SQLiteIndex::OpenWithBaseline(const std::string& deltaFilePath, const std::string& baselineFilePath, OpenDisposition disposition) + { + AICLI_LOG(Repo, Info, << "Opening delta index [" << deltaFilePath << "] with baseline [" << baselineFilePath << "]"); + + // The combined form is presented through views over a union of two databases, so there is + // nothing here that could be written back to. + THROW_HR_IF(E_INVALIDARG, disposition == OpenDisposition::ReadWrite); + + std::filesystem::path baselinePath{ Utility::ConvertToUTF16(baselineFilePath) }; + THROW_HR_IF(E_INVALIDARG, baselinePath.empty() || baselinePath.is_relative()); + + SQLiteIndex result{ SQLite::DatabaseSpecifier{ deltaFilePath, disposition }, {} }; + + // The interface for the delta's schema version establishes the combined view. + result.m_interface->SetupDeltaReadMode(result.m_dbconn, SQLite::DatabaseSpecifier{ baselineFilePath, disposition }); + + return result; + } + + SQLiteIndex::SQLiteIndex(const std::string& target, const SQLite::Version& version, CreateOptions options) : SQLiteStorageBase(target, version, GetPageSizeFromOptions(options)) + { + m_dbconn.EnableICU(); + m_interface = Schema::CreateISQLiteIndex(version); + m_version = m_interface->GetVersion(); + SetDatabaseFilePath(target); + } + + SQLiteIndex::SQLiteIndex(const std::string& target, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile) : + SQLiteIndex(SQLite::DatabaseSpecifier{ target, disposition }, std::move(indexFile)) + { + } + + SQLiteIndex::SQLiteIndex(const SQLite::DatabaseSpecifier& specifier, Utility::ManagedFile&& indexFile) : + SQLiteStorageBase(specifier, std::move(indexFile)) + { + m_dbconn.EnableICU(); + AICLI_LOG(Repo, Info, << "Opened SQLite Index with version [" << m_version << "], last write [" << GetLastWriteTime() << "]"); + m_interface = Schema::CreateISQLiteIndex(m_version); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_CANNOT_WRITE_TO_UPLEVEL_INDEX, specifier.Disposition() == SQLiteStorageBase::OpenDisposition::ReadWrite && m_version != m_interface->GetVersion()); + SetDatabaseFilePath(specifier.Path()); + } + + SQLiteIndex::SQLiteIndex(const std::string& target, SQLiteIndex& source) : + SQLiteStorageBase(target, source) + { + m_dbconn.EnableICU(); + m_interface = Schema::CreateISQLiteIndex(m_version); + SetDatabaseFilePath(target); + } + + void SQLiteIndex::SetDatabaseFilePath(const std::string& target) + { + if (target != SQLITE_MEMORY_DB_CONNECTION_TARGET) + { + m_contextData.Add(Utility::ConvertToUTF16(target)); + } + } + +#ifndef AICLI_DISABLE_TEST_HOOKS + void SQLiteIndex::ForceVersion(const SQLite::Version& version) + { + m_interface = Schema::CreateISQLiteIndex(version); + } + + SQLite::Version SQLiteIndex::GetLatestVersion() + { + return Schema::CreateISQLiteIndex(SQLite::Version::Latest())->GetVersion(); + } + + const Schema::SQLiteIndexContextData& SQLiteIndex::GetContextData() const + { + return m_contextData; + } +#endif + + SQLiteIndex::IdType SQLiteIndex::AddManifest(const std::filesystem::path& manifestPath, const std::filesystem::path& relativePath) + { + AICLI_LOG(Repo, Verbose, << "Adding manifest from file [" << manifestPath << "]"); + + Manifest::Manifest manifest = Manifest::YamlParser::CreateFromPath(manifestPath); + return AddManifestInternal(manifest, relativePath); + } + + SQLiteIndex::IdType SQLiteIndex::AddManifest(const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) + { + return AddManifestInternal(manifest, relativePath); + } + + SQLiteIndex::IdType SQLiteIndex::AddManifest(const Manifest::Manifest& manifest) + { + return AddManifestInternal(manifest, {}); + } + + SQLiteIndex::IdType SQLiteIndex::AddManifestInternal(const Manifest::Manifest& manifest, const std::optional& relativePath) + { + std::lock_guard lockInterface{ *m_interfaceLock }; + return AddManifestInternalHoldingLock(manifest, relativePath); + } + + SQLiteIndex::IdType SQLiteIndex::AddManifestInternalHoldingLock(const Manifest::Manifest& manifest, const std::optional& relativePath) + { + AICLI_LOG(Repo, Verbose, << "Adding manifest for [" << manifest.Id << ", " << manifest.Version << "] at relative path [" << relativePath.value_or("") << "]"); + + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "sqliteindex_addmanifest"); + + IdType result = m_interface->AddManifest(m_dbconn, manifest, relativePath); + + SetLastWriteTime(); + + savepoint.Commit(); + + return result; + } + + bool SQLiteIndex::UpdateManifest(const std::filesystem::path& manifestPath, const std::filesystem::path& relativePath) + { + AICLI_LOG(Repo, Verbose, << "Updating manifest from file [" << manifestPath << "]"); + + Manifest::Manifest manifest = Manifest::YamlParser::CreateFromPath(manifestPath); + return UpdateManifestInternal(manifest, relativePath); + } + + bool SQLiteIndex::UpdateManifest(const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) + { + return UpdateManifestInternal(manifest, relativePath); + } + + bool SQLiteIndex::UpdateManifest(const Manifest::Manifest& manifest) + { + return UpdateManifestInternal(manifest, {}); + } + + bool SQLiteIndex::UpdateManifestInternal(const Manifest::Manifest& manifest, const std::optional& relativePath) + { + std::lock_guard lockInterface{ *m_interfaceLock }; + return UpdateManifestInternalHoldingLock(manifest, relativePath); + } + + bool SQLiteIndex::UpdateManifestInternalHoldingLock(const Manifest::Manifest& manifest, const std::optional& relativePath) + { + AICLI_LOG(Repo, Verbose, << "Updating manifest for [" << manifest.Id << ", " << manifest.Version << "] at relative path [" << relativePath.value_or("") << "]"); + + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "sqliteindex_updatemanifest"); + + bool result = m_interface->UpdateManifest(m_dbconn, manifest, relativePath).first; + + if (result) + { + SetLastWriteTime(); + + savepoint.Commit(); + } + + return result; + } + + bool SQLiteIndex::AddOrUpdateManifest(const std::filesystem::path& manifestPath, const std::filesystem::path& relativePath) + { + AICLI_LOG(Repo, Verbose, << "Adding or Updating manifest from file [" << manifestPath << "]"); + + Manifest::Manifest manifest = Manifest::YamlParser::CreateFromPath(manifestPath); + return AddOrUpdateManifestInternal(manifest, relativePath); + } + + bool SQLiteIndex::AddOrUpdateManifest(const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) + { + return AddOrUpdateManifestInternal(manifest, relativePath); + } + + bool SQLiteIndex::AddOrUpdateManifest(const Manifest::Manifest& manifest) + { + return AddOrUpdateManifestInternal(manifest, {}); + } + + bool SQLiteIndex::AddOrUpdateManifestInternal(const Manifest::Manifest& manifest, const std::optional& relativePath) + { + std::lock_guard lockInterface{ *m_interfaceLock }; + AICLI_LOG(Repo, Verbose, << "Adding or Updating manifest for [" << manifest.Id << ", " << manifest.Version << "] at relative path [" << relativePath.value_or("") << "]"); + + if (m_interface->GetManifestIdByManifest(m_dbconn, manifest)) + { + UpdateManifestInternalHoldingLock(manifest, relativePath); + return false; + } + else + { + AddManifestInternalHoldingLock(manifest, relativePath); + return true; + } + } + + void SQLiteIndex::RemoveManifest(const std::filesystem::path& manifestPath, const std::filesystem::path& relativePath) + { + AICLI_LOG(Repo, Verbose, << "Removing manifest from file [" << manifestPath << "]"); + + Manifest::Manifest manifest = Manifest::YamlParser::CreateFromPath(manifestPath); + RemoveManifest(manifest, relativePath); + } + + void SQLiteIndex::RemoveManifest(const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) + { + AICLI_LOG(Repo, Verbose, << "Removing manifest for [" << manifest.Id << ", " << manifest.Version << "] at relative path [" << relativePath << "]"); + RemoveManifest(manifest); + } + + void SQLiteIndex::RemoveManifest(const Manifest::Manifest& manifest) + { + std::lock_guard lockInterface{ *m_interfaceLock }; + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "sqliteindex_removemanifest"); + + m_interface->RemoveManifest(m_dbconn, manifest); + + SetLastWriteTime(); + + savepoint.Commit(); + } + + void SQLiteIndex::RemoveManifestById(IdType manifestId) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "SQLiteIndex_RemoveManifestById"); + + m_interface->RemoveManifestById(m_dbconn, manifestId); + + SetLastWriteTime(); + + savepoint.Commit(); + } + + void SQLiteIndex::PrepareForPackaging() + { + std::lock_guard lockInterface{ *m_interfaceLock }; + AICLI_LOG(Repo, Info, << "Preparing index for packaging"); + + m_interface->PrepareForPackaging(Schema::SQLiteIndexContext{ m_dbconn, m_contextData }); + } + + void SQLiteIndex::MarkAsBaseline() + { + std::lock_guard lockInterface{ *m_interfaceLock }; + AICLI_LOG(Repo, Info, << "Marking index as a delta baseline"); + + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "sqliteindex_markasbaseline"); + + m_interface->MarkAsBaseline(m_dbconn); + + SetLastWriteTime(); + + savepoint.Commit(); + } + + bool SQLiteIndex::CheckConsistency(bool log) const + { + std::lock_guard lockInterface{ *m_interfaceLock }; + AICLI_LOG(Repo, Info, << "Checking index consistency..."); + + bool result = m_interface->CheckConsistency(m_dbconn, log); + + AICLI_LOG(Repo, Info, << "...index *WAS" << (result ? "*" : " NOT*") << " consistent."); + + return result; + } + + Schema::ISQLiteIndex::SearchResult SQLiteIndex::Search(const SearchRequest& request) const + { + std::lock_guard lockInterface{ *m_interfaceLock }; + AICLI_LOG(Repo, Verbose, << "Performing search: " << request.ToString()); + + return m_interface->Search(m_dbconn, request); + } + + std::optional SQLiteIndex::GetPropertyByPrimaryId(IdType primaryId, PackageVersionProperty property) const + { + std::lock_guard lockInterface{ *m_interfaceLock }; + return m_interface->GetPropertyByPrimaryId(m_dbconn, primaryId, property); + } + + std::vector SQLiteIndex::GetMultiPropertyByPrimaryId(IdType primaryId, PackageVersionMultiProperty property) const + { + std::lock_guard lockInterface{ *m_interfaceLock }; + return m_interface->GetMultiPropertyByPrimaryId(m_dbconn, primaryId, property); + } + + std::optional SQLiteIndex::GetManifestIdByKey(IdType id, std::string_view version, std::string_view channel) const + { + std::lock_guard lockInterface{ *m_interfaceLock }; + return m_interface->GetManifestIdByKey(m_dbconn, id, version, channel); + } + + std::optional SQLiteIndex::GetManifestIdByManifest(const Manifest::Manifest& manifest) const + { + return m_interface->GetManifestIdByManifest(m_dbconn, manifest); + } + + std::vector SQLiteIndex::GetVersionKeysById(IdType id) const + { + std::lock_guard lockInterface{ *m_interfaceLock }; + return m_interface->GetVersionKeysById(m_dbconn, id); + } + + SQLiteIndex::MetadataResult SQLiteIndex::GetMetadataByManifestId(SQLite::rowid_t manifestId) const + { + std::lock_guard lockInterface{ *m_interfaceLock }; + return m_interface->GetMetadataByManifestId(m_dbconn, manifestId); + } + + void SQLiteIndex::SetMetadataByManifestId(IdType manifestId, PackageVersionMetadata metadata, std::string_view value) + { + std::lock_guard lockInterface{ *m_interfaceLock }; + m_interface->SetMetadataByManifestId(m_dbconn, manifestId, metadata, value); + } + + Utility::NormalizedName SQLiteIndex::NormalizeName(std::string_view name, std::string_view publisher) const + { + std::lock_guard lockInterface{ *m_interfaceLock }; + return m_interface->NormalizeName(name, publisher); + } + + std::set> SQLiteIndex::GetDependenciesByManifestRowId(SQLite::rowid_t manifestRowId) const + { + return m_interface->GetDependenciesByManifestRowId(m_dbconn, manifestRowId); + } + + std::vector> SQLiteIndex::GetDependentsById(AppInstaller::Manifest::string_t packageId) const + { + return m_interface->GetDependentsById(m_dbconn, packageId); + } + + bool SQLiteIndex::MigrateTo(SQLite::Version version) + { + std::lock_guard lockInterface{ *m_interfaceLock }; + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "sqliteindex_migrate_to"); + + AICLI_LOG(Repo, Info, << "Attempting to migrate index from [" << m_interface->GetVersion() << "] to [" << version << "]..."); + std::unique_ptr newInterface = Schema::CreateISQLiteIndex(version); + + bool result = newInterface->MigrateFrom(m_dbconn, m_interface.get()); + + AICLI_LOG(Repo, Info, << "...migration was " << (result ? "" : "NOT ") << "successful"); + if (result) + { + version.SetSchemaVersion(m_dbconn); + SetLastWriteTime(); + savepoint.Commit(); + + m_version = version; + m_interface = std::move(newInterface); + } + + return result; + } + + void SQLiteIndex::SetProperty(Property property, const std::string& value) + { + std::lock_guard lockInterface{ *m_interfaceLock }; + + switch (property) + { + case Property::PackageUpdateTrackingBaseTime: + m_interface->SetProperty(m_dbconn, Schema::Property::PackageUpdateTrackingBaseTime, value); + break; + case Property::IntermediateFileOutputPath: + { + std::filesystem::path pathValue{ Utility::ConvertToUTF16(value) }; + THROW_HR_IF(E_INVALIDARG, pathValue.empty() || pathValue.is_relative()); + m_contextData.Add(std::move(pathValue)); + } + break; + case Property::DeltaBaselineIndexPath: + { + std::filesystem::path pathValue{ Utility::ConvertToUTF16(value) }; + THROW_HR_IF(E_INVALIDARG, pathValue.empty() || pathValue.is_relative()); + m_contextData.Add(std::move(pathValue)); + } + break; + case Property::DeltaOutputPath: + { + std::filesystem::path pathValue{ Utility::ConvertToUTF16(value) }; + THROW_HR_IF(E_INVALIDARG, pathValue.empty() || pathValue.is_relative()); + m_contextData.Add(std::move(pathValue)); + } + break; + } + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h index be63be9632..aa90e762a2 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h @@ -55,6 +55,12 @@ namespace AppInstaller::Repository::Microsoft // Creates a copy of the given index. static SQLiteIndex CopyFrom(const std::string& filePath, SQLiteIndex& source); + // Opens a delta index combined with its baseline for reading. + // The delta is the main connection; the baseline is ATTACHed and TEMP VIEWs are created + // so that existing search code operates transparently across both. + // The disposition applies to both files, because the pair is only meaningful as a unit. + static SQLiteIndex OpenWithBaseline(const std::string& deltaFilePath, const std::string& baselineFilePath, OpenDisposition disposition = OpenDisposition::Read); + #ifndef AICLI_DISABLE_TEST_HOOKS // Changes the version of the interface being used to operate on the database. // Should only be used for testing. @@ -121,6 +127,11 @@ namespace AppInstaller::Repository::Microsoft // Removes data that is no longer needed for an index that is to be published. void PrepareForPackaging(); + // Designates this index as a baseline that delta indexes may be generated against. + // Should be called on an index that has been prepared for packaging, as that is the form + // that will be published and that a delta will later be paired with. + void MarkAsBaseline(); + // Checks the consistency of the index to ensure that every referenced row exists. // Returns true if index is consistent; false if it is not. bool CheckConsistency(bool log = false) const; @@ -167,6 +178,8 @@ namespace AppInstaller::Repository::Microsoft { PackageUpdateTrackingBaseTime, IntermediateFileOutputPath, + DeltaBaselineIndexPath, + DeltaOutputPath, }; // Sets the given property. @@ -180,6 +193,8 @@ namespace AppInstaller::Repository::Microsoft // Constructor used to open an existing index. SQLiteIndex(const std::string& target, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile); + SQLiteIndex(const SQLite::DatabaseSpecifier& specifier, Utility::ManagedFile&& indexFile); + // Constructor used to copy the given index. SQLiteIndex(const std::string& target, SQLiteIndex& source); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/Interface.h index 361fc28e8d..f74a86bf5b 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/Interface.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/Interface.h @@ -4,6 +4,7 @@ #include "Microsoft/Schema/ISQLiteIndex.h" #include "Microsoft/Schema/2_0/SearchResultsTable.h" #include "Microsoft/Schema/2_0/OneToManyTableWithMap.h" +#include "Microsoft/Schema/2_0/PackageUpdateTrackingTable.h" #include #include @@ -74,6 +75,12 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 // Prepares for packaging, optionally vacuuming the database. virtual void PrepareForPackaging(const SQLiteIndexContext& context, bool vacuum); + // Extends PrepareForPackaging at the point where the 2.0 tables have been populated but + // the update tracking and internal 1.7 tables have not yet been dropped. That is the only + // window in which both the packaged and the pre-packaged forms of the data exist, so any + // output that must correlate the two has to be produced here. Does nothing by default. + virtual void CreateAdditionalPackagingOutput(const SQLiteIndexContext& context); + // Force the database to shrink the file size. // This *must* be done outside of an active transaction. void Vacuum(const SQLite::Connection& connection); @@ -89,6 +96,13 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 // If EnsureInternalInterface has been called. mutable bool m_internalInterfaceChecked = false; + // Determines how the removal of a package is recorded in the update tracking table. + PackageUpdateTrackingTable::RemovalBehavior m_trackingRemovalBehavior = PackageUpdateTrackingTable::RemovalBehavior::Delete; + + // Set when the tables that this interface reads are the merged views over a delta and its + // baseline rather than tables of this database. + mutable bool m_isDeltaReadMode = false; + // Interface to the data before PrepareForPackaging is called. mutable std::unique_ptr m_internalInterface; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/Interface_2_0.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/Interface_2_0.cpp index 99c25fe319..c3974e5b7b 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/Interface_2_0.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/Interface_2_0.cpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "pch.h" #include +#include #include "Microsoft/Schema/2_0/Interface.h" #include "Microsoft/Schema/2_0/PackagesTable.h" @@ -16,6 +17,7 @@ #include "Microsoft/Schema/2_0/SearchResultsTable.h" #include "Microsoft/Schema/2_0/PackageUpdateTrackingTable.h" +#include "Microsoft/Schema/1_0/IdTable.h" #include @@ -105,7 +107,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 { EnsureInternalInterface(connection, true); SQLite::rowid_t manifestId = m_internalInterface->AddManifest(connection, manifest, relativePath); - PackageUpdateTrackingTable::Update(connection, m_internalInterface.get(), m_internalInterface->GetPropertyByPrimaryId(connection, manifestId, PackageVersionProperty::Id).value()); + PackageUpdateTrackingTable::Update(connection, m_internalInterface.get(), m_internalInterface->GetPropertyByPrimaryId(connection, manifestId, PackageVersionProperty::Id).value(), m_trackingRemovalBehavior); return manifestId; } @@ -115,7 +117,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 std::pair result = m_internalInterface->UpdateManifest(connection, manifest, relativePath); if (result.first) { - PackageUpdateTrackingTable::Update(connection, m_internalInterface.get(), m_internalInterface->GetPropertyByPrimaryId(connection, result.second, PackageVersionProperty::Id).value()); + PackageUpdateTrackingTable::Update(connection, m_internalInterface.get(), m_internalInterface->GetPropertyByPrimaryId(connection, result.second, PackageVersionProperty::Id).value(), m_trackingRemovalBehavior); } return result; } @@ -138,10 +140,21 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 { EnsureInternalInterface(connection, true); std::optional identifier = m_internalInterface->GetPropertyByPrimaryId(connection, manifestId, PackageVersionProperty::Id); + + // Resolve the rowid the package occupies while it is still present. If this removes its + // last version the ids row goes with it, and the value becomes unrecoverable; the tracking + // table needs it in order to record which rowid was vacated. + std::optional packageRowId; + + if (identifier && m_trackingRemovalBehavior == PackageUpdateTrackingTable::RemovalBehavior::Record) + { + packageRowId = V1_0::IdTable::SelectIdByValue(connection, identifier.value(), true); + } + m_internalInterface->RemoveManifestById(connection, manifestId); if (identifier) { - PackageUpdateTrackingTable::Update(connection, m_internalInterface.get(), identifier.value()); + PackageUpdateTrackingTable::Update(connection, m_internalInterface.get(), identifier.value(), m_trackingRemovalBehavior, true, packageRowId); } } @@ -172,7 +185,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 if (m_internalInterface) { AICLI_CHECK_CONSISTENCY(m_internalInterface->CheckConsistency(connection, log)); - AICLI_CHECK_CONSISTENCY(PackageUpdateTrackingTable::CheckConsistency(connection, m_internalInterface.get(), log)); + AICLI_CHECK_CONSISTENCY(PackageUpdateTrackingTable::CheckConsistency(connection, m_internalInterface.get(), m_trackingRemovalBehavior, log)); return result; } @@ -396,14 +409,14 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "migrate_from_v2_0"); // We only need to insert all of the existing packages into the update tracking table. - PackageUpdateTrackingTable::EnsureExists(connection); + PackageUpdateTrackingTable::EnsureExists(connection, m_trackingRemovalBehavior); SearchResult allPackages = current->Search(connection, {}); for (const auto& packageMatch : allPackages.Matches) { std::vector versionKeys = current->GetVersionKeysById(connection, packageMatch.first); ISQLiteIndex::VersionKey& latestVersionKey = versionKeys[0]; - PackageUpdateTrackingTable::Update(connection, current, current->GetPropertyByPrimaryId(connection, latestVersionKey.ManifestId, PackageVersionProperty::Id).value(), false); + PackageUpdateTrackingTable::Update(connection, current, current->GetPropertyByPrimaryId(connection, latestVersionKey.ManifestId, PackageVersionProperty::Id).value(), m_trackingRemovalBehavior, false); } savepoint.Commit(); @@ -434,6 +447,10 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 } } + void Interface::CreateAdditionalPackagingOutput(const SQLiteIndexContext&) + { + } + std::unique_ptr Interface::CreateSearchResultsTable(const SQLite::Connection& connection) const { return std::make_unique(connection); @@ -644,7 +661,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 THROW_WIN32_IF(ERROR_INVALID_STATE, baseOutputDirectory.empty() || baseOutputDirectory.is_relative()); // Output all of the changed package version manifests since the base time to the target location - for (const auto& packageData : PackageUpdateTrackingTable::GetUpdatesSince(connection, updateBaseTime)) + for (const auto& packageData : PackageUpdateTrackingTable::GetUpdatesSince(connection, updateBaseTime, m_trackingRemovalBehavior)) { std::filesystem::path packageDirectory = baseOutputDirectory / Manifest::PackageVersionDataManifest::GetRelativeDirectoryPath(packageData.PackageIdentifier, Utility::SHA256::ConvertToString(packageData.Hash)); @@ -712,9 +729,12 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 addIfPresent(PackagesTable::ARPMinVersionColumn::Name, m_internalInterface->GetPropertyByPrimaryId(connection, latestVersionKey.ManifestId, PackageVersionProperty::ArpMinVersion).value()); addIfPresent(PackagesTable::ARPMaxVersionColumn::Name, m_internalInterface->GetPropertyByPrimaryId(connection, latestVersionKey.ManifestId, PackageVersionProperty::ArpMaxVersion).value()); - SQLite::rowid_t packageId = PackagesTable::Insert(connection, packageData); + auto idRowId = V1_0::IdTable::SelectIdByValue(connection, packageIdentifier); + THROW_HR_IF(E_NOT_VALID_STATE, !idRowId); - PackagesTable::UpdateValueIdById(connection, packageId, PackageUpdateTrackingTable::GetDataHash(connection, packageIdentifier)); + SQLite::rowid_t packageId = PackagesTable::Insert(connection, packageData, idRowId); + + PackagesTable::UpdateValueIdById(connection, packageId, PackageUpdateTrackingTable::GetDataHash(connection, packageIdentifier, m_trackingRemovalBehavior)); for (const auto& versionKey : versionKeys) { @@ -729,6 +749,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 } } + CreateAdditionalPackagingOutput(context); + PackagesTable::PrepareForPackaging< PackagesTable::IdColumn, PackagesTable::NameColumn, @@ -770,7 +792,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 { if (!m_internalInterfaceChecked) { - if (!PackagesTable::Exists(connection)) + // In delta read mode the TEMP VIEWs are already set up; no internal interface needed. + if (!m_isDeltaReadMode && !PackagesTable::Exists(connection)) { m_internalInterface = CreateInternalInterface(); } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/OneToManyTableWithMap.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/OneToManyTableWithMap.cpp index 76f650db82..2739282285 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/OneToManyTableWithMap.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/OneToManyTableWithMap.cpp @@ -18,6 +18,18 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 static constexpr std::string_view s_OneToManyTableWithMap_MapTable_IndexSuffix = "_index"sv; static constexpr std::string_view s_OneToManyTableWithMap_PrimaryKeyIndexSuffix = "_pkindex"sv; + std::string_view OneToManyTableGetManifestColumnName() + { + return s_OneToManyTableWithMap_MapTable_PrimaryName; + } + + std::string OneToManyTableGetMapTableName(std::string_view tableName) + { + std::string result{ tableName }; + result += s_OneToManyTableWithMap_MapTable_Suffix; + return result; + } + namespace anon { // Create the mapping table insert statement for multiple use. diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackageUpdateTrackingTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackageUpdateTrackingTable.cpp index cf253fd8a1..9539ad2dbb 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackageUpdateTrackingTable.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackageUpdateTrackingTable.cpp @@ -2,8 +2,10 @@ // Licensed under the MIT License. #include "pch.h" #include "PackageUpdateTrackingTable.h" +#include "Microsoft/Schema/1_0/IdTable.h" #include #include +#include using namespace AppInstaller::SQLite; @@ -12,17 +14,171 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 using namespace std::string_view_literals; static constexpr std::string_view s_PUTT_Table_Name = "update_tracking"sv; static constexpr std::string_view s_PUTT_WriteTimeIndex_Name = "update_tracking_write_idx"sv; + static constexpr std::string_view s_PUTT_LiveIndex_Name = "update_tracking_live_idx"sv; static constexpr std::string_view s_PUTT_Package = "package"sv; static constexpr std::string_view s_PUTT_WriteTime = "write_time"sv; static constexpr std::string_view s_PUTT_Manifest = "manifest"sv; static constexpr std::string_view s_PUTT_Hash = "hash"sv; + static constexpr std::string_view s_PUTT_IsRemoved = "is_removed"sv; + static constexpr std::string_view s_PUTT_PackageRowId = "package_rowid"sv; + static constexpr std::string_view s_PUTT_ChangeSequence = "change_seq"sv; + static constexpr std::string_view s_PUTT_ChangeSequenceIndex_Name = "update_tracking_change_seq_idx"sv; + + namespace + { + // Finds the rowid that the package occupies in the index. + std::optional GetPackageRowIdInIndex(const SQLite::Connection& connection, const std::string& packageIdentifier) + { + return V1_0::IdTable::SelectIdByValue(connection, packageIdentifier, true); + } + + // The tombstones, as the identifier recorded for the package and the rowid it vacated. + std::vector> GetRemovedRows( + const SQLite::Connection& connection, + PackageUpdateTrackingTable::RemovalBehavior removals) + { + std::vector> result; + + if (removals == PackageUpdateTrackingTable::RemovalBehavior::Delete) + { + return result; + } + + Builder::StatementBuilder builder; + builder.Select({ s_PUTT_Package, s_PUTT_PackageRowId }).From(s_PUTT_Table_Name). + Where(s_PUTT_IsRemoved).Equals(1); + + Statement statement = builder.Prepare(connection); + + while (statement.Step()) + { + result.emplace_back(statement.GetColumn(0), statement.GetColumn(1)); + } + + return result; + } + + // The sequence to stamp on the row about to be written. + int64_t GetNextChangeSequence(const SQLite::Connection& connection) + { + Builder::StatementBuilder builder; + builder.Select().Column(Builder::Aggregate::Max, s_PUTT_ChangeSequence).From(s_PUTT_Table_Name); + + Statement statement = builder.Prepare(connection); + + // The aggregate produces a single row holding null when the table is empty. + if (statement.Step() && !statement.GetColumnIsNull(0)) + { + return statement.GetColumn(0) + 1; + } + + return 1; + } + + // The rows written after the given point, as measured by the given column. + // The boundary is exclusive for the sequence and inclusive for the write time. + std::vector GetUpdates( + const SQLite::Connection& connection, + std::string_view boundaryColumn, + int64_t boundaryValue, + bool exclusive, + PackageUpdateTrackingTable::RemovalBehavior removals) + { + bool recordingRemovals = (removals == PackageUpdateTrackingTable::RemovalBehavior::Record); + + Builder::StatementBuilder builder; + + if (recordingRemovals) + { + builder.Select({ RowIDName, s_PUTT_Package, s_PUTT_WriteTime, s_PUTT_Manifest, s_PUTT_Hash, s_PUTT_PackageRowId }); + } + else + { + builder.Select({ RowIDName, s_PUTT_Package, s_PUTT_WriteTime, s_PUTT_Manifest, s_PUTT_Hash }); + } + + builder.From(s_PUTT_Table_Name).Where(boundaryColumn); + + if (exclusive) + { + builder.IsGreaterThan(boundaryValue); + } + else + { + builder.IsGreaterThanOrEqualTo(boundaryValue); + } + + if (recordingRemovals) + { + // Removals are reported separately, so that this remains the set of packages that + // have data to write out, exactly as it is when removals delete their row. + builder.And(s_PUTT_IsRemoved).Equals(0); + } + + Statement select = builder.Prepare(connection); + + std::vector result; + + while (select.Step()) + { + PackageUpdateTrackingTable::PackageData item; + item.RowID = select.GetColumn(0); + item.PackageIdentifier = select.GetColumn(1); + item.WriteTime = select.GetColumn(2); + item.Manifest = select.GetColumn(3); + item.Hash = select.GetColumn(4); + + if (recordingRemovals) + { + item.PackageRowId = select.GetColumn(5); + } + + result.emplace_back(std::move(item)); + } + + return result; + } + + // The rowids vacated after the given point, as measured by the given column. + std::set GetRemovals( + const SQLite::Connection& connection, + std::string_view boundaryColumn, + int64_t boundaryValue, + bool exclusive) + { + Builder::StatementBuilder builder; + builder.Select(s_PUTT_PackageRowId).From(s_PUTT_Table_Name).Where(boundaryColumn); + + if (exclusive) + { + builder.IsGreaterThan(boundaryValue); + } + else + { + builder.IsGreaterThanOrEqualTo(boundaryValue); + } + + builder.And(s_PUTT_IsRemoved).Equals(1); + + Statement select = builder.Prepare(connection); + + std::set result; + + while (select.Step()) + { + result.emplace(select.GetColumn(0)); + } + + return result; + } + } std::string_view PackageUpdateTrackingTable::TableName() { return s_PUTT_Table_Name; } - void PackageUpdateTrackingTable::Create(SQLite::Connection& connection) + void PackageUpdateTrackingTable::Create(SQLite::Connection& connection, RemovalBehavior removals) { using namespace Builder; @@ -32,8 +188,15 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 builder.Column(IntegerPrimaryKey()); builder.Column(ColumnBuilder(s_PUTT_Package, Type::Text).NotNull()); builder.Column(ColumnBuilder(s_PUTT_WriteTime, Type::Int64).NotNull()); - builder.Column(ColumnBuilder(s_PUTT_Manifest, Type::Blob).NotNull()); - builder.Column(ColumnBuilder(s_PUTT_Hash, Type::Blob).NotNull()); + builder.Column(ColumnBuilder(s_PUTT_Manifest, Type::Blob)); + builder.Column(ColumnBuilder(s_PUTT_Hash, Type::Blob)); + + if (removals == RemovalBehavior::Record) + { + builder.Column(ColumnBuilder(s_PUTT_IsRemoved, Type::Int64).NotNull().Default(0)); + builder.Column(ColumnBuilder(s_PUTT_PackageRowId, Type::Int64).NotNull()); + builder.Column(ColumnBuilder(s_PUTT_ChangeSequence, Type::Int64).NotNull()); + } builder.EndColumns(); @@ -42,13 +205,48 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 StatementBuilder indexBuilder; indexBuilder.CreateIndex(s_PUTT_WriteTimeIndex_Name).On(s_PUTT_Table_Name).Columns(s_PUTT_WriteTime); indexBuilder.Execute(connection); + + if (removals == RemovalBehavior::Record) + { + CreateLiveRowIndex(connection); + CreateChangeSequenceIndex(connection); + } + } + + void PackageUpdateTrackingTable::CreateChangeSequenceIndex(SQLite::Connection& connection) + { + // Serves both the range scan that reports changes and the maximum that allocates the next + // sequence. Not unique: a row updated in place takes a new sequence and leaves none behind, + // but nothing depends on two rows never sharing one, and the migration backfills every + // existing row with the same value. + Builder::StatementBuilder builder; + builder.CreateIndex(s_PUTT_ChangeSequenceIndex_Name).On(s_PUTT_Table_Name).Columns(s_PUTT_ChangeSequence); + builder.Execute(connection); + } + + void PackageUpdateTrackingTable::CreateLiveRowIndex(SQLite::Connection& connection) + { + // A package occupies exactly one rowid at a time, so at most one row per rowid can be + // live. Tombstones are excluded because a rowid vacated by one package can be taken by + // another, leaving the old package's tombstone and the new package's live row sharing it. + // + // The constraint is deliberately on the rowid rather than the identifier: a unique index + // cannot use LIKE, and no available collation matches it. NOCASE is ASCII-only, while + // LIKE here is the ICU implementation registered by sqlite3IcuInit, so NOCASE would + // disagree with every other accessor on non-ASCII identifiers. An ICU collation cannot be + // used either, since it would bake the ICU version into a published index file and make + // the table unreadable to any connection that had not registered it. + Builder::StatementBuilder builder; + builder.CreateUniqueIndex(s_PUTT_LiveIndex_Name).On(s_PUTT_Table_Name).Columns(s_PUTT_PackageRowId). + Where(s_PUTT_IsRemoved).EqualsLiteral(0); + builder.Execute(connection); } - void PackageUpdateTrackingTable::EnsureExists(SQLite::Connection& connection) + void PackageUpdateTrackingTable::EnsureExists(SQLite::Connection& connection, RemovalBehavior removals) { if (!Exists(connection)) { - Create(connection); + Create(connection, removals); } } @@ -71,11 +269,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 return statement.GetColumn(0) != 0; } - void PackageUpdateTrackingTable::Update(SQLite::Connection& connection, const ISQLiteIndex* internalIndex, const std::string& packageIdentifier, bool ensureTable) + void PackageUpdateTrackingTable::Update(SQLite::Connection& connection, const ISQLiteIndex* internalIndex, const std::string& packageIdentifier, RemovalBehavior removals, bool ensureTable, std::optional removedPackageRowId) { if (ensureTable) { - EnsureExists(connection); + EnsureExists(connection, removals); } SearchRequest request; @@ -84,11 +282,46 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 if (result.Matches.empty()) { - // Remove any existing package update row - Builder::StatementBuilder deleteBuilder; - deleteBuilder.DeleteFrom(s_PUTT_Table_Name).Where(s_PUTT_Package).LikeWithEscape(packageIdentifier); + if (removals == RemovalBehavior::Delete) + { + // Remove any existing package update row + Builder::StatementBuilder deleteBuilder; + deleteBuilder.DeleteFrom(s_PUTT_Table_Name).Where(s_PUTT_Package).LikeWithEscape(packageIdentifier); - deleteBuilder.Execute(connection); + deleteBuilder.Execute(connection); + } + else + { + // Mark the package as removed rather than deleting the row, clearing the data columns. + // The row is found by the rowid the package occupied rather than by its identifier. + THROW_HR_IF(E_NOT_VALID_STATE, !removedPackageRowId); + + int64_t currentTime = Utility::GetCurrentUnixEpoch(); + int64_t changeSequence = GetNextChangeSequence(connection); + + Builder::StatementBuilder updateBuilder; + updateBuilder.Update(s_PUTT_Table_Name).Set(). + Column(s_PUTT_WriteTime).Equals(currentTime). + Column(s_PUTT_ChangeSequence).Equals(changeSequence). + Column(s_PUTT_Manifest).AssignValue(nullptr). + Column(s_PUTT_Hash).AssignValue(nullptr). + Column(s_PUTT_IsRemoved).Equals(1). + Where(s_PUTT_PackageRowId).Equals(removedPackageRowId.value()). + And(s_PUTT_IsRemoved).Equals(0); + updateBuilder.Execute(connection); + + if (connection.GetChanges() == 0) + { + // The package was added and removed without an intervening tracking checkpoint, + // so there is no row to mark. Record the removal so that a delta built against an + // older baseline still learns that the rowid was vacated. + Builder::StatementBuilder insertBuilder; + insertBuilder.InsertInto(s_PUTT_Table_Name). + Columns({ s_PUTT_Package, s_PUTT_WriteTime, s_PUTT_IsRemoved, s_PUTT_PackageRowId, s_PUTT_ChangeSequence }). + Values(packageIdentifier, currentTime, 1, removedPackageRowId.value(), changeSequence); + insertBuilder.Execute(connection); + } + } } else { @@ -120,34 +353,91 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 Utility::SHA256::HashBuffer manifestHash = Utility::SHA256::ComputeHash(compressedManifest); int64_t currentTime = Utility::GetCurrentUnixEpoch(); + // The rowid the package occupies is the identity that a delta is keyed on, so it is + // recorded alongside the data. It is resolved here rather than at removal time because + // the package is still in the index at this point. + SQLite::rowid_t packageRowId = 0; + int64_t changeSequence = 0; + + if (removals == RemovalBehavior::Record) + { + std::optional indexRowId = GetPackageRowIdInIndex(connection, packageIdentifier); + THROW_HR_IF(E_NOT_VALID_STATE, !indexRowId); + packageRowId = indexRowId.value(); + changeSequence = GetNextChangeSequence(connection); + } + // First attempt to update the row and then insert it if no modification occurred. Builder::StatementBuilder updateBuilder; updateBuilder.Update(s_PUTT_Table_Name).Set(). Column(s_PUTT_WriteTime).Equals(currentTime). Column(s_PUTT_Manifest).Equals(compressedManifest). - Column(s_PUTT_Hash).Equals(manifestHash). - Where(s_PUTT_Package).LikeWithEscape(packageIdentifier); + Column(s_PUTT_Hash).Equals(manifestHash); + + if (removals == RemovalBehavior::Record) + { + // Clear the flag in case this package was previously removed and is now being re-added. + updateBuilder.Column(s_PUTT_IsRemoved).Equals(0); + updateBuilder.Column(s_PUTT_ChangeSequence).Equals(changeSequence); + } + + updateBuilder.Where(s_PUTT_Package).LikeWithEscape(packageIdentifier); + + if (removals == RemovalBehavior::Record) + { + // Match on the rowid as well, so that a re-add only revives the tombstone that + // refers to the rowid the package now occupies. A package removed and re-added + // elsewhere leaves its old tombstone intact, which is what tells a delta to + // suppress the baseline row it still holds. Restricting by rowid also prevents + // this update from overwriting a different package's tombstone that happens to + // name the same rowid. + updateBuilder.And(s_PUTT_PackageRowId).Equals(packageRowId); + } updateBuilder.Execute(connection); if (connection.GetChanges() == 0) { Builder::StatementBuilder insertBuilder; - insertBuilder.InsertInto(s_PUTT_Table_Name). - Columns({ s_PUTT_Package, s_PUTT_WriteTime, s_PUTT_Manifest, s_PUTT_Hash }). - Values(packageIdentifier, currentTime, compressedManifest, manifestHash); + insertBuilder.InsertInto(s_PUTT_Table_Name).BeginColumns(); + + insertBuilder.Column(s_PUTT_Package); + insertBuilder.Column(s_PUTT_WriteTime); + insertBuilder.Column(s_PUTT_Manifest); + insertBuilder.Column(s_PUTT_Hash); + + if (removals == RemovalBehavior::Record) + { + insertBuilder.Column(s_PUTT_PackageRowId); + insertBuilder.Column(s_PUTT_ChangeSequence); + } + + insertBuilder.EndColumns().BeginValues(); + + insertBuilder.Value(packageIdentifier); + insertBuilder.Value(currentTime); + insertBuilder.Value(compressedManifest); + insertBuilder.Value(manifestHash); + + if (removals == RemovalBehavior::Record) + { + insertBuilder.Value(packageRowId); + insertBuilder.Value(changeSequence); + } + + insertBuilder.EndValues(); insertBuilder.Execute(connection); } } } - bool PackageUpdateTrackingTable::CheckConsistency(const SQLite::Connection& connection, ISQLiteIndex* internalIndex, bool log) + bool PackageUpdateTrackingTable::CheckConsistency(const SQLite::Connection& connection, ISQLiteIndex* internalIndex, RemovalBehavior removals, bool log) { bool result = true; // Ensure that all data in the update table matches the internal index - for (const PackageData& packageData : GetUpdatesSince(connection, 0)) + for (const PackageData& packageData : GetUpdatesSince(connection, 0, removals)) { auto manifestHash = Utility::SHA256::ComputeHash(packageData.Manifest); if (!Utility::SHA256::AreEqual(packageData.Hash, manifestHash)) @@ -178,10 +468,56 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 } } + // A package recorded as removed must no longer occupy the rowid it vacated. + for (const auto& [packageIdentifier, vacatedRowId] : GetRemovedRows(connection, removals)) + { + std::optional indexRowId = GetPackageRowIdInIndex(connection, packageIdentifier); + + if (indexRowId && indexRowId.value() == vacatedRowId) + { + if (!log) + { + return false; + } + + result = false; + AICLI_LOG(Repo, Info, << " [INVALID] value [" << s_PUTT_Package << "] in table [" << s_PUTT_Table_Name << + "]; the package [" << packageIdentifier << "] is marked as having vacated rowid [" << vacatedRowId << + "] but still occupies it in the internal index"); + } + } + + // Every live row must name the rowid that the package actually occupies. + if (removals == RemovalBehavior::Record) + { + for (const PackageData& packageData : GetUpdatesSince(connection, 0, removals)) + { + std::optional indexRowId = GetPackageRowIdInIndex(connection, packageData.PackageIdentifier); + + if (!indexRowId || indexRowId.value() != packageData.PackageRowId) + { + if (!log) + { + return false; + } + + result = false; + AICLI_LOG(Repo, Info, << " [INVALID] value [" << s_PUTT_PackageRowId << "] in table [" << s_PUTT_Table_Name << + "] at row [" << packageData.RowID << "]; the package [" << packageData.PackageIdentifier << "] records rowid [" << + packageData.PackageRowId << "] but occupies [" << (indexRowId ? std::to_string(indexRowId.value()) : "none") << "]"); + } + } + } + // Ensure that all packages in the internal index are present in the update table Builder::StatementBuilder builder; builder.Select(Builder::RowCount).From(s_PUTT_Table_Name).Where(s_PUTT_Package).Like(Builder::Unbound).Escape(EscapeCharForLike); + if (removals == RemovalBehavior::Record) + { + builder.And(s_PUTT_IsRemoved).Equals(0); + } + Statement select = builder.Prepare(connection); for (const auto& packageMatch : internalIndex->Search(connection, {}).Matches) @@ -203,47 +539,135 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 } result = false; - AICLI_LOG(Repo, Info, << " [INVALID] value [" << packageIdentifier << "] in the internal index was not found in [" << s_PUTT_Table_Name << "]"); + AICLI_LOG(Repo, Info, << " [INVALID] value [" << packageIdentifier << "] in the internal index was not found as a non-removed entry in [" << s_PUTT_Table_Name << "]"); } } return result; } - std::vector PackageUpdateTrackingTable::GetUpdatesSince(const SQLite::Connection& connection, int64_t updateBaseTime) + std::vector PackageUpdateTrackingTable::GetUpdatesSince(const SQLite::Connection& connection, int64_t updateBaseTime, RemovalBehavior removals) { - Builder::StatementBuilder builder; - builder.Select({ RowIDName, s_PUTT_Package, s_PUTT_WriteTime, s_PUTT_Manifest, s_PUTT_Hash }). - From(s_PUTT_Table_Name).Where(s_PUTT_WriteTime).IsGreaterThanOrEqualTo(updateBaseTime); + return GetUpdates(connection, s_PUTT_WriteTime, updateBaseTime, false, removals); + } - Statement select = builder.Prepare(connection); + std::vector PackageUpdateTrackingTable::GetUpdatesSinceSequence(const SQLite::Connection& connection, int64_t baseSequence, RemovalBehavior removals) + { + // Only the recording form has the column, and only a delta asks this question. + THROW_HR_IF(E_NOT_VALID_STATE, removals != RemovalBehavior::Record); - std::vector result; + return GetUpdates(connection, s_PUTT_ChangeSequence, baseSequence, true, removals); + } - while (select.Step()) + std::set PackageUpdateTrackingTable::GetRemovalsSince(const SQLite::Connection& connection, int64_t updateBaseTime, RemovalBehavior removals) + { + if (removals == RemovalBehavior::Delete) { - PackageData item; - item.RowID = select.GetColumn(0); - item.PackageIdentifier = select.GetColumn(1); - item.WriteTime = select.GetColumn(2); - item.Manifest = select.GetColumn(3); - item.Hash = select.GetColumn(4); + // Removals delete their row, so there is nothing to report. + return {}; + } + + return GetRemovals(connection, s_PUTT_WriteTime, updateBaseTime, false); + } + + std::set PackageUpdateTrackingTable::GetRemovalsSinceSequence(const SQLite::Connection& connection, int64_t baseSequence, RemovalBehavior removals) + { + THROW_HR_IF(E_NOT_VALID_STATE, removals != RemovalBehavior::Record); + + return GetRemovals(connection, s_PUTT_ChangeSequence, baseSequence, true); + } + + int64_t PackageUpdateTrackingTable::GetCurrentChangeSequence(const SQLite::Connection& connection, RemovalBehavior removals) + { + THROW_HR_IF(E_NOT_VALID_STATE, removals != RemovalBehavior::Record); - result.emplace_back(std::move(item)); + if (!Exists(connection)) + { + // The table is created on demand, so an index to which no manifest has ever been + // written has none. Nothing has been recorded, so nothing has been sequenced. + return 0; } - return result; + return GetNextChangeSequence(connection) - 1; } - SQLite::blob_t PackageUpdateTrackingTable::GetDataHash(const SQLite::Connection& connection, const std::string& packageIdentifier) + SQLite::blob_t PackageUpdateTrackingTable::GetDataHash(const SQLite::Connection& connection, const std::string& packageIdentifier, RemovalBehavior removals) { Builder::StatementBuilder builder; builder.Select(s_PUTT_Hash).From(s_PUTT_Table_Name).Where(s_PUTT_Package).LikeWithEscape(packageIdentifier); + if (removals == RemovalBehavior::Record) + { + // A tombstone has its hash cleared, so restrict to the live row or this would be + // liable to return a null hash for a package that was removed and re-added. + builder.And(s_PUTT_IsRemoved).Equals(0); + } + Statement select = builder.Prepare(connection); THROW_HR_IF(E_NOT_SET, !select.Step()); return select.GetColumn(0); } + + void PackageUpdateTrackingTable::AddRemovalTrackingColumns(SQLite::Connection& connection) + { + // The table is created on demand, so an index that has never had a manifest written + // to it will not have one yet. It will be created with the columns when it is needed. + if (!Exists(connection)) + { + return; + } + + Builder::StatementBuilder isRemovedBuilder; + isRemovedBuilder.AlterTable(s_PUTT_Table_Name).Add(Builder::ColumnBuilder(s_PUTT_IsRemoved, Builder::Type::Int64).NotNull().Default(0)); + isRemovedBuilder.Execute(connection); + + Builder::StatementBuilder packageRowIdBuilder; + packageRowIdBuilder.AlterTable(s_PUTT_Table_Name).Add(Builder::ColumnBuilder(s_PUTT_PackageRowId, Builder::Type::Int64).NotNull().Default(0)); + packageRowIdBuilder.Execute(connection); + + // Every existing row is backfilled with the same sequence, and the next one issued is 1. + Builder::StatementBuilder changeSequenceBuilder; + changeSequenceBuilder.AlterTable(s_PUTT_Table_Name).Add(Builder::ColumnBuilder(s_PUTT_ChangeSequence, Builder::Type::Int64).NotNull().Default(0)); + changeSequenceBuilder.Execute(connection); + + // Backfill the rowid for the rows already present. Every one of them is live: 2.0 deletes + // the row when a package is removed, so a table being migrated has no tombstones and every + // package it names is still in the index. + Builder::StatementBuilder selectBuilder; + selectBuilder.Select({ RowIDName, s_PUTT_Package }).From(s_PUTT_Table_Name); + + std::vector> rows; + + { + Statement select = selectBuilder.Prepare(connection); + + while (select.Step()) + { + rows.emplace_back(select.GetColumn(0), select.GetColumn(1)); + } + } + + Builder::StatementBuilder updateBuilder; + updateBuilder.Update(s_PUTT_Table_Name).Set(). + Column(s_PUTT_PackageRowId).Equals(Builder::Unbound). + Where(RowIDName).Equals(Builder::Unbound); + + Statement update = updateBuilder.Prepare(connection); + + for (const auto& row : rows) + { + std::optional packageRowId = GetPackageRowIdInIndex(connection, row.second); + THROW_HR_IF(E_NOT_VALID_STATE, !packageRowId); + + update.Reset(); + update.Bind(1, packageRowId.value()); + update.Bind(2, row.first); + update.Execute(); + } + + CreateLiveRowIndex(connection); + CreateChangeSequenceIndex(connection); + } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackageUpdateTrackingTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackageUpdateTrackingTable.h index 5cf429bd1f..2cc518cb62 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackageUpdateTrackingTable.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackageUpdateTrackingTable.h @@ -11,14 +11,27 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 // only the necessary package manifests. struct PackageUpdateTrackingTable { + // Determines how the removal of a package is reflected in the table. + enum class RemovalBehavior + { + Delete, + Record, + }; + // Get the table name. static std::string_view TableName(); // Creates the table. - static void Create(SQLite::Connection& connection); + static void Create(SQLite::Connection& connection, RemovalBehavior removals); + + // Creates the unique index that allows at most one live row per package rowid. + static void CreateLiveRowIndex(SQLite::Connection& connection); + + // Creates the index over the change sequence. + static void CreateChangeSequenceIndex(SQLite::Connection& connection); // Creates the table if it does not exist. - static void EnsureExists(SQLite::Connection& connection); + static void EnsureExists(SQLite::Connection& connection, RemovalBehavior removals); // Drops the table. static void Drop(SQLite::Connection& connection); @@ -27,11 +40,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 static bool Exists(const SQLite::Connection& connection); // Updates the tracking table for the given package identifier in the internal index. - static void Update(SQLite::Connection& connection, const ISQLiteIndex* internalIndex, const std::string& packageIdentifier, bool ensureTable = true); + static void Update(SQLite::Connection& connection, const ISQLiteIndex* internalIndex, const std::string& packageIdentifier, RemovalBehavior removals, bool ensureTable = true, std::optional removedPackageRowId = {}); // Checks the consistency of the index to ensure that every referenced row exists. // Returns true if index is consistent; false if it is not. - static bool CheckConsistency(const SQLite::Connection& connection, ISQLiteIndex* internalIndex, bool log); + static bool CheckConsistency(const SQLite::Connection& connection, ISQLiteIndex* internalIndex, RemovalBehavior removals, bool log); // Data on a single row in the table. struct PackageData @@ -41,12 +54,34 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 int64_t WriteTime = 0; SQLite::blob_t Manifest; SQLite::blob_t Hash; + // The rowid the package occupies in the index, or 0 when it is not known. + SQLite::rowid_t PackageRowId = 0; }; // Gets the data on updates that have been written since the given base time. - static std::vector GetUpdatesSince(const SQLite::Connection& connection, int64_t updateBaseTime); + // Removed packages are never included; use GetRemovalsSince to retrieve those. + static std::vector GetUpdatesSince(const SQLite::Connection& connection, int64_t updateBaseTime, RemovalBehavior removals); + + // Gets the rowids vacated by packages removed since the given base time. + // Only meaningful when removals are being recorded; always empty otherwise. + static std::set GetRemovalsSince(const SQLite::Connection& connection, int64_t updateBaseTime, RemovalBehavior removals); + + // Gets the data on updates written after the given change sequence, exclusive. + // Only available when removals are being recorded, as only then does the column exist. + static std::vector GetUpdatesSinceSequence(const SQLite::Connection& connection, int64_t baseSequence, RemovalBehavior removals); + + // Gets the rowids vacated by packages removed after the given change sequence, exclusive. + // Only available when removals are being recorded, as only then does the column exist. + static std::set GetRemovalsSinceSequence(const SQLite::Connection& connection, int64_t baseSequence, RemovalBehavior removals); + + // Gets the most recently issued change sequence, or 0 if nothing has been written. + // Only available when removals are being recorded, as only then does the column exist. + static int64_t GetCurrentChangeSequence(const SQLite::Connection& connection, RemovalBehavior removals); // Gets the data hash for the given package identifier. - static SQLite::blob_t GetDataHash(const SQLite::Connection& connection, const std::string& packageIdentifier); + static SQLite::blob_t GetDataHash(const SQLite::Connection& connection, const std::string& packageIdentifier, RemovalBehavior removals); + + // Adds the columns needed to record removals to an existing table that does not have them. + static void AddRemovalTrackingColumns(SQLite::Connection& connection); }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackagesTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackagesTable.cpp index c58a8bfb0b..a30c92c0e4 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackagesTable.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackagesTable.cpp @@ -213,11 +213,16 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 savepoint.Commit(); } - SQLite::rowid_t PackagesTable::Insert(SQLite::Connection& connection, const std::vector& values) + SQLite::rowid_t PackagesTable::Insert(SQLite::Connection& connection, const std::vector& values, std::optional rowid) { SQLite::Builder::StatementBuilder builder; builder.InsertInto(s_PackagesTable_Table_Name).BeginColumns(); + if (rowid) + { + builder.Column(SQLite::RowIDName); + } + for (const NameValuePair& value : values) { builder.Column(value.Name); @@ -225,6 +230,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 builder.EndColumns().BeginValues(); + if (rowid) + { + builder.Value(rowid.value()); + } + for (const NameValuePair& value : values) { builder.Value(value.Value); @@ -234,7 +244,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 builder.Execute(connection); - return connection.GetLastInsertRowID(); + return rowid ? rowid.value() : connection.GetLastInsertRowID(); } bool PackagesTable::ExistsById(const SQLite::Connection& connection, SQLite::rowid_t id) diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackagesTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackagesTable.h index cb879092b9..7f44f400b9 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackagesTable.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackagesTable.h @@ -144,7 +144,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 }; // Insert the given values into the table. - static SQLite::rowid_t Insert(SQLite::Connection& connection, const std::vector& values); + static SQLite::rowid_t Insert(SQLite::Connection& connection, const std::vector& values, std::optional rowid = std::nullopt); // Gets a value indicating whether the package with rowid exists. static bool ExistsById(const SQLite::Connection& connection, SQLite::rowid_t rowid); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.cpp new file mode 100644 index 0000000000..1cdae2edd7 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.cpp @@ -0,0 +1,395 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/Schema/2_1/DeltaGeneration.h" +#include "Microsoft/Schema/2_1/DeltaTables.h" +#include "Microsoft/Schema/2_1/Interface.h" + +#include "Microsoft/Schema/2_0/PackagesTable.h" +#include "Microsoft/Schema/2_0/OneToManyTableWithMap.h" +#include "Microsoft/Schema/2_0/SystemReferenceStringTable.h" + +#include +#include +#include + + +namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta +{ + using namespace SQLite::Builder; + + namespace + { + struct DeltaDatabase : public SQLite::SQLiteStorageBase + { + DeltaDatabase(const std::filesystem::path& path, const SQLite::Version& version) : + SQLiteStorageBase(path.u8string(), version) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "delta_createdatabase_v2_1"); + + m_version.SetSchemaVersion(m_dbconn); + CreateTables(m_dbconn); + SetLastWriteTime(); + + savepoint.Commit(); + } + + SQLite::Connection& GetConnection() { return m_dbconn; } + }; + + // Gets the rowid of the given package identifier in the packages table, if it is present. + std::optional SelectPackageRowId(const SQLite::Connection& connection, const std::string& packageIdentifier) + { + StatementBuilder builder; + builder.Select(SQLite::RowIDName).From(V2_0::PackagesTable::TableName()). + Where(V2_0::PackagesTable::IdColumn::Name).LikeWithEscape(packageIdentifier); + + SQLite::Statement statement = builder.Prepare(connection); + + if (statement.Step()) + { + return statement.GetColumn(0); + } + + return {}; + } + + // Gets the identifier of the package at the given rowid, if there is one. + std::optional SelectPackageIdByRowId(const SQLite::Connection& connection, SQLite::rowid_t packageRowId) + { + StatementBuilder builder; + builder.Select(V2_0::PackagesTable::IdColumn::Name).From(V2_0::PackagesTable::TableName()). + Where(SQLite::RowIDName).Equals(packageRowId); + + SQLite::Statement statement = builder.Prepare(connection); + + if (statement.Step()) + { + return statement.GetColumn(0); + } + + return {}; + } + + // Gets the rowid of the given value in a one to many data table, if it is present. + std::optional SelectValueRowId( + const SQLite::Connection& connection, + std::string_view tableName, + std::string_view valueName, + const std::string& value) + { + StatementBuilder builder; + builder.Select(SQLite::RowIDName).From(tableName).Where(valueName).Equals(value); + + SQLite::Statement statement = builder.Prepare(connection); + + if (statement.Step()) + { + return statement.GetColumn(0); + } + + return {}; + } + + // Gets the largest rowid in the given table, or 0 when it is empty. + SQLite::rowid_t GetMaximumRowId(const SQLite::Connection& connection, std::string_view tableName) + { + StatementBuilder builder; + builder.Select().Column(Aggregate::Max, SQLite::RowIDName).From(tableName); + + SQLite::Statement statement = builder.Prepare(connection); + + // The aggregate produces a single row holding null when the table is empty. + if (statement.Step() && !statement.GetColumnIsNull(0)) + { + return statement.GetColumn(0); + } + + return 0; + } + + // Determines the values that are in the first collection but not the second. + std::vector GetValuesNotIn(const std::vector& values, const std::vector& other) + { + std::vector result; + + for (const std::string& value : values) + { + if (std::find(other.begin(), other.end(), value) == other.end()) + { + result.emplace_back(value); + } + } + + return result; + } + + // Records that the package with the given identifier is no longer present. + void WriteRemovedPackage(SQLite::Connection& deltaConnection, SQLite::rowid_t packageRowId, const std::string& packageIdentifier) + { + std::string tableName = GetTableName(V2_0::PackagesTable::TableName()); + + StatementBuilder builder; + builder.InsertInto(tableName). + Columns({ SQLite::RowIDName, V2_0::PackagesTable::IdColumn::Name, IsRemovedColumnName() }). + Values(packageRowId, packageIdentifier, 1); + + builder.Execute(deltaConnection); + } + + // Copies the current state of a package into the delta. + void WriteChangedPackage(SQLite::Connection& deltaConnection, const SQLite::Connection& sourceConnection, SQLite::rowid_t packageRowId) + { + auto [id, name, moniker, latestVersion, arpMinVersion, arpMaxVersion, hash] = + V2_0::PackagesTable::GetValuesById< + V2_0::PackagesTable::IdColumn, + V2_0::PackagesTable::NameColumn, + V2_0::PackagesTable::MonikerColumn, + V2_0::PackagesTable::LatestVersionColumn, + V2_0::PackagesTable::ARPMinVersionColumn, + V2_0::PackagesTable::ARPMaxVersionColumn, + V2_0::PackagesTable::HashColumn + >(sourceConnection, packageRowId); + + std::string tableName = GetTableName(V2_0::PackagesTable::TableName()); + + StatementBuilder builder; + builder.InsertInto(tableName). + Columns({ + SQLite::RowIDName, + V2_0::PackagesTable::IdColumn::Name, + V2_0::PackagesTable::NameColumn::Name, + V2_0::PackagesTable::MonikerColumn::Name, + V2_0::PackagesTable::LatestVersionColumn::Name, + V2_0::PackagesTable::ARPMinVersionColumn::Name, + V2_0::PackagesTable::ARPMaxVersionColumn::Name, + V2_0::PackagesTable::HashColumn::Name, + IsRemovedColumnName() }). + Values(packageRowId, id, name, moniker, latestVersion, arpMinVersion, arpMaxVersion, hash, 0); + + builder.Execute(deltaConnection); + } + + // Records a single system reference value as added or removed for a package. + void WriteSystemReferenceValue( + SQLite::Connection& deltaConnection, + const ValueTableInfo& table, + const std::string& value, + SQLite::rowid_t packageRowId, + bool isRemoved) + { + std::string tableName = GetTableName(table.TableName); + + StatementBuilder builder; + builder.InsertInto(tableName). + Columns({ table.ValueName, V2_0::details::SystemReferenceStringTableGetPrimaryColumnName(), IsRemovedColumnName() }). + Values(value, packageRowId, isRemoved ? 1 : 0); + + builder.Execute(deltaConnection); + } + + // Records only the system reference values that changed for the package. + void WriteSystemReferenceDifference( + SQLite::Connection& deltaConnection, + const SQLite::Connection& sourceConnection, + const SQLite::Connection& baselineConnection, + const ValueTableInfo& table, + SQLite::rowid_t packageRowId) + { + std::vector currentValues = V2_0::details::SystemReferenceStringTableGetValuesByPrimaryId( + sourceConnection, table.TableName, table.ValueName, packageRowId); + std::vector baselineValues = V2_0::details::SystemReferenceStringTableGetValuesByPrimaryId( + baselineConnection, table.TableName, table.ValueName, packageRowId); + + for (const std::string& value : GetValuesNotIn(currentValues, baselineValues)) + { + WriteSystemReferenceValue(deltaConnection, table, value, packageRowId, false); + } + + for (const std::string& value : GetValuesNotIn(baselineValues, currentValues)) + { + WriteSystemReferenceValue(deltaConnection, table, value, packageRowId, true); + } + } + + // Gets a rowid that identifies the value in the merged data table, creating one if needed. + SQLite::rowid_t EnsureValueRowId( + SQLite::Connection& deltaConnection, + const SQLite::Connection& baselineConnection, + const ValueTableInfo& table, + const std::string& value, + SQLite::rowid_t& nextValueRowId) + { + std::optional baselineRowId = SelectValueRowId(baselineConnection, table.TableName, table.ValueName, value); + + if (baselineRowId) + { + return baselineRowId.value(); + } + + std::string tableName = GetTableName(table.TableName); + + std::optional deltaRowId = SelectValueRowId(deltaConnection, tableName, table.ValueName, value); + + if (deltaRowId) + { + return deltaRowId.value(); + } + + SQLite::rowid_t newRowId = ++nextValueRowId; + + StatementBuilder builder; + builder.InsertInto(tableName). + Columns({ SQLite::RowIDName, table.ValueName }). + Values(newRowId, value); + + builder.Execute(deltaConnection); + + return newRowId; + } + + // Records a single map entry as added or removed for a package. + void WriteOneToManyValue( + SQLite::Connection& deltaConnection, + const ValueTableInfo& table, + SQLite::rowid_t valueRowId, + SQLite::rowid_t packageRowId, + bool isRemoved) + { + std::string mapTableName = GetMapTableName(table.TableName); + + StatementBuilder builder; + builder.InsertInto(mapTableName). + Columns({ table.ValueName, V2_0::details::OneToManyTableGetManifestColumnName(), IsRemovedColumnName() }). + Values(valueRowId, packageRowId, isRemoved ? 1 : 0); + + builder.Execute(deltaConnection); + } + + // Records only the map entries that changed for the package. + void WriteOneToManyDifference( + SQLite::Connection& deltaConnection, + const SQLite::Connection& sourceConnection, + const SQLite::Connection& baselineConnection, + const ValueTableInfo& table, + SQLite::rowid_t packageRowId, + SQLite::rowid_t& nextValueRowId) + { + std::vector currentValues = V2_0::details::OneToManyTableWithMapGetValuesByPrimaryId( + sourceConnection, table.TableName, table.ValueName, packageRowId); + std::vector baselineValues = V2_0::details::OneToManyTableWithMapGetValuesByPrimaryId( + baselineConnection, table.TableName, table.ValueName, packageRowId); + + for (const std::string& value : GetValuesNotIn(currentValues, baselineValues)) + { + SQLite::rowid_t valueRowId = EnsureValueRowId(deltaConnection, baselineConnection, table, value, nextValueRowId); + WriteOneToManyValue(deltaConnection, table, valueRowId, packageRowId, false); + } + + for (const std::string& value : GetValuesNotIn(baselineValues, currentValues)) + { + // The value came from the baseline's own data table, so it must be found there. + std::optional valueRowId = SelectValueRowId(baselineConnection, table.TableName, table.ValueName, value); + THROW_HR_IF(E_NOT_VALID_STATE, !valueRowId); + + WriteOneToManyValue(deltaConnection, table, valueRowId.value(), packageRowId, true); + } + } + } + + void Generate( + const SQLite::Connection& sourceConnection, + const SQLite::Connection& baselineConnection, + const std::filesystem::path& deltaOutputPath, + const SQLite::Version& version, + const std::vector& changedPackages, + const std::set& removedPackages) + { + AICLI_LOG(Repo, Info, << "Generating delta index at [" << deltaOutputPath << "] for " << changedPackages.size() << + " changed and " << removedPackages.size() << " removed packages"); + + // A delta is only meaningful alongside the exact baseline it was computed from, so the + // baseline has to be one that was designated as such and can therefore be named. + std::optional baselineIdentifier = + SQLite::MetadataTable::TryGetNamedValue(baselineConnection, s_MetadataValueName_BaselineIdentifier); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_INDEX_INTEGRITY_COMPROMISED, !baselineIdentifier || baselineIdentifier->empty()); + + DeltaDatabase deltaDatabase{ deltaOutputPath, version }; + SQLite::Connection& deltaConnection = deltaDatabase.GetConnection(); + + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(deltaConnection, "delta_generate_v2_1"); + + SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselineIdentifier, baselineIdentifier.value()); + + std::map nextValueRowIds; + + for (const auto& table : OneToManyTables()) + { + nextValueRowIds[table.TableName] = GetMaximumRowId(baselineConnection, table.TableName); + } + + // Every rowid already written to the delta's packages table. The changed packages are + // written first so that a removal can tell whether the rowid it is about to vacate has + // since been taken, and each removal joins the set so that two tombstones resolving to one + // baseline rowid cannot both be written. + std::set writtenRowIds; + + for (const auto& package : changedPackages) + { + // The rowid comes from the source rather than the baseline so that packages new to this + // delta are covered by the same lookup; rowid stability is what makes the two agree. + std::optional packageRowId = SelectPackageRowId(sourceConnection, package.PackageIdentifier); + THROW_HR_IF(E_NOT_VALID_STATE, !packageRowId); + + AICLI_LOG(Repo, Verbose, << "Delta: recording change to [" << package.PackageIdentifier << "] (rowid " << packageRowId.value() << ")"); + + writtenRowIds.insert(packageRowId.value()); + WriteChangedPackage(deltaConnection, sourceConnection, packageRowId.value()); + + for (const auto& table : SystemReferenceTables()) + { + WriteSystemReferenceDifference(deltaConnection, sourceConnection, baselineConnection, table, packageRowId.value()); + } + + for (const auto& table : OneToManyTables()) + { + WriteOneToManyDifference(deltaConnection, sourceConnection, baselineConnection, table, packageRowId.value(), nextValueRowIds[table.TableName]); + } + } + + for (SQLite::rowid_t removedRowId : removedPackages) + { + // The rowid is resolved against the baseline directly, which is exact and is a primary + // key lookup. Whatever identifier the tracking table recorded is irrelevant here: what + // the delta suppresses is the baseline row at this rowid, so that row is also where the + // identifier stored alongside the tombstone comes from. + std::optional baselinePackageId = SelectPackageIdByRowId(baselineConnection, removedRowId); + + if (!baselinePackageId) + { + // The rowid was allocated after the baseline was produced, so as far as the + // baseline is concerned it never held anything and there is nothing to suppress. + AICLI_LOG(Repo, Verbose, << "Delta: rowid " << removedRowId << " was vacated but is not in the baseline"); + continue; + } + + if (writtenRowIds.count(removedRowId)) + { + // The rowid has already been written by a package that has since taken it.. + AICLI_LOG(Repo, Verbose, << "Delta: rowid " << removedRowId << " was vacated but has already been written"); + continue; + } + + AICLI_LOG(Repo, Verbose, << "Delta: recording removal of [" << baselinePackageId.value() << "] (rowid " << removedRowId << ")"); + + WriteRemovedPackage(deltaConnection, removedRowId, baselinePackageId.value()); + } + + savepoint.Commit(); + + // Outside the savepoint, since this vacuums. + PrepareTablesForPackaging(deltaConnection); + + AICLI_LOG(Repo, Info, << "Delta index generation complete"); + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.h new file mode 100644 index 0000000000..d6fcda5b7a --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.h @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "Microsoft/Schema/2_0/PackageUpdateTrackingTable.h" +#include +#include +#include +#include +#include +#include + + +namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta +{ + // Writes a delta database describing the difference between a baseline index and the index + // that is currently being packaged. + // + // This must run while the index being packaged still holds its update tracking table, as that + // is the only record of which packages have changed since the baseline was produced. + // + // The source and the baseline assign the same rowid to a given package, so a package that + // exists in both is described by rows that carry its baseline rowid, and a package that is new + // to the source carries a rowid that the baseline cannot have used. + // + // The version is recorded as the delta's own schema version, so that opening the delta selects + // the interface that knows how to merge it with a baseline. + void Generate( + const SQLite::Connection& sourceConnection, + const SQLite::Connection& baselineConnection, + const std::filesystem::path& deltaOutputPath, + const SQLite::Version& version, + const std::vector& changedPackages, + const std::set& removedPackages); +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaTables.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaTables.cpp new file mode 100644 index 0000000000..c35f97744d --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaTables.cpp @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/Schema/2_1/DeltaTables.h" + +#include "Microsoft/Schema/2_0/PackagesTable.h" +#include "Microsoft/Schema/2_0/TagsTable.h" +#include "Microsoft/Schema/2_0/CommandsTable.h" +#include "Microsoft/Schema/2_0/PackageFamilyNameTable.h" +#include "Microsoft/Schema/2_0/ProductCodeTable.h" +#include "Microsoft/Schema/2_0/NormalizedPackageNameTable.h" +#include "Microsoft/Schema/2_0/NormalizedPackagePublisherTable.h" +#include "Microsoft/Schema/2_0/UpgradeCodeTable.h" + +#include + + +namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta +{ + using namespace std::string_view_literals; + + namespace + { + constexpr std::string_view s_Delta_TablePrefix = "delta_"sv; + constexpr std::string_view s_Delta_MapTableSuffix = "_map"sv; + constexpr std::string_view s_Delta_ValueIndexSuffix = "_pkindex"sv; + constexpr std::string_view s_Delta_IsRemovedColumn = "is_removed"sv; + + template + ValueTableInfo MakeValueTableInfo() + { + return { Table::TableName(), Table::ValueName() }; + } + } + + std::vector SystemReferenceTables() + { + return std::vector{ + MakeValueTableInfo(), + MakeValueTableInfo(), + MakeValueTableInfo(), + MakeValueTableInfo(), + MakeValueTableInfo(), + }; + } + + std::vector OneToManyTables() + { + return std::vector{ + MakeValueTableInfo(), + MakeValueTableInfo(), + }; + } + + std::string GetTableName(std::string_view baseTableName) + { + auto result = std::string{ s_Delta_TablePrefix }; + result.append(baseTableName); + return result; + } + + std::string GetMapTableName(std::string_view baseTableName) + { + auto result = GetTableName(baseTableName); + result.append(s_Delta_MapTableSuffix); + return result; + } + + std::string_view IsRemovedColumnName() + { + return s_Delta_IsRemovedColumn; + } + + void CreateTables(SQLite::Connection& connection) + { + using namespace SQLite::Builder; + + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "delta_createtables_v2_1"); + + // The packages table mirrors its 2.0 counterpart, with the addition of the removal flag. + // Every column other than the identifier is nullable here, because a row that records a + // removal carries no data beyond the identity of what was removed. + { + std::string tableName = GetTableName(V2_0::PackagesTable::TableName()); + + StatementBuilder builder; + builder.CreateTable(tableName).Columns({ + IntegerPrimaryKey(), + ColumnBuilder(V2_0::PackagesTable::IdColumn::Name, Type::Text).NotNull(), + ColumnBuilder(V2_0::PackagesTable::NameColumn::Name, Type::Text), + ColumnBuilder(V2_0::PackagesTable::MonikerColumn::Name, Type::Text), + ColumnBuilder(V2_0::PackagesTable::LatestVersionColumn::Name, Type::Text), + ColumnBuilder(V2_0::PackagesTable::ARPMinVersionColumn::Name, Type::Text), + ColumnBuilder(V2_0::PackagesTable::ARPMaxVersionColumn::Name, Type::Text), + ColumnBuilder(V2_0::PackagesTable::HashColumn::Name, Type::Blob), + ColumnBuilder(s_Delta_IsRemovedColumn, Type::Int64).NotNull() + }); + builder.Execute(connection); + } + + // The system reference tables hold the value itself, so the delta only adds the removal flag. + for (const auto& table : SystemReferenceTables()) + { + std::string tableName = GetTableName(table.TableName); + + StatementBuilder builder; + builder.CreateTable(tableName).Columns({ + ColumnBuilder(table.ValueName, Type::Text).NotNull(), + ColumnBuilder(V2_0::details::SystemReferenceStringTableGetPrimaryColumnName(), Type::RowId).NotNull(), + ColumnBuilder(s_Delta_IsRemovedColumn, Type::Int64).NotNull(), + PrimaryKeyBuilder({ table.ValueName, V2_0::details::SystemReferenceStringTableGetPrimaryColumnName() }) + }).WithoutRowID(); + builder.Execute(connection); + } + + for (const auto& table : OneToManyTables()) + { + // The data table holds only values that the baseline does not already have. There is no + // removal flag because a value is only unreferenced once every map entry naming it is + // removed, which the map table already records. + { + std::string tableName = GetTableName(table.TableName); + + StatementBuilder builder; + builder.CreateTable(tableName).Columns({ + IntegerPrimaryKey(), + ColumnBuilder(table.ValueName, Type::Text).NotNull() + }); + builder.Execute(connection); + + // Generation looks values up by string to reuse an already allocated rowid. + StatementBuilder indexBuilder; + indexBuilder.CreateUniqueIndex({ tableName, s_Delta_ValueIndexSuffix }). + On(tableName).Columns(table.ValueName); + indexBuilder.Execute(connection); + } + + { + std::string mapTableName = GetMapTableName(table.TableName); + + StatementBuilder builder; + builder.CreateTable(mapTableName).Columns({ + ColumnBuilder(table.ValueName, Type::Int64).NotNull(), + ColumnBuilder(V2_0::details::OneToManyTableGetManifestColumnName(), Type::Int64).NotNull(), + ColumnBuilder(s_Delta_IsRemovedColumn, Type::Int64).NotNull(), + PrimaryKeyBuilder({ table.ValueName, V2_0::details::OneToManyTableGetManifestColumnName() }) + }).WithoutRowID(); + builder.Execute(connection); + } + } + + savepoint.Commit(); + } + + void PrepareTablesForPackaging(SQLite::Connection& connection) + { + using namespace SQLite::Builder; + + // Every index here exists only to serve generation: those on the one to many data tables + // let generation find the rowid it already allocated for a value. + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "delta_preparetables_v2_1"); + + for (const auto& table : OneToManyTables()) + { + StatementBuilder builder; + auto tableName = GetTableName(table.TableName); + builder.DropIndex({ tableName, s_Delta_ValueIndexSuffix }); + builder.Execute(connection); + } + + savepoint.Commit(); + } + + StatementBuilder vacuumBuilder; + vacuumBuilder.Vacuum(); + vacuumBuilder.Execute(connection); + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaTables.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaTables.h new file mode 100644 index 0000000000..5d2fa0179e --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaTables.h @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include +#include +#include +#include + + +namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta +{ + // Describes a 2.0 table that associates string values with packages. + struct ValueTableInfo + { + std::string_view TableName; + std::string_view ValueName; + }; + + // The tables that store a value directly alongside the package that it refers to. + std::vector SystemReferenceTables(); + + // The tables that store values in a data table, associated with packages through a map table. + std::vector OneToManyTables(); + + // Gets the name of the delta table that mirrors the given 2.0 table. + // The delta tables are named distinctly so that a delta database can be attached alongside + // a baseline, and so that the merged views can take the 2.0 names for themselves. + std::string GetTableName(std::string_view baseTableName); + + // Gets the name of the delta map table that mirrors the map table of the given 2.0 table. + std::string GetMapTableName(std::string_view baseTableName); + + // The column that records a row as representing the removal of the data that it identifies. + std::string_view IsRemovedColumnName(); + + // Creates the full set of delta tables in the given database. + void CreateTables(SQLite::Connection& connection); + + // Performs the necessary packaging steps for the delta tables. + void PrepareTablesForPackaging(SQLite::Connection& connection); +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaViews.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaViews.cpp new file mode 100644 index 0000000000..1a60bacc6a --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaViews.cpp @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/Schema/2_1/DeltaViews.h" +#include "Microsoft/Schema/2_1/DeltaTables.h" +#include "Microsoft/Schema/2_1/Interface.h" + +#include "Microsoft/Schema/2_0/PackagesTable.h" +#include "Microsoft/Schema/2_0/OneToManyTableWithMap.h" +#include "Microsoft/Schema/2_0/SystemReferenceStringTable.h" + +#include +#include + + +namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta +{ + using namespace std::string_view_literals; + using namespace SQLite::Builder; + + namespace + { + // The schema name that the baseline database is attached under. + constexpr std::string_view s_Delta_BaselineSchema = "baseline"sv; + + // Aliases for the two sides of a merge. The correlated subqueries below name the same + // table on both sides, so the aliases are what keep the two references apart. + constexpr std::string_view s_Delta_BaselineAlias = "b"sv; + constexpr std::string_view s_Delta_DeltaAlias = "d"sv; + constexpr std::string_view s_Delta_PackagesAlias = "p"sv; + + // Appends a test that the package owning the current baseline row still exists. + // + // When a package is removed, the delta records that fact once, in its packages table; it + // does not write a removal row for each of the package's associations. This is therefore + // the only thing standing between a removed package and its associations continuing to + // appear in the merged data. + void AppendPackageNotRemoved(StatementBuilder& builder, std::string_view packageColumn) + { + builder.NotExists().BeginParenthetical(). + Select(SQLite::RowIDName). + From(GetTableName(V2_0::PackagesTable::TableName())).As(s_Delta_PackagesAlias). + Where(QualifiedColumn{ s_Delta_PackagesAlias, SQLite::RowIDName }). + Equals(QualifiedColumn{ s_Delta_BaselineAlias, packageColumn }). + And(QualifiedColumn{ s_Delta_PackagesAlias, IsRemovedColumnName() }).EqualsLiteral(1). + EndParenthetical(); + } + + // Creates the view that merges the packages themselves. + // + // The delta holds a row for every package that changed and every package that was removed, + // both keyed by the rowid that the package also has in the baseline. A baseline package is + // therefore superseded whenever the delta mentions its rowid at all: the delta row replaces + // it when it changed, and stands for its absence when it was removed. + void CreatePackagesView(SQLite::Connection& connection) + { + std::string deltaPackages = GetTableName(V2_0::PackagesTable::TableName()); + + StatementBuilder builder; + builder.CreateTempView(V2_0::PackagesTable::TableName()). + Select({ + SQLite::RowIDName, + V2_0::PackagesTable::IdColumn::Name, + V2_0::PackagesTable::NameColumn::Name, + V2_0::PackagesTable::MonikerColumn::Name, + V2_0::PackagesTable::LatestVersionColumn::Name, + V2_0::PackagesTable::ARPMinVersionColumn::Name, + V2_0::PackagesTable::ARPMaxVersionColumn::Name, + V2_0::PackagesTable::HashColumn::Name }). + From(deltaPackages). + Where(IsRemovedColumnName()).EqualsLiteral(0). + UnionAll(). + Select({ + QualifiedColumn{ s_Delta_BaselineAlias, SQLite::RowIDName }, + QualifiedColumn{ s_Delta_BaselineAlias, V2_0::PackagesTable::IdColumn::Name }, + QualifiedColumn{ s_Delta_BaselineAlias, V2_0::PackagesTable::NameColumn::Name }, + QualifiedColumn{ s_Delta_BaselineAlias, V2_0::PackagesTable::MonikerColumn::Name }, + QualifiedColumn{ s_Delta_BaselineAlias, V2_0::PackagesTable::LatestVersionColumn::Name }, + QualifiedColumn{ s_Delta_BaselineAlias, V2_0::PackagesTable::ARPMinVersionColumn::Name }, + QualifiedColumn{ s_Delta_BaselineAlias, V2_0::PackagesTable::ARPMaxVersionColumn::Name }, + QualifiedColumn{ s_Delta_BaselineAlias, V2_0::PackagesTable::HashColumn::Name } }). + From(QualifiedTable{ s_Delta_BaselineSchema, V2_0::PackagesTable::TableName() }).As(s_Delta_BaselineAlias). + Where().NotExists().BeginParenthetical(). + Select(SQLite::RowIDName).From(deltaPackages).As(s_Delta_DeltaAlias). + Where(QualifiedColumn{ s_Delta_DeltaAlias, SQLite::RowIDName }). + Equals(QualifiedColumn{ s_Delta_BaselineAlias, SQLite::RowIDName }). + EndParenthetical(); + + builder.Execute(connection); + } + + // Creates the view that merges a table associating values with packages. This covers both + // the system reference tables, which hold the value inline, and the one to many map tables, + // which hold a reference to it; the two have the same shape as far as merging is concerned. + // + // The delta records only the associations that changed, rather than the full current set + // for a changed package. A baseline association therefore survives unless the delta names + // that exact pair, or the package it belongs to has gone away entirely. + void CreateAssociationView( + SQLite::Connection& connection, + std::string_view viewName, + const std::string& deltaTableName, + std::string_view valueColumn, + std::string_view packageColumn) + { + StatementBuilder builder; + builder.CreateTempView(viewName). + Select({ valueColumn, packageColumn }). + From(deltaTableName). + Where(IsRemovedColumnName()).EqualsLiteral(0). + UnionAll(). + Select({ + QualifiedColumn{ s_Delta_BaselineAlias, valueColumn }, + QualifiedColumn{ s_Delta_BaselineAlias, packageColumn } }). + From(QualifiedTable{ s_Delta_BaselineSchema, viewName }).As(s_Delta_BaselineAlias). + Where().NotExists().BeginParenthetical(). + Select(packageColumn).From(deltaTableName).As(s_Delta_DeltaAlias). + Where(QualifiedColumn{ s_Delta_DeltaAlias, valueColumn }). + Equals(QualifiedColumn{ s_Delta_BaselineAlias, valueColumn }). + And(QualifiedColumn{ s_Delta_DeltaAlias, packageColumn }). + Equals(QualifiedColumn{ s_Delta_BaselineAlias, packageColumn }). + EndParenthetical(). + And(); + + AppendPackageNotRemoved(builder, packageColumn); + + builder.Execute(connection); + } + + // Creates the view that merges a one to many data table. + // + // Nothing is ever suppressed here. The delta only contains values that the baseline has + // never held, and it numbers them above the baseline's highest rowid, so the two sets are + // disjoint. A value that no package refers to any more is left in place; the map table + // governs what is visible, and an unreferenced value simply never appears. + void CreateValueView(SQLite::Connection& connection, std::string_view tableName, std::string_view valueColumn) + { + StatementBuilder builder; + builder.CreateTempView(tableName). + Select({ SQLite::RowIDName, valueColumn }). + From(GetTableName(tableName)). + UnionAll(). + Select({ SQLite::RowIDName, valueColumn }). + From(QualifiedTable{ s_Delta_BaselineSchema, tableName }); + + builder.Execute(connection); + } + + // Verifies that the baseline is the one that the delta was generated against. + void ValidateBaselineAffinity(const SQLite::Connection& connection, const SQLite::DatabaseSpecifier& baseline) + { + std::optional expected = + SQLite::MetadataTable::TryGetNamedValue(connection, s_MetadataValueName_DeltaBaselineIdentifier); + + SQLite::Connection baselineConnection = SQLite::Connection::Create(baseline); + std::optional actual = + SQLite::MetadataTable::TryGetNamedValue(baselineConnection, s_MetadataValueName_BaselineIdentifier); + + if (!expected || !actual || expected.value() != actual.value()) + { + AICLI_LOG(Repo, Error, << "Delta expects baseline [" << expected.value_or("") << + "] but was given [" << actual.value_or("") << "]"); + THROW_HR(APPINSTALLER_CLI_ERROR_INDEX_INTEGRITY_COMPROMISED); + } + } + } + + void SetupReadMode(SQLite::Connection& connection, const SQLite::DatabaseSpecifier& baseline) + { + AICLI_LOG(Repo, Info, << "Setting up delta read mode with baseline [" << baseline.Path() << "]"); + + ValidateBaselineAffinity(connection, baseline); + + { + StatementBuilder builder; + builder.Attach(baseline, s_Delta_BaselineSchema); + builder.Execute(connection); + } + + CreatePackagesView(connection); + + for (const auto& table : SystemReferenceTables()) + { + CreateAssociationView( + connection, + table.TableName, + GetTableName(table.TableName), + table.ValueName, + V2_0::details::SystemReferenceStringTableGetPrimaryColumnName()); + } + + for (const auto& table : OneToManyTables()) + { + CreateValueView(connection, table.TableName, table.ValueName); + + CreateAssociationView( + connection, + V2_0::details::OneToManyTableGetMapTableName(table.TableName), + GetMapTableName(table.TableName), + table.ValueName, + V2_0::details::OneToManyTableGetManifestColumnName()); + } + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaViews.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaViews.h new file mode 100644 index 0000000000..0bded48d1b --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaViews.h @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include + + +namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta +{ + // Attaches the given baseline database to the delta database that the connection is open on, + // then defines a temporary view for each 2.0 table that presents the combination of the two. + // + // The views take the 2.0 table names for themselves, and the delta's own tables are named + // distinctly, so every 2.0 read path operates on the merged data without knowing that it is + // merged. The views are temporary, so they last only as long as the connection. + void SetupReadMode(SQLite::Connection& connection, const SQLite::DatabaseSpecifier& baseline); +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface.h new file mode 100644 index 0000000000..5d66e4dfbc --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface.h @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "Microsoft/Schema/2_0/Interface.h" + +namespace AppInstaller::Repository::Microsoft::Schema::V2_1 +{ + // The change sequence from which the next delta generated against this index should be computed. + static constexpr std::string_view s_MetadataValueName_DeltaBaselineSequence = "deltaBaselineSequence"sv; + + // Identifies this index as a baseline that deltas may be generated against. + static constexpr std::string_view s_MetadataValueName_BaselineIdentifier = "baselineIdentifier"sv; + + // Written into a delta, naming the baseline that it was generated against. A delta is only + // meaningful when paired with that exact baseline, so this is checked when the two are opened. + static constexpr std::string_view s_MetadataValueName_DeltaBaselineIdentifier = "deltaBaselineIdentifier"sv; + + // Interface to schema version 2.1 exposed through ISQLiteIndex. + // Version 2.1 adds the is_removed column to the update_tracking table, + // enabling delta index generation that can represent package removals. + struct Interface : public V2_0::Interface + { + Interface(Utility::NormalizationVersion normVersion = Utility::NormalizationVersion::Initial); + + // Version 1.0 + SQLite::Version GetVersion() const override; + + // Version 2.0 + bool MigrateFrom(SQLite::Connection& connection, const ISQLiteIndex* current) override; + + // Version 2.1 + + // Designates this index as a baseline that deltas may be generated against. + void MarkAsBaseline(SQLite::Connection& connection) override; + + // Sets this index up to read the combination of a delta and the baseline it was generated + // against. Attaches the baseline and defines the merged views, after which every inherited + // read path operates on the combination. Must be called before any read. + void SetupDeltaReadMode(SQLite::Connection& connection, const SQLite::DatabaseSpecifier& baseline) override; + + protected: + // Records the baseline sequence for this index, and generates a delta index against a previous + // baseline when the caller has supplied the paths to do so. + void CreateAdditionalPackagingOutput(const SQLiteIndexContext& context) override; + }; +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp new file mode 100644 index 0000000000..71511e52a0 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Interface.h" +#include "Microsoft/Schema/2_0/PackageUpdateTrackingTable.h" +#include "Microsoft/Schema/2_1/DeltaGeneration.h" +#include "Microsoft/Schema/2_1/DeltaViews.h" + +#include +#include + +namespace AppInstaller::Repository::Microsoft::Schema::V2_1 +{ + Interface::Interface(Utility::NormalizationVersion normVersion) : V2_0::Interface(normVersion) + { + // Removals are recorded rather than deleted, so that delta generation can see which + // packages have gone away. + m_trackingRemovalBehavior = V2_0::PackageUpdateTrackingTable::RemovalBehavior::Record; + } + + SQLite::Version Interface::GetVersion() const + { + return { 2, 1 }; + } + + bool Interface::MigrateFrom(SQLite::Connection& connection, const ISQLiteIndex* current) + { + THROW_HR_IF_NULL(E_POINTER, current); + + auto currentVersion = current->GetVersion(); + + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "migrate_from_v2_1"); + + // Attempt a migration to 2.0 first, which will only return true if it actually performed a migration + bool v2result = V2_0::Interface::MigrateFrom(connection, current); + + // Migration from 2.0 → 2.1 + if (v2result || (currentVersion.MajorVersion == 2 && currentVersion.MinorVersion == 0)) + { + V2_0::PackageUpdateTrackingTable::AddRemovalTrackingColumns(connection); + savepoint.Commit(); + return true; + } + + savepoint.Rollback(true); + return false; + } + + void Interface::MarkAsBaseline(SQLite::Connection& connection) + { + GUID baselineIdentifier; + THROW_IF_FAILED(CoCreateGuid(&baselineIdentifier)); + + std::ostringstream stream; + stream << baselineIdentifier; + std::string value = stream.str(); + + AICLI_LOG(Repo, Info, << "Marking index as a delta baseline with identifier [" << value << "]"); + + SQLite::MetadataTable::SetNamedValue(connection, s_MetadataValueName_BaselineIdentifier, value); + } + + void Interface::SetupDeltaReadMode(SQLite::Connection& connection, const SQLite::DatabaseSpecifier& baseline) + { + Delta::SetupReadMode(connection, baseline); + + // The merged data is presented through views rather than tables, so the checks that the + // base makes to decide whether this index has been packaged cannot see it. Record that the + // question is already settled: a delta is only ever read, and only in its packaged form. + m_isDeltaReadMode = true; + m_internalInterfaceChecked = true; + } + + void Interface::CreateAdditionalPackagingOutput(const SQLiteIndexContext& context) + { + SQLite::Connection& connection = context.Connection; + + int64_t currentSequence = V2_0::PackageUpdateTrackingTable::GetCurrentChangeSequence(connection, m_trackingRemovalBehavior); + SQLite::MetadataTable::SetNamedValue(connection, s_MetadataValueName_DeltaBaselineSequence, std::to_string(currentSequence)); + + if (!context.Data.Contains(Property::DeltaBaselineIndexPath) || + !context.Data.Contains(Property::DeltaOutputPath)) + { + return; + } + + std::filesystem::path baselinePath = context.Data.Get(); + std::filesystem::path deltaOutputPath = context.Data.Get(); + + AICLI_LOG(Repo, Info, << "Generating a delta index against baseline [" << baselinePath << "]"); + + SQLite::Connection baselineConnection = SQLite::Connection::Create(baselinePath.u8string(), SQLite::Connection::OpenDisposition::ReadOnly); + + // The baseline must be a previous version of this database. + std::string databaseIdentifier = SQLite::MetadataTable::GetNamedValue(connection, SQLite::s_MetadataValueName_DatabaseIdentifier); + std::string baselineDatabaseIdentifier = SQLite::MetadataTable::GetNamedValue(baselineConnection, SQLite::s_MetadataValueName_DatabaseIdentifier); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_INDEX_INTEGRITY_COMPROMISED, databaseIdentifier != baselineDatabaseIdentifier); + + // Ensure that the baseline is earlier in the sequence. + int64_t baselineSequence = 0; + std::optional baselineSequenceString = SQLite::MetadataTable::TryGetNamedValue(baselineConnection, s_MetadataValueName_DeltaBaselineSequence); + if (baselineSequenceString && !baselineSequenceString->empty()) + { + baselineSequence = std::stoll(baselineSequenceString.value()); + } + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_INDEX_INTEGRITY_COMPROMISED, currentSequence < baselineSequence); + + auto changedPackages = V2_0::PackageUpdateTrackingTable::GetUpdatesSinceSequence(connection, baselineSequence, m_trackingRemovalBehavior); + auto removedPackages = V2_0::PackageUpdateTrackingTable::GetRemovalsSinceSequence(connection, baselineSequence, m_trackingRemovalBehavior); + + Delta::Generate( + connection, + baselineConnection, + deltaOutputPath, + GetVersion(), + changedPackages, + removedPackages); + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp index 4e21d059ca..efa1558155 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp @@ -12,6 +12,7 @@ #include "Microsoft/Schema/1_6/Interface.h" #include "Microsoft/Schema/1_7/Interface.h" #include "Microsoft/Schema/2_0/Interface.h" +#include "Microsoft/Schema/2_1/Interface.h" namespace AppInstaller::Repository::Microsoft::Schema { @@ -25,6 +26,16 @@ namespace AppInstaller::Repository::Microsoft::Schema THROW_WIN32(ERROR_NOT_SUPPORTED); } + void ISQLiteIndex::MarkAsBaseline(SQLite::Connection&) + { + THROW_WIN32(ERROR_NOT_SUPPORTED); + } + + void ISQLiteIndex::SetupDeltaReadMode(SQLite::Connection&, const SQLite::DatabaseSpecifier&) + { + THROW_WIN32(ERROR_NOT_SUPPORTED); + } + std::unique_ptr CreateISQLiteIndex(const SQLite::Version& version) { if (version.MajorVersion == 1 || @@ -45,13 +56,14 @@ namespace AppInstaller::Repository::Microsoft::Schema return versionCreatorMap[std::min(static_cast(version.MinorVersion), versionCreatorMap.size() - 1)](); } - // Version 2.0 is designed solely for minimizing the size of the index for transport. + // Version 2.* is designed solely for minimizing the size of the index for transport. // Unless it is prepared for packaging, it will be identical to a 1.N index. if (version.MajorVersion == 2) { - constexpr std::array(*)(), 1> versionCreatorMap = + constexpr std::array(*)(), 2> versionCreatorMap = { []() { return std::unique_ptr(std::make_unique()); }, + []() { return std::unique_ptr(std::make_unique()); }, }; return versionCreatorMap[std::min(static_cast(version.MinorVersion), versionCreatorMap.size() - 1)](); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h index 12de8e786f..5680aed3c1 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h @@ -151,6 +151,17 @@ namespace AppInstaller::Repository::Microsoft::Schema // Set the property value. virtual void SetProperty(SQLite::Connection& connection, Property property, const std::string& value); + + // Version 2.1 + + // Designates this index as a baseline that delta indexes may be generated against, giving + // it an identity that a delta can name. + virtual void MarkAsBaseline(SQLite::Connection& connection); + + // Sets this index up to read the combination of a delta and the baseline that it was + // generated against, so that every subsequent read sees the merged data. Must be called + // before any read. + virtual void SetupDeltaReadMode(SQLite::Connection& connection, const SQLite::DatabaseSpecifier& baseline); }; DEFINE_ENUM_FLAG_OPERATORS(ISQLiteIndex::CreateOptions); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/SQLiteIndexContextData.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/SQLiteIndexContextData.h index e8d5670649..039c6e5ed8 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/SQLiteIndexContextData.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/SQLiteIndexContextData.h @@ -13,6 +13,8 @@ namespace AppInstaller::Repository::Microsoft::Schema PackageUpdateTrackingBaseTime, IntermediateFileOutputPath, DatabaseFilePath, + DeltaBaselineIndexPath, + DeltaOutputPath, Max }; @@ -44,6 +46,20 @@ namespace AppInstaller::Repository::Microsoft::Schema using value_t = std::filesystem::path; static constexpr bool SetThroughInterface = false; }; + + template <> + struct PropertyMapping + { + using value_t = std::filesystem::path; + static constexpr bool SetThroughInterface = false; + }; + + template <> + struct PropertyMapping + { + using value_t = std::filesystem::path; + static constexpr bool SetThroughInterface = false; + }; } using SQLiteIndexContextData = EnumBasedVariantMap; diff --git a/src/AppInstallerSharedLib/Public/winget/SQLiteStatementBuilder.h b/src/AppInstallerSharedLib/Public/winget/SQLiteStatementBuilder.h index 0ef543a87a..f3b6e722a9 100644 --- a/src/AppInstallerSharedLib/Public/winget/SQLiteStatementBuilder.h +++ b/src/AppInstallerSharedLib/Public/winget/SQLiteStatementBuilder.h @@ -253,6 +253,9 @@ namespace AppInstaller::SQLite::Builder StatementBuilder& Where(std::string_view column); StatementBuilder& Where(const QualifiedColumn& column); + // Begin a filter clause that is not rooted at a column, such as one using `exists`. + StatementBuilder& Where(); + // A full filter clause looking for an embedded null character. // Is extremely specific to consistency checks, and so a more detailed construct is not required. StatementBuilder& WhereValueContainsEmbeddedNullCharacter(std::string_view column); @@ -284,6 +287,16 @@ namespace AppInstaller::SQLite::Builder StatementBuilder& Equals(); StatementBuilder& Equals(const QualifiedColumn& column); + // Assigns NULL to the current column in the `set` portion of an update statement. + // Only valid in an update; `Equals(nullptr)` is intentionally blocked because + // `value = NULL` is always false when used as a filter. Use IsNull for filtering. + StatementBuilder& AssignValue(std::nullptr_t); + + // Compares the current column against the value written directly into the statement + // rather than bound to it. A view definition cannot contain bound parameters, so a + // comparison within one has to be expressed this way. + StatementBuilder& EqualsLiteral(int64_t value); + template StatementBuilder& IsGreaterThan(const ValueType& value) { @@ -311,6 +324,12 @@ namespace AppInstaller::SQLite::Builder // Appends a set of value binders for the In clause. StatementBuilder& In(size_t count); + // Begins an `exists` subquery clause; follow with a parenthetical select. + StatementBuilder& Exists(); + + // Begins a `not exists` subquery clause; follow with a parenthetical select. + StatementBuilder& NotExists(); + // IsNull(true) means the value is null; IsNull(false) means the value is not null. StatementBuilder& IsNull(bool isNull = true); StatementBuilder& IsNotNull() { return IsNull(false); } @@ -320,6 +339,10 @@ namespace AppInstaller::SQLite::Builder StatementBuilder& And(const QualifiedColumn& column); StatementBuilder& Or(const QualifiedColumn& column); + // Continues a filter clause with a condition that is not introduced by a column, + // such as a further `not exists` subquery. + StatementBuilder& And(); + // Begin a join clause. // The initializer_list form enables the table name to be constructed from multiple parts. StatementBuilder& Join(std::string_view table); @@ -423,7 +446,10 @@ namespace AppInstaller::SQLite::Builder StatementBuilder& AlterTable(std::initializer_list table); // Complete an alter table statement by adding a column. + // The SubBuilder form allows for constraints such as `not null` and a default value; + // SQLite requires a non-null default when adding a column declared as not null. StatementBuilder& Add(std::string_view column, Type type); + StatementBuilder& Add(const details::SubBuilder& column); // Begin a table deletion statement. // The initializer_list form enables the table name to be constructed from multiple parts. @@ -437,6 +463,16 @@ namespace AppInstaller::SQLite::Builder StatementBuilder& DropTableIfExists(QualifiedTable table); StatementBuilder& DropTableIfExists(std::initializer_list table); + // Begin a temporary view creation statement, terminated by the `as` keyword. + // Temporary views live only for the duration of the connection, and are the only + // form of view that can reference tables in an attached database. + // The initializer_list form enables the view name to be constructed from multiple parts. + StatementBuilder& CreateTempView(std::string_view view); + StatementBuilder& CreateTempView(std::initializer_list view); + + // Combine the preceding select statement with the one that follows, retaining duplicate rows. + StatementBuilder& UnionAll(); + // Begin an index creation statement. // The initializer_list form enables the index name to be constructed from multiple parts. StatementBuilder& CreateIndex(std::string_view table); @@ -483,6 +519,12 @@ namespace AppInstaller::SQLite::Builder // Output the set portion of an update statement. StatementBuilder& Vacuum(); + // Attaches another database to the connection under the given alias. + // The target is bound as a parameter rather than embedded in the statement text. + // The alias remains valid only for the connection that executes the statement, and + // the attachment is released when that connection is closed. + StatementBuilder& Attach(const DatabaseSpecifier& specifier, std::string_view alias); + // General purpose functions to begin and end a parenthetical expression. StatementBuilder& BeginParenthetical(); StatementBuilder& EndParenthetical(); diff --git a/src/AppInstallerSharedLib/Public/winget/SQLiteStorageBase.h b/src/AppInstallerSharedLib/Public/winget/SQLiteStorageBase.h index 8b4958343d..4b8a659d2d 100644 --- a/src/AppInstallerSharedLib/Public/winget/SQLiteStorageBase.h +++ b/src/AppInstallerSharedLib/Public/winget/SQLiteStorageBase.h @@ -14,15 +14,7 @@ namespace AppInstaller::SQLite struct SQLiteStorageBase { // The disposition for opening the database. - enum class OpenDisposition - { - // Open for read only. - Read, - // Open for read and write. - ReadWrite, - // The database will not change while in use; open for immutable read. - Immutable, - }; + using OpenDisposition = DatabaseDisposition; // Gets the last write time for the database. std::chrono::system_clock::time_point GetLastWriteTime() const; @@ -43,6 +35,8 @@ namespace AppInstaller::SQLite SQLiteStorageBase(const std::string& filePath, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile); + SQLiteStorageBase(const DatabaseSpecifier& specifier, Utility::ManagedFile&& indexFile); + SQLiteStorageBase(const std::string& target, SQLiteStorageBase& source); // Sets the last write time metadata value in the database. diff --git a/src/AppInstallerSharedLib/Public/winget/SQLiteWrapper.h b/src/AppInstallerSharedLib/Public/winget/SQLiteWrapper.h index 0f100b4394..50d355e488 100644 --- a/src/AppInstallerSharedLib/Public/winget/SQLiteWrapper.h +++ b/src/AppInstallerSharedLib/Public/winget/SQLiteWrapper.h @@ -206,6 +206,7 @@ namespace AppInstaller::SQLite }; struct Statement; + struct DatabaseSpecifier; // The connection to a database. struct Connection @@ -234,6 +235,9 @@ namespace AppInstaller::SQLite static Connection Create(const std::string& target, OpenDisposition disposition, OpenFlags flags = OpenFlags::None); + // Creates a connection to the database that the specifier identifies. + static Connection Create(const DatabaseSpecifier& specifier); + Connection() = default; Connection(const Connection&) = delete; @@ -282,6 +286,46 @@ namespace AppInstaller::SQLite std::shared_ptr m_dbconn; }; + // How a database is intended to be used. + enum class DatabaseDisposition + { + // Open for read only. + Read, + // Open for read and write. + ReadWrite, + // The database will not change while in use; open for immutable read. + Immutable, + }; + + // Identifies a database and how it is to be used, translating that into the target string and + // flags that SQLite requires. + // + // The translation cannot live inside the act of opening a connection, because `ATTACH` takes + // the same target string as `sqlite3_open_v2` and has to be given the identical value. Anything + // that needs to name a database therefore takes one of these rather than a bare path. + struct DatabaseSpecifier + { + DatabaseSpecifier(std::string path, DatabaseDisposition disposition); + + // The path to the database file, as given. + const std::string& Path() const { return m_path; } + + DatabaseDisposition Disposition() const { return m_disposition; } + + // The value to hand to SQLite, which is a URI when the disposition requires query + // parameters to express it and the path itself otherwise. + const std::string& Target() const { return m_target; } + + // The connection level disposition and flags that carry this disposition. + Connection::OpenDisposition ConnectionDisposition() const; + Connection::OpenFlags ConnectionFlags() const; + + private: + std::string m_path; + std::string m_target; + DatabaseDisposition m_disposition; + }; + // A SQL statement. struct Statement { diff --git a/src/AppInstallerSharedLib/SQLiteStatementBuilder.cpp b/src/AppInstallerSharedLib/SQLiteStatementBuilder.cpp index 4277ebf1b5..22ed296422 100644 --- a/src/AppInstallerSharedLib/SQLiteStatementBuilder.cpp +++ b/src/AppInstallerSharedLib/SQLiteStatementBuilder.cpp @@ -324,6 +324,12 @@ namespace AppInstaller::SQLite::Builder return *this; } + StatementBuilder& StatementBuilder::Where() + { + m_stream << " WHERE"; + return *this; + } + StatementBuilder& StatementBuilder::WhereValueContainsEmbeddedNullCharacter(std::string_view column) { OutputColumns(m_stream, " WHERE instr(", column); @@ -348,10 +354,23 @@ namespace AppInstaller::SQLite::Builder { // This is almost certainly not what you want. // In SQL, value = NULL is always false. - // Use StatementBuilder::IsNull instead. + // Use StatementBuilder::IsNull instead, or StatementBuilder::AssignValue + // to assign NULL in the set portion of an update statement. THROW_HR(E_NOTIMPL); } + StatementBuilder& StatementBuilder::AssignValue(std::nullptr_t) + { + m_stream << " = NULL"; + return *this; + } + + StatementBuilder& StatementBuilder::EqualsLiteral(int64_t value) + { + m_stream << " = " << value; + return *this; + } + StatementBuilder& StatementBuilder::Equals() { m_stream << " ="; @@ -420,6 +439,18 @@ namespace AppInstaller::SQLite::Builder return *this; } + StatementBuilder& StatementBuilder::Exists() + { + m_stream << " EXISTS"; + return *this; + } + + StatementBuilder& StatementBuilder::NotExists() + { + m_stream << " NOT EXISTS"; + return *this; + } + StatementBuilder& StatementBuilder::IsNull(bool isNull) { m_stream << " IS " << (isNull ? "" : "NOT ") << "NULL"; @@ -444,6 +475,12 @@ namespace AppInstaller::SQLite::Builder return *this; } + StatementBuilder& StatementBuilder::And() + { + m_stream << " AND"; + return *this; + } + StatementBuilder& StatementBuilder::Join(std::string_view table) { OutputOperationAndTable(m_stream, " JOIN", table); @@ -740,6 +777,12 @@ namespace AppInstaller::SQLite::Builder return *this; } + StatementBuilder& StatementBuilder::Add(const details::SubBuilder& column) + { + m_stream << " ADD " << column; + return *this; + } + StatementBuilder& StatementBuilder::DropTable(std::string_view table) { OutputOperationAndTable(m_stream, "DROP TABLE", table); @@ -776,6 +819,26 @@ namespace AppInstaller::SQLite::Builder return *this; } + StatementBuilder& StatementBuilder::CreateTempView(std::string_view view) + { + OutputOperationAndTable(m_stream, "CREATE TEMP VIEW", view); + m_stream << " AS "; + return *this; + } + + StatementBuilder& StatementBuilder::CreateTempView(std::initializer_list view) + { + OutputOperationAndTable(m_stream, "CREATE TEMP VIEW", view); + m_stream << " AS "; + return *this; + } + + StatementBuilder& StatementBuilder::UnionAll() + { + m_stream << " UNION ALL "; + return *this; + } + StatementBuilder& StatementBuilder::CreateIndex(std::string_view table) { OutputOperationAndTable(m_stream, "CREATE INDEX", table); @@ -909,6 +972,14 @@ namespace AppInstaller::SQLite::Builder return *this; } + StatementBuilder& StatementBuilder::Attach(const DatabaseSpecifier& specifier, std::string_view alias) + { + m_stream << "ATTACH DATABASE ?"; + AddBindFunctor(m_bindIndex++, specifier.Target()); + OutputOperationAndTable(m_stream, " AS", alias); + return *this; + } + StatementBuilder& StatementBuilder::BeginParenthetical() { m_stream << '('; diff --git a/src/AppInstallerSharedLib/SQLiteStorageBase.cpp b/src/AppInstallerSharedLib/SQLiteStorageBase.cpp index 61d50dfb78..dff989519f 100644 --- a/src/AppInstallerSharedLib/SQLiteStorageBase.cpp +++ b/src/AppInstallerSharedLib/SQLiteStorageBase.cpp @@ -39,8 +39,6 @@ namespace AppInstaller::SQLite } } - // One method for converting open disposition to proper open disposition - // another method for obtaining the right flags void SQLiteStorageBase::SetLastWriteTime() { MetadataTable::SetNamedValue(m_dbconn, s_MetadataValueName_LastWriteTime, Utility::GetCurrentUnixEpoch()); @@ -91,67 +89,16 @@ namespace AppInstaller::SQLite } SQLiteStorageBase::SQLiteStorageBase(const std::string& filePath, OpenDisposition disposition, Utility::ManagedFile&& file) : - m_indexFile(std::move(file)) + SQLiteStorageBase(DatabaseSpecifier{ filePath, disposition }, std::move(file)) { - AICLI_LOG(Repo, Info, << "Opening database for " << GetOpenDispositionString(disposition) << " at '" << filePath << "'"); - switch (disposition) - { - case OpenDisposition::Read: - m_dbconn = SQLite::Connection::Create(filePath, SQLite::Connection::OpenDisposition::ReadOnly, SQLite::Connection::OpenFlags::None); - break; - case OpenDisposition::ReadWrite: - m_dbconn = SQLite::Connection::Create(filePath, SQLite::Connection::OpenDisposition::ReadWrite, SQLite::Connection::OpenFlags::None); - break; - case OpenDisposition::Immutable: - { - // Following the algorithm set forth at https://sqlite.org/uri.html [3.1] to convert to a URI path - // The execution order builds out the string so that it shouldn't require any moves (other than growing) - std::string target; - // Add an 'arbitrary' growth size to prevent the majority of needing to grow (adding 'file:/' and '?immutable=1') - target.reserve(filePath.size() + 20); - - target += "file:"; - - bool wasLastCharSlash = false; - - if (filePath.size() >= 2 && filePath[1] == ':' && - ((filePath[0] >= 'a' && filePath[0] <= 'z') || - (filePath[0] >= 'A' && filePath[0] <= 'Z'))) - { - target += '/'; - wasLastCharSlash = true; - } - - for (char c : filePath) - { - bool wasThisCharSlash = false; - switch (c) - { - case '?': target += "%3f"; break; - case '#': target += "%23"; break; - case '\\': - case '/': - { - wasThisCharSlash = true; - if (!wasLastCharSlash) - { - target += '/'; - } - break; - } - default: target += c; break; - } + } - wasLastCharSlash = wasThisCharSlash; - } + SQLiteStorageBase::SQLiteStorageBase(const DatabaseSpecifier& specifier, Utility::ManagedFile&& file) : + m_indexFile(std::move(file)) + { + AICLI_LOG(Repo, Info, << "Opening database for " << GetOpenDispositionString(specifier.Disposition()) << " at '" << specifier.Path() << "'"); - target += "?immutable=1"; - m_dbconn = SQLite::Connection::Create(filePath, SQLite::Connection::OpenDisposition::ReadOnly, SQLite::Connection::OpenFlags::Uri); - break; - } - default: - THROW_HR(E_UNEXPECTED); - } + m_dbconn = SQLite::Connection::Create(specifier); m_version = Version::GetSchemaVersion(m_dbconn); } diff --git a/src/AppInstallerSharedLib/SQLiteWrapper.cpp b/src/AppInstallerSharedLib/SQLiteWrapper.cpp index 78df7a5b9c..5e00ef872d 100644 --- a/src/AppInstallerSharedLib/SQLiteWrapper.cpp +++ b/src/AppInstallerSharedLib/SQLiteWrapper.cpp @@ -213,6 +213,11 @@ namespace AppInstaller::SQLite return result; } + Connection Connection::Create(const DatabaseSpecifier& specifier) + { + return Create(specifier.Target(), specifier.ConnectionDisposition(), specifier.ConnectionFlags()); + } + void Connection::EnableICU() { AICLI_LOG(SQL, Verbose, << "Enabling ICU"); @@ -265,6 +270,82 @@ namespace AppInstaller::SQLite return m_dbconn; } + DatabaseSpecifier::DatabaseSpecifier(std::string path, DatabaseDisposition disposition) : + m_path(std::move(path)), m_disposition(disposition) + { + if (m_disposition != DatabaseDisposition::Immutable) + { + m_target = m_path; + return; + } + + // Following the algorithm set forth at https://sqlite.org/uri.html [3.1] to convert to a URI path. + // The execution order builds out the string so that it shouldn't require any moves (other than growing). + // Add an 'arbitrary' growth size to prevent the majority of needing to grow (adding 'file:/' and '?immutable=1'). + m_target.reserve(m_path.size() + 20); + + m_target += "file:"; + + bool wasLastCharSlash = false; + + if (m_path.size() >= 2 && m_path[1] == ':' && + ((m_path[0] >= 'a' && m_path[0] <= 'z') || + (m_path[0] >= 'A' && m_path[0] <= 'Z'))) + { + m_target += '/'; + wasLastCharSlash = true; + } + + for (char c : m_path) + { + bool wasThisCharSlash = false; + switch (c) + { + case '?': m_target += "%3f"; break; + case '#': m_target += "%23"; break; + case '\\': + case '/': + { + wasThisCharSlash = true; + if (!wasLastCharSlash) + { + m_target += '/'; + } + break; + } + default: m_target += c; break; + } + + wasLastCharSlash = wasThisCharSlash; + } + + m_target += "?immutable=1"; + } + + Connection::OpenDisposition DatabaseSpecifier::ConnectionDisposition() const + { + switch (m_disposition) + { + case DatabaseDisposition::Read: + case DatabaseDisposition::Immutable: + return Connection::OpenDisposition::ReadOnly; + case DatabaseDisposition::ReadWrite: + return Connection::OpenDisposition::ReadWrite; + default: + THROW_HR(E_UNEXPECTED); + } + } + + Connection::OpenFlags DatabaseSpecifier::ConnectionFlags() const + { + // URI handling is enabled unconditionally, not only for the dispositions that produce one. + // SQLite decides whether a name is a URI when the connection is opened and applies that + // same decision to every later `ATTACH`, so a connection that did not ask for URIs cannot + // attach one. A name that does not begin with "file:" is always taken literally, so this + // costs nothing for the dispositions that hand over a plain path. + return Connection::OpenFlags::Uri; + } + Statement::Statement(const Connection& connection, std::string_view sql) { m_dbconn = connection.GetSharedConnection();