Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions Cli/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,13 @@ private static int RunUpgrade(CliOptions options)
case SchemaAction.UpgradeInPlace:
DatabaseMaintenance.UpgradeInPlace(options.UpgradeDbPath);
Console.WriteLine($"Upgraded database schema from v{before.Major}.{before.Minor} to {current}.");
var applied = DatabaseSchemaInfo.ChangesSince(before.Major, before.Minor);
if (applied.Count > 0)
{
Console.WriteLine("Applied (views/indexes re-created):");
foreach (var change in applied)
Console.WriteLine($" • {change}");
}
return 0;
}

Expand Down
2 changes: 2 additions & 0 deletions Cli/SchemaGate.cs
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ private static void HandleUpgradeInPlace(SchemaStatus status, string current, st
{
DatabaseMaintenance.UpgradeInPlace(status.DatabasePath);
Console.Error.WriteLine($"Upgraded database schema to {current}.");
foreach (var change in DatabaseSchemaInfo.ChangesSince(status.Major, status.Minor))
Console.Error.WriteLine($" • {change}");
}
else
{
Expand Down
28 changes: 28 additions & 0 deletions Core/ExportDestination/DuckDbExportDestination.cs
Original file line number Diff line number Diff line change
Expand Up @@ -667,6 +667,34 @@ FROM native_objects b
WHERE b.native_type_name = 'AssetBundle'
GROUP BY b.native_object_index, b.name, b.size_bytes, b.resident_size_bytes, b.is_destroyed;

-- One row per (AssetBundle, loaded native object) pair: the exploded, per-asset companion to
-- v_assetbundle_utilization (which is the per-bundle aggregate). The refs CTE is the SAME filter the
-- utilization view uses, so the bundle's own native self-reference (to_index = from_index) and its
-- managed wrapper(s) (excluded by to_kind = 'native_object' + native_connection) are left out — every
-- row is a genuine OTHER asset the bundle keeps loaded, with no magic numbers.
CREATE OR REPLACE VIEW v_assetbundle_loaded_assets AS
WITH refs AS (
SELECT DISTINCT c.from_index AS bundle_index, c.to_index AS asset_index
FROM connections c
JOIN native_objects b ON b.native_object_index = c.from_index AND b.native_type_name = 'AssetBundle'
WHERE c.from_kind = 'native_object' AND c.to_kind = 'native_object'
AND c.connection_type = 'native_connection' AND c.to_index <> c.from_index
)
SELECT
b.native_object_index AS bundle_index,
b.name AS bundle_name,
b.size_bytes AS bundle_size_bytes,
b.resident_size_bytes AS bundle_resident_bytes,
o.native_object_index AS asset_index,
o.name AS asset_name,
o.native_type_name AS asset_type_name,
o.size_bytes AS asset_size_bytes,
o.resident_size_bytes AS asset_resident_bytes,
o.is_destroyed AS asset_is_destroyed
FROM refs r
JOIN native_objects b ON b.native_object_index = r.bundle_index
JOIN native_objects o ON o.native_object_index = r.asset_index;

CREATE OR REPLACE MACRO region_allocations(region_name) AS TABLE
SELECT * FROM v_allocation_enriched WHERE system_region_name = region_name;

Expand Down
30 changes: 30 additions & 0 deletions Core/ExportDestination/SqliteWriter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -808,6 +808,7 @@ private static long QueryCount(SqliteConnection connection, string sql)
#endregion

private const string SchemaTablesScript = """
DROP VIEW IF EXISTS v_assetbundle_loaded_assets;
DROP VIEW IF EXISTS v_assetbundle_utilization;
DROP VIEW IF EXISTS v_connection_edges;
DROP VIEW IF EXISTS v_region_owner_breakdown;
Expand Down Expand Up @@ -936,6 +937,7 @@ resident_available INTEGER NOT NULL
// Drops the analysis views so CreateViewsScript (which uses CREATE VIEW, not CREATE OR REPLACE,
// for SQLite) is re-runnable by the in-place upgrade path. Order does not matter with IF EXISTS.
private const string DropViewsScript = """
DROP VIEW IF EXISTS v_assetbundle_loaded_assets;
DROP VIEW IF EXISTS v_assetbundle_utilization;
DROP VIEW IF EXISTS v_connection_edges;
DROP VIEW IF EXISTS v_region_owner_breakdown;
Expand Down Expand Up @@ -1042,6 +1044,34 @@ FROM native_objects b
LEFT JOIN native_objects o ON o.native_object_index = r.ref_index
WHERE b.native_type_name = 'AssetBundle'
GROUP BY b.native_object_index, b.name, b.size_bytes, b.resident_size_bytes, b.is_destroyed;

-- One row per (AssetBundle, loaded native object) pair: the exploded, per-asset companion to
-- v_assetbundle_utilization (which is the per-bundle aggregate). The refs CTE is the SAME filter the
-- utilization view uses, so the bundle's own native self-reference (to_index = from_index) and its
-- managed wrapper(s) (excluded by to_kind = 'native_object' + native_connection) are left out — every
-- row is a genuine OTHER asset the bundle keeps loaded, with no magic numbers.
CREATE VIEW v_assetbundle_loaded_assets AS
WITH refs AS (
SELECT DISTINCT c.from_index AS bundle_index, c.to_index AS asset_index
FROM connections c
JOIN native_objects b ON b.native_object_index = c.from_index AND b.native_type_name = 'AssetBundle'
WHERE c.from_kind = 'native_object' AND c.to_kind = 'native_object'
AND c.connection_type = 'native_connection' AND c.to_index <> c.from_index
)
SELECT
b.native_object_index AS bundle_index,
b.name AS bundle_name,
b.size_bytes AS bundle_size_bytes,
b.resident_size_bytes AS bundle_resident_bytes,
o.native_object_index AS asset_index,
o.name AS asset_name,
o.native_type_name AS asset_type_name,
o.size_bytes AS asset_size_bytes,
o.resident_size_bytes AS asset_resident_bytes,
o.is_destroyed AS asset_is_destroyed
FROM refs r
JOIN native_objects b ON b.native_object_index = r.bundle_index
JOIN native_objects o ON o.native_object_index = r.asset_index;
""";
}

38 changes: 37 additions & 1 deletion Core/Models/DatabaseSchemaInfo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ public static class DatabaseSchemaInfo
public const int SchemaMajor = 1;

/// <summary>Current minor schema version (views/indexes). A lower minor can be upgraded in place.</summary>
public const int SchemaMinor = 2;
public const int SchemaMinor = 3;

/// <summary>Name used in advisories to refer to the CLI tool.</summary>
public const string ToolName = "MemorySnapshotDataTools";
Expand All @@ -71,6 +71,42 @@ public static class DatabaseSchemaInfo
?? typeof(DatabaseSchemaInfo).Assembly.GetName().Version?.ToString()
?? "unknown";

/// <summary>
/// Version-by-version summary of what each schema version introduced, mirroring the version table
/// in <c>docs/database-schema.md</c>. Ordered oldest → newest. Used by <see cref="ChangesSince"/>
/// so the <c>upgrade</c> command can tell the user exactly what an in-place upgrade applied. Add a
/// row here whenever you bump <see cref="SchemaMajor"/> or <see cref="SchemaMinor"/>.
/// </summary>
public static readonly IReadOnlyList<(int Major, int Minor, string Summary)> Changelog = new[]
{
(1, 0, "Initial versioned schema: schema_meta, page_size, region analysis views/macros."),
(1, 1, "Added v_connection_edges and v_assetbundle_utilization views."),
(1, 2, "Reformulated v_connection_edges joins so filtered queries hash-join (much faster)."),
(1, 3, "Added v_assetbundle_loaded_assets view (the assets each AssetBundle keeps loaded)."),
};

/// <summary>
/// The <see cref="Changelog"/> summaries for every schema version newer than the supplied
/// (<paramref name="major"/>, <paramref name="minor"/>) up to and including this build's version,
/// each prefixed with its version (e.g. <c>"v1.3: …"</c>). Describes what an in-place upgrade from
/// the stored version applied; empty when nothing newer exists.
/// </summary>
/// <param name="major">The database's stored major version.</param>
/// <param name="minor">The database's stored minor version.</param>
public static IReadOnlyList<string> ChangesSince(int major, int minor)
{
var changes = new List<string>();
foreach (var (m, n, summary) in Changelog)
{
var newerThanStored = m > major || (m == major && n > minor);
var atMostCurrent = m < SchemaMajor || (m == SchemaMajor && n <= SchemaMinor);
if (newerThanStored && atMostCurrent)
changes.Add($"v{m}.{n}: {summary}");
}

return changes;
}

/// <summary>Classifies a database's stored (major, minor) version against this build.</summary>
/// <param name="major">Value from <c>schema_meta.schema_version_major</c>, or 0 if the table is absent.</param>
/// <param name="minor">Value from <c>schema_meta.schema_version_minor</c>, or 0 if absent.</param>
Expand Down
24 changes: 24 additions & 0 deletions Tests/DatabaseSchemaInfoTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -53,4 +53,28 @@ public void BuildReExportCommand_FormatsCommand(bool sqlite, string expected)
{
Assert.Equal(expected, DatabaseSchemaInfo.BuildReExportCommand("/snaps/a.snap", "/dbs/a.duckdb", sqlite));
}

[Fact]
public void Changelog_CoversCurrentVersion()
{
// Guard: bumping SchemaMajor/SchemaMinor without adding a Changelog row would leave the
// upgrade command unable to describe the new version.
Assert.Contains(DatabaseSchemaInfo.Changelog,
e => e.Major == DatabaseSchemaInfo.SchemaMajor && e.Minor == DatabaseSchemaInfo.SchemaMinor);
}

[Fact]
public void ChangesSince_CurrentVersion_IsEmpty()
{
Assert.Empty(DatabaseSchemaInfo.ChangesSince(DatabaseSchemaInfo.SchemaMajor, DatabaseSchemaInfo.SchemaMinor));
}

[Fact]
public void ChangesSince_OneMinorBehind_DescribesCurrentVersion()
{
// A database one minor behind should be told exactly which (single) version the upgrade applied.
var changes = DatabaseSchemaInfo.ChangesSince(DatabaseSchemaInfo.SchemaMajor, DatabaseSchemaInfo.SchemaMinor - 1);
var change = Assert.Single(changes);
Assert.StartsWith($"v{DatabaseSchemaInfo.SchemaMajor}.{DatabaseSchemaInfo.SchemaMinor}:", change);
}
}
56 changes: 54 additions & 2 deletions docs/database-schema.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ Reset minor to 0 whenever you bump major.
| 1.0 | First versioned schema: `schema_meta`, `snapshot_info.page_size`, region analysis views/macros. |
| 1.1 | Added `v_connection_edges` and `v_assetbundle_utilization` views (minor — upgradeable in place). |
| 1.2 | Reformulated `v_connection_edges` joins (kind check folded into the join key) so DuckDB hash-joins instead of nested-loop — `SELECT … WHERE from_type=…` drops from minutes to sub-second (minor). |
| 1.3 | Added `v_assetbundle_loaded_assets` view — one row per (AssetBundle, loaded native object) it references (minor — upgradeable in place). |

**What the CLI does.** Before a read command (`report`, `summary`, `validate`),
`DatabaseSchemaInfo.Evaluate(major, minor)` classifies the database and the CLI acts:
Expand All @@ -58,8 +59,16 @@ MemorySnapshotDataTools upgrade <database.duckdb|.db>
```

This re-applies indexes and views (`DatabaseMaintenance.UpgradeInPlace`) and bumps the stored minor
version. It refuses major-version gaps and tells you to re-export instead. Non-interactive sessions
(stdin redirected) never auto-modify a database — they only print the advisory and command.
version, then lists which schema versions were applied (from `DatabaseSchemaInfo.ChangesSince`, the
same per-version summaries as the [version table](#schema-version) below). It refuses major-version
gaps and tells you to re-export instead. Non-interactive sessions (stdin redirected) never auto-modify
a database — they only print the advisory and command.

```text
Upgraded database schema from v1.2 to v1.3.
Applied (views/indexes re-created):
• v1.3: Added v_assetbundle_loaded_assets view (the assets each AssetBundle keeps loaded).
```

The stored version is also **displayed in output**: `summary` prints a `Schema` field, the HTML
`report` shows a *Schema Version* row in Snapshot Info, and `multi-report` shows it per database
Expand Down Expand Up @@ -288,6 +297,49 @@ WHERE references_loaded_assets ORDER BY referenced_object_count DESC;
> flattened bundle→contained-object edges, so this is comprehensive for bundles; it is not transitive
> retained size, and the same shared asset may be counted under more than one bundle.

### `v_assetbundle_loaded_assets` (view)
The **exploded, per-asset companion** to `v_assetbundle_utilization`: one row per *(AssetBundle, loaded
native object)* pair — i.e. every asset an `AssetBundle` keeps loaded in memory, attributed to the
bundle that references it. Use it to **list the actual assets** held by bundles (the utilization view
only counts/sums them per bundle). It applies the **same edge filter** as `v_assetbundle_utilization`
(`native_object → native_object` `native_connection` edges with `to_index <> from_index`), so the
bundle's own native self-reference and its managed wrapper(s) are excluded — every row is a genuine
*other* loaded asset, with no magic numbers. A bundle that holds nothing (an "empty" bundle) produces
**no rows** here. A shared asset referenced by N bundles appears in N rows.

| Column | Type | Notes |
|--------|------|-------|
| `bundle_index` | INTEGER | The `AssetBundle`'s `native_objects.native_object_index`. |
| `bundle_name` | VARCHAR | The bundle's object name. |
| `bundle_size_bytes` | BIGINT | The bundle object's own native size. |
| `bundle_resident_bytes` | BIGINT | The bundle's resident bytes (format ≥ 17), else NULL. |
| `asset_index` | INTEGER | The loaded object's `native_objects.native_object_index`. |
| `asset_name` | VARCHAR | The loaded object's name. |
| `asset_type_name` | VARCHAR | The loaded object's native type (e.g. `Texture2D`, `Mesh`). |
| `asset_size_bytes` | BIGINT | The loaded object's own native size. |
| `asset_resident_bytes` | BIGINT | The loaded object's resident bytes (format ≥ 17), else NULL. |
| `asset_is_destroyed` | BOOLEAN | Loaded object marked destroyed but still resident. |

```sql
-- Every asset a specific bundle keeps loaded, biggest first.
SELECT asset_type_name, asset_name, ROUND(asset_size_bytes / 1048576.0, 2) AS asset_mb
FROM v_assetbundle_loaded_assets
WHERE bundle_name = 'characters' ORDER BY asset_size_bytes DESC;

-- What kinds of assets do bundles pull into memory, and how much?
SELECT asset_type_name, COUNT(*) AS assets,
ROUND(SUM(asset_size_bytes) / 1048576.0, 1) AS total_mb
FROM v_assetbundle_loaded_assets GROUP BY 1 ORDER BY total_mb DESC;

-- Assets shared across multiple bundles (counted under each).
SELECT asset_name, asset_type_name, COUNT(DISTINCT bundle_index) AS bundles
FROM v_assetbundle_loaded_assets GROUP BY 1, 2 HAVING bundles > 1 ORDER BY bundles DESC;
```

> Like `v_assetbundle_utilization`, this reflects Unity's flattened bundle→contained-object edges:
> it is the set of **directly-referenced** loaded objects (comprehensive for bundles), not transitive
> retained reachability, and `asset_size_bytes` is each object's **own** size.

### `region_allocations(region_name)` (DuckDB macro)
All `v_allocation_enriched` rows for one OS region: `SELECT * FROM region_allocations('MALLOC_NANO');`

Expand Down
Loading