From d9119512c9fd287920ce89c0708f2d6ac7f43a5c Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 14 Sep 2026 13:37:50 +0100 Subject: [PATCH 1/9] JIT: cost fragmented vector-sized struct copies Physical promotion does not currently charge for decomposing overlapping whole-struct stores. Account for the extra moves when a vector-sized struct is split entirely into sub-native fields, with more distinct primitive access entries than native-sized copy parts. Reuse existing store counters and register-move costs, amortize the original vector move across fields, and exclude call-result readbacks already costed separately. This is a per- field approximation, not joint evaluation of the final replacement set. Validation against this change's base on updated main a652cdfed564: - Windows x64 Checked and Release and Linux x64 Release JIT builds. - 18 existing physical-promotion regression runs pass with normal, forced promotion and JitStress=2 settings. - Existing promotion facts pass on both JITs with tiering/PGO enabled. - Eleven SuperPMI collections: 983,234 comparable contexts; 238,457,491 -> 238,457,477 bytes (14 saved). 2 shrink; 2 grow by 32 bytes total. Zero compilation failures; 190 missing-recording contexts excluded. The 190 recording gaps affect both sides. - Release PIN JIT instructions, benchmarks: +0.00629%. - Release PIN JIT instructions, HashSet PGO: +0.00266%. Standalone changes are two 23-byte Markdig reductions and two 16-byte StringBuilder increases; all four have higher static PerfScore. With address propagation enabled, the controlled HashSet comparison saves 1,174 bytes; the largest method is 2,878 -> 2,268 bytes with a lower PerfScore. Distinct access entries approximate the final replacement set. Code size, static PerfScore and JIT instruction counts do not establish application execution speed. No ARM64 or x86 execution is claimed. Related to #133833. --- src/coreclr/jit/promotion.cpp | 63 +++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/src/coreclr/jit/promotion.cpp b/src/coreclr/jit/promotion.cpp index 2ecad18d74760f..d5ae03e29b7ba9 100644 --- a/src/coreclr/jit/promotion.cpp +++ b/src/coreclr/jit/promotion.cpp @@ -98,20 +98,20 @@ struct Access weight_t CountCallArgsWtd = 0; weight_t CountRegCallArgsWtd = 0; -#ifdef DEBUG // Number of times this access is the source of a store. unsigned CountStoreSource = 0; // Number of times this access is the destination of a store. unsigned CountStoreDestination = 0; - unsigned CountReturns = 0; // Number of times this is stored by being passed as the retbuf. // These stores need a readback unsigned CountPassedAsRetbuf = 0; weight_t CountStoreSourceWtd = 0; weight_t CountStoreDestinationWtd = 0; - weight_t CountReturnsWtd = 0; weight_t CountPassedAsRetbufWtd = 0; +#ifdef DEBUG + unsigned CountReturns = 0; + weight_t CountReturnsWtd = 0; #endif Access(unsigned offset, var_types accessType, ClassLayout* layout) @@ -146,15 +146,15 @@ struct Access enum class AccessKindFlags : uint32_t { - None = 0, - IsCallArg = 1, - IsRegCallArg = 2, - IsStoredFromCall = 4, - IsCallRetBuf = 8, -#ifdef DEBUG + None = 0, + IsCallArg = 1, + IsRegCallArg = 2, + IsStoredFromCall = 4, + IsCallRetBuf = 8, IsStoreSource = 16, IsStoreDestination = 32, - IsReturned = 64, +#ifdef DEBUG + IsReturned = 64, #endif }; @@ -402,7 +402,6 @@ class LocalUses access->CountStoredFromCallWtd += weight; } -#ifdef DEBUG if ((flags & AccessKindFlags::IsCallRetBuf) != AccessKindFlags::None) { access->CountPassedAsRetbuf++; @@ -421,6 +420,7 @@ class LocalUses access->CountStoreDestinationWtd += weight; } +#ifdef DEBUG if ((flags & AccessKindFlags::IsReturned) != AccessKindFlags::None) { access->CountReturns++; @@ -707,7 +707,11 @@ class LocalUses weight_t countOverlappedCallArgWtd = 0; weight_t countOverlappedStoredFromCallWtd = 0; - bool overlap = false; + unsigned countVectorCopies = 0; + weight_t countVectorCopiesWtd = 0; + unsigned primitiveAccessCount = 1; + bool costVectorCopies = + (genTypeSize(access.AccessType) < TARGET_POINTER_SIZE) && varTypeIsSIMD(layout->GetRegisterType()); for (const Access& otherAccess : m_accesses) { if (&otherAccess == &access) @@ -715,6 +719,12 @@ class LocalUses continue; } + if (otherAccess.AccessType != TYP_STRUCT) + { + primitiveAccessCount++; + costVectorCopies &= genTypeSize(otherAccess.AccessType) < TARGET_POINTER_SIZE; + } + if (!otherAccess.Overlaps(access.Offset, genTypeSize(access.AccessType))) { continue; @@ -731,6 +741,15 @@ class LocalUses countOverlappedCallArgWtd += otherAccess.CountCallArgsWtd; countOverlappedStoredFromCallWtd += otherAccess.CountStoredFromCallWtd; + if (costVectorCopies && (otherAccess.GetAccessSize() == layout->GetSize())) + { + // Call-result read-backs are already costed separately. + countVectorCopies += otherAccess.CountStoreSource + otherAccess.CountStoreDestination + + otherAccess.CountPassedAsRetbuf - otherAccess.CountStoredFromCall; + countVectorCopiesWtd += otherAccess.CountStoreSourceWtd + otherAccess.CountStoreDestinationWtd + + otherAccess.CountPassedAsRetbufWtd - otherAccess.CountStoredFromCallWtd; + } + if (otherAccess.CountRegCallArgs > 0) { // The call argument will be decomposed and will not require a @@ -842,7 +861,18 @@ class LocalUses costWith += countWriteBacksWtd * writeBackCost; sizeWith += countWriteBacks * writeBackSize; - // Overlapping stores are decomposable so we don't cost them as + // Charge for fragmenting vector-sized copies into sub-native fields. + // Keep smaller splits and mixed-width copies unpenalized. + unsigned nativeParts = layout->GetSize() / TARGET_POINTER_SIZE; + if (costVectorCopies && (primitiveAccessCount > nativeParts)) + { + // Amortize the original vector move over its fields. + weight_t extraMoves = 1 - (weight_t)genTypeSize(access.AccessType) / layout->GetSize(); + costWith += countVectorCopiesWtd * extraMoves * COST_REG_ACCESS_CYCLES; + sizeWith += countVectorCopies * extraMoves * COST_REG_ACCESS_SIZE; + } + + // Other overlapping stores are decomposable so we don't cost them as // being more expensive than their unpromoted counterparts (i.e. we // don't consider them at all). However, we should do something more // clever here, since: @@ -1526,7 +1556,7 @@ class LocalsUseVisitor : public GenTreeVisitor AccessKindFlags flags = AccessKindFlags::None; if (lcl->OperIsLocalStore()) { - INDEBUG(flags |= AccessKindFlags::IsStoreDestination); + flags |= AccessKindFlags::IsStoreDestination; if (lcl->AsLclVarCommon()->Data()->gtEffectiveVal()->IsCall()) { @@ -1565,12 +1595,12 @@ class LocalsUseVisitor : public GenTreeVisitor } } -#ifdef DEBUG if (user->OperIsStore() && (user->Data()->gtEffectiveVal() == lcl)) { flags |= AccessKindFlags::IsStoreSource; } +#ifdef DEBUG if (user->OperIs(GT_RETURN, GT_SWIFT_ERROR_RET)) { flags |= AccessKindFlags::IsReturned; @@ -2408,8 +2438,7 @@ GenTreeFieldList* ReplaceVisitor::CreateFieldListForStructLocal(GenTreeLclVarCom // argNode - The argument node // // Returns: -// True if the call argument was replaced with a FIELD_LIST; false if the -// argument could not be represented as a FIELD_LIST. +// True if the argument was replaced; false if write-backs are required. // bool ReplaceVisitor::ReplaceCallArgWithFieldList(GenTreeCall* call, GenTree** use, GenTreeLclVarCommon* argNode) { From afaadebc9391b958c8b64b2d0b9556213d666e3f Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 14 Sep 2026 16:15:14 +0100 Subject: [PATCH 2/9] JIT: reuse induced accesses when costing shared vector copies Record whole-local copies that induce matching accesses from already selected replacements. Exclude those copies from the induced field fragmentation charge so the second endpoint does not pay again for an existing decomposition. Keep one-sided and partial-copy costs. Reuse the existing induced-promotion retry pass and its access records. Add SharedVectorCopyCost with behavioral and Windows x64 codegen coverage. It restores main codegen: 340 to 302 bytes, 86 to 81 instructions and 96 to 80 local frame bytes versus the previous model. Restore the existing FIELD_LIST return-contract comment. Validation: Windows Checked/Release and Linux Release JIT builds; focused normal, promotion, JitStress=2 and PGO runs; Windows codegen check passes and fails on the previous model; Linux execution; 18 existing regression runs. All passed. SuperPMI: 983,234 comparable contexts, no compilation failures, 190 shared recording gaps. Net 14 bytes saved, unchanged from the previous model. Controlled HashSet savings remain 1,174 bytes with address propagation. Release JIT instructions versus main increase 0.00638% on benchmarks and 0.00196% on HashSet. Application execution speed and ARM64/x86 execution were not measured. --- src/coreclr/jit/promotion.cpp | 57 +++++++++++---- src/tests/JIT/Directed/Directed_do.csproj | 1 + .../physicalpromotion/SharedVectorCopyCost.cs | 73 +++++++++++++++++++ .../SharedVectorCopyCost.csproj | 14 ++++ 4 files changed, 132 insertions(+), 13 deletions(-) create mode 100644 src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs create mode 100644 src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.csproj diff --git a/src/coreclr/jit/promotion.cpp b/src/coreclr/jit/promotion.cpp index d5ae03e29b7ba9..8e06833e074a80 100644 --- a/src/coreclr/jit/promotion.cpp +++ b/src/coreclr/jit/promotion.cpp @@ -311,8 +311,10 @@ AggregateInfo* AggregateInfoMap::Lookup(unsigned lclNum) struct PrimitiveAccess { - unsigned Count = 0; - weight_t CountWtd = 0; + unsigned Count = 0; + weight_t CountWtd = 0; + unsigned CountFullCopies = 0; + weight_t CountFullCopiesWtd = 0; unsigned Offset; var_types AccessType; @@ -437,12 +439,13 @@ class LocalUses // offs - The offset being accessed // accessType - The type of the access // weight - Weight of the block containing the access + // isFullCopy - Whether the copy covers both whole locals // // Remarks: // Induced accesses are accesses that are induced by physical promotion // due to store decompositon. They are always of primitive type. // - void RecordInducedAccess(unsigned offs, var_types accessType, weight_t weight) + void RecordInducedAccess(unsigned offs, var_types accessType, weight_t weight, bool isFullCopy) { PrimitiveAccess* access = nullptr; @@ -477,6 +480,11 @@ class LocalUses access->Count++; access->CountWtd += weight; + if (isFullCopy) + { + access->CountFullCopies++; + access->CountFullCopiesWtd += weight; + } } //------------------------------------------------------------------------ @@ -512,7 +520,7 @@ class LocalUses continue; } - if (!EvaluateReplacement(comp, lclNum, access, 0, 0)) + if (!EvaluateReplacement(comp, lclNum, access, nullptr)) { continue; } @@ -601,14 +609,14 @@ class LocalUses if (access == nullptr) { Access fakeAccess(inducedAccess.Offset, inducedAccess.AccessType, nullptr); - if (!EvaluateReplacement(comp, lclNum, fakeAccess, inducedAccess.Count, inducedAccess.CountWtd)) + if (!EvaluateReplacement(comp, lclNum, fakeAccess, &inducedAccess)) { continue; } } else { - if (!EvaluateReplacement(comp, lclNum, *access, inducedAccess.Count, inducedAccess.CountWtd)) + if (!EvaluateReplacement(comp, lclNum, *access, &inducedAccess)) { continue; } @@ -661,13 +669,15 @@ class LocalUses // comp - Compiler instance // lclNum - Local num for this struct local // access - Access information for the candidate. - // inducedCountWtd - Additional weighted count due to induced accesses. + // inducedAccess - Additional accesses induced by already selected replacements, or nullptr. // // Returns: // True if we should promote this access and create a replacement; otherwise false. // - bool EvaluateReplacement( - Compiler* comp, unsigned lclNum, const Access& access, unsigned inducedCount, weight_t inducedCountWtd) + bool EvaluateReplacement(Compiler* comp, + unsigned lclNum, + const Access& access, + const PrimitiveAccess* inducedAccess) { // Verify that this replacement has proper GC ness compared to the // layout. While reinterpreting GC fields to integers can be considered @@ -701,6 +711,9 @@ class LocalUses } } + unsigned inducedCount = inducedAccess == nullptr ? 0 : inducedAccess->Count; + weight_t inducedCountWtd = inducedAccess == nullptr ? 0 : inducedAccess->CountWtd; + unsigned countOverlappedCallArg = 0; unsigned countOverlappedStoredFromCall = 0; @@ -866,6 +879,14 @@ class LocalUses unsigned nativeParts = layout->GetSize() / TARGET_POINTER_SIZE; if (costVectorCopies && (primitiveAccessCount > nativeParts)) { + // Matching fields already promoted at the other endpoint have fragmented these copies. + // Do not charge for the same decomposition when promoting the induced accesses. + if (inducedAccess != nullptr) + { + assert(inducedAccess->CountFullCopies <= countVectorCopies); + countVectorCopies -= inducedAccess->CountFullCopies; + countVectorCopiesWtd = max(0.0, countVectorCopiesWtd - inducedAccess->CountFullCopiesWtd); + } // Amortize the original vector move over its fields. weight_t extraMoves = 1 - (weight_t)genTypeSize(access.AccessType) / layout->GetSize(); costWith += countVectorCopiesWtd * extraMoves * COST_REG_ACCESS_CYCLES; @@ -1493,6 +1514,9 @@ class LocalsUseVisitor : public GenTreeVisitor AggregateInfo* inducerAgg = aggregates.Lookup(inducer->GetLclNum()); if (inducerAgg != nullptr) { + bool isFullCopy = (candOffs == 0) && (inducerOffs == 0) && + (size == m_compiler->lvaGetDesc(candidate)->GetLayout()->GetSize()) && + (size == m_compiler->lvaGetDesc(inducer)->GetLayout()->GetSize()); Replacement* firstRep; Replacement* endRep; if (inducerAgg->OverlappingReplacements(inducerOffs, size, &firstRep, &endRep)) @@ -1503,7 +1527,7 @@ class LocalsUseVisitor : public GenTreeVisitor (rep->Offset + genTypeSize(rep->AccessType) <= (inducerOffs + size))) { InduceAccess(aggregates, candidate->GetLclNum(), candOffs + (rep->Offset - inducerOffs), - rep->AccessType, block); + rep->AccessType, block, isFullCopy); } } } @@ -1521,8 +1545,14 @@ class LocalsUseVisitor : public GenTreeVisitor // offset - Offset at which the induced access starts. // type - Type of the induced access. // block - The block with the induced access. + // isFullCopy - Whether the inducing copy covers both whole locals. // - void InduceAccess(AggregateInfoMap& aggregates, unsigned lclNum, unsigned offset, var_types type, BasicBlock* block) + void InduceAccess(AggregateInfoMap& aggregates, + unsigned lclNum, + unsigned offset, + var_types type, + BasicBlock* block, + bool isFullCopy = false) { AggregateInfo* agg = aggregates.Lookup(lclNum); if (agg != nullptr) @@ -1535,7 +1565,7 @@ class LocalsUseVisitor : public GenTreeVisitor } LocalUses* uses = GetOrCreateUses(lclNum); - uses->RecordInducedAccess(offset, type, block->getBBWeight(m_compiler)); + uses->RecordInducedAccess(offset, type, block->getBBWeight(m_compiler), isFullCopy); } //------------------------------------------------------------------------ @@ -2438,7 +2468,8 @@ GenTreeFieldList* ReplaceVisitor::CreateFieldListForStructLocal(GenTreeLclVarCom // argNode - The argument node // // Returns: -// True if the argument was replaced; false if write-backs are required. +// True if the call argument was replaced with a FIELD_LIST; false if the +// argument could not be represented as a FIELD_LIST. // bool ReplaceVisitor::ReplaceCallArgWithFieldList(GenTreeCall* call, GenTree** use, GenTreeLclVarCommon* argNode) { diff --git a/src/tests/JIT/Directed/Directed_do.csproj b/src/tests/JIT/Directed/Directed_do.csproj index 9ef81e4ada860d..165acd79d9a73c 100644 --- a/src/tests/JIT/Directed/Directed_do.csproj +++ b/src/tests/JIT/Directed/Directed_do.csproj @@ -15,6 +15,7 @@ + diff --git a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs new file mode 100644 index 00000000000000..1207f7d817e257 --- /dev/null +++ b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs @@ -0,0 +1,73 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Xunit; + +public class SharedVectorCopyCost +{ + // Five fields prevent regular promotion of this vector-sized struct. + [StructLayout(LayoutKind.Explicit, Size = 16)] + private struct S + { + [FieldOffset(0)] public int A; + [FieldOffset(4)] public int B; + [FieldOffset(8)] public int C; + [FieldOffset(12)] public short D; + [FieldOffset(14)] public short E; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static S Create(int value) => + new S { A = value, B = value + 1, C = value + 2, D = (short)(value + 3), E = (short)(value + 4) }; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void Fields(int a, int b, int c, int d, int e) { } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void Consume(S value) { } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int Copy(int input) + { + // After the last copy, reuse the promoted fields instead of reloading the shorts. + // X64-WINDOWS: call {{.*}}SharedVectorCopyCost:Consume + // X64-WINDOWS: call {{.*}}SharedVectorCopyCost:Consume + // X64-WINDOWS: call {{.*}}SharedVectorCopyCost:Consume + // X64-WINDOWS: call {{.*}}SharedVectorCopyCost:Consume + // X64-WINDOWS-NOT: movsx + // X64-WINDOWS: ret + S src = Create(input); + Fields(src.A, src.B, src.C, src.D, src.E); + Fields(src.A, src.B, src.C, src.D, src.E); + Fields(src.A, src.B, src.C, src.D, src.E); + + // The source is profitable to promote independently. Its replacements + // already fragment these copies, so promoting matching destination + // fields must not pay the fragmentation cost again. + S dst = src; + dst.A++; + Consume(dst); + dst = src; + dst.A++; + Consume(dst); + dst = src; + dst.A++; + Consume(dst); + dst = src; + dst.A++; + Consume(dst); + + Fields(dst.A, dst.B, dst.C, dst.D, dst.E); + return dst.A + dst.B + dst.C + dst.D + dst.E; + } + + [Fact] + public static void TestEntryPoint() + { + Assert.Equal(61, Copy(10)); + Assert.Equal(-39, Copy(-10)); + Assert.Equal(32769, Copy(32766)); + } +} diff --git a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.csproj b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.csproj new file mode 100644 index 00000000000000..994533b25961dd --- /dev/null +++ b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.csproj @@ -0,0 +1,14 @@ + + + true + true + True + + + + true + + + + + From c514c297aa2731e403cb05f835d8da8935f466c0 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 14 Sep 2026 16:30:08 +0100 Subject: [PATCH 3/9] JIT: cover shared vector-copy costs on Unix x64 Add a stack-argument variant with six preceding integer arguments, reusing the existing struct and an inlined copy body. Run the assembly assertions with the X64 prefix on both Windows and Unix. Validation: Windows baseline/candidate behavioral runs in normal, forced-promotion, JitStress=2 and PGO modes; Linux optimized and PGO runs; actual Release test-project build. All pass. Assembly checks pass with the revised model and fail with the previous cost model on both platforms. For the final shared-body test, Windows code shrinks from 518 to 478 bytes (115 to 110 instructions); Linux shrinks from 441 to 409 bytes (109 to 101 instructions). Both restore main codegen. These are generated-code comparisons, not measured application throughput. Compiler code is unchanged. --- .../physicalpromotion/SharedVectorCopyCost.cs | 58 +++++++++++++++---- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs index 1207f7d817e257..3e08cde3709e7c 100644 --- a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs +++ b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs @@ -28,16 +28,51 @@ private static void Fields(int a, int b, int c, int d, int e) { } [MethodImpl(MethodImplOptions.NoInlining)] private static void Consume(S value) { } + // Six integer arguments exhaust the Unix x64 argument registers. + [MethodImpl(MethodImplOptions.NoInlining)] + private static void ConsumeStackArgs(int a, int b, int c, int d, int e, int f, S value) { } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Consume(S value, bool stackArgs) + { + if (stackArgs) + { + ConsumeStackArgs(1, 2, 3, 4, 5, 6, value); + } + else + { + Consume(value); + } + } + [MethodImpl(MethodImplOptions.NoInlining)] private static int Copy(int input) { // After the last copy, reuse the promoted fields instead of reloading the shorts. - // X64-WINDOWS: call {{.*}}SharedVectorCopyCost:Consume - // X64-WINDOWS: call {{.*}}SharedVectorCopyCost:Consume - // X64-WINDOWS: call {{.*}}SharedVectorCopyCost:Consume - // X64-WINDOWS: call {{.*}}SharedVectorCopyCost:Consume - // X64-WINDOWS-NOT: movsx - // X64-WINDOWS: ret + // X64: call {{.*}}SharedVectorCopyCost:Consume + // X64: call {{.*}}SharedVectorCopyCost:Consume + // X64: call {{.*}}SharedVectorCopyCost:Consume + // X64: call {{.*}}SharedVectorCopyCost:Consume + // X64-NOT: movsx + // X64: ret + return CopyCore(input, false); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int CopyStackArgs(int input) + { + // X64: call {{.*}}SharedVectorCopyCost:ConsumeStackArgs + // X64: call {{.*}}SharedVectorCopyCost:ConsumeStackArgs + // X64: call {{.*}}SharedVectorCopyCost:ConsumeStackArgs + // X64: call {{.*}}SharedVectorCopyCost:ConsumeStackArgs + // X64-NOT: movsx + // X64: ret + return CopyCore(input, true); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int CopyCore(int input, bool stackArgs) + { S src = Create(input); Fields(src.A, src.B, src.C, src.D, src.E); Fields(src.A, src.B, src.C, src.D, src.E); @@ -48,16 +83,16 @@ private static int Copy(int input) // fields must not pay the fragmentation cost again. S dst = src; dst.A++; - Consume(dst); + Consume(dst, stackArgs); dst = src; dst.A++; - Consume(dst); + Consume(dst, stackArgs); dst = src; dst.A++; - Consume(dst); + Consume(dst, stackArgs); dst = src; dst.A++; - Consume(dst); + Consume(dst, stackArgs); Fields(dst.A, dst.B, dst.C, dst.D, dst.E); return dst.A + dst.B + dst.C + dst.D + dst.E; @@ -69,5 +104,8 @@ public static void TestEntryPoint() Assert.Equal(61, Copy(10)); Assert.Equal(-39, Copy(-10)); Assert.Equal(32769, Copy(32766)); + Assert.Equal(61, CopyStackArgs(10)); + Assert.Equal(-39, CopyStackArgs(-10)); + Assert.Equal(32769, CopyStackArgs(32766)); } } From f0b46071351e386d58d119f114c9e36f4a5998cb Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 14 Sep 2026 17:13:45 +0100 Subject: [PATCH 4/9] JIT: preserve constant copies and credit regular promotion Charge vector-copy fragmentation only for fields with scalar reads, using existing access and store-destination counts. Definition-only fields can propagate into struct copies and eliminate the local. Pass whole-local copy information from regular promotion and require the classification at every InduceAccess call. Add regular-endpoint, vector array-copy and constant-Guid coverage to SharedVectorCopyCost. Register its mobile fallback through the existing Directed wrapper conditions. Validation against main a652cdfed564: - 983,234 comparable SuperPMI contexts in 11 Windows x64 collections: identical assembly, zero failures; 190 shared missing recordings. - Windows optimized, forced-promotion, JitStress=2 and Tier1 tests pass; Linux behavioral and codegen checks pass; 18 existing regression runs pass. - Array-copy check fails on main and passes here; constant-Guid check rejects the previous cost model and passes here. - Windows Checked/Release and Linux Release JIT builds pass. - Five target/runtime MSBuild evaluations register the test exactly once. - Focused readonly Color record: Windows FullOpts 394 -> 271 bytes, Tier1/PGO 329 -> 269; Linux FullOpts unchanged at 573 bytes. - PIN JIT instruction counts: benchmarks +0.00805%, HashSet +0.00224%. No application-speed measurement or mobile/ARM64/x86 execution is claimed. --- src/coreclr/jit/promotion.cpp | 15 ++- src/tests/JIT/Directed/Directed_do.csproj | 3 +- .../physicalpromotion/SharedVectorCopyCost.cs | 109 ++++++++++++++++++ 3 files changed, 122 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/promotion.cpp b/src/coreclr/jit/promotion.cpp index 8e06833e074a80..7efa35b47c69f0 100644 --- a/src/coreclr/jit/promotion.cpp +++ b/src/coreclr/jit/promotion.cpp @@ -723,8 +723,11 @@ class LocalUses unsigned countVectorCopies = 0; weight_t countVectorCopiesWtd = 0; unsigned primitiveAccessCount = 1; - bool costVectorCopies = - (genTypeSize(access.AccessType) < TARGET_POINTER_SIZE) && varTypeIsSIMD(layout->GetRegisterType()); + // With only field definitions, promotion can propagate their values into the + // whole-struct copies and eliminate the local. Do not assume those copies fragment. + bool costVectorCopies = (access.Count > access.CountStoreDestination) && + (genTypeSize(access.AccessType) < TARGET_POINTER_SIZE) && + varTypeIsSIMD(layout->GetRegisterType()); for (const Access& otherAccess : m_accesses) { if (&otherAccess == &access) @@ -1477,6 +1480,10 @@ class LocalsUseVisitor : public GenTreeVisitor unsigned size = regPromLcl->GetLayout(m_compiler)->GetSize(); LclVarDsc* regPromDsc = m_compiler->lvaGetDesc(regPromLcl); + + bool isFullCopy = (candidateOffs == 0) && (regPromOffs == 0) && + (size == m_compiler->lvaGetDesc(candidateLcl)->GetLayout()->GetSize()) && + (size == regPromDsc->GetLayout()->GetSize()); for (unsigned fieldLcl = regPromDsc->lvFieldLclStart, i = 0; i < regPromDsc->lvFieldCnt; fieldLcl++, i++) { LclVarDsc* fieldDsc = m_compiler->lvaGetDesc(fieldLcl); @@ -1485,7 +1492,7 @@ class LocalsUseVisitor : public GenTreeVisitor { InduceAccess(aggregates, candidateLcl->GetLclNum(), candidateLcl->GetLclOffs() + (fieldDsc->lvFldOffset - regPromOffs), fieldDsc->lvType, - block); + block, isFullCopy); } } } @@ -1552,7 +1559,7 @@ class LocalsUseVisitor : public GenTreeVisitor unsigned offset, var_types type, BasicBlock* block, - bool isFullCopy = false) + bool isFullCopy) { AggregateInfo* agg = aggregates.Lookup(lclNum); if (agg != nullptr) diff --git a/src/tests/JIT/Directed/Directed_do.csproj b/src/tests/JIT/Directed/Directed_do.csproj index 165acd79d9a73c..13e93b9177fcbe 100644 --- a/src/tests/JIT/Directed/Directed_do.csproj +++ b/src/tests/JIT/Directed/Directed_do.csproj @@ -15,7 +15,7 @@ - + @@ -87,6 +87,7 @@ + diff --git a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs index 3e08cde3709e7c..984adc88cec190 100644 --- a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs +++ b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs @@ -98,6 +98,95 @@ private static int CopyCore(int input, bool stackArgs) return dst.A + dst.B + dst.C + dst.D + dst.E; } + [StructLayout(LayoutKind.Explicit, Size = 16)] + private struct ArrayValue + { + [FieldOffset(0)] public int A; + [FieldOffset(4)] public int B; + [FieldOffset(8)] public int C; + [FieldOffset(12)] public int D; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ArrayValue CreateArrayValue(int value) => + new ArrayValue { A = value, B = value + 1, C = value + 2, D = value + 3 }; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void CheckFields(int a, int b, int c, int d) + { + Assert.Equal(a + 1, b); + Assert.Equal(b + 1, c); + Assert.Equal(c + 1, d); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void Consume(ArrayValue value) { } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int CopyToRegularlyPromotedLocal(int input) + { + // The four-field destination is eligible for regular promotion. Whole-local + // copies must credit that endpoint just as they credit physical promotion. + ArrayValue source = CreateArrayValue(input); + CheckFields(source.A, source.B, source.C, source.D); + CheckFields(source.A, source.B, source.C, source.D); + CheckFields(source.A, source.B, source.C, source.D); + CheckFields(source.A, source.B, source.C, source.D); + Consume(source); + Consume(source); + Consume(source); + Consume(source); + ArrayValue destination = source; + destination.A++; + Consume(destination); + destination = source; + destination.A++; + Consume(destination); + destination = source; + destination.A++; + Consume(destination); + destination = source; + destination.A++; + Consume(destination); + return destination.A + destination.B + destination.C + destination.D; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void CopyToArray(int input, ArrayValue[] destination) + { + // Preserve vector copies instead of splitting each array store into four field stores. + // X64-WINDOWS: call {{.*}}SharedVectorCopyCost:CheckFields + // X64-WINDOWS: call {{.*}}SharedVectorCopyCost:CheckFields + // X64-WINDOWS: {{v?movups}} xmm{{[0-9]+}}, xmmword ptr + // X64-WINDOWS: {{v?movups}} xmmword ptr + // X64-WINDOWS: {{v?movups}} xmm{{[0-9]+}}, xmmword ptr + // X64-WINDOWS: {{v?movups}} xmmword ptr + // X64-WINDOWS: {{v?movups}} xmm{{[0-9]+}}, xmmword ptr + // X64-WINDOWS: {{v?movups}} xmmword ptr + // X64-WINDOWS: {{v?movups}} xmm{{[0-9]+}}, xmmword ptr + // X64-WINDOWS: {{v?movups}} xmmword ptr + ArrayValue source = CreateArrayValue(input); + CheckFields(source.A, source.B, source.C, source.D); + CheckFields(source.A, source.B, source.C, source.D); + destination[0] = source; + destination[1] = source; + destination[2] = source; + destination[3] = source; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void CopyConstantToArray(System.Guid[] destination) + { + // Promotion should allow constant fields to propagate into the copies. + // X64-WINDOWS-NOT: xmmword ptr [rsp + // X64-WINDOWS: ret + System.Guid source = new System.Guid(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); + destination[0] = source; + destination[1] = source; + destination[2] = source; + destination[3] = source; + } + [Fact] public static void TestEntryPoint() { @@ -107,5 +196,25 @@ public static void TestEntryPoint() Assert.Equal(61, CopyStackArgs(10)); Assert.Equal(-39, CopyStackArgs(-10)); Assert.Equal(32769, CopyStackArgs(32766)); + Assert.Equal(47, CopyToRegularlyPromotedLocal(10)); + Assert.Equal(-33, CopyToRegularlyPromotedLocal(-10)); + + ArrayValue[] values = new ArrayValue[4]; + foreach (int input in new[] { 10, -10 }) + { + CopyToArray(input, values); + foreach (ArrayValue value in values) + { + Assert.Equal(input, value.A); + CheckFields(value.A, value.B, value.C, value.D); + } + } + + System.Guid[] constants = new System.Guid[4]; + CopyConstantToArray(constants); + foreach (System.Guid value in constants) + { + Assert.Equal(new System.Guid(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), value); + } } } From f9f5ff83abad734f9186365ed744951b5f7534ca Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 14 Sep 2026 18:11:00 +0100 Subject: [PATCH 5/9] JIT: exclude definition-only siblings from copy costing Count only primitive accesses with scalar reads when deciding whether a vector-sized copy fragments. Apply the same filter to the mixed-width gate, using the existing access and store-destination counters. Add an X64 codegen assertion for CopyToRegularlyPromotedLocal. It passes on Windows and Linux. Disabling only regular-endpoint copy credit makes the assertion fail on Windows; Linux already selects the checked shape. Validation: 983,234 comparable contexts across eleven SuperPMI collections produce identical assembly to f0b46071351, with zero compilation failures and 190 shared missing recordings. Windows optimized, forced-promotion, JitStress=2 and Tier1/PGO tests pass; Linux tests and codegen checks pass. Windows Checked/Release and Linux Release JIT builds pass. The focused Color example remains 271 bytes FullOpts and 269 bytes Tier1 on Windows. --- src/coreclr/jit/promotion.cpp | 2 +- .../JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/promotion.cpp b/src/coreclr/jit/promotion.cpp index 7efa35b47c69f0..41faecff521a0f 100644 --- a/src/coreclr/jit/promotion.cpp +++ b/src/coreclr/jit/promotion.cpp @@ -735,7 +735,7 @@ class LocalUses continue; } - if (otherAccess.AccessType != TYP_STRUCT) + if ((otherAccess.AccessType != TYP_STRUCT) && (otherAccess.Count > otherAccess.CountStoreDestination)) { primitiveAccessCount++; costVectorCopies &= genTypeSize(otherAccess.AccessType) < TARGET_POINTER_SIZE; diff --git a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs index 984adc88cec190..468ae1ebf1d489 100644 --- a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs +++ b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs @@ -127,6 +127,13 @@ private static int CopyToRegularlyPromotedLocal(int input) { // The four-field destination is eligible for regular promotion. Whole-local // copies must credit that endpoint just as they credit physical promotion. + // X64: call {{.*}}SharedVectorCopyCost:CheckFields + // X64: call {{.*}}SharedVectorCopyCost:CheckFields + // X64: call {{.*}}SharedVectorCopyCost:CheckFields + // X64: call {{.*}}SharedVectorCopyCost:CheckFields + // Keep the source fields promoted instead of copying their stack storage. + // X64-NOT: xmmword ptr + // X64: ret ArrayValue source = CreateArrayValue(input); CheckFields(source.A, source.B, source.C, source.D); CheckFields(source.A, source.B, source.C, source.D); From 3ea37128b0e2db3704a0ffe9030d47f33dcf73e2 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 14 Sep 2026 20:29:52 +0100 Subject: [PATCH 6/9] JIT: share whole-copy classification during promotion Use IsFullCopy in both induction paths and reuse Compiler::IsEntireAccess for each endpoint. Preserve each caller's copy size and partial-copy classification, and remove the unused candidate offset. Validation: isolated Windows x64 Checked build; focused behavior and codegen checks in optimized, forced promotion, JitStress=2 and Tier1/PGO modes. Three SuperPMI collections have identical assembly across 87,586 matched contexts, no compilation failures and six shared recording misses. Changed-line clang-format and git diff --check pass. Linux and the full suite were not rerun for this refactor. --- src/coreclr/jit/promotion.cpp | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/coreclr/jit/promotion.cpp b/src/coreclr/jit/promotion.cpp index 41faecff521a0f..57a712789b54a9 100644 --- a/src/coreclr/jit/promotion.cpp +++ b/src/coreclr/jit/promotion.cpp @@ -1457,6 +1457,23 @@ class LocalsUseVisitor : public GenTreeVisitor return m_uses[lclNum]; } + //------------------------------------------------------------------------ + // IsFullCopy: Check whether a copy covers both struct locals in their entirety. + // + // Parameters: + // first - One endpoint of the copy. + // second - The other endpoint of the copy. + // size - The number of bytes copied. + // + // Returns: + // True if both accesses cover their entire local. + // + bool IsFullCopy(GenTreeLclVarCommon* first, GenTreeLclVarCommon* second, unsigned size) + { + return m_compiler->IsEntireAccess(first->GetLclNum(), first->GetLclOffs(), ValueSize(size)) && + m_compiler->IsEntireAccess(second->GetLclNum(), second->GetLclOffs(), ValueSize(size)); + } + //------------------------------------------------------------------------ // InduceAccessesFromRegularlyPromotedStruct: // Create induced accesses based on the fact that there is a store @@ -1475,15 +1492,12 @@ class LocalsUseVisitor : public GenTreeVisitor GenTreeLclVarCommon* regPromLcl, BasicBlock* block) { - unsigned regPromOffs = regPromLcl->GetLclOffs(); - unsigned candidateOffs = candidateLcl->GetLclOffs(); - unsigned size = regPromLcl->GetLayout(m_compiler)->GetSize(); + unsigned regPromOffs = regPromLcl->GetLclOffs(); + unsigned size = regPromLcl->GetLayout(m_compiler)->GetSize(); LclVarDsc* regPromDsc = m_compiler->lvaGetDesc(regPromLcl); - bool isFullCopy = (candidateOffs == 0) && (regPromOffs == 0) && - (size == m_compiler->lvaGetDesc(candidateLcl)->GetLayout()->GetSize()) && - (size == regPromDsc->GetLayout()->GetSize()); + bool isFullCopy = IsFullCopy(candidateLcl, regPromLcl, size); for (unsigned fieldLcl = regPromDsc->lvFieldLclStart, i = 0; i < regPromDsc->lvFieldCnt; fieldLcl++, i++) { LclVarDsc* fieldDsc = m_compiler->lvaGetDesc(fieldLcl); @@ -1521,9 +1535,7 @@ class LocalsUseVisitor : public GenTreeVisitor AggregateInfo* inducerAgg = aggregates.Lookup(inducer->GetLclNum()); if (inducerAgg != nullptr) { - bool isFullCopy = (candOffs == 0) && (inducerOffs == 0) && - (size == m_compiler->lvaGetDesc(candidate)->GetLayout()->GetSize()) && - (size == m_compiler->lvaGetDesc(inducer)->GetLayout()->GetSize()); + bool isFullCopy = IsFullCopy(candidate, inducer, size); Replacement* firstRep; Replacement* endRep; if (inducerAgg->OverlappingReplacements(inducerOffs, size, &firstRep, &endRep)) From 8f8a73620e7cc3e8b3d3751a53624bef440b0d43 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 14 Sep 2026 21:25:24 +0100 Subject: [PATCH 7/9] JIT: account for induced fields in vector copy costing Include induced accesses in fragmentation eligibility and field counting, reusing FindAccess to avoid counting existing scalar reads twice. Preserve full-copy credit and scan induced siblings only for eligible accesses. Exclude struct initializations from the existing destination counters so initialization does not incur a copy charge; no per-access fields are added. Add X64 coverage for a partial copy into a destination with only induced fields and for a native-width field alongside smaller fields. Explicitly enable physical promotion in the test project. Validation: Windows x64 Checked and Linux x64 Release builds; focused Windows optimized, forced-promotion, JitStress=2 and Tier1/PGO behavior; Linux behavior and both platforms' codegen checks. Negative controls fail on the prior JIT and, for mixed-width coverage, with only that guard removed. The induced-copy example shrinks 472 to 373 bytes on Windows and 503 to 367 on Linux; the mixed-width example is unchanged. All eleven SuperPMI collections: 983,234 matched contexts, identical assembly and 238,457,491 bytes on both sides, 190 shared misses and no new misses or compilation failures. MSBuild confirms process isolation and the promotion setting. Formatting, diff and ASCII checks pass. No runtime benchmark, new throughput, full-tree, x86 or ARM run is claimed. --- src/coreclr/jit/promotion.cpp | 38 +++++-- .../physicalpromotion/SharedVectorCopyCost.cs | 102 ++++++++++++++++++ .../SharedVectorCopyCost.csproj | 1 + 3 files changed, 135 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/promotion.cpp b/src/coreclr/jit/promotion.cpp index 57a712789b54a9..69a3b12859532d 100644 --- a/src/coreclr/jit/promotion.cpp +++ b/src/coreclr/jit/promotion.cpp @@ -100,7 +100,7 @@ struct Access // Number of times this access is the source of a store. unsigned CountStoreSource = 0; - // Number of times this access is the destination of a store. + // Number of times this access is a store destination, excluding struct initializations. unsigned CountStoreDestination = 0; // Number of times this is stored by being passed as the retbuf. // These stores need a readback @@ -153,6 +153,7 @@ enum class AccessKindFlags : uint32_t IsCallRetBuf = 8, IsStoreSource = 16, IsStoreDestination = 32, + IsInit = 128, #ifdef DEBUG IsReturned = 64, #endif @@ -416,7 +417,8 @@ class LocalUses access->CountStoreSourceWtd += weight; } - if ((flags & AccessKindFlags::IsStoreDestination) != AccessKindFlags::None) + if (((flags & AccessKindFlags::IsStoreDestination) != AccessKindFlags::None) && + ((flags & AccessKindFlags::IsInit) == AccessKindFlags::None)) { access->CountStoreDestination++; access->CountStoreDestinationWtd += weight; @@ -723,9 +725,9 @@ class LocalUses unsigned countVectorCopies = 0; weight_t countVectorCopiesWtd = 0; unsigned primitiveAccessCount = 1; - // With only field definitions, promotion can propagate their values into the - // whole-struct copies and eliminate the local. Do not assume those copies fragment. - bool costVectorCopies = (access.Count > access.CountStoreDestination) && + // Without field reads or induced accesses, promotion can propagate definitions + // into the whole-struct copies and eliminate the local. Do not assume those copies fragment. + bool costVectorCopies = ((access.Count > access.CountStoreDestination) || (inducedCount > 0)) && (genTypeSize(access.AccessType) < TARGET_POINTER_SIZE) && varTypeIsSIMD(layout->GetRegisterType()); for (const Access& otherAccess : m_accesses) @@ -759,7 +761,7 @@ class LocalUses if (costVectorCopies && (otherAccess.GetAccessSize() == layout->GetSize())) { - // Call-result read-backs are already costed separately. + // Initializations are not copies; call-result read-backs are costed separately. countVectorCopies += otherAccess.CountStoreSource + otherAccess.CountStoreDestination + otherAccess.CountPassedAsRetbuf - otherAccess.CountStoredFromCall; countVectorCopiesWtd += otherAccess.CountStoreSourceWtd + otherAccess.CountStoreDestinationWtd + @@ -775,6 +777,25 @@ class LocalUses } } + if (costVectorCopies) + { + // Count induced-only fields too, without counting an existing scalar read twice. + for (const PrimitiveAccess& otherInduced : m_inducedAccesses) + { + if ((otherInduced.Offset == access.Offset) && (otherInduced.AccessType == access.AccessType)) + { + continue; + } + + const Access* existing = FindAccess(otherInduced.Offset, otherInduced.AccessType); + if ((existing == nullptr) || (existing->Count <= existing->CountStoreDestination)) + { + primitiveAccessCount++; + costVectorCopies &= genTypeSize(otherInduced.AccessType) < TARGET_POINTER_SIZE; + } + } + } + // We cost any normal access (which is a struct load or store) without promotion at 3 cycles. const weight_t COST_STRUCT_ACCESS_CYCLES = 3; // And at 4 bytes size @@ -1607,6 +1628,11 @@ class LocalsUseVisitor : public GenTreeVisitor { flags |= AccessKindFlags::IsStoreDestination; + if (lcl->TypeIs(TYP_STRUCT) && lcl->Data()->gtEffectiveVal()->IsIntegralConst()) + { + flags |= AccessKindFlags::IsInit; + } + if (lcl->AsLclVarCommon()->Data()->gtEffectiveVal()->IsCall()) { flags |= AccessKindFlags::IsStoredFromCall; diff --git a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs index 468ae1ebf1d489..bbd2c6f8dab4a2 100644 --- a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs +++ b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs @@ -194,6 +194,83 @@ private static void CopyConstantToArray(System.Guid[] destination) destination[3] = source; } + [StructLayout(LayoutKind.Explicit, Size = 24)] + private struct OuterValue + { + [FieldOffset(0)] public long Prefix; + [FieldOffset(8)] public ArrayValue Value; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static OuterValue CreateOuterValue(int input) => + new OuterValue { Prefix = input, Value = new ArrayValue { A = input, B = input + 1, C = input + 2, D = input + 3 } }; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void CopyInducedToArray(int input, ArrayValue[] destination) + { + // Copy a subrange of the promoted source into a local with no explicit field reads. + // Its induced fields must still pay for fragmenting the subsequent array copies. + // X64: call {{.*}}SharedVectorCopyCost:CheckFields + // X64: call {{.*}}SharedVectorCopyCost:CheckFields + // X64: call {{.*}}SharedVectorCopyCost:CheckFields + // X64: {{v?movups}} xmm{{[0-9]+}}, xmmword ptr + // X64: {{v?movups}} xmmword ptr + // X64: {{v?movups}} xmm{{[0-9]+}}, xmmword ptr + // X64: {{v?movups}} xmmword ptr + OuterValue source = CreateOuterValue(input); + CheckFields(source.Value.A, source.Value.B, source.Value.C, source.Value.D); + CheckFields(source.Value.A, source.Value.B, source.Value.C, source.Value.D); + CheckFields(source.Value.A, source.Value.B, source.Value.C, source.Value.D); + ArrayValue value = source.Value; + destination[0] = value; + destination[1] = value; + destination[2] = value; + destination[3] = value; + destination[4] = value; + destination[5] = value; + destination[6] = value; + destination[7] = value; + } + + [StructLayout(LayoutKind.Explicit, Size = 16)] + private struct MixedValue + { + [FieldOffset(0)] public long A; + [FieldOffset(8)] public short B; + [FieldOffset(10)] public short C; + [FieldOffset(12)] public short D; + [FieldOffset(14)] public short E; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static MixedValue CreateMixedValue(int value) => + new MixedValue { A = value, B = (short)value, C = (short)value, D = (short)value, E = (short)value }; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void MixedFields(long a, int b, int c, int d, int e) { } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void CopyMixedToArray(int input, MixedValue[] destination) + { + // The native-width read keeps this mixed-width copy outside the fragmentation charge. + // X64: call {{.*}}SharedVectorCopyCost:MixedFields + // X64: call {{.*}}SharedVectorCopyCost:MixedFields + // X64-NOT: xmmword ptr + // X64-NOT: ptr [rsp + // X64: ret + MixedValue source = CreateMixedValue(input); + MixedFields(source.A, source.B, source.C, source.D, source.E); + MixedFields(source.A, source.B, source.C, source.D, source.E); + destination[0] = source; + destination[1] = source; + destination[2] = source; + destination[3] = source; + destination[4] = source; + destination[5] = source; + destination[6] = source; + destination[7] = source; + } + [Fact] public static void TestEntryPoint() { @@ -217,6 +294,31 @@ public static void TestEntryPoint() } } + ArrayValue[] inducedValues = new ArrayValue[8]; + foreach (int input in new[] { 10, -10 }) + { + CopyInducedToArray(input, inducedValues); + foreach (ArrayValue value in inducedValues) + { + Assert.Equal(input, value.A); + CheckFields(value.A, value.B, value.C, value.D); + } + } + + MixedValue[] mixedValues = new MixedValue[8]; + foreach (int input in new[] { 10, -10, 32768 }) + { + CopyMixedToArray(input, mixedValues); + foreach (MixedValue value in mixedValues) + { + Assert.Equal((long)input, value.A); + Assert.Equal((short)input, value.B); + Assert.Equal((short)input, value.C); + Assert.Equal((short)input, value.D); + Assert.Equal((short)input, value.E); + } + } + System.Guid[] constants = new System.Guid[4]; CopyConstantToArray(constants); foreach (System.Guid value in constants) diff --git a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.csproj b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.csproj index 994533b25961dd..030af284e91008 100644 --- a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.csproj +++ b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.csproj @@ -8,6 +8,7 @@ true + From c1340aa5853a4bcc67932b60d872499c27d2de94 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 14 Sep 2026 22:27:29 +0100 Subject: [PATCH 8/9] JIT: refine vector-copy costing guards and target coverage Use IsInitVal to exclude nonzero struct initialization from copy counts. Skip the induced-field summary when the existing full-copy count is zero. Add a nonzero initblk test and ARM64 vector-copy / x86 native-field exclusion checks, without adding cached state or per-access counters. Windows and Linux x64 behavior and FileCheck pass. Matching-source crossgen2 checks pass for Windows-target ARM64 and x86 code generation; ARM64 CopyInducedToArray improves from 448 to 380 bytes against main, and its check fails on main. x86 remains at 409 bytes, as intended. No ARM64/x86 execution is claimed. Full Windows x64 SuperPMI: 983,234 comparable contexts, identical assembly and 238,457,491 bytes on both sides; 190 shared misses, no compile failures. Release-JIT PIN counts versus main increase 0.03082% on benchmarks.run (51,617 contexts) and 0.01260% on HashSet (1,307 contexts). Formatting, diff and added-source ASCII checks pass. The full JIT tree and generated-code execution-speed benchmarks were not run. --- src/coreclr/jit/promotion.cpp | 4 +-- .../physicalpromotion/SharedVectorCopyCost.cs | 33 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/promotion.cpp b/src/coreclr/jit/promotion.cpp index 69a3b12859532d..1b137833eb7b11 100644 --- a/src/coreclr/jit/promotion.cpp +++ b/src/coreclr/jit/promotion.cpp @@ -777,7 +777,7 @@ class LocalUses } } - if (costVectorCopies) + if (costVectorCopies && (countVectorCopies > 0)) { // Count induced-only fields too, without counting an existing scalar read twice. for (const PrimitiveAccess& otherInduced : m_inducedAccesses) @@ -1628,7 +1628,7 @@ class LocalsUseVisitor : public GenTreeVisitor { flags |= AccessKindFlags::IsStoreDestination; - if (lcl->TypeIs(TYP_STRUCT) && lcl->Data()->gtEffectiveVal()->IsIntegralConst()) + if (lcl->TypeIs(TYP_STRUCT) && lcl->Data()->gtEffectiveVal()->IsInitVal()) { flags |= AccessKindFlags::IsInit; } diff --git a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs index bbd2c6f8dab4a2..064901ba8c6742 100644 --- a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs +++ b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs @@ -210,6 +210,16 @@ private static void CopyInducedToArray(int input, ArrayValue[] destination) { // Copy a subrange of the promoted source into a local with no explicit field reads. // Its induced fields must still pay for fragmenting the subsequent array copies. + // ARM64: SharedVectorCopyCost:CheckFields + // ARM64: SharedVectorCopyCost:CheckFields + // ARM64: SharedVectorCopyCost:CheckFields + // ARM64: ldr q{{[0-9]+}}, [{{fp|sp}} + // ARM64: {{stp|str}} q{{[0-9]+}}, + // ARM64: ret + // On x86 these int fields are native-sized and must remain uncharged. + // X86: call {{.*}}SharedVectorCopyCost:CheckFields + // X86-NOT: xmmword ptr + // X86: ret // X64: call {{.*}}SharedVectorCopyCost:CheckFields // X64: call {{.*}}SharedVectorCopyCost:CheckFields // X64: call {{.*}}SharedVectorCopyCost:CheckFields @@ -271,9 +281,32 @@ private static void CopyMixedToArray(int input, MixedValue[] destination) destination[7] = source; } + [MethodImpl(MethodImplOptions.NoInlining)] + private static int InitializeNonzero(int input) + { + // Nonzero initblk is initialization, not a copy to fragment. + // X64-NOT: xmmword ptr + // X64: call {{.*}}SharedVectorCopyCost:Fields + // X64: ret + // X86-NOT: xmmword ptr + // X86: call {{.*}}SharedVectorCopyCost:Fields + // X86: ret + // ARM64-NOT: {{stp|str}} q{{[0-9]+}} + // ARM64: SharedVectorCopyCost:Fields + // ARM64: ret + S value; + Unsafe.SkipInit(out value); + Unsafe.InitBlockUnaligned(ref Unsafe.As(ref value), 0xAB, (uint)Unsafe.SizeOf()); + value.A = input; + Fields(value.A, value.B, value.C, value.D, value.E); + return value.A + value.B + value.C + value.D + value.E; + } + [Fact] public static void TestEntryPoint() { + Assert.Equal(unchecked(10 + 2 * (int)0xABABABAB + 2 * (short)0xABAB), InitializeNonzero(10)); + Assert.Equal(unchecked(-10 + 2 * (int)0xABABABAB + 2 * (short)0xABAB), InitializeNonzero(-10)); Assert.Equal(61, Copy(10)); Assert.Equal(-39, Copy(-10)); Assert.Equal(32769, Copy(32766)); From 36ea665682f2d0dc5ab830b7ca691b4d63640a46 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Tue, 15 Sep 2026 00:13:32 +0100 Subject: [PATCH 9/9] Strengthen vector-copy codegen coverage Check all eight induced array copies on x64 and ARM64 and cover direct array copies on ARM64. Bound store checks at the first return so fallback copies cannot mask missing fast-path stores, without requiring repeated source loads. SuperFileCheck passes against retained Windows/Linux x64 and ARM64 PR output; x86 checks remain passing. All 26 individual fast-path store-removal mutations fail, and ARM64 main fails the vector-store checks. Only test directives changed; no native ARM64 execution is claimed. --- .../physicalpromotion/SharedVectorCopyCost.cs | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs index 064901ba8c6742..ba2b3842e868e0 100644 --- a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs +++ b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs @@ -162,16 +162,19 @@ private static int CopyToRegularlyPromotedLocal(int input) private static void CopyToArray(int input, ArrayValue[] destination) { // Preserve vector copies instead of splitting each array store into four field stores. + // ARM64 pairs the four array copies into two vector stores. + // ARM64: SharedVectorCopyCost:CheckFields + // ARM64: SharedVectorCopyCost:CheckFields + // ARM64: stp q{{[0-9]+}}, q{{[0-9]+}}, + // ARM64: stp q{{[0-9]+}}, q{{[0-9]+}}, + // ARM64-LABEL: {{^ *ret}} // X64-WINDOWS: call {{.*}}SharedVectorCopyCost:CheckFields // X64-WINDOWS: call {{.*}}SharedVectorCopyCost:CheckFields - // X64-WINDOWS: {{v?movups}} xmm{{[0-9]+}}, xmmword ptr // X64-WINDOWS: {{v?movups}} xmmword ptr - // X64-WINDOWS: {{v?movups}} xmm{{[0-9]+}}, xmmword ptr // X64-WINDOWS: {{v?movups}} xmmword ptr - // X64-WINDOWS: {{v?movups}} xmm{{[0-9]+}}, xmmword ptr // X64-WINDOWS: {{v?movups}} xmmword ptr - // X64-WINDOWS: {{v?movups}} xmm{{[0-9]+}}, xmmword ptr // X64-WINDOWS: {{v?movups}} xmmword ptr + // X64-WINDOWS-LABEL: {{^ *ret}} ArrayValue source = CreateArrayValue(input); CheckFields(source.A, source.B, source.C, source.D); CheckFields(source.A, source.B, source.C, source.D); @@ -213,9 +216,13 @@ private static void CopyInducedToArray(int input, ArrayValue[] destination) // ARM64: SharedVectorCopyCost:CheckFields // ARM64: SharedVectorCopyCost:CheckFields // ARM64: SharedVectorCopyCost:CheckFields - // ARM64: ldr q{{[0-9]+}}, [{{fp|sp}} - // ARM64: {{stp|str}} q{{[0-9]+}}, - // ARM64: ret + // Check all eight copies before the first return, excluding the fallback path. + // ARM64 pairs the copies; do not require a reload for each store. + // ARM64: stp q{{[0-9]+}}, q{{[0-9]+}}, + // ARM64: stp q{{[0-9]+}}, q{{[0-9]+}}, + // ARM64: stp q{{[0-9]+}}, q{{[0-9]+}}, + // ARM64: stp q{{[0-9]+}}, q{{[0-9]+}}, + // ARM64-LABEL: {{^ *ret}} // On x86 these int fields are native-sized and must remain uncharged. // X86: call {{.*}}SharedVectorCopyCost:CheckFields // X86-NOT: xmmword ptr @@ -223,10 +230,15 @@ private static void CopyInducedToArray(int input, ArrayValue[] destination) // X64: call {{.*}}SharedVectorCopyCost:CheckFields // X64: call {{.*}}SharedVectorCopyCost:CheckFields // X64: call {{.*}}SharedVectorCopyCost:CheckFields - // X64: {{v?movups}} xmm{{[0-9]+}}, xmmword ptr // X64: {{v?movups}} xmmword ptr - // X64: {{v?movups}} xmm{{[0-9]+}}, xmmword ptr // X64: {{v?movups}} xmmword ptr + // X64: {{v?movups}} xmmword ptr + // X64: {{v?movups}} xmmword ptr + // X64: {{v?movups}} xmmword ptr + // X64: {{v?movups}} xmmword ptr + // X64: {{v?movups}} xmmword ptr + // X64: {{v?movups}} xmmword ptr + // X64-LABEL: {{^ *ret}} OuterValue source = CreateOuterValue(input); CheckFields(source.Value.A, source.Value.B, source.Value.C, source.Value.D); CheckFields(source.Value.A, source.Value.B, source.Value.C, source.Value.D);