diff --git a/src/coreclr/jit/promotion.cpp b/src/coreclr/jit/promotion.cpp index 2ecad18d74760f..1b137833eb7b11 100644 --- a/src/coreclr/jit/promotion.cpp +++ b/src/coreclr/jit/promotion.cpp @@ -98,20 +98,20 @@ struct Access weight_t CountCallArgsWtd = 0; weight_t CountRegCallArgsWtd = 0; -#ifdef DEBUG // Number of times this access is the source of a store. unsigned CountStoreSource = 0; - // Number of times this access is the destination of a store. + // Number of times this access is a store destination, excluding struct initializations. unsigned CountStoreDestination = 0; - unsigned CountReturns = 0; // Number of times this is stored by being passed as the retbuf. // These stores need a readback unsigned CountPassedAsRetbuf = 0; weight_t CountStoreSourceWtd = 0; weight_t CountStoreDestinationWtd = 0; - weight_t CountReturnsWtd = 0; weight_t CountPassedAsRetbufWtd = 0; +#ifdef DEBUG + unsigned CountReturns = 0; + weight_t CountReturnsWtd = 0; #endif Access(unsigned offset, var_types accessType, ClassLayout* layout) @@ -146,15 +146,16 @@ struct Access enum class AccessKindFlags : uint32_t { - None = 0, - IsCallArg = 1, - IsRegCallArg = 2, - IsStoredFromCall = 4, - IsCallRetBuf = 8, -#ifdef DEBUG + None = 0, + IsCallArg = 1, + IsRegCallArg = 2, + IsStoredFromCall = 4, + IsCallRetBuf = 8, IsStoreSource = 16, IsStoreDestination = 32, - IsReturned = 64, + IsInit = 128, +#ifdef DEBUG + IsReturned = 64, #endif }; @@ -311,8 +312,10 @@ AggregateInfo* AggregateInfoMap::Lookup(unsigned lclNum) struct PrimitiveAccess { - unsigned Count = 0; - weight_t CountWtd = 0; + unsigned Count = 0; + weight_t CountWtd = 0; + unsigned CountFullCopies = 0; + weight_t CountFullCopiesWtd = 0; unsigned Offset; var_types AccessType; @@ -402,7 +405,6 @@ class LocalUses access->CountStoredFromCallWtd += weight; } -#ifdef DEBUG if ((flags & AccessKindFlags::IsCallRetBuf) != AccessKindFlags::None) { access->CountPassedAsRetbuf++; @@ -415,12 +417,14 @@ class LocalUses access->CountStoreSourceWtd += weight; } - if ((flags & AccessKindFlags::IsStoreDestination) != AccessKindFlags::None) + if (((flags & AccessKindFlags::IsStoreDestination) != AccessKindFlags::None) && + ((flags & AccessKindFlags::IsInit) == AccessKindFlags::None)) { access->CountStoreDestination++; access->CountStoreDestinationWtd += weight; } +#ifdef DEBUG if ((flags & AccessKindFlags::IsReturned) != AccessKindFlags::None) { access->CountReturns++; @@ -437,12 +441,13 @@ class LocalUses // offs - The offset being accessed // accessType - The type of the access // weight - Weight of the block containing the access + // isFullCopy - Whether the copy covers both whole locals // // Remarks: // Induced accesses are accesses that are induced by physical promotion // due to store decompositon. They are always of primitive type. // - void RecordInducedAccess(unsigned offs, var_types accessType, weight_t weight) + void RecordInducedAccess(unsigned offs, var_types accessType, weight_t weight, bool isFullCopy) { PrimitiveAccess* access = nullptr; @@ -477,6 +482,11 @@ class LocalUses access->Count++; access->CountWtd += weight; + if (isFullCopy) + { + access->CountFullCopies++; + access->CountFullCopiesWtd += weight; + } } //------------------------------------------------------------------------ @@ -512,7 +522,7 @@ class LocalUses continue; } - if (!EvaluateReplacement(comp, lclNum, access, 0, 0)) + if (!EvaluateReplacement(comp, lclNum, access, nullptr)) { continue; } @@ -601,14 +611,14 @@ class LocalUses if (access == nullptr) { Access fakeAccess(inducedAccess.Offset, inducedAccess.AccessType, nullptr); - if (!EvaluateReplacement(comp, lclNum, fakeAccess, inducedAccess.Count, inducedAccess.CountWtd)) + if (!EvaluateReplacement(comp, lclNum, fakeAccess, &inducedAccess)) { continue; } } else { - if (!EvaluateReplacement(comp, lclNum, *access, inducedAccess.Count, inducedAccess.CountWtd)) + if (!EvaluateReplacement(comp, lclNum, *access, &inducedAccess)) { continue; } @@ -661,13 +671,15 @@ class LocalUses // comp - Compiler instance // lclNum - Local num for this struct local // access - Access information for the candidate. - // inducedCountWtd - Additional weighted count due to induced accesses. + // inducedAccess - Additional accesses induced by already selected replacements, or nullptr. // // Returns: // True if we should promote this access and create a replacement; otherwise false. // - bool EvaluateReplacement( - Compiler* comp, unsigned lclNum, const Access& access, unsigned inducedCount, weight_t inducedCountWtd) + bool EvaluateReplacement(Compiler* comp, + unsigned lclNum, + const Access& access, + const PrimitiveAccess* inducedAccess) { // Verify that this replacement has proper GC ness compared to the // layout. While reinterpreting GC fields to integers can be considered @@ -701,13 +713,23 @@ class LocalUses } } + unsigned inducedCount = inducedAccess == nullptr ? 0 : inducedAccess->Count; + weight_t inducedCountWtd = inducedAccess == nullptr ? 0 : inducedAccess->CountWtd; + unsigned countOverlappedCallArg = 0; unsigned countOverlappedStoredFromCall = 0; weight_t countOverlappedCallArgWtd = 0; weight_t countOverlappedStoredFromCallWtd = 0; - bool overlap = false; + unsigned countVectorCopies = 0; + weight_t countVectorCopiesWtd = 0; + unsigned primitiveAccessCount = 1; + // Without field reads or induced accesses, promotion can propagate definitions + // into the whole-struct copies and eliminate the local. Do not assume those copies fragment. + bool costVectorCopies = ((access.Count > access.CountStoreDestination) || (inducedCount > 0)) && + (genTypeSize(access.AccessType) < TARGET_POINTER_SIZE) && + varTypeIsSIMD(layout->GetRegisterType()); for (const Access& otherAccess : m_accesses) { if (&otherAccess == &access) @@ -715,6 +737,12 @@ class LocalUses continue; } + if ((otherAccess.AccessType != TYP_STRUCT) && (otherAccess.Count > otherAccess.CountStoreDestination)) + { + primitiveAccessCount++; + costVectorCopies &= genTypeSize(otherAccess.AccessType) < TARGET_POINTER_SIZE; + } + if (!otherAccess.Overlaps(access.Offset, genTypeSize(access.AccessType))) { continue; @@ -731,6 +759,15 @@ class LocalUses countOverlappedCallArgWtd += otherAccess.CountCallArgsWtd; countOverlappedStoredFromCallWtd += otherAccess.CountStoredFromCallWtd; + if (costVectorCopies && (otherAccess.GetAccessSize() == layout->GetSize())) + { + // Initializations are not copies; call-result read-backs are costed separately. + countVectorCopies += otherAccess.CountStoreSource + otherAccess.CountStoreDestination + + otherAccess.CountPassedAsRetbuf - otherAccess.CountStoredFromCall; + countVectorCopiesWtd += otherAccess.CountStoreSourceWtd + otherAccess.CountStoreDestinationWtd + + otherAccess.CountPassedAsRetbufWtd - otherAccess.CountStoredFromCallWtd; + } + if (otherAccess.CountRegCallArgs > 0) { // The call argument will be decomposed and will not require a @@ -740,6 +777,25 @@ class LocalUses } } + if (costVectorCopies && (countVectorCopies > 0)) + { + // Count induced-only fields too, without counting an existing scalar read twice. + for (const PrimitiveAccess& otherInduced : m_inducedAccesses) + { + if ((otherInduced.Offset == access.Offset) && (otherInduced.AccessType == access.AccessType)) + { + continue; + } + + const Access* existing = FindAccess(otherInduced.Offset, otherInduced.AccessType); + if ((existing == nullptr) || (existing->Count <= existing->CountStoreDestination)) + { + primitiveAccessCount++; + costVectorCopies &= genTypeSize(otherInduced.AccessType) < TARGET_POINTER_SIZE; + } + } + } + // We cost any normal access (which is a struct load or store) without promotion at 3 cycles. const weight_t COST_STRUCT_ACCESS_CYCLES = 3; // And at 4 bytes size @@ -842,7 +898,26 @@ class LocalUses costWith += countWriteBacksWtd * writeBackCost; sizeWith += countWriteBacks * writeBackSize; - // Overlapping stores are decomposable so we don't cost them as + // Charge for fragmenting vector-sized copies into sub-native fields. + // Keep smaller splits and mixed-width copies unpenalized. + unsigned nativeParts = layout->GetSize() / TARGET_POINTER_SIZE; + if (costVectorCopies && (primitiveAccessCount > nativeParts)) + { + // Matching fields already promoted at the other endpoint have fragmented these copies. + // Do not charge for the same decomposition when promoting the induced accesses. + if (inducedAccess != nullptr) + { + assert(inducedAccess->CountFullCopies <= countVectorCopies); + countVectorCopies -= inducedAccess->CountFullCopies; + countVectorCopiesWtd = max(0.0, countVectorCopiesWtd - inducedAccess->CountFullCopiesWtd); + } + // Amortize the original vector move over its fields. + weight_t extraMoves = 1 - (weight_t)genTypeSize(access.AccessType) / layout->GetSize(); + costWith += countVectorCopiesWtd * extraMoves * COST_REG_ACCESS_CYCLES; + sizeWith += countVectorCopies * extraMoves * COST_REG_ACCESS_SIZE; + } + + // Other overlapping stores are decomposable so we don't cost them as // being more expensive than their unpromoted counterparts (i.e. we // don't consider them at all). However, we should do something more // clever here, since: @@ -1403,6 +1478,23 @@ class LocalsUseVisitor : public GenTreeVisitor return m_uses[lclNum]; } + //------------------------------------------------------------------------ + // IsFullCopy: Check whether a copy covers both struct locals in their entirety. + // + // Parameters: + // first - One endpoint of the copy. + // second - The other endpoint of the copy. + // size - The number of bytes copied. + // + // Returns: + // True if both accesses cover their entire local. + // + bool IsFullCopy(GenTreeLclVarCommon* first, GenTreeLclVarCommon* second, unsigned size) + { + return m_compiler->IsEntireAccess(first->GetLclNum(), first->GetLclOffs(), ValueSize(size)) && + m_compiler->IsEntireAccess(second->GetLclNum(), second->GetLclOffs(), ValueSize(size)); + } + //------------------------------------------------------------------------ // InduceAccessesFromRegularlyPromotedStruct: // Create induced accesses based on the fact that there is a store @@ -1421,11 +1513,12 @@ class LocalsUseVisitor : public GenTreeVisitor GenTreeLclVarCommon* regPromLcl, BasicBlock* block) { - unsigned regPromOffs = regPromLcl->GetLclOffs(); - unsigned candidateOffs = candidateLcl->GetLclOffs(); - unsigned size = regPromLcl->GetLayout(m_compiler)->GetSize(); + unsigned regPromOffs = regPromLcl->GetLclOffs(); + unsigned size = regPromLcl->GetLayout(m_compiler)->GetSize(); LclVarDsc* regPromDsc = m_compiler->lvaGetDesc(regPromLcl); + + bool isFullCopy = IsFullCopy(candidateLcl, regPromLcl, size); for (unsigned fieldLcl = regPromDsc->lvFieldLclStart, i = 0; i < regPromDsc->lvFieldCnt; fieldLcl++, i++) { LclVarDsc* fieldDsc = m_compiler->lvaGetDesc(fieldLcl); @@ -1434,7 +1527,7 @@ class LocalsUseVisitor : public GenTreeVisitor { InduceAccess(aggregates, candidateLcl->GetLclNum(), candidateLcl->GetLclOffs() + (fieldDsc->lvFldOffset - regPromOffs), fieldDsc->lvType, - block); + block, isFullCopy); } } } @@ -1463,6 +1556,7 @@ class LocalsUseVisitor : public GenTreeVisitor AggregateInfo* inducerAgg = aggregates.Lookup(inducer->GetLclNum()); if (inducerAgg != nullptr) { + bool isFullCopy = IsFullCopy(candidate, inducer, size); Replacement* firstRep; Replacement* endRep; if (inducerAgg->OverlappingReplacements(inducerOffs, size, &firstRep, &endRep)) @@ -1473,7 +1567,7 @@ class LocalsUseVisitor : public GenTreeVisitor (rep->Offset + genTypeSize(rep->AccessType) <= (inducerOffs + size))) { InduceAccess(aggregates, candidate->GetLclNum(), candOffs + (rep->Offset - inducerOffs), - rep->AccessType, block); + rep->AccessType, block, isFullCopy); } } } @@ -1491,8 +1585,14 @@ class LocalsUseVisitor : public GenTreeVisitor // offset - Offset at which the induced access starts. // type - Type of the induced access. // block - The block with the induced access. + // isFullCopy - Whether the inducing copy covers both whole locals. // - void InduceAccess(AggregateInfoMap& aggregates, unsigned lclNum, unsigned offset, var_types type, BasicBlock* block) + void InduceAccess(AggregateInfoMap& aggregates, + unsigned lclNum, + unsigned offset, + var_types type, + BasicBlock* block, + bool isFullCopy) { AggregateInfo* agg = aggregates.Lookup(lclNum); if (agg != nullptr) @@ -1505,7 +1605,7 @@ class LocalsUseVisitor : public GenTreeVisitor } LocalUses* uses = GetOrCreateUses(lclNum); - uses->RecordInducedAccess(offset, type, block->getBBWeight(m_compiler)); + uses->RecordInducedAccess(offset, type, block->getBBWeight(m_compiler), isFullCopy); } //------------------------------------------------------------------------ @@ -1526,7 +1626,12 @@ class LocalsUseVisitor : public GenTreeVisitor AccessKindFlags flags = AccessKindFlags::None; if (lcl->OperIsLocalStore()) { - INDEBUG(flags |= AccessKindFlags::IsStoreDestination); + flags |= AccessKindFlags::IsStoreDestination; + + if (lcl->TypeIs(TYP_STRUCT) && lcl->Data()->gtEffectiveVal()->IsInitVal()) + { + flags |= AccessKindFlags::IsInit; + } if (lcl->AsLclVarCommon()->Data()->gtEffectiveVal()->IsCall()) { @@ -1565,12 +1670,12 @@ class LocalsUseVisitor : public GenTreeVisitor } } -#ifdef DEBUG if (user->OperIsStore() && (user->Data()->gtEffectiveVal() == lcl)) { flags |= AccessKindFlags::IsStoreSource; } +#ifdef DEBUG if (user->OperIs(GT_RETURN, GT_SWIFT_ERROR_RET)) { flags |= AccessKindFlags::IsReturned; diff --git a/src/tests/JIT/Directed/Directed_do.csproj b/src/tests/JIT/Directed/Directed_do.csproj index 9ef81e4ada860d..13e93b9177fcbe 100644 --- a/src/tests/JIT/Directed/Directed_do.csproj +++ b/src/tests/JIT/Directed/Directed_do.csproj @@ -15,6 +15,7 @@ + @@ -86,6 +87,7 @@ + diff --git a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs new file mode 100644 index 00000000000000..ba2b3842e868e0 --- /dev/null +++ b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.cs @@ -0,0 +1,374 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Xunit; + +public class SharedVectorCopyCost +{ + // Five fields prevent regular promotion of this vector-sized struct. + [StructLayout(LayoutKind.Explicit, Size = 16)] + private struct S + { + [FieldOffset(0)] public int A; + [FieldOffset(4)] public int B; + [FieldOffset(8)] public int C; + [FieldOffset(12)] public short D; + [FieldOffset(14)] public short E; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static S Create(int value) => + new S { A = value, B = value + 1, C = value + 2, D = (short)(value + 3), E = (short)(value + 4) }; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void Fields(int a, int b, int c, int d, int e) { } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void Consume(S value) { } + + // Six integer arguments exhaust the Unix x64 argument registers. + [MethodImpl(MethodImplOptions.NoInlining)] + private static void ConsumeStackArgs(int a, int b, int c, int d, int e, int f, S value) { } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Consume(S value, bool stackArgs) + { + if (stackArgs) + { + ConsumeStackArgs(1, 2, 3, 4, 5, 6, value); + } + else + { + Consume(value); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int Copy(int input) + { + // After the last copy, reuse the promoted fields instead of reloading the shorts. + // X64: call {{.*}}SharedVectorCopyCost:Consume + // X64: call {{.*}}SharedVectorCopyCost:Consume + // X64: call {{.*}}SharedVectorCopyCost:Consume + // X64: call {{.*}}SharedVectorCopyCost:Consume + // X64-NOT: movsx + // X64: ret + return CopyCore(input, false); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int CopyStackArgs(int input) + { + // X64: call {{.*}}SharedVectorCopyCost:ConsumeStackArgs + // X64: call {{.*}}SharedVectorCopyCost:ConsumeStackArgs + // X64: call {{.*}}SharedVectorCopyCost:ConsumeStackArgs + // X64: call {{.*}}SharedVectorCopyCost:ConsumeStackArgs + // X64-NOT: movsx + // X64: ret + return CopyCore(input, true); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int CopyCore(int input, bool stackArgs) + { + S src = Create(input); + Fields(src.A, src.B, src.C, src.D, src.E); + Fields(src.A, src.B, src.C, src.D, src.E); + Fields(src.A, src.B, src.C, src.D, src.E); + + // The source is profitable to promote independently. Its replacements + // already fragment these copies, so promoting matching destination + // fields must not pay the fragmentation cost again. + S dst = src; + dst.A++; + Consume(dst, stackArgs); + dst = src; + dst.A++; + Consume(dst, stackArgs); + dst = src; + dst.A++; + Consume(dst, stackArgs); + dst = src; + dst.A++; + Consume(dst, stackArgs); + + Fields(dst.A, dst.B, dst.C, dst.D, dst.E); + return dst.A + dst.B + dst.C + dst.D + dst.E; + } + + [StructLayout(LayoutKind.Explicit, Size = 16)] + private struct ArrayValue + { + [FieldOffset(0)] public int A; + [FieldOffset(4)] public int B; + [FieldOffset(8)] public int C; + [FieldOffset(12)] public int D; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ArrayValue CreateArrayValue(int value) => + new ArrayValue { A = value, B = value + 1, C = value + 2, D = value + 3 }; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void CheckFields(int a, int b, int c, int d) + { + Assert.Equal(a + 1, b); + Assert.Equal(b + 1, c); + Assert.Equal(c + 1, d); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void Consume(ArrayValue value) { } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int CopyToRegularlyPromotedLocal(int input) + { + // The four-field destination is eligible for regular promotion. Whole-local + // copies must credit that endpoint just as they credit physical promotion. + // X64: call {{.*}}SharedVectorCopyCost:CheckFields + // X64: call {{.*}}SharedVectorCopyCost:CheckFields + // X64: call {{.*}}SharedVectorCopyCost:CheckFields + // X64: call {{.*}}SharedVectorCopyCost:CheckFields + // Keep the source fields promoted instead of copying their stack storage. + // X64-NOT: xmmword ptr + // X64: ret + ArrayValue source = CreateArrayValue(input); + CheckFields(source.A, source.B, source.C, source.D); + CheckFields(source.A, source.B, source.C, source.D); + CheckFields(source.A, source.B, source.C, source.D); + CheckFields(source.A, source.B, source.C, source.D); + Consume(source); + Consume(source); + Consume(source); + Consume(source); + ArrayValue destination = source; + destination.A++; + Consume(destination); + destination = source; + destination.A++; + Consume(destination); + destination = source; + destination.A++; + Consume(destination); + destination = source; + destination.A++; + Consume(destination); + return destination.A + destination.B + destination.C + destination.D; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void CopyToArray(int input, ArrayValue[] destination) + { + // Preserve vector copies instead of splitting each array store into four field stores. + // ARM64 pairs the four array copies into two vector stores. + // ARM64: SharedVectorCopyCost:CheckFields + // ARM64: SharedVectorCopyCost:CheckFields + // ARM64: stp q{{[0-9]+}}, q{{[0-9]+}}, + // ARM64: stp q{{[0-9]+}}, q{{[0-9]+}}, + // ARM64-LABEL: {{^ *ret}} + // X64-WINDOWS: call {{.*}}SharedVectorCopyCost:CheckFields + // X64-WINDOWS: call {{.*}}SharedVectorCopyCost:CheckFields + // X64-WINDOWS: {{v?movups}} xmmword ptr + // X64-WINDOWS: {{v?movups}} xmmword ptr + // X64-WINDOWS: {{v?movups}} xmmword ptr + // X64-WINDOWS: {{v?movups}} xmmword ptr + // X64-WINDOWS-LABEL: {{^ *ret}} + ArrayValue source = CreateArrayValue(input); + CheckFields(source.A, source.B, source.C, source.D); + CheckFields(source.A, source.B, source.C, source.D); + destination[0] = source; + destination[1] = source; + destination[2] = source; + destination[3] = source; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void CopyConstantToArray(System.Guid[] destination) + { + // Promotion should allow constant fields to propagate into the copies. + // X64-WINDOWS-NOT: xmmword ptr [rsp + // X64-WINDOWS: ret + System.Guid source = new System.Guid(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); + destination[0] = source; + destination[1] = source; + destination[2] = source; + destination[3] = source; + } + + [StructLayout(LayoutKind.Explicit, Size = 24)] + private struct OuterValue + { + [FieldOffset(0)] public long Prefix; + [FieldOffset(8)] public ArrayValue Value; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static OuterValue CreateOuterValue(int input) => + new OuterValue { Prefix = input, Value = new ArrayValue { A = input, B = input + 1, C = input + 2, D = input + 3 } }; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void CopyInducedToArray(int input, ArrayValue[] destination) + { + // Copy a subrange of the promoted source into a local with no explicit field reads. + // Its induced fields must still pay for fragmenting the subsequent array copies. + // ARM64: SharedVectorCopyCost:CheckFields + // ARM64: SharedVectorCopyCost:CheckFields + // ARM64: SharedVectorCopyCost:CheckFields + // Check all eight copies before the first return, excluding the fallback path. + // ARM64 pairs the copies; do not require a reload for each store. + // ARM64: stp q{{[0-9]+}}, q{{[0-9]+}}, + // ARM64: stp q{{[0-9]+}}, q{{[0-9]+}}, + // ARM64: stp q{{[0-9]+}}, q{{[0-9]+}}, + // ARM64: stp q{{[0-9]+}}, q{{[0-9]+}}, + // ARM64-LABEL: {{^ *ret}} + // On x86 these int fields are native-sized and must remain uncharged. + // X86: call {{.*}}SharedVectorCopyCost:CheckFields + // X86-NOT: xmmword ptr + // X86: ret + // X64: call {{.*}}SharedVectorCopyCost:CheckFields + // X64: call {{.*}}SharedVectorCopyCost:CheckFields + // X64: call {{.*}}SharedVectorCopyCost:CheckFields + // X64: {{v?movups}} xmmword ptr + // X64: {{v?movups}} xmmword ptr + // X64: {{v?movups}} xmmword ptr + // X64: {{v?movups}} xmmword ptr + // X64: {{v?movups}} xmmword ptr + // X64: {{v?movups}} xmmword ptr + // X64: {{v?movups}} xmmword ptr + // X64: {{v?movups}} xmmword ptr + // X64-LABEL: {{^ *ret}} + OuterValue source = CreateOuterValue(input); + CheckFields(source.Value.A, source.Value.B, source.Value.C, source.Value.D); + CheckFields(source.Value.A, source.Value.B, source.Value.C, source.Value.D); + CheckFields(source.Value.A, source.Value.B, source.Value.C, source.Value.D); + ArrayValue value = source.Value; + destination[0] = value; + destination[1] = value; + destination[2] = value; + destination[3] = value; + destination[4] = value; + destination[5] = value; + destination[6] = value; + destination[7] = value; + } + + [StructLayout(LayoutKind.Explicit, Size = 16)] + private struct MixedValue + { + [FieldOffset(0)] public long A; + [FieldOffset(8)] public short B; + [FieldOffset(10)] public short C; + [FieldOffset(12)] public short D; + [FieldOffset(14)] public short E; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static MixedValue CreateMixedValue(int value) => + new MixedValue { A = value, B = (short)value, C = (short)value, D = (short)value, E = (short)value }; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void MixedFields(long a, int b, int c, int d, int e) { } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void CopyMixedToArray(int input, MixedValue[] destination) + { + // The native-width read keeps this mixed-width copy outside the fragmentation charge. + // X64: call {{.*}}SharedVectorCopyCost:MixedFields + // X64: call {{.*}}SharedVectorCopyCost:MixedFields + // X64-NOT: xmmword ptr + // X64-NOT: ptr [rsp + // X64: ret + MixedValue source = CreateMixedValue(input); + MixedFields(source.A, source.B, source.C, source.D, source.E); + MixedFields(source.A, source.B, source.C, source.D, source.E); + destination[0] = source; + destination[1] = source; + destination[2] = source; + destination[3] = source; + destination[4] = source; + destination[5] = source; + destination[6] = source; + destination[7] = source; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int InitializeNonzero(int input) + { + // Nonzero initblk is initialization, not a copy to fragment. + // X64-NOT: xmmword ptr + // X64: call {{.*}}SharedVectorCopyCost:Fields + // X64: ret + // X86-NOT: xmmword ptr + // X86: call {{.*}}SharedVectorCopyCost:Fields + // X86: ret + // ARM64-NOT: {{stp|str}} q{{[0-9]+}} + // ARM64: SharedVectorCopyCost:Fields + // ARM64: ret + S value; + Unsafe.SkipInit(out value); + Unsafe.InitBlockUnaligned(ref Unsafe.As(ref value), 0xAB, (uint)Unsafe.SizeOf()); + value.A = input; + Fields(value.A, value.B, value.C, value.D, value.E); + return value.A + value.B + value.C + value.D + value.E; + } + + [Fact] + public static void TestEntryPoint() + { + Assert.Equal(unchecked(10 + 2 * (int)0xABABABAB + 2 * (short)0xABAB), InitializeNonzero(10)); + Assert.Equal(unchecked(-10 + 2 * (int)0xABABABAB + 2 * (short)0xABAB), InitializeNonzero(-10)); + Assert.Equal(61, Copy(10)); + Assert.Equal(-39, Copy(-10)); + Assert.Equal(32769, Copy(32766)); + Assert.Equal(61, CopyStackArgs(10)); + Assert.Equal(-39, CopyStackArgs(-10)); + Assert.Equal(32769, CopyStackArgs(32766)); + Assert.Equal(47, CopyToRegularlyPromotedLocal(10)); + Assert.Equal(-33, CopyToRegularlyPromotedLocal(-10)); + + ArrayValue[] values = new ArrayValue[4]; + foreach (int input in new[] { 10, -10 }) + { + CopyToArray(input, values); + foreach (ArrayValue value in values) + { + Assert.Equal(input, value.A); + CheckFields(value.A, value.B, value.C, value.D); + } + } + + ArrayValue[] inducedValues = new ArrayValue[8]; + foreach (int input in new[] { 10, -10 }) + { + CopyInducedToArray(input, inducedValues); + foreach (ArrayValue value in inducedValues) + { + Assert.Equal(input, value.A); + CheckFields(value.A, value.B, value.C, value.D); + } + } + + MixedValue[] mixedValues = new MixedValue[8]; + foreach (int input in new[] { 10, -10, 32768 }) + { + CopyMixedToArray(input, mixedValues); + foreach (MixedValue value in mixedValues) + { + Assert.Equal((long)input, value.A); + Assert.Equal((short)input, value.B); + Assert.Equal((short)input, value.C); + Assert.Equal((short)input, value.D); + Assert.Equal((short)input, value.E); + } + } + + System.Guid[] constants = new System.Guid[4]; + CopyConstantToArray(constants); + foreach (System.Guid value in constants) + { + Assert.Equal(new System.Guid(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), value); + } + } +} diff --git a/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.csproj b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.csproj new file mode 100644 index 00000000000000..030af284e91008 --- /dev/null +++ b/src/tests/JIT/Directed/physicalpromotion/SharedVectorCopyCost.csproj @@ -0,0 +1,15 @@ + + + true + true + True + + + + true + + + + + +