diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md index 72e1b2a12ee1d..62a08433412c3 100644 --- a/llvm/docs/ReleaseNotes.md +++ b/llvm/docs/ReleaseNotes.md @@ -131,6 +131,12 @@ Makes programs 10x faster by doing Special New Thing. ### Changes to the CodeGen infrastructure +* The fast register allocator can consume SSA machine IR, lowering PHI nodes and + tied operands itself. X86 and AArch64 enable this, so at `-O0` their pipelines + no longer run `PHIElimination` and `TwoAddressInstructionPass`. Command lines + that name either pass in `-start-before`, `-start-after`, `-stop-before`, + `-stop-after` or `-run-pass` need `-regalloc-fast-ssa=0`, which restores them. + ### Changes to the Metadata Info ### Changes to the Debug Info diff --git a/llvm/include/llvm/CodeGen/RegAllocFast.h b/llvm/include/llvm/CodeGen/RegAllocFast.h index 82d7f5ba914a1..e88e000539425 100644 --- a/llvm/include/llvm/CodeGen/RegAllocFast.h +++ b/llvm/include/llvm/CodeGen/RegAllocFast.h @@ -27,20 +27,13 @@ class RegAllocFastPass : public RequiredPassInfoMixin { RegAllocFastPass(Options Opts = Options()) : Opts(std::move(Opts)) {} - MachineFunctionProperties getRequiredProperties() const { - return MachineFunctionProperties().setNoPHIs(); - } - MachineFunctionProperties getSetProperties() const { - if (Opts.ClearVRegs) { - return MachineFunctionProperties().setNoVRegs(); - } - - return MachineFunctionProperties(); - } - - MachineFunctionProperties getClearedProperties() const { - return MachineFunctionProperties().setIsSSA(); + MachineFunctionProperties P; + P.setNoPHIs(); + P.setTiedOpsRewritten(); + if (Opts.ClearVRegs) + P.setNoVRegs(); + return P; } LLVM_ABI PreservedAnalyses run(MachineFunction &MF, diff --git a/llvm/include/llvm/CodeGen/TargetPassConfig.h b/llvm/include/llvm/CodeGen/TargetPassConfig.h index 5e0e641a981f9..9dbb2b7ce4b25 100644 --- a/llvm/include/llvm/CodeGen/TargetPassConfig.h +++ b/llvm/include/llvm/CodeGen/TargetPassConfig.h @@ -478,6 +478,11 @@ class LLVM_ABI TargetPassConfig : public ImmutablePass { LLVM_ABI void registerCodeGenCallback(PassInstrumentationCallbacks &PIC, TargetMachine &); +/// Whether the fast register allocator consumes SSA MachineIR for \p TM, +/// resolving the target default against -regalloc-fast-ssa. Shared by both +/// codegen pipelines. +LLVM_ABI bool useSSAFastRegAlloc(const TargetMachine &TM); + } // end namespace llvm #endif // LLVM_CODEGEN_TARGETPASSCONFIG_H diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h index 11e448dd7f78b..0ea4ca5919e6a 100644 --- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h +++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h @@ -1221,8 +1221,12 @@ CodeGenPassBuilder::addRegAssignAndRewriteOptimized( template Error CodeGenPassBuilder::addFastRegAlloc( PassManagerWrapper &PMW) const { - addMachineFunctionPass(PHIEliminationPass(), PMW); - addMachineFunctionPass(TwoAddressInstructionPass(), PMW); + // When the fast allocator consumes SSA MachineIR it lowers PHIs and tied + // operands itself, detecting the input form per function from IsSSA. + if (!useSSAFastRegAlloc(TM)) { + addMachineFunctionPass(PHIEliminationPass(), PMW); + addMachineFunctionPass(TwoAddressInstructionPass(), PMW); + } return derived().addRegAssignAndRewriteFast(PMW); } diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h index d8f497358f89c..22d59b03cf824 100644 --- a/llvm/include/llvm/Target/TargetMachine.h +++ b/llvm/include/llvm/Target/TargetMachine.h @@ -120,6 +120,7 @@ class LLVM_ABI TargetMachine { unsigned RequireStructuredCFG : 1; unsigned O0WantsFastISel : 1; + unsigned EnableSSAFastRegAlloc : 1; // PGO related tunables. std::optional PGOOption; @@ -260,6 +261,14 @@ class LLVM_ABI TargetMachine { bool requiresStructuredCFG() const { return RequireStructuredCFG; } void setRequiresStructuredCFG(bool Value) { RequireStructuredCFG = Value; } + /// Whether the fast register allocator consumes SSA MachineIR, lowering + /// PHIs and tied operands itself instead of running PHIElimination and + /// TwoAddressInstructionPass. + /// TODO: AMDGPU inserts passes anchored on those pass IDs, must leave this + /// false. + bool enableSSAFastRegAlloc() const { return EnableSSAFastRegAlloc; } + void setEnableSSAFastRegAlloc(bool Value) { EnableSSAFastRegAlloc = Value; } + /// Returns the code generation relocation model. The choices are static, PIC, /// and dynamic-no-pic, and target default. Reloc::Model getRelocationModel() const; diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp index f10283272f1d9..b24a22bf9ac8e 100644 --- a/llvm/lib/CodeGen/RegAllocFast.cpp +++ b/llvm/lib/CodeGen/RegAllocFast.cpp @@ -6,21 +6,32 @@ // //===----------------------------------------------------------------------===// // -/// \file This register allocator allocates registers to a basic block at a -/// time, attempting to keep values in registers and reusing registers as -/// appropriate. +// This register allocator allocates registers to a basic block at a time, +// attempting to keep values in registers and reusing registers as appropriate. +// A value living across a block boundary gets a stack slot of its own, spilled +// at the end of a block and reloaded where next used. +// +// Where the target enables it, the input is SSA MachineIR: PHIElimination +// and TwoAddressInstructionPass are left out of the pipeline and this pass +// lowers PHIs and tied operands itself. Most PHIs then cost nothing, the +// incoming value being written straight into the destination's stack slot; +// only where that write could escape on another edge out of the predecessor +// does a virtual register carry it instead, as PHIElimination does. // //===----------------------------------------------------------------------===// #include "llvm/CodeGen/RegAllocFast.h" +#include "PHIEliminationUtils.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SCCIterator.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/SparseSet.h" #include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" @@ -34,6 +45,7 @@ #include "llvm/CodeGen/RegisterClassInfo.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetOpcodes.h" +#include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/InitializePasses.h" @@ -53,6 +65,7 @@ using namespace llvm; STATISTIC(NumStores, "Number of stores added"); STATISTIC(NumLoads, "Number of loads added"); STATISTIC(NumCoalesced, "Number of copies coalesced"); +STATISTIC(NumPHIsViaVReg, "Number of PHIs lowered through a virtual register"); // FIXME: Remove this switch when all testcases are fixed! static cl::opt IgnoreMissingDefs("rafast-ignore-missing-defs", @@ -61,6 +74,10 @@ static cl::opt IgnoreMissingDefs("rafast-ignore-missing-defs", static RegisterRegAlloc fastRegAlloc("fast", "fast register allocator", createFastRegisterAllocator); +// Budget for walking a virtual register's use or def list when deciding +// whether it stays within one basic block. +constexpr unsigned BlockLocalScanLimit = 8; + namespace { /// Assign ascending index for instructions in machine basic block. The index @@ -331,10 +348,20 @@ class RegAllocFastImpl { public: bool ClearVirtRegs; + /// The input is SSA MachineIR (the pipeline did not run PHIElimination and + /// TwoAddressInstructionPass): lower PHIs and tied operands here. + /// Determined per function from the input's properties. + bool SSAInput = false; + bool runOnMachineFunction(MachineFunction &MF); private: void allocateBasicBlock(MachineBasicBlock &MBB); + bool canWritePHISlotInPreds(const MachineInstr &PHI, + ArrayRef SCCId) const; + void lowerPHIs(MachineFunction &MF); + void lowerPHIEdgeCopies(MachineBasicBlock &MBB); + void expandSSAPseudo(MachineInstr &MI); void addRegClassDefCounts(MutableArrayRef RegClassDefCounts, Register Reg) const; @@ -421,20 +448,13 @@ class RegAllocFast : public MachineFunctionPass { MachineFunctionPass::getAnalysisUsage(AU); } - MachineFunctionProperties getRequiredProperties() const override { - return MachineFunctionProperties().setNoPHIs(); - } - MachineFunctionProperties getSetProperties() const override { - if (Impl.ClearVirtRegs) { - return MachineFunctionProperties().setNoVRegs(); - } - - return MachineFunctionProperties(); - } - - MachineFunctionProperties getClearedProperties() const override { - return MachineFunctionProperties().setIsSSA(); + MachineFunctionProperties P; + P.setNoPHIs(); + P.setTiedOpsRewritten(); + if (Impl.ClearVirtRegs) + P.setNoVRegs(); + return P; } }; @@ -554,12 +574,10 @@ bool RegAllocFastImpl::mayLiveOut(Register VirtReg) { } } - // See if the first \p Limit uses of the register are all in the current - // block. - static const unsigned Limit = 8; + // See if the first few uses of the register are all in the current block. unsigned C = 0; for (const MachineInstr &UseInst : MRI->use_nodbg_instructions(VirtReg)) { - if (UseInst.getParent() != MBB || ++C >= Limit) { + if (UseInst.getParent() != MBB || ++C >= BlockLocalScanLimit) { MayLiveAcrossBlocks.set(VirtReg.virtRegIndex()); // Cannot be live-out if there are no successors. return !MBB->succ_empty(); @@ -584,11 +602,10 @@ bool RegAllocFastImpl::mayLiveIn(Register VirtReg) { if (MayLiveAcrossBlocks.test(VirtReg.virtRegIndex())) return !MBB->pred_empty(); - // See if the first \p Limit def of the register are all in the current block. - static const unsigned Limit = 8; + // See if the first few defs of the register are all in the current block. unsigned C = 0; for (const MachineInstr &DefInst : MRI->def_instructions(VirtReg)) { - if (DefInst.getParent() != MBB || ++C >= Limit) { + if (DefInst.getParent() != MBB || ++C >= BlockLocalScanLimit) { MayLiveAcrossBlocks.set(VirtReg.virtRegIndex()); return !MBB->pred_empty(); } @@ -677,7 +694,8 @@ MachineBasicBlock::iterator RegAllocFastImpl::getMBBBeginInsertionPoint( MachineBasicBlock &MBB, SmallSet &PrologLiveIns) const { MachineBasicBlock::iterator I = MBB.begin(); while (I != MBB.end()) { - if (I->isLabel()) { + // PHIs must stay first: successor scans and PHI deletion use MBB.phis(). + if (I->isLabel() || I->isPHI()) { ++I; continue; } @@ -733,7 +751,7 @@ void RegAllocFastImpl::reloadAtBegin(MachineBasicBlock &MBB) { // FIXME: Theoretically this should use an insert point skipping labels // but I'm not sure how labels should interact with prolog instruction // that need reloads. - reload(MBB.begin(), LR.VirtReg, PhysReg); + reload(MBB.getFirstNonPHI(), LR.VirtReg, PhysReg); } else reload(InsertBefore, LR.VirtReg, PhysReg); } @@ -888,34 +906,51 @@ void RegAllocFastImpl::assignVirtToPhysReg(MachineInstr &AtMI, LiveReg &LR, assignDanglingDebugValues(AtMI, VirtReg, PhysReg); } -static bool isCoalescable(const MachineInstr &MI) { return MI.isFullCopy(); } - Register RegAllocFastImpl::traceCopyChain(Register Reg) const { static const unsigned ChainLengthLimit = 3; - unsigned C = 0; - do { + for (unsigned C = 0; C <= ChainLengthLimit; ++C) { if (Reg.isPhysical()) return Reg; - assert(Reg.isVirtual()); - - MachineInstr *VRegDef = MRI->getUniqueVRegDef(Reg); - if (!VRegDef || !isCoalescable(*VRegDef)) - return 0; - Reg = VRegDef->getOperand(1).getReg(); - } while (++C <= ChainLengthLimit); - return 0; + const MachineOperand *DefMO = MRI->getOneDef(Reg); + if (!DefMO) + return {}; + const MachineInstr *Def = DefMO->getParent(); + if (Def->isFullCopy()) { + Reg = Def->getOperand(1).getReg(); + continue; + } + if (!SSAInput) + return {}; + // In SSA form a two-address instruction's def and tied use end up in the + // same register, so the tie continues the chain. + if (!DefMO->isTied() || DefMO->getSubReg()) + return {}; + Reg = Def->getOperand(Def->findTiedOperandIdx(DefMO->getOperandNo())) + .getReg(); + } + return {}; } /// Check if any of \p VirtReg's definitions is a copy. If it is follow the /// chain of copies to check whether we reach a physical register we can /// coalesce with. Register RegAllocFastImpl::traceCopies(Register VirtReg) const { + if (SSAInput) { + // A single def: enter the chain past a copy so the walk keeps the same + // reach as the multi-def loop below. + const MachineOperand *DefMO = MRI->getOneDef(VirtReg); + if (!DefMO) + return {}; + const MachineInstr *Def = DefMO->getParent(); + return Def->isFullCopy() ? traceCopyChain(Def->getOperand(1).getReg()) + : traceCopyChain(VirtReg); + } + static const unsigned DefLimit = 3; unsigned C = 0; for (const MachineInstr &MI : MRI->def_instructions(VirtReg)) { - if (isCoalescable(MI)) { - Register Reg = MI.getOperand(1).getReg(); - Reg = traceCopyChain(Reg); + if (MI.isFullCopy()) { + Register Reg = traceCopyChain(MI.getOperand(1).getReg()); if (Reg.isValid()) return Reg; } @@ -1178,8 +1213,61 @@ bool RegAllocFastImpl::useVirtReg(MachineInstr &MI, MachineOperand &MO, assert((!MO.isKill() || LRI->LastUse == &MI) && "Invalid kill flag"); } - // If necessary allocate a register. - if (LRI->PhysReg == 0) { + // SSA two-address lowering: the tied use is a distinct value whose range + // ends here (the tied def's range ended at this instruction in the + // backward walk). + const MachineOperand *TiedDefMO = + SSAInput && MO.isTied() + ? &MI.getOperand(MI.findTiedOperandIdx(MO.getOperandNo())) + : nullptr; + if (TiedDefMO && TiedDefMO->getReg().isPhysical()) { + const MachineOperand &DefMO = *TiedDefMO; + Register DefReg = DefMO.getReg(); + unsigned SubReg = MO.getSubReg(); + if (LRI->PhysReg == 0) { + // Take over the def's register. A register outside this operand's class + // cannot hold the value, as the def's class may be the wider one. A + // reserved register (e.g. the base pointer, tied to an inline asm + // operand) may not hold a virtual register. An early-clobber def that + // also reads this value through another operand needs that operand in a + // different register than the def. A subregister read is a truncation, + // so the value the def wants is a part of this one. All four allocate + // elsewhere and copy below. + bool MustCopy = SubReg || MRI->isReserved(DefReg) || + !MRI->getRegClass(VirtReg)->contains(DefReg); + if (DefMO.isEarlyClobber()) + for (const MachineOperand &O : MI.operands()) + if (&O != &MO && O.isReg() && O.isUse() && O.getReg() == VirtReg) + MustCopy = true; + if (!MustCopy) { + freePhysReg(DefReg.asMCReg()); + assignVirtToPhysReg(MI, *LRI, DefReg.asMCReg()); + } else { + allocVirtReg(MI, *LRI, 0, false); + } + } + MCRegister SrcReg = LRI->PhysReg; + if (SubReg) + SrcReg = TRI->getSubReg(SrcReg, SubReg); + if (Register(SrcReg) != DefReg) { + // The value lives in its own register (for other uses or code below); + // satisfy the tie the way TwoAddressInstructionPass does: copy it into + // the def's register right before the instruction and read it there. + BuildMI(*MBB, MI.getIterator(), MI.getDebugLoc(), + TII->get(TargetOpcode::COPY), DefReg) + .addReg(SrcReg); + LRI->LastUse = &MI; + markRegUsedInInstr(LRI->PhysReg); + MO.setReg(DefReg); + MO.setSubReg(0); + // A reserved register may not be renamed. + MO.setIsRenamable(!MRI->isReserved(DefReg)); + // The tied def's value range ends here; the "free def operands" step + // keeps tied defs for the same-vreg case, so release it explicitly. + freePhysReg(DefReg.asMCReg()); + return false; + } + } else if (LRI->PhysReg == 0) { assert(!MO.isTied() && "tied op should be allocated"); Register Hint; if (MI.isCopy() && MI.getOperand(1).getSubReg() == 0) { @@ -1802,8 +1890,14 @@ void RegAllocFastImpl::allocateBasicBlock(MachineBasicBlock &MBB) { Coalesced.clear(); - // Traverse block in reverse order allocating instructions one by one. - for (MachineInstr &MI : reverse(MBB)) { + // Traverse block in reverse order allocating instructions one by one. The + // iterator is advanced before the instruction is handled, since allocating + // it inserts copies and reloads ahead of it. + for (auto It = MBB.rbegin(), E = MBB.rend(); It != E;) { + MachineInstr &MI = *It; + ++It; + if (MI.isPHI()) + continue; LLVM_DEBUG(dbgs() << "\n>> " << MI << "Regs:"; dumpState()); // Special handling for debug values. Note that they are not allowed to @@ -1850,6 +1944,300 @@ void RegAllocFastImpl::allocateBasicBlock(MachineBasicBlock &MBB) { LLVM_DEBUG(MBB.dump()); } +/// Expand REG_SEQUENCE and INSERT_SUBREG into subregister COPYs: the +/// lowering TwoAddressInstructionPass performs when it runs before +/// allocation, plus the base-value copy its tie-processing provides. +void RegAllocFastImpl::expandSSAPseudo(MachineInstr &MI) { + MachineBasicBlock &MBB = *MI.getParent(); + const DebugLoc &DL = MI.getDebugLoc(); + if (MI.isInsertSubreg()) { + // %d = INSERT_SUBREG %base, %sub, idx -> %d = COPY %base + // %d.idx = COPY %sub + Register Dst = MI.getOperand(0).getReg(); + const MachineOperand &BaseMO = MI.getOperand(1); + const MachineOperand &SubMO = MI.getOperand(2); + unsigned SubIdx = MI.getOperand(3).getImm(); + assert(MI.getOperand(0).getSubReg() == 0 && "Unexpected subreg idx"); + if (!BaseMO.isUndef()) + BuildMI(MBB, MI.getIterator(), DL, TII->get(TargetOpcode::COPY), Dst) + .addReg(BaseMO.getReg(), RegState::NoFlags, BaseMO.getSubReg()); + BuildMI(MBB, MI.getIterator(), DL, TII->get(TargetOpcode::COPY)) + .addReg(Dst, RegState::Define | getUndefRegState(BaseMO.isUndef()), + SubIdx) + .addReg(SubMO.getReg(), getKillRegState(SubMO.isKill()), + SubMO.getSubReg()); + MI.eraseFromParent(); + return; + } + + // %d = REG_SEQUENCE %s1, idx1, ... -> undef %d.idx1 = COPY %s1 + // %d.idx2 = COPY %s2 ... + assert(MI.isRegSequence()); + Register Dst = MI.getOperand(0).getReg(); + bool DefEmitted = false; + for (unsigned I = 1, E = MI.getNumOperands(); I + 1 < E; I += 2) { + MachineOperand &SrcMO = MI.getOperand(I); + unsigned SubIdx = MI.getOperand(I + 1).getImm(); + BuildMI(MBB, MI.getIterator(), DL, TII->get(TargetOpcode::COPY)) + .addReg(Dst, RegState::Define | getUndefRegState(!DefEmitted), SubIdx) + .addReg(SrcMO.getReg(), RegState::NoFlags, SrcMO.getSubReg()); + DefEmitted = true; + } + MI.eraseFromParent(); +} + +// Whether this PHI can be lowered by having every predecessor write the +// incoming value straight into the destination's own stack slot. Yes if such +// escaped writes cannot be observed. +bool RegAllocFastImpl::canWritePHISlotInPreds(const MachineInstr &PHI, + ArrayRef SCCId) const { + const MachineBasicBlock *MBB = PHI.getParent(); + Register Dst = PHI.getOperand(0).getReg(); + + for (unsigned I = 1, E = PHI.getNumOperands(); I != E; I += 2) { + const MachineBasicBlock *Pred = PHI.getOperand(I + 1).getMBB(); + // Different SCC means MBB cannot reach Pred, so Dst, defined by MBB, + // is not live at Pred. Early write at Pred is ok. + if (SCCId[Pred->getNumber()] != SCCId[MBB->getNumber()]) + continue; + // If Pred has another successor, the early write may clobber a value of + // Dst that is still read after leaving Pred that way. When MBB is a landing + // pad: findPHICopyInsertPoint then puts the write before the call that may + // throw rather than before the terminator, so it runs even when the call + // returns normally. + if (Pred->succ_size() == 1 && !MBB->isEHPad()) + continue; + // Unless every read of Dst is in MBB: an edge into MBB rewrites the slot, + // so the escaped write is never observed. A PHI operand does not count, + // being read on the edge, in the predecessor. Last as it walks the uses. + return all_of(MRI->use_nodbg_instructions(Dst), + [&](const MachineInstr &UseMI) { + return !UseMI.isPHI() && UseMI.getParent() == MBB; + }); + } + return true; +} + +/// Whether one stack slot can serve both register classes, so that a PHI +/// source may take the destination's. getStackSpaceFor() creates the slot for +/// one class and every access uses the class of the register it moves, so only +/// the geometry has to agree. +static bool sharesStackSlot(const TargetRegisterInfo &TRI, + const TargetRegisterClass &A, + const TargetRegisterClass &B) { + return &A == &B || (TRI.getSpillSize(A) == TRI.getSpillSize(B) && + TRI.getSpillAlign(A) == TRI.getSpillAlign(B) && + TRI.getSpillStackID(A) == TRI.getSpillStackID(B)); +} + +/// Lower the PHIs of \p MF to copies at the end of the predecessors. Those +/// canWritePHISlotInPreds() rejects go through an extra virtual register, as +/// PHIElimination does, so that a copy on a not-taken edge cannot be +/// observed; the rest have the copy define the destination itself. A source +/// dying in its predecessor is given the destination's stack slot instead, +/// making its spill the transfer, so that edge needs no copy at all. +/// +/// Only registers the PHIs already reference are looked up in +/// StackSlotForVirtReg, so the caller need only have sized it for those. +void RegAllocFastImpl::lowerPHIs(MachineFunction &MF) { + SmallVector SCCId(MF.getNumBlockIDs(), ~0u); + unsigned Id = 0; + for (scc_iterator I = scc_begin(&MF); !I.isAtEnd(); + ++I, ++Id) + for (MachineBasicBlock *MBB : *I) + SCCId[MBB->getNumber()] = Id; + + // Determine PHIs that cannot be lowered by writing the destination's slot in + // the predecessors. Collected first: the copies below move the end of + // MBB.phis(), and the queries above need the still-SSA form. + SmallVector Unsafe; + for (MachineBasicBlock &MBB : MF) + for (MachineInstr &PHI : MBB.phis()) + if (!canWritePHISlotInPreds(PHI, SCCId)) + Unsafe.push_back(&PHI); + + // Isolate such unsafe PHIs the way PHIElimination does: a fresh register + // carries the value over every edge, and a copy at the top of the block moves + // it to the destination. Read at that one place, it cannot be observed when + // written on a not-taken edge. + // + // This destroys SSA: Incoming is defined once per predecessor. leaveSSA() has + // run and the allocator proper is built for multiple definitions, but nothing + // past this point may assume a unique def. + SmallPtrSet InsertedInto; + for (MachineInstr *PHI : Unsafe) { + MachineBasicBlock &MBB = *PHI->getParent(); + Register Dst = PHI->getOperand(0).getReg(); + Register Incoming = MRI->createVirtualRegister(MRI->getRegClass(Dst)); + TII->createPHIDestinationCopy(MBB, MBB.SkipPHIsAndLabels(MBB.begin()), + PHI->getDebugLoc(), Incoming, Dst); + InsertedInto.clear(); + for (unsigned I = 1, E = PHI->getNumOperands(); I != E; I += 2) { + const MachineOperand &SrcMO = PHI->getOperand(I); + MachineBasicBlock &Pred = *PHI->getOperand(I + 1).getMBB(); + // Duplicate predecessors carry the same value; one copy suffices. + if (!InsertedInto.insert(&Pred).second) + continue; + // No debug location: the copy lands in another block than the PHI. + TII->createPHISourceCopy( + Pred, findPHICopyInsertPoint(&Pred, &MBB, SrcMO.getReg()), DebugLoc(), + SrcMO.getReg(), SrcMO.getSubReg(), Incoming); + } + PHI->eraseFromParent(); + ++NumPHIsViaVReg; + } + + // For each source of a safe PHI node, perform a conservative interference + // test to try sharing slot with Dst. Each source is decided on its own, so + // the order blocks are visited in does not matter. Sharing makes Src's spill + // the edge's write of Dst's slot; canWritePHISlotInPreds() ruled out escapes + // assuming every edge writes that slot in its own predecessor, so sharing + // must not move the write out of one. + SmallPtrSet DstReadIn; + for (MachineBasicBlock &MBB : MF) { + for (MachineInstr &PHI : MBB.phis()) { + Register Dst = PHI.getOperand(0).getReg(); + // Blocks reading Dst. A PHI reads its operand on the edge, in the + // predecessor, though its parent is the successor. + DstReadIn.clear(); + for (const MachineInstr &UseMI : MRI->use_nodbg_instructions(Dst)) { + DstReadIn.insert(UseMI.getParent()); + if (UseMI.isPHI()) + for (unsigned J = 1, JE = UseMI.getNumOperands(); J != JE; J += 2) + if (UseMI.getOperand(J).getReg() == Dst) + DstReadIn.insert(UseMI.getOperand(J + 1).getMBB()); + } + for (unsigned I = 1, E = PHI.getNumOperands(); I != E; I += 2) { + MachineOperand &SrcMO = PHI.getOperand(I); + Register Src = SrcMO.getReg(); + if (!Src.isVirtual() || SrcMO.getSubReg()) + continue; + if (StackSlotForVirtReg[Src] != -1 || + !sharesStackSlot(*TRI, *MRI->getRegClass(Src), + *MRI->getRegClass(Dst))) + continue; + MachineBasicBlock *Pred = PHI.getOperand(I + 1).getMBB(); + const MachineInstr *Def = MRI->getUniqueVRegDef(Src); + // If the value is defined and only used (except the PHI) in Pred, its + // live range lies entirely inside Pred. The use-list walk follows + // mayLiveOut(). A PHI is not a write point: lowering moves it to the + // ends of Pred's own predecessors. + if (!Def || Def->getParent() != Pred || Def->isPHI()) + continue; + // Nor may Src reach MBB from another predecessor: those edges get no + // copy either, and their write would sit in Pred. + bool FromPredOnly = true; + for (unsigned J = 1; J != E; J += 2) + if (PHI.getOperand(J).getReg() == Src && + PHI.getOperand(J + 1).getMBB() != Pred) { + FromPredOnly = false; + break; + } + if (!FromPredOnly) + continue; + bool DiesInPred = true; + unsigned C = 0; + for (const MachineInstr &UseMI : MRI->use_nodbg_instructions(Src)) { + if (++C >= BlockLocalScanLimit || + (&UseMI != &PHI && + (UseMI.getParent() != Pred || UseMI.isPHI()))) { + DiesInPred = false; + break; + } + } + if (!DiesInPred) + continue; + // If Dst is not read in Pred, Src and Dst don't overlap. + if (!DstReadIn.contains(Pred)) + StackSlotForVirtReg[Src] = getStackSpaceFor(Dst); + } + } + } + + for (MachineBasicBlock &MBB : MF) + lowerPHIEdgeCopies(MBB); +} + +/// Emit the copies carrying the PHI values of \p MBB's successors on the edges +/// leaving \p MBB, before the terminators. A source sharing the destination's +/// slot needs none, its spill being the transfer. The rest are ordered so that +/// a destination is written only once nothing left reads it; a cycle is broken +/// by copying one value aside first. An operand a copy has taken over is +/// rewritten to the destination, so that the PHI no longer keeps the source +/// alive across the edge. +void RegAllocFastImpl::lowerPHIEdgeCopies(MachineBasicBlock &MBB) { + struct EdgeCopy { + Register Dst, Src; + MachineOperand *PHIOp; + }; + SmallPtrSet SeenSucc; + SmallVector Copies; + SmallDenseMap ReadCount; + for (MachineBasicBlock *Succ : MBB.successors()) { + if (!SeenSucc.insert(Succ).second) + continue; + Copies.clear(); + ReadCount.clear(); + // Not MBB.phis(): a copy below can land inside its range when MBB is its + // own successor. + for (MachineInstr &PHI : *Succ) { + if (!PHI.isPHI()) + break; + Register Dst = PHI.getOperand(0).getReg(); + for (unsigned I = 1, E = PHI.getNumOperands(); I != E; I += 2) { + if (PHI.getOperand(I + 1).getMBB() != &MBB) + continue; + MachineOperand &SrcMO = PHI.getOperand(I); + Register Src = SrcMO.getReg(); + if (!Src.isVirtual() || StackSlotForVirtReg[Src] == -1 || + StackSlotForVirtReg[Src] != StackSlotForVirtReg[Dst]) { + Copies.push_back({Dst, Src, &SrcMO}); + ++ReadCount[Src]; + } + // Duplicate predecessors carry the same value; one copy suffices. + break; + } + } + if (Copies.empty()) + continue; + + MachineBasicBlock::iterator IP = + findPHICopyInsertPoint(&MBB, Succ, Copies.front().Src); + while (!Copies.empty()) { + bool Progress = false; + for (unsigned I = 0; I != Copies.size(); ++I) { + if (ReadCount.lookup(Copies[I].Dst)) + continue; + const EdgeCopy &C = Copies[I]; + // No debug location: the copy lands in another block than the PHI. + TII->createPHISourceCopy(MBB, IP, DebugLoc(), C.Src, + C.PHIOp->getSubReg(), C.Dst); + // The copy is the transfer now. The PHI's own read of the source would + // keep it alive across the edge, costing a spill, so point it at the + // destination instead. + C.PHIOp->setReg(C.Dst); + C.PHIOp->setSubReg(0); + --ReadCount[C.Src]; + Copies.erase(Copies.begin() + I); + Progress = true; + break; + } + if (Progress) + continue; + // Every destination left is still read, so they form a cycle. Copy one + // value aside and let its readers take that register instead. + EdgeCopy &C = Copies.front(); + Register Tmp = MRI->createVirtualRegister(MRI->getRegClass(C.Dst)); + TII->createPHISourceCopy(MBB, IP, DebugLoc(), C.Dst, 0, Tmp); + ReadCount[Tmp] = ReadCount[C.Dst]; + ReadCount[C.Dst] = 0; + for (EdgeCopy &O : Copies) + if (O.Src == C.Dst) + O.Src = Tmp; + } + } +} + bool RegAllocFastImpl::runOnMachineFunction(MachineFunction &MF) { LLVM_DEBUG(dbgs() << "********** FAST REGISTER ALLOCATION **********\n" << "********** Function: " << MF.getName() << '\n'); @@ -1864,6 +2252,38 @@ bool RegAllocFastImpl::runOnMachineFunction(MachineFunction &MF) { InstrGen = 0; UsedInInstr.assign(NumRegUnits, 0); + // For SSA input, lower PHI and tied operands ourselves, skipping the + // SSA-destroying PHIElimination and TwoAddressInstructionPass. We want to + // require SSAInput in the future. The currently hasIsSSA() supports partial + // pipelines (-run-pass, -start-before). + SSAInput = + MF.getProperties().hasIsSSA() && useSSAFastRegAlloc(MF.getTarget()); + assert((SSAInput || none_of(MF, + [](const MachineBasicBlock &MBB) { + return !MBB.empty() && MBB.begin()->isPHI(); + })) && + "PHIs in non-SSA input: PHIElimination did not run and this target " + "does not consume SSA MachineIR"); + // A target that has not opted in may still be handed SSA MachineIR. + MRI->leaveSSA(); + + bool HasPHIs = false; + if (SSAInput) { + HasPHIs = any_of(MF, [](const MachineBasicBlock &MBB) { + return !MBB.empty() && MBB.begin()->isPHI(); + }); + if (HasPHIs) { + // Only the registers the PHIs reference are looked up, so this sizing is + // enough. + StackSlotForVirtReg.resize(MRI->getNumVirtRegs()); + lowerPHIs(MF); + } + for (MachineBasicBlock &MBB : MF) + for (MachineInstr &MI : make_early_inc_range(MBB)) + if (MI.isRegSequence() || MI.isInsertSubreg()) + expandSSAPseudo(MI); + } + // initialize the virtual->physical register map to have a 'null' // mapping for all virtual registers unsigned NumVirtRegs = MRI->getNumVirtRegs(); @@ -1872,10 +2292,24 @@ bool RegAllocFastImpl::runOnMachineFunction(MachineFunction &MF) { MayLiveAcrossBlocks.clear(); MayLiveAcrossBlocks.resize(NumVirtRegs); + // A PHI destination lives across the edge by construction: the copies + // lowerPHIEdgeCopies() inserted define it at the end of the predecessors, + // and it is read in the PHI's own block. The use-list walk in mayLiveOut() + // cannot tell, the PHI itself not being a use, so say so directly. + if (HasPHIs) + for (MachineBasicBlock &MBB : MF) + for (MachineInstr &PHI : MBB.phis()) + MayLiveAcrossBlocks.set(PHI.getOperand(0).getReg().virtRegIndex()); + // Loop over all of the basic blocks, eliminating virtual register references for (MachineBasicBlock &MBB : MF) allocateBasicBlock(MBB); + if (HasPHIs) + for (MachineBasicBlock &MBB : MF) + for (MachineInstr &MI : make_early_inc_range(MBB.phis())) + MI.eraseFromParent(); + if (ClearVirtRegs) { // All machine operands and other references to virtual registers have been // replaced. Remove the virtual registers. diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp index f5ea9086f6109..19f229e2d7007 100644 --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -57,6 +57,14 @@ using namespace llvm; +static cl::opt EnableRegAllocFastSSA( + "regalloc-fast-ssa", + cl::desc("Have the fast register allocator consume SSA MachineIR, " + "lowering PHIs and tied operands itself instead of running " + "PHIElimination and TwoAddressInstructionPass (overrides the " + "target's default)"), + cl::Hidden); + static cl::opt EnableIPRA("enable-ipra", cl::init(false), cl::Hidden, cl::desc("Enable interprocedural register allocation " @@ -1467,12 +1475,21 @@ bool TargetPassConfig::usingDefaultRegAlloc() const { /// Add the minimum set of target-independent passes that are required for /// register allocation. No coalescing or scheduling. void TargetPassConfig::addFastRegAlloc() { - addPass(&PHIEliminationID); - addPass(&TwoAddressInstructionPassID); + if (!useSSAFastRegAlloc(*TM)) { + addPass(&PHIEliminationID); + addPass(&TwoAddressInstructionPassID); + } addRegAssignAndRewriteFast(); } +bool llvm::useSSAFastRegAlloc(const TargetMachine &TM) { + if (EnableRegAllocFastSSA != cl::boolOrDefault::BOU_UNSET) + return EnableRegAllocFastSSA == cl::boolOrDefault::BOU_TRUE; + // Otherwise, respect TargetMachine preference. + return TM.enableSSAFastRegAlloc(); +} + /// Add standard target-independent passes that are tightly coupled with /// optimized register allocation, including coalescing, machine instruction /// scheduling, and register allocation itself. diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp index dee61a4f2ac8c..895f75492fb34 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -422,6 +422,8 @@ AArch64TargetMachine::AArch64TargetMachine(const Target &T, const Triple &TT, // AArch64 supports the debug entry values. setSupportsDebugEntryValues(true); + setEnableSSAFastRegAlloc(true); + // AArch64 supports fixing up the DWARF unwind information. if (!getMCAsmInfo().usesWindowsCFI()) setCFIFixup(true); diff --git a/llvm/lib/Target/TargetMachine.cpp b/llvm/lib/Target/TargetMachine.cpp index 46c02df9e2620..92a9dbaacb9e6 100644 --- a/llvm/lib/Target/TargetMachine.cpp +++ b/llvm/lib/Target/TargetMachine.cpp @@ -43,7 +43,7 @@ TargetMachine::TargetMachine(const Target &T, StringRef DataLayoutString, : TheTarget(T), DL(DataLayoutString), TargetTriple(TT), TargetCPU(std::string(CPU)), TargetFS(std::string(FS)), AsmInfo(nullptr), MRI(nullptr), MII(nullptr), STI(nullptr), RequireStructuredCFG(false), - O0WantsFastISel(false), Options(Options) {} + O0WantsFastISel(false), EnableSSAFastRegAlloc(false), Options(Options) {} TargetMachine::~TargetMachine() = default; diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp index 932669b5cbac6..ee0b7c695f898 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -205,6 +205,7 @@ X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT, } setMachineOutliner(true); + setEnableSSAFastRegAlloc(true); // x86 supports the debug entry values. setSupportsDebugEntryValues(true); diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-outline_atomics.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-outline_atomics.ll index f91e54be0bca6..c43e64cff4e03 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-outline_atomics.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-outline_atomics.ll @@ -2148,8 +2148,8 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_monotonic(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_relax -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_aligned_monotonic: ; -O1: ldxrb w8, [x0] @@ -2165,8 +2165,8 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acquire(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_acq -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_aligned_acquire: ; -O1: ldaxrb w8, [x0] @@ -2182,8 +2182,8 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_release(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_aligned_release: ; -O1: ldxrb w8, [x0] @@ -2199,8 +2199,8 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_aligned_acq_rel: ; -O1: ldaxrb w8, [x0] @@ -2216,8 +2216,8 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_aligned_seq_cst: ; -O1: ldaxrb w8, [x0] @@ -2233,7 +2233,7 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_monotonic(ptr %ptr, i16 %value) ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas2_relax -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_nand_i16_aligned_monotonic: ; -O1: ldxrh w8, [x0] @@ -2249,7 +2249,7 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acquire(ptr %ptr, i16 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas2_acq -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_nand_i16_aligned_acquire: ; -O1: ldaxrh w8, [x0] @@ -2265,7 +2265,7 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_release(ptr %ptr, i16 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas2_rel -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_nand_i16_aligned_release: ; -O1: ldxrh w8, [x0] @@ -2281,7 +2281,7 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acq_rel(ptr %ptr, i16 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_nand_i16_aligned_acq_rel: ; -O1: ldaxrh w8, [x0] @@ -2297,7 +2297,7 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_seq_cst(ptr %ptr, i16 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_nand_i16_aligned_seq_cst: ; -O1: ldaxrh w8, [x0] @@ -2313,7 +2313,7 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_monotonic(ptr %ptr, i32 %value) ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas4_relax -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_nand_i32_aligned_monotonic: ; -O1: ldxr w8, [x0] @@ -2329,7 +2329,7 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acquire(ptr %ptr, i32 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas4_acq -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_nand_i32_aligned_acquire: ; -O1: ldaxr w8, [x0] @@ -2345,7 +2345,7 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_release(ptr %ptr, i32 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas4_rel -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_nand_i32_aligned_release: ; -O1: ldxr w8, [x0] @@ -2361,7 +2361,7 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acq_rel(ptr %ptr, i32 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_nand_i32_aligned_acq_rel: ; -O1: ldaxr w8, [x0] @@ -2377,7 +2377,7 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_seq_cst(ptr %ptr, i32 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_nand_i32_aligned_seq_cst: ; -O1: ldaxr w8, [x0] @@ -2393,7 +2393,7 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_monotonic(ptr %ptr, i64 %value) ; -O0: and x8, x0, x8 ; -O0: mvn x1, x8 ; -O0: bl __aarch64_cas8_relax -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_nand_i64_aligned_monotonic: ; -O1: ldxr x0, [x8] @@ -2409,7 +2409,7 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acquire(ptr %ptr, i64 %value) { ; -O0: and x8, x0, x8 ; -O0: mvn x1, x8 ; -O0: bl __aarch64_cas8_acq -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_nand_i64_aligned_acquire: ; -O1: ldaxr x0, [x8] @@ -2425,7 +2425,7 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_release(ptr %ptr, i64 %value) { ; -O0: and x8, x0, x8 ; -O0: mvn x1, x8 ; -O0: bl __aarch64_cas8_rel -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_nand_i64_aligned_release: ; -O1: ldxr x0, [x8] @@ -2441,7 +2441,7 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acq_rel(ptr %ptr, i64 %value) { ; -O0: and x8, x0, x8 ; -O0: mvn x1, x8 ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_nand_i64_aligned_acq_rel: ; -O1: ldaxr x0, [x8] @@ -2457,7 +2457,7 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_seq_cst(ptr %ptr, i64 %value) { ; -O0: and x8, x0, x8 ; -O0: mvn x1, x8 ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_nand_i64_aligned_seq_cst: ; -O1: ldaxr x0, [x8] @@ -2583,8 +2583,8 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_monotonic(ptr %ptr, i8 %value) ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_relax -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_unaligned_monotonic: ; -O1: ldxrb w8, [x0] @@ -2600,8 +2600,8 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acquire(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_acq -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acquire: ; -O1: ldaxrb w8, [x0] @@ -2617,8 +2617,8 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_release(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_unaligned_release: ; -O1: ldxrb w8, [x0] @@ -2634,8 +2634,8 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acq_rel: ; -O1: ldaxrb w8, [x0] @@ -2651,8 +2651,8 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_unaligned_seq_cst: ; -O1: ldaxrb w8, [x0] @@ -4014,8 +4014,8 @@ define dso_local i8 @atomicrmw_max_i8_aligned_monotonic(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_relax -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_aligned_monotonic: ; -O1: ldxrb w9, [x0] @@ -4033,8 +4033,8 @@ define dso_local i8 @atomicrmw_max_i8_aligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_acq -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_aligned_acquire: ; -O1: ldaxrb w9, [x0] @@ -4052,8 +4052,8 @@ define dso_local i8 @atomicrmw_max_i8_aligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_aligned_release: ; -O1: ldxrb w9, [x0] @@ -4071,8 +4071,8 @@ define dso_local i8 @atomicrmw_max_i8_aligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_aligned_acq_rel: ; -O1: ldaxrb w9, [x0] @@ -4090,8 +4090,8 @@ define dso_local i8 @atomicrmw_max_i8_aligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_aligned_seq_cst: ; -O1: ldaxrb w9, [x0] @@ -4109,7 +4109,7 @@ define dso_local i16 @atomicrmw_max_i16_aligned_monotonic(ptr %ptr, i16 %value) ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas2_relax -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_max_i16_aligned_monotonic: ; -O1: ldxrh w9, [x0] @@ -4127,7 +4127,7 @@ define dso_local i16 @atomicrmw_max_i16_aligned_acquire(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas2_acq -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_max_i16_aligned_acquire: ; -O1: ldaxrh w9, [x0] @@ -4145,7 +4145,7 @@ define dso_local i16 @atomicrmw_max_i16_aligned_release(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas2_rel -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_max_i16_aligned_release: ; -O1: ldxrh w9, [x0] @@ -4163,7 +4163,7 @@ define dso_local i16 @atomicrmw_max_i16_aligned_acq_rel(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_max_i16_aligned_acq_rel: ; -O1: ldaxrh w9, [x0] @@ -4181,7 +4181,7 @@ define dso_local i16 @atomicrmw_max_i16_aligned_seq_cst(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_max_i16_aligned_seq_cst: ; -O1: ldaxrh w9, [x0] @@ -4198,7 +4198,7 @@ define dso_local i32 @atomicrmw_max_i32_aligned_monotonic(ptr %ptr, i32 %value) ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas4_relax -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_max_i32_aligned_monotonic: ; -O1: ldxr w8, [x0] @@ -4214,7 +4214,7 @@ define dso_local i32 @atomicrmw_max_i32_aligned_acquire(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas4_acq -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_max_i32_aligned_acquire: ; -O1: ldaxr w8, [x0] @@ -4230,7 +4230,7 @@ define dso_local i32 @atomicrmw_max_i32_aligned_release(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas4_rel -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_max_i32_aligned_release: ; -O1: ldxr w8, [x0] @@ -4246,7 +4246,7 @@ define dso_local i32 @atomicrmw_max_i32_aligned_acq_rel(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_max_i32_aligned_acq_rel: ; -O1: ldaxr w8, [x0] @@ -4262,7 +4262,7 @@ define dso_local i32 @atomicrmw_max_i32_aligned_seq_cst(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_max_i32_aligned_seq_cst: ; -O1: ldaxr w8, [x0] @@ -4278,7 +4278,7 @@ define dso_local i64 @atomicrmw_max_i64_aligned_monotonic(ptr %ptr, i64 %value) ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, gt ; -O0: bl __aarch64_cas8_relax -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_max_i64_aligned_monotonic: ; -O1: ldxr x0, [x8] @@ -4294,7 +4294,7 @@ define dso_local i64 @atomicrmw_max_i64_aligned_acquire(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, gt ; -O0: bl __aarch64_cas8_acq -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_max_i64_aligned_acquire: ; -O1: ldaxr x0, [x8] @@ -4310,7 +4310,7 @@ define dso_local i64 @atomicrmw_max_i64_aligned_release(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, gt ; -O0: bl __aarch64_cas8_rel -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_max_i64_aligned_release: ; -O1: ldxr x0, [x8] @@ -4326,7 +4326,7 @@ define dso_local i64 @atomicrmw_max_i64_aligned_acq_rel(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, gt ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_max_i64_aligned_acq_rel: ; -O1: ldaxr x0, [x8] @@ -4342,7 +4342,7 @@ define dso_local i64 @atomicrmw_max_i64_aligned_seq_cst(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, gt ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_max_i64_aligned_seq_cst: ; -O1: ldaxr x0, [x8] @@ -4484,8 +4484,8 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_monotonic(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_relax -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_unaligned_monotonic: ; -O1: ldxrb w9, [x0] @@ -4503,8 +4503,8 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_acq -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_unaligned_acquire: ; -O1: ldaxrb w9, [x0] @@ -4522,8 +4522,8 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_unaligned_release: ; -O1: ldxrb w9, [x0] @@ -4541,8 +4541,8 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_unaligned_acq_rel: ; -O1: ldaxrb w9, [x0] @@ -4560,8 +4560,8 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_unaligned_seq_cst: ; -O1: ldaxrb w9, [x0] @@ -4909,8 +4909,8 @@ define dso_local i8 @atomicrmw_min_i8_aligned_monotonic(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_relax -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_aligned_monotonic: ; -O1: ldxrb w9, [x0] @@ -4928,8 +4928,8 @@ define dso_local i8 @atomicrmw_min_i8_aligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_acq -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_aligned_acquire: ; -O1: ldaxrb w9, [x0] @@ -4947,8 +4947,8 @@ define dso_local i8 @atomicrmw_min_i8_aligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_aligned_release: ; -O1: ldxrb w9, [x0] @@ -4966,8 +4966,8 @@ define dso_local i8 @atomicrmw_min_i8_aligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_aligned_acq_rel: ; -O1: ldaxrb w9, [x0] @@ -4985,8 +4985,8 @@ define dso_local i8 @atomicrmw_min_i8_aligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_aligned_seq_cst: ; -O1: ldaxrb w9, [x0] @@ -5004,7 +5004,7 @@ define dso_local i16 @atomicrmw_min_i16_aligned_monotonic(ptr %ptr, i16 %value) ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas2_relax -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_min_i16_aligned_monotonic: ; -O1: ldxrh w9, [x0] @@ -5022,7 +5022,7 @@ define dso_local i16 @atomicrmw_min_i16_aligned_acquire(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas2_acq -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_min_i16_aligned_acquire: ; -O1: ldaxrh w9, [x0] @@ -5040,7 +5040,7 @@ define dso_local i16 @atomicrmw_min_i16_aligned_release(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas2_rel -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_min_i16_aligned_release: ; -O1: ldxrh w9, [x0] @@ -5058,7 +5058,7 @@ define dso_local i16 @atomicrmw_min_i16_aligned_acq_rel(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_min_i16_aligned_acq_rel: ; -O1: ldaxrh w9, [x0] @@ -5076,7 +5076,7 @@ define dso_local i16 @atomicrmw_min_i16_aligned_seq_cst(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_min_i16_aligned_seq_cst: ; -O1: ldaxrh w9, [x0] @@ -5093,7 +5093,7 @@ define dso_local i32 @atomicrmw_min_i32_aligned_monotonic(ptr %ptr, i32 %value) ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas4_relax -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_min_i32_aligned_monotonic: ; -O1: ldxr w8, [x0] @@ -5109,7 +5109,7 @@ define dso_local i32 @atomicrmw_min_i32_aligned_acquire(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas4_acq -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_min_i32_aligned_acquire: ; -O1: ldaxr w8, [x0] @@ -5125,7 +5125,7 @@ define dso_local i32 @atomicrmw_min_i32_aligned_release(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas4_rel -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_min_i32_aligned_release: ; -O1: ldxr w8, [x0] @@ -5141,7 +5141,7 @@ define dso_local i32 @atomicrmw_min_i32_aligned_acq_rel(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_min_i32_aligned_acq_rel: ; -O1: ldaxr w8, [x0] @@ -5157,7 +5157,7 @@ define dso_local i32 @atomicrmw_min_i32_aligned_seq_cst(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_min_i32_aligned_seq_cst: ; -O1: ldaxr w8, [x0] @@ -5173,7 +5173,7 @@ define dso_local i64 @atomicrmw_min_i64_aligned_monotonic(ptr %ptr, i64 %value) ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, le ; -O0: bl __aarch64_cas8_relax -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_min_i64_aligned_monotonic: ; -O1: ldxr x0, [x8] @@ -5189,7 +5189,7 @@ define dso_local i64 @atomicrmw_min_i64_aligned_acquire(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, le ; -O0: bl __aarch64_cas8_acq -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_min_i64_aligned_acquire: ; -O1: ldaxr x0, [x8] @@ -5205,7 +5205,7 @@ define dso_local i64 @atomicrmw_min_i64_aligned_release(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, le ; -O0: bl __aarch64_cas8_rel -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_min_i64_aligned_release: ; -O1: ldxr x0, [x8] @@ -5221,7 +5221,7 @@ define dso_local i64 @atomicrmw_min_i64_aligned_acq_rel(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, le ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_min_i64_aligned_acq_rel: ; -O1: ldaxr x0, [x8] @@ -5237,7 +5237,7 @@ define dso_local i64 @atomicrmw_min_i64_aligned_seq_cst(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, le ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_min_i64_aligned_seq_cst: ; -O1: ldaxr x0, [x8] @@ -5379,8 +5379,8 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_monotonic(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_relax -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_unaligned_monotonic: ; -O1: ldxrb w9, [x0] @@ -5398,8 +5398,8 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_acq -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_unaligned_acquire: ; -O1: ldaxrb w9, [x0] @@ -5417,8 +5417,8 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_unaligned_release: ; -O1: ldxrb w9, [x0] @@ -5436,8 +5436,8 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_unaligned_acq_rel: ; -O1: ldaxrb w9, [x0] @@ -5455,8 +5455,8 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_unaligned_seq_cst: ; -O1: ldaxrb w9, [x0] @@ -5804,8 +5804,8 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_monotonic(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_relax -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umax_i8_aligned_monotonic: ; -O1: and w9, w1, #0xff @@ -5823,8 +5823,8 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_acq -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umax_i8_aligned_acquire: ; -O1: and w9, w1, #0xff @@ -5842,8 +5842,8 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umax_i8_aligned_release: ; -O1: and w9, w1, #0xff @@ -5861,8 +5861,8 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umax_i8_aligned_acq_rel: ; -O1: and w9, w1, #0xff @@ -5880,8 +5880,8 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umax_i8_aligned_seq_cst: ; -O1: and w9, w1, #0xff @@ -5898,7 +5898,7 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_monotonic(ptr %ptr, i16 %value) ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas2_relax -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_umax_i16_aligned_monotonic: ; -O1: and w9, w1, #0xffff @@ -5915,7 +5915,7 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_acquire(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas2_acq -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_umax_i16_aligned_acquire: ; -O1: and w9, w1, #0xffff @@ -5932,7 +5932,7 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_release(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas2_rel -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_umax_i16_aligned_release: ; -O1: and w9, w1, #0xffff @@ -5949,7 +5949,7 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_acq_rel(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_umax_i16_aligned_acq_rel: ; -O1: and w9, w1, #0xffff @@ -5966,7 +5966,7 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_seq_cst(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_umax_i16_aligned_seq_cst: ; -O1: and w9, w1, #0xffff @@ -5983,7 +5983,7 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_monotonic(ptr %ptr, i32 %value) ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas4_relax -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_umax_i32_aligned_monotonic: ; -O1: ldxr w8, [x0] @@ -5999,7 +5999,7 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_acquire(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas4_acq -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_umax_i32_aligned_acquire: ; -O1: ldaxr w8, [x0] @@ -6015,7 +6015,7 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_release(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas4_rel -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_umax_i32_aligned_release: ; -O1: ldxr w8, [x0] @@ -6031,7 +6031,7 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_acq_rel(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_umax_i32_aligned_acq_rel: ; -O1: ldaxr w8, [x0] @@ -6047,7 +6047,7 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_seq_cst(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_umax_i32_aligned_seq_cst: ; -O1: ldaxr w8, [x0] @@ -6063,7 +6063,7 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_monotonic(ptr %ptr, i64 %value) ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, hi ; -O0: bl __aarch64_cas8_relax -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_umax_i64_aligned_monotonic: ; -O1: ldxr x0, [x8] @@ -6079,7 +6079,7 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_acquire(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, hi ; -O0: bl __aarch64_cas8_acq -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_umax_i64_aligned_acquire: ; -O1: ldaxr x0, [x8] @@ -6095,7 +6095,7 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_release(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, hi ; -O0: bl __aarch64_cas8_rel -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_umax_i64_aligned_release: ; -O1: ldxr x0, [x8] @@ -6111,7 +6111,7 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_acq_rel(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, hi ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_umax_i64_aligned_acq_rel: ; -O1: ldaxr x0, [x8] @@ -6127,7 +6127,7 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_seq_cst(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, hi ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_umax_i64_aligned_seq_cst: ; -O1: ldaxr x0, [x8] @@ -6269,8 +6269,8 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_monotonic(ptr %ptr, i8 %value) ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_relax -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umax_i8_unaligned_monotonic: ; -O1: and w9, w1, #0xff @@ -6288,8 +6288,8 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_acq -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umax_i8_unaligned_acquire: ; -O1: and w9, w1, #0xff @@ -6307,8 +6307,8 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umax_i8_unaligned_release: ; -O1: and w9, w1, #0xff @@ -6326,8 +6326,8 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umax_i8_unaligned_acq_rel: ; -O1: and w9, w1, #0xff @@ -6345,8 +6345,8 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umax_i8_unaligned_seq_cst: ; -O1: and w9, w1, #0xff @@ -6689,8 +6689,8 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_monotonic(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_relax -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umin_i8_aligned_monotonic: ; -O1: and w9, w1, #0xff @@ -6708,8 +6708,8 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_acq -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umin_i8_aligned_acquire: ; -O1: and w9, w1, #0xff @@ -6727,8 +6727,8 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umin_i8_aligned_release: ; -O1: and w9, w1, #0xff @@ -6746,8 +6746,8 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umin_i8_aligned_acq_rel: ; -O1: and w9, w1, #0xff @@ -6765,8 +6765,8 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umin_i8_aligned_seq_cst: ; -O1: and w9, w1, #0xff @@ -6783,7 +6783,7 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_monotonic(ptr %ptr, i16 %value) ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas2_relax -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_umin_i16_aligned_monotonic: ; -O1: and w9, w1, #0xffff @@ -6800,7 +6800,7 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_acquire(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas2_acq -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_umin_i16_aligned_acquire: ; -O1: and w9, w1, #0xffff @@ -6817,7 +6817,7 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_release(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas2_rel -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_umin_i16_aligned_release: ; -O1: and w9, w1, #0xffff @@ -6834,7 +6834,7 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_acq_rel(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_umin_i16_aligned_acq_rel: ; -O1: and w9, w1, #0xffff @@ -6851,7 +6851,7 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_seq_cst(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w8, w9, uxth +; -O0: subs w9, w9, w0, uxth ; ; -O1-LABEL: atomicrmw_umin_i16_aligned_seq_cst: ; -O1: and w9, w1, #0xffff @@ -6868,7 +6868,7 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_monotonic(ptr %ptr, i32 %value) ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas4_relax -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_umin_i32_aligned_monotonic: ; -O1: ldxr w8, [x0] @@ -6884,7 +6884,7 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_acquire(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas4_acq -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_umin_i32_aligned_acquire: ; -O1: ldaxr w8, [x0] @@ -6900,7 +6900,7 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_release(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas4_rel -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_umin_i32_aligned_release: ; -O1: ldxr w8, [x0] @@ -6916,7 +6916,7 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_acq_rel(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_umin_i32_aligned_acq_rel: ; -O1: ldaxr w8, [x0] @@ -6932,7 +6932,7 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_seq_cst(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w8, w9 +; -O0: subs w9, w0, w9 ; ; -O1-LABEL: atomicrmw_umin_i32_aligned_seq_cst: ; -O1: ldaxr w8, [x0] @@ -6948,7 +6948,7 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_monotonic(ptr %ptr, i64 %value) ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, ls ; -O0: bl __aarch64_cas8_relax -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_umin_i64_aligned_monotonic: ; -O1: ldxr x0, [x8] @@ -6964,7 +6964,7 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_acquire(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, ls ; -O0: bl __aarch64_cas8_acq -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_umin_i64_aligned_acquire: ; -O1: ldaxr x0, [x8] @@ -6980,7 +6980,7 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_release(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, ls ; -O0: bl __aarch64_cas8_rel -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_umin_i64_aligned_release: ; -O1: ldxr x0, [x8] @@ -6996,7 +6996,7 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_acq_rel(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, ls ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_umin_i64_aligned_acq_rel: ; -O1: ldaxr x0, [x8] @@ -7012,7 +7012,7 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_seq_cst(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, ls ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x8, x9 +; -O0: subs x9, x0, x9 ; ; -O1-LABEL: atomicrmw_umin_i64_aligned_seq_cst: ; -O1: ldaxr x0, [x8] @@ -7154,8 +7154,8 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_monotonic(ptr %ptr, i8 %value) ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_relax -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umin_i8_unaligned_monotonic: ; -O1: and w9, w1, #0xff @@ -7173,8 +7173,8 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_acq -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umin_i8_unaligned_acquire: ; -O1: and w9, w1, #0xff @@ -7192,8 +7192,8 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umin_i8_unaligned_release: ; -O1: and w9, w1, #0xff @@ -7211,8 +7211,8 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umin_i8_unaligned_acq_rel: ; -O1: and w9, w1, #0xff @@ -7230,8 +7230,8 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_acq_rel -; -O0: and w8, w0, #0xff -; -O0: subs w8, w8, w9, uxtb +; -O0: and w9, w8, #0xff +; -O0: subs w9, w9, w10, uxtb ; ; -O1-LABEL: atomicrmw_umin_i8_unaligned_seq_cst: ; -O1: and w9, w1, #0xff diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-store-outline_atomics.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-store-outline_atomics.ll index fd9038e85a5cc..eb27f194dd296 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-store-outline_atomics.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-store-outline_atomics.ll @@ -118,8 +118,8 @@ define dso_local void @store_atomic_i64_aligned_seq_cst(i64 %value, ptr %ptr) { define dso_local void @store_atomic_i128_aligned_unordered(i128 %value, ptr %ptr) { ; -O0-LABEL: store_atomic_i128_aligned_unordered: ; -O0: bl __aarch64_cas16_relax -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: store_atomic_i128_aligned_unordered: ; -O1: ldxp xzr, x8, [x2] @@ -131,8 +131,8 @@ define dso_local void @store_atomic_i128_aligned_unordered(i128 %value, ptr %ptr define dso_local void @store_atomic_i128_aligned_monotonic(i128 %value, ptr %ptr) { ; -O0-LABEL: store_atomic_i128_aligned_monotonic: ; -O0: bl __aarch64_cas16_relax -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: store_atomic_i128_aligned_monotonic: ; -O1: ldxp xzr, x8, [x2] @@ -144,8 +144,8 @@ define dso_local void @store_atomic_i128_aligned_monotonic(i128 %value, ptr %ptr define dso_local void @store_atomic_i128_aligned_release(i128 %value, ptr %ptr) { ; -O0-LABEL: store_atomic_i128_aligned_release: ; -O0: bl __aarch64_cas16_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: store_atomic_i128_aligned_release: ; -O1: ldxp xzr, x8, [x2] @@ -157,8 +157,8 @@ define dso_local void @store_atomic_i128_aligned_release(i128 %value, ptr %ptr) define dso_local void @store_atomic_i128_aligned_seq_cst(i128 %value, ptr %ptr) { ; -O0-LABEL: store_atomic_i128_aligned_seq_cst: ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: store_atomic_i128_aligned_seq_cst: ; -O1: ldaxp xzr, x8, [x2] diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-outline_atomics.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-outline_atomics.ll index 785689529d5cd..cb99f38a64282 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-outline_atomics.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-outline_atomics.ll @@ -146,8 +146,8 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_seq_cst(ptr %ptr, i64 %value) { define dso_local i128 @atomicrmw_xchg_i128_aligned_monotonic(ptr %ptr, i128 %value) { ; -O0-LABEL: atomicrmw_xchg_i128_aligned_monotonic: ; -O0: bl __aarch64_cas16_relax -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_xchg_i128_aligned_monotonic: ; -O1: ldxp x8, x1, [x0] @@ -159,8 +159,8 @@ define dso_local i128 @atomicrmw_xchg_i128_aligned_monotonic(ptr %ptr, i128 %val define dso_local i128 @atomicrmw_xchg_i128_aligned_acquire(ptr %ptr, i128 %value) { ; -O0-LABEL: atomicrmw_xchg_i128_aligned_acquire: ; -O0: bl __aarch64_cas16_acq -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_xchg_i128_aligned_acquire: ; -O1: ldaxp x8, x1, [x0] @@ -172,8 +172,8 @@ define dso_local i128 @atomicrmw_xchg_i128_aligned_acquire(ptr %ptr, i128 %value define dso_local i128 @atomicrmw_xchg_i128_aligned_release(ptr %ptr, i128 %value) { ; -O0-LABEL: atomicrmw_xchg_i128_aligned_release: ; -O0: bl __aarch64_cas16_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_xchg_i128_aligned_release: ; -O1: ldxp x8, x1, [x0] @@ -185,8 +185,8 @@ define dso_local i128 @atomicrmw_xchg_i128_aligned_release(ptr %ptr, i128 %value define dso_local i128 @atomicrmw_xchg_i128_aligned_acq_rel(ptr %ptr, i128 %value) { ; -O0-LABEL: atomicrmw_xchg_i128_aligned_acq_rel: ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_xchg_i128_aligned_acq_rel: ; -O1: ldaxp x8, x1, [x0] @@ -198,8 +198,8 @@ define dso_local i128 @atomicrmw_xchg_i128_aligned_acq_rel(ptr %ptr, i128 %value define dso_local i128 @atomicrmw_xchg_i128_aligned_seq_cst(ptr %ptr, i128 %value) { ; -O0-LABEL: atomicrmw_xchg_i128_aligned_seq_cst: ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_xchg_i128_aligned_seq_cst: ; -O1: ldaxp x8, x1, [x0] @@ -527,8 +527,8 @@ define dso_local i128 @atomicrmw_add_i128_aligned_monotonic(ptr %ptr, i128 %valu ; -O0-LABEL: atomicrmw_add_i128_aligned_monotonic: ; -O0: adds x3, x1, x9 ; -O0: bl __aarch64_cas16_relax -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_add_i128_aligned_monotonic: ; -O1: ldxp x0, x1, [x8] @@ -542,8 +542,8 @@ define dso_local i128 @atomicrmw_add_i128_aligned_acquire(ptr %ptr, i128 %value) ; -O0-LABEL: atomicrmw_add_i128_aligned_acquire: ; -O0: adds x3, x1, x9 ; -O0: bl __aarch64_cas16_acq -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_add_i128_aligned_acquire: ; -O1: ldaxp x0, x1, [x8] @@ -557,8 +557,8 @@ define dso_local i128 @atomicrmw_add_i128_aligned_release(ptr %ptr, i128 %value) ; -O0-LABEL: atomicrmw_add_i128_aligned_release: ; -O0: adds x3, x1, x9 ; -O0: bl __aarch64_cas16_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_add_i128_aligned_release: ; -O1: ldxp x0, x1, [x8] @@ -572,8 +572,8 @@ define dso_local i128 @atomicrmw_add_i128_aligned_acq_rel(ptr %ptr, i128 %value) ; -O0-LABEL: atomicrmw_add_i128_aligned_acq_rel: ; -O0: adds x3, x1, x9 ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_add_i128_aligned_acq_rel: ; -O1: ldaxp x0, x1, [x8] @@ -587,8 +587,8 @@ define dso_local i128 @atomicrmw_add_i128_aligned_seq_cst(ptr %ptr, i128 %value) ; -O0-LABEL: atomicrmw_add_i128_aligned_seq_cst: ; -O0: adds x3, x1, x9 ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_add_i128_aligned_seq_cst: ; -O1: ldaxp x0, x1, [x8] @@ -1102,8 +1102,8 @@ define dso_local i128 @atomicrmw_sub_i128_aligned_monotonic(ptr %ptr, i128 %valu ; -O0-LABEL: atomicrmw_sub_i128_aligned_monotonic: ; -O0: subs x3, x1, x9 ; -O0: bl __aarch64_cas16_relax -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_sub_i128_aligned_monotonic: ; -O1: ldxp x0, x1, [x8] @@ -1117,8 +1117,8 @@ define dso_local i128 @atomicrmw_sub_i128_aligned_acquire(ptr %ptr, i128 %value) ; -O0-LABEL: atomicrmw_sub_i128_aligned_acquire: ; -O0: subs x3, x1, x9 ; -O0: bl __aarch64_cas16_acq -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_sub_i128_aligned_acquire: ; -O1: ldaxp x0, x1, [x8] @@ -1132,8 +1132,8 @@ define dso_local i128 @atomicrmw_sub_i128_aligned_release(ptr %ptr, i128 %value) ; -O0-LABEL: atomicrmw_sub_i128_aligned_release: ; -O0: subs x3, x1, x9 ; -O0: bl __aarch64_cas16_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_sub_i128_aligned_release: ; -O1: ldxp x0, x1, [x8] @@ -1147,8 +1147,8 @@ define dso_local i128 @atomicrmw_sub_i128_aligned_acq_rel(ptr %ptr, i128 %value) ; -O0-LABEL: atomicrmw_sub_i128_aligned_acq_rel: ; -O0: subs x3, x1, x9 ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_sub_i128_aligned_acq_rel: ; -O1: ldaxp x0, x1, [x8] @@ -1162,8 +1162,8 @@ define dso_local i128 @atomicrmw_sub_i128_aligned_seq_cst(ptr %ptr, i128 %value) ; -O0-LABEL: atomicrmw_sub_i128_aligned_seq_cst: ; -O0: subs x3, x1, x9 ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_sub_i128_aligned_seq_cst: ; -O1: ldaxp x0, x1, [x8] @@ -1718,8 +1718,8 @@ define dso_local i128 @atomicrmw_and_i128_aligned_monotonic(ptr %ptr, i128 %valu ; -O0: and x2, x0, x9 ; -O0: and x3, x1, x8 ; -O0: bl __aarch64_cas16_relax -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_and_i128_aligned_monotonic: ; -O1: ldxp x0, x1, [x8] @@ -1735,8 +1735,8 @@ define dso_local i128 @atomicrmw_and_i128_aligned_acquire(ptr %ptr, i128 %value) ; -O0: and x2, x0, x9 ; -O0: and x3, x1, x8 ; -O0: bl __aarch64_cas16_acq -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_and_i128_aligned_acquire: ; -O1: ldaxp x0, x1, [x8] @@ -1752,8 +1752,8 @@ define dso_local i128 @atomicrmw_and_i128_aligned_release(ptr %ptr, i128 %value) ; -O0: and x2, x0, x9 ; -O0: and x3, x1, x8 ; -O0: bl __aarch64_cas16_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_and_i128_aligned_release: ; -O1: ldxp x0, x1, [x8] @@ -1769,8 +1769,8 @@ define dso_local i128 @atomicrmw_and_i128_aligned_acq_rel(ptr %ptr, i128 %value) ; -O0: and x2, x0, x9 ; -O0: and x3, x1, x8 ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_and_i128_aligned_acq_rel: ; -O1: ldaxp x0, x1, [x8] @@ -1786,8 +1786,8 @@ define dso_local i128 @atomicrmw_and_i128_aligned_seq_cst(ptr %ptr, i128 %value) ; -O0: and x2, x0, x9 ; -O0: and x3, x1, x8 ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_and_i128_aligned_seq_cst: ; -O1: ldaxp x0, x1, [x8] @@ -2118,7 +2118,7 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_monotonic(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_relax -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_aligned_monotonic: ; -O1: ldxrb w8, [x0] @@ -2134,7 +2134,7 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acquire(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_acq -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_aligned_acquire: ; -O1: ldaxrb w8, [x0] @@ -2150,7 +2150,7 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_release(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_aligned_release: ; -O1: ldxrb w8, [x0] @@ -2166,7 +2166,7 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_aligned_acq_rel: ; -O1: ldaxrb w8, [x0] @@ -2182,7 +2182,7 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_aligned_seq_cst: ; -O1: ldaxrb w8, [x0] @@ -2198,7 +2198,7 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_monotonic(ptr %ptr, i16 %value) ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas2_relax -; -O0: subs w8, w0, w8, uxth +; -O0: subs w9, w8, w9, uxth ; ; -O1-LABEL: atomicrmw_nand_i16_aligned_monotonic: ; -O1: ldxrh w8, [x0] @@ -2214,7 +2214,7 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acquire(ptr %ptr, i16 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas2_acq -; -O0: subs w8, w0, w8, uxth +; -O0: subs w9, w8, w9, uxth ; ; -O1-LABEL: atomicrmw_nand_i16_aligned_acquire: ; -O1: ldaxrh w8, [x0] @@ -2230,7 +2230,7 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_release(ptr %ptr, i16 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas2_rel -; -O0: subs w8, w0, w8, uxth +; -O0: subs w9, w8, w9, uxth ; ; -O1-LABEL: atomicrmw_nand_i16_aligned_release: ; -O1: ldxrh w8, [x0] @@ -2246,7 +2246,7 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acq_rel(ptr %ptr, i16 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w0, w8, uxth +; -O0: subs w9, w8, w9, uxth ; ; -O1-LABEL: atomicrmw_nand_i16_aligned_acq_rel: ; -O1: ldaxrh w8, [x0] @@ -2262,7 +2262,7 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_seq_cst(ptr %ptr, i16 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w0, w8, uxth +; -O0: subs w9, w8, w9, uxth ; ; -O1-LABEL: atomicrmw_nand_i16_aligned_seq_cst: ; -O1: ldaxrh w8, [x0] @@ -2278,7 +2278,7 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_monotonic(ptr %ptr, i32 %value) ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas4_relax -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_nand_i32_aligned_monotonic: ; -O1: ldxr w8, [x0] @@ -2294,7 +2294,7 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acquire(ptr %ptr, i32 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas4_acq -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_nand_i32_aligned_acquire: ; -O1: ldaxr w8, [x0] @@ -2310,7 +2310,7 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_release(ptr %ptr, i32 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas4_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_nand_i32_aligned_release: ; -O1: ldxr w8, [x0] @@ -2326,7 +2326,7 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acq_rel(ptr %ptr, i32 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_nand_i32_aligned_acq_rel: ; -O1: ldaxr w8, [x0] @@ -2342,7 +2342,7 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_seq_cst(ptr %ptr, i32 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_nand_i32_aligned_seq_cst: ; -O1: ldaxr w8, [x0] @@ -2358,7 +2358,7 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_monotonic(ptr %ptr, i64 %value) ; -O0: and x8, x0, x8 ; -O0: mvn x1, x8 ; -O0: bl __aarch64_cas8_relax -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_nand_i64_aligned_monotonic: ; -O1: ldxr x0, [x8] @@ -2374,7 +2374,7 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acquire(ptr %ptr, i64 %value) { ; -O0: and x8, x0, x8 ; -O0: mvn x1, x8 ; -O0: bl __aarch64_cas8_acq -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_nand_i64_aligned_acquire: ; -O1: ldaxr x0, [x8] @@ -2390,7 +2390,7 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_release(ptr %ptr, i64 %value) { ; -O0: and x8, x0, x8 ; -O0: mvn x1, x8 ; -O0: bl __aarch64_cas8_rel -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_nand_i64_aligned_release: ; -O1: ldxr x0, [x8] @@ -2406,7 +2406,7 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acq_rel(ptr %ptr, i64 %value) { ; -O0: and x8, x0, x8 ; -O0: mvn x1, x8 ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_nand_i64_aligned_acq_rel: ; -O1: ldaxr x0, [x8] @@ -2422,7 +2422,7 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_seq_cst(ptr %ptr, i64 %value) { ; -O0: and x8, x0, x8 ; -O0: mvn x1, x8 ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_nand_i64_aligned_seq_cst: ; -O1: ldaxr x0, [x8] @@ -2440,8 +2440,8 @@ define dso_local i128 @atomicrmw_nand_i128_aligned_monotonic(ptr %ptr, i128 %val ; -O0: mvn x2, x9 ; -O0: mvn x3, x8 ; -O0: bl __aarch64_cas16_relax -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_nand_i128_aligned_monotonic: ; -O1: ldxp x0, x1, [x8] @@ -2461,8 +2461,8 @@ define dso_local i128 @atomicrmw_nand_i128_aligned_acquire(ptr %ptr, i128 %value ; -O0: mvn x2, x9 ; -O0: mvn x3, x8 ; -O0: bl __aarch64_cas16_acq -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_nand_i128_aligned_acquire: ; -O1: ldaxp x0, x1, [x8] @@ -2482,8 +2482,8 @@ define dso_local i128 @atomicrmw_nand_i128_aligned_release(ptr %ptr, i128 %value ; -O0: mvn x2, x9 ; -O0: mvn x3, x8 ; -O0: bl __aarch64_cas16_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_nand_i128_aligned_release: ; -O1: ldxp x0, x1, [x8] @@ -2503,8 +2503,8 @@ define dso_local i128 @atomicrmw_nand_i128_aligned_acq_rel(ptr %ptr, i128 %value ; -O0: mvn x2, x9 ; -O0: mvn x3, x8 ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_nand_i128_aligned_acq_rel: ; -O1: ldaxp x0, x1, [x8] @@ -2524,8 +2524,8 @@ define dso_local i128 @atomicrmw_nand_i128_aligned_seq_cst(ptr %ptr, i128 %value ; -O0: mvn x2, x9 ; -O0: mvn x3, x8 ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_nand_i128_aligned_seq_cst: ; -O1: ldaxp x0, x1, [x8] @@ -2543,7 +2543,7 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_monotonic(ptr %ptr, i8 %value) ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_relax -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_unaligned_monotonic: ; -O1: ldxrb w8, [x0] @@ -2559,7 +2559,7 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acquire(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_acq -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acquire: ; -O1: ldaxrb w8, [x0] @@ -2575,7 +2575,7 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_release(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_unaligned_release: ; -O1: ldxrb w8, [x0] @@ -2591,7 +2591,7 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acq_rel: ; -O1: ldaxrb w8, [x0] @@ -2607,7 +2607,7 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: and w8, w0, w8 ; -O0: mvn w1, w8 ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_nand_i8_unaligned_seq_cst: ; -O1: ldaxrb w8, [x0] @@ -3068,8 +3068,8 @@ define dso_local i128 @atomicrmw_or_i128_aligned_monotonic(ptr %ptr, i128 %value ; -O0: orr x2, x0, x9 ; -O0: orr x3, x1, x8 ; -O0: bl __aarch64_cas16_relax -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_or_i128_aligned_monotonic: ; -O1: ldxp x0, x1, [x8] @@ -3085,8 +3085,8 @@ define dso_local i128 @atomicrmw_or_i128_aligned_acquire(ptr %ptr, i128 %value) ; -O0: orr x2, x0, x9 ; -O0: orr x3, x1, x8 ; -O0: bl __aarch64_cas16_acq -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_or_i128_aligned_acquire: ; -O1: ldaxp x0, x1, [x8] @@ -3102,8 +3102,8 @@ define dso_local i128 @atomicrmw_or_i128_aligned_release(ptr %ptr, i128 %value) ; -O0: orr x2, x0, x9 ; -O0: orr x3, x1, x8 ; -O0: bl __aarch64_cas16_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_or_i128_aligned_release: ; -O1: ldxp x0, x1, [x8] @@ -3119,8 +3119,8 @@ define dso_local i128 @atomicrmw_or_i128_aligned_acq_rel(ptr %ptr, i128 %value) ; -O0: orr x2, x0, x9 ; -O0: orr x3, x1, x8 ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_or_i128_aligned_acq_rel: ; -O1: ldaxp x0, x1, [x8] @@ -3136,8 +3136,8 @@ define dso_local i128 @atomicrmw_or_i128_aligned_seq_cst(ptr %ptr, i128 %value) ; -O0: orr x2, x0, x9 ; -O0: orr x3, x1, x8 ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_or_i128_aligned_seq_cst: ; -O1: ldaxp x0, x1, [x8] @@ -3583,8 +3583,8 @@ define dso_local i128 @atomicrmw_xor_i128_aligned_monotonic(ptr %ptr, i128 %valu ; -O0: eor x2, x0, x9 ; -O0: eor x3, x1, x8 ; -O0: bl __aarch64_cas16_relax -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_xor_i128_aligned_monotonic: ; -O1: ldxp x0, x1, [x8] @@ -3600,8 +3600,8 @@ define dso_local i128 @atomicrmw_xor_i128_aligned_acquire(ptr %ptr, i128 %value) ; -O0: eor x2, x0, x9 ; -O0: eor x3, x1, x8 ; -O0: bl __aarch64_cas16_acq -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_xor_i128_aligned_acquire: ; -O1: ldaxp x0, x1, [x8] @@ -3617,8 +3617,8 @@ define dso_local i128 @atomicrmw_xor_i128_aligned_release(ptr %ptr, i128 %value) ; -O0: eor x2, x0, x9 ; -O0: eor x3, x1, x8 ; -O0: bl __aarch64_cas16_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_xor_i128_aligned_release: ; -O1: ldxp x0, x1, [x8] @@ -3634,8 +3634,8 @@ define dso_local i128 @atomicrmw_xor_i128_aligned_acq_rel(ptr %ptr, i128 %value) ; -O0: eor x2, x0, x9 ; -O0: eor x3, x1, x8 ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_xor_i128_aligned_acq_rel: ; -O1: ldaxp x0, x1, [x8] @@ -3651,8 +3651,8 @@ define dso_local i128 @atomicrmw_xor_i128_aligned_seq_cst(ptr %ptr, i128 %value) ; -O0: eor x2, x0, x9 ; -O0: eor x3, x1, x8 ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_xor_i128_aligned_seq_cst: ; -O1: ldaxp x0, x1, [x8] @@ -3959,7 +3959,7 @@ define dso_local i8 @atomicrmw_max_i8_aligned_monotonic(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_relax -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_aligned_monotonic: ; -O1: ldxrb w9, [x0] @@ -3977,7 +3977,7 @@ define dso_local i8 @atomicrmw_max_i8_aligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_acq -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_aligned_acquire: ; -O1: ldaxrb w9, [x0] @@ -3995,7 +3995,7 @@ define dso_local i8 @atomicrmw_max_i8_aligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_aligned_release: ; -O1: ldxrb w9, [x0] @@ -4013,7 +4013,7 @@ define dso_local i8 @atomicrmw_max_i8_aligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_aligned_acq_rel: ; -O1: ldaxrb w9, [x0] @@ -4031,7 +4031,7 @@ define dso_local i8 @atomicrmw_max_i8_aligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_aligned_seq_cst: ; -O1: ldaxrb w9, [x0] @@ -4049,7 +4049,7 @@ define dso_local i16 @atomicrmw_max_i16_aligned_monotonic(ptr %ptr, i16 %value) ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas2_relax -; -O0: subs w8, w0, w8, uxth +; -O0: subs w9, w8, w9, uxth ; ; -O1-LABEL: atomicrmw_max_i16_aligned_monotonic: ; -O1: ldxrh w9, [x0] @@ -4067,7 +4067,7 @@ define dso_local i16 @atomicrmw_max_i16_aligned_acquire(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas2_acq -; -O0: subs w8, w0, w8, uxth +; -O0: subs w9, w8, w9, uxth ; ; -O1-LABEL: atomicrmw_max_i16_aligned_acquire: ; -O1: ldaxrh w9, [x0] @@ -4085,7 +4085,7 @@ define dso_local i16 @atomicrmw_max_i16_aligned_release(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas2_rel -; -O0: subs w8, w0, w8, uxth +; -O0: subs w9, w8, w9, uxth ; ; -O1-LABEL: atomicrmw_max_i16_aligned_release: ; -O1: ldxrh w9, [x0] @@ -4103,7 +4103,7 @@ define dso_local i16 @atomicrmw_max_i16_aligned_acq_rel(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w0, w8, uxth +; -O0: subs w9, w8, w9, uxth ; ; -O1-LABEL: atomicrmw_max_i16_aligned_acq_rel: ; -O1: ldaxrh w9, [x0] @@ -4121,7 +4121,7 @@ define dso_local i16 @atomicrmw_max_i16_aligned_seq_cst(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w0, w8, uxth +; -O0: subs w9, w8, w9, uxth ; ; -O1-LABEL: atomicrmw_max_i16_aligned_seq_cst: ; -O1: ldaxrh w9, [x0] @@ -4138,7 +4138,7 @@ define dso_local i32 @atomicrmw_max_i32_aligned_monotonic(ptr %ptr, i32 %value) ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas4_relax -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_max_i32_aligned_monotonic: ; -O1: ldxr w8, [x0] @@ -4154,7 +4154,7 @@ define dso_local i32 @atomicrmw_max_i32_aligned_acquire(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas4_acq -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_max_i32_aligned_acquire: ; -O1: ldaxr w8, [x0] @@ -4170,7 +4170,7 @@ define dso_local i32 @atomicrmw_max_i32_aligned_release(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas4_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_max_i32_aligned_release: ; -O1: ldxr w8, [x0] @@ -4186,7 +4186,7 @@ define dso_local i32 @atomicrmw_max_i32_aligned_acq_rel(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_max_i32_aligned_acq_rel: ; -O1: ldaxr w8, [x0] @@ -4202,7 +4202,7 @@ define dso_local i32 @atomicrmw_max_i32_aligned_seq_cst(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_max_i32_aligned_seq_cst: ; -O1: ldaxr w8, [x0] @@ -4218,7 +4218,7 @@ define dso_local i64 @atomicrmw_max_i64_aligned_monotonic(ptr %ptr, i64 %value) ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, gt ; -O0: bl __aarch64_cas8_relax -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_max_i64_aligned_monotonic: ; -O1: ldxr x0, [x8] @@ -4234,7 +4234,7 @@ define dso_local i64 @atomicrmw_max_i64_aligned_acquire(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, gt ; -O0: bl __aarch64_cas8_acq -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_max_i64_aligned_acquire: ; -O1: ldaxr x0, [x8] @@ -4250,7 +4250,7 @@ define dso_local i64 @atomicrmw_max_i64_aligned_release(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, gt ; -O0: bl __aarch64_cas8_rel -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_max_i64_aligned_release: ; -O1: ldxr x0, [x8] @@ -4266,7 +4266,7 @@ define dso_local i64 @atomicrmw_max_i64_aligned_acq_rel(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, gt ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_max_i64_aligned_acq_rel: ; -O1: ldaxr x0, [x8] @@ -4282,7 +4282,7 @@ define dso_local i64 @atomicrmw_max_i64_aligned_seq_cst(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, gt ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_max_i64_aligned_seq_cst: ; -O1: ldaxr x0, [x8] @@ -4299,8 +4299,8 @@ define dso_local i128 @atomicrmw_max_i128_aligned_monotonic(ptr %ptr, i128 %valu ; -O0: csel x2, x0, x9, lt ; -O0: csel x3, x1, x8, lt ; -O0: bl __aarch64_cas16_relax -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_max_i128_aligned_monotonic: ; -O1: ldxp x0, x1, [x8] @@ -4318,8 +4318,8 @@ define dso_local i128 @atomicrmw_max_i128_aligned_acquire(ptr %ptr, i128 %value) ; -O0: csel x2, x0, x9, lt ; -O0: csel x3, x1, x8, lt ; -O0: bl __aarch64_cas16_acq -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_max_i128_aligned_acquire: ; -O1: ldaxp x0, x1, [x8] @@ -4337,8 +4337,8 @@ define dso_local i128 @atomicrmw_max_i128_aligned_release(ptr %ptr, i128 %value) ; -O0: csel x2, x0, x9, lt ; -O0: csel x3, x1, x8, lt ; -O0: bl __aarch64_cas16_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_max_i128_aligned_release: ; -O1: ldxp x0, x1, [x8] @@ -4356,8 +4356,8 @@ define dso_local i128 @atomicrmw_max_i128_aligned_acq_rel(ptr %ptr, i128 %value) ; -O0: csel x2, x0, x9, lt ; -O0: csel x3, x1, x8, lt ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_max_i128_aligned_acq_rel: ; -O1: ldaxp x0, x1, [x8] @@ -4375,8 +4375,8 @@ define dso_local i128 @atomicrmw_max_i128_aligned_seq_cst(ptr %ptr, i128 %value) ; -O0: csel x2, x0, x9, lt ; -O0: csel x3, x1, x8, lt ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_max_i128_aligned_seq_cst: ; -O1: ldaxp x0, x1, [x8] @@ -4394,7 +4394,7 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_monotonic(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_relax -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_unaligned_monotonic: ; -O1: ldxrb w9, [x0] @@ -4412,7 +4412,7 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_acq -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_unaligned_acquire: ; -O1: ldaxrb w9, [x0] @@ -4430,7 +4430,7 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_unaligned_release: ; -O1: ldxrb w9, [x0] @@ -4448,7 +4448,7 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_unaligned_acq_rel: ; -O1: ldaxrb w9, [x0] @@ -4466,7 +4466,7 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, gt ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_max_i8_unaligned_seq_cst: ; -O1: ldaxrb w9, [x0] @@ -4789,7 +4789,7 @@ define dso_local i8 @atomicrmw_min_i8_aligned_monotonic(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_relax -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_aligned_monotonic: ; -O1: ldxrb w9, [x0] @@ -4807,7 +4807,7 @@ define dso_local i8 @atomicrmw_min_i8_aligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_acq -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_aligned_acquire: ; -O1: ldaxrb w9, [x0] @@ -4825,7 +4825,7 @@ define dso_local i8 @atomicrmw_min_i8_aligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_aligned_release: ; -O1: ldxrb w9, [x0] @@ -4843,7 +4843,7 @@ define dso_local i8 @atomicrmw_min_i8_aligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_aligned_acq_rel: ; -O1: ldaxrb w9, [x0] @@ -4861,7 +4861,7 @@ define dso_local i8 @atomicrmw_min_i8_aligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_aligned_seq_cst: ; -O1: ldaxrb w9, [x0] @@ -4879,7 +4879,7 @@ define dso_local i16 @atomicrmw_min_i16_aligned_monotonic(ptr %ptr, i16 %value) ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas2_relax -; -O0: subs w8, w0, w8, uxth +; -O0: subs w9, w8, w9, uxth ; ; -O1-LABEL: atomicrmw_min_i16_aligned_monotonic: ; -O1: ldxrh w9, [x0] @@ -4897,7 +4897,7 @@ define dso_local i16 @atomicrmw_min_i16_aligned_acquire(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas2_acq -; -O0: subs w8, w0, w8, uxth +; -O0: subs w9, w8, w9, uxth ; ; -O1-LABEL: atomicrmw_min_i16_aligned_acquire: ; -O1: ldaxrh w9, [x0] @@ -4915,7 +4915,7 @@ define dso_local i16 @atomicrmw_min_i16_aligned_release(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas2_rel -; -O0: subs w8, w0, w8, uxth +; -O0: subs w9, w8, w9, uxth ; ; -O1-LABEL: atomicrmw_min_i16_aligned_release: ; -O1: ldxrh w9, [x0] @@ -4933,7 +4933,7 @@ define dso_local i16 @atomicrmw_min_i16_aligned_acq_rel(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w0, w8, uxth +; -O0: subs w9, w8, w9, uxth ; ; -O1-LABEL: atomicrmw_min_i16_aligned_acq_rel: ; -O1: ldaxrh w9, [x0] @@ -4951,7 +4951,7 @@ define dso_local i16 @atomicrmw_min_i16_aligned_seq_cst(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, sxth ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w0, w8, uxth +; -O0: subs w9, w8, w9, uxth ; ; -O1-LABEL: atomicrmw_min_i16_aligned_seq_cst: ; -O1: ldaxrh w9, [x0] @@ -4968,7 +4968,7 @@ define dso_local i32 @atomicrmw_min_i32_aligned_monotonic(ptr %ptr, i32 %value) ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas4_relax -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_min_i32_aligned_monotonic: ; -O1: ldxr w8, [x0] @@ -4984,7 +4984,7 @@ define dso_local i32 @atomicrmw_min_i32_aligned_acquire(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas4_acq -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_min_i32_aligned_acquire: ; -O1: ldaxr w8, [x0] @@ -5000,7 +5000,7 @@ define dso_local i32 @atomicrmw_min_i32_aligned_release(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas4_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_min_i32_aligned_release: ; -O1: ldxr w8, [x0] @@ -5016,7 +5016,7 @@ define dso_local i32 @atomicrmw_min_i32_aligned_acq_rel(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_min_i32_aligned_acq_rel: ; -O1: ldaxr w8, [x0] @@ -5032,7 +5032,7 @@ define dso_local i32 @atomicrmw_min_i32_aligned_seq_cst(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_min_i32_aligned_seq_cst: ; -O1: ldaxr w8, [x0] @@ -5048,7 +5048,7 @@ define dso_local i64 @atomicrmw_min_i64_aligned_monotonic(ptr %ptr, i64 %value) ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, le ; -O0: bl __aarch64_cas8_relax -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_min_i64_aligned_monotonic: ; -O1: ldxr x0, [x8] @@ -5064,7 +5064,7 @@ define dso_local i64 @atomicrmw_min_i64_aligned_acquire(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, le ; -O0: bl __aarch64_cas8_acq -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_min_i64_aligned_acquire: ; -O1: ldaxr x0, [x8] @@ -5080,7 +5080,7 @@ define dso_local i64 @atomicrmw_min_i64_aligned_release(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, le ; -O0: bl __aarch64_cas8_rel -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_min_i64_aligned_release: ; -O1: ldxr x0, [x8] @@ -5096,7 +5096,7 @@ define dso_local i64 @atomicrmw_min_i64_aligned_acq_rel(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, le ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_min_i64_aligned_acq_rel: ; -O1: ldaxr x0, [x8] @@ -5112,7 +5112,7 @@ define dso_local i64 @atomicrmw_min_i64_aligned_seq_cst(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, le ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_min_i64_aligned_seq_cst: ; -O1: ldaxr x0, [x8] @@ -5129,8 +5129,8 @@ define dso_local i128 @atomicrmw_min_i128_aligned_monotonic(ptr %ptr, i128 %valu ; -O0: csel x2, x0, x9, ge ; -O0: csel x3, x1, x8, ge ; -O0: bl __aarch64_cas16_relax -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_min_i128_aligned_monotonic: ; -O1: ldxp x0, x1, [x8] @@ -5148,8 +5148,8 @@ define dso_local i128 @atomicrmw_min_i128_aligned_acquire(ptr %ptr, i128 %value) ; -O0: csel x2, x0, x9, ge ; -O0: csel x3, x1, x8, ge ; -O0: bl __aarch64_cas16_acq -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_min_i128_aligned_acquire: ; -O1: ldaxp x0, x1, [x8] @@ -5167,8 +5167,8 @@ define dso_local i128 @atomicrmw_min_i128_aligned_release(ptr %ptr, i128 %value) ; -O0: csel x2, x0, x9, ge ; -O0: csel x3, x1, x8, ge ; -O0: bl __aarch64_cas16_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_min_i128_aligned_release: ; -O1: ldxp x0, x1, [x8] @@ -5186,8 +5186,8 @@ define dso_local i128 @atomicrmw_min_i128_aligned_acq_rel(ptr %ptr, i128 %value) ; -O0: csel x2, x0, x9, ge ; -O0: csel x3, x1, x8, ge ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_min_i128_aligned_acq_rel: ; -O1: ldaxp x0, x1, [x8] @@ -5205,8 +5205,8 @@ define dso_local i128 @atomicrmw_min_i128_aligned_seq_cst(ptr %ptr, i128 %value) ; -O0: csel x2, x0, x9, ge ; -O0: csel x3, x1, x8, ge ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_min_i128_aligned_seq_cst: ; -O1: ldaxp x0, x1, [x8] @@ -5224,7 +5224,7 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_monotonic(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_relax -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_unaligned_monotonic: ; -O1: ldxrb w9, [x0] @@ -5242,7 +5242,7 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_acq -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_unaligned_acquire: ; -O1: ldaxrb w9, [x0] @@ -5260,7 +5260,7 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_unaligned_release: ; -O1: ldxrb w9, [x0] @@ -5278,7 +5278,7 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_unaligned_acq_rel: ; -O1: ldaxrb w9, [x0] @@ -5296,7 +5296,7 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, sxtb ; -O0: csel w1, w0, w8, le ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8, uxtb +; -O0: subs w9, w8, w9, uxtb ; ; -O1-LABEL: atomicrmw_min_i8_unaligned_seq_cst: ; -O1: ldaxrb w9, [x0] @@ -5619,7 +5619,7 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_monotonic(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_relax -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i8_aligned_monotonic: ; -O1: and w9, w1, #0xff @@ -5637,7 +5637,7 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_acq -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i8_aligned_acquire: ; -O1: and w9, w1, #0xff @@ -5655,7 +5655,7 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i8_aligned_release: ; -O1: and w9, w1, #0xff @@ -5673,7 +5673,7 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i8_aligned_acq_rel: ; -O1: and w9, w1, #0xff @@ -5691,7 +5691,7 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i8_aligned_seq_cst: ; -O1: and w9, w1, #0xff @@ -5709,7 +5709,7 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_monotonic(ptr %ptr, i16 %value) ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas2_relax -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i16_aligned_monotonic: ; -O1: and w9, w1, #0xffff @@ -5727,7 +5727,7 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_acquire(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas2_acq -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i16_aligned_acquire: ; -O1: and w9, w1, #0xffff @@ -5745,7 +5745,7 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_release(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas2_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i16_aligned_release: ; -O1: and w9, w1, #0xffff @@ -5763,7 +5763,7 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_acq_rel(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i16_aligned_acq_rel: ; -O1: and w9, w1, #0xffff @@ -5781,7 +5781,7 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_seq_cst(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i16_aligned_seq_cst: ; -O1: and w9, w1, #0xffff @@ -5798,7 +5798,7 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_monotonic(ptr %ptr, i32 %value) ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas4_relax -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i32_aligned_monotonic: ; -O1: ldxr w8, [x0] @@ -5814,7 +5814,7 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_acquire(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas4_acq -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i32_aligned_acquire: ; -O1: ldaxr w8, [x0] @@ -5830,7 +5830,7 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_release(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas4_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i32_aligned_release: ; -O1: ldxr w8, [x0] @@ -5846,7 +5846,7 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_acq_rel(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i32_aligned_acq_rel: ; -O1: ldaxr w8, [x0] @@ -5862,7 +5862,7 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_seq_cst(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i32_aligned_seq_cst: ; -O1: ldaxr w8, [x0] @@ -5878,7 +5878,7 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_monotonic(ptr %ptr, i64 %value) ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, hi ; -O0: bl __aarch64_cas8_relax -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_umax_i64_aligned_monotonic: ; -O1: ldxr x0, [x8] @@ -5894,7 +5894,7 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_acquire(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, hi ; -O0: bl __aarch64_cas8_acq -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_umax_i64_aligned_acquire: ; -O1: ldaxr x0, [x8] @@ -5910,7 +5910,7 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_release(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, hi ; -O0: bl __aarch64_cas8_rel -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_umax_i64_aligned_release: ; -O1: ldxr x0, [x8] @@ -5926,7 +5926,7 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_acq_rel(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, hi ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_umax_i64_aligned_acq_rel: ; -O1: ldaxr x0, [x8] @@ -5942,7 +5942,7 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_seq_cst(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, hi ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_umax_i64_aligned_seq_cst: ; -O1: ldaxr x0, [x8] @@ -5959,8 +5959,8 @@ define dso_local i128 @atomicrmw_umax_i128_aligned_monotonic(ptr %ptr, i128 %val ; -O0: csel x2, x0, x9, lo ; -O0: csel x3, x1, x8, lo ; -O0: bl __aarch64_cas16_relax -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_umax_i128_aligned_monotonic: ; -O1: ldxp x0, x1, [x8] @@ -5978,8 +5978,8 @@ define dso_local i128 @atomicrmw_umax_i128_aligned_acquire(ptr %ptr, i128 %value ; -O0: csel x2, x0, x9, lo ; -O0: csel x3, x1, x8, lo ; -O0: bl __aarch64_cas16_acq -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_umax_i128_aligned_acquire: ; -O1: ldaxp x0, x1, [x8] @@ -5997,8 +5997,8 @@ define dso_local i128 @atomicrmw_umax_i128_aligned_release(ptr %ptr, i128 %value ; -O0: csel x2, x0, x9, lo ; -O0: csel x3, x1, x8, lo ; -O0: bl __aarch64_cas16_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_umax_i128_aligned_release: ; -O1: ldxp x0, x1, [x8] @@ -6016,8 +6016,8 @@ define dso_local i128 @atomicrmw_umax_i128_aligned_acq_rel(ptr %ptr, i128 %value ; -O0: csel x2, x0, x9, lo ; -O0: csel x3, x1, x8, lo ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_umax_i128_aligned_acq_rel: ; -O1: ldaxp x0, x1, [x8] @@ -6035,8 +6035,8 @@ define dso_local i128 @atomicrmw_umax_i128_aligned_seq_cst(ptr %ptr, i128 %value ; -O0: csel x2, x0, x9, lo ; -O0: csel x3, x1, x8, lo ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_umax_i128_aligned_seq_cst: ; -O1: ldaxp x0, x1, [x8] @@ -6054,7 +6054,7 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_monotonic(ptr %ptr, i8 %value) ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_relax -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i8_unaligned_monotonic: ; -O1: and w9, w1, #0xff @@ -6072,7 +6072,7 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_acq -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i8_unaligned_acquire: ; -O1: and w9, w1, #0xff @@ -6090,7 +6090,7 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i8_unaligned_release: ; -O1: and w9, w1, #0xff @@ -6108,7 +6108,7 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i8_unaligned_acq_rel: ; -O1: and w9, w1, #0xff @@ -6126,7 +6126,7 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, hi ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umax_i8_unaligned_seq_cst: ; -O1: and w9, w1, #0xff @@ -6449,7 +6449,7 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_monotonic(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_relax -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i8_aligned_monotonic: ; -O1: and w9, w1, #0xff @@ -6467,7 +6467,7 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_acq -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i8_aligned_acquire: ; -O1: and w9, w1, #0xff @@ -6485,7 +6485,7 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i8_aligned_release: ; -O1: and w9, w1, #0xff @@ -6503,7 +6503,7 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i8_aligned_acq_rel: ; -O1: and w9, w1, #0xff @@ -6521,7 +6521,7 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i8_aligned_seq_cst: ; -O1: and w9, w1, #0xff @@ -6539,7 +6539,7 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_monotonic(ptr %ptr, i16 %value) ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas2_relax -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i16_aligned_monotonic: ; -O1: and w9, w1, #0xffff @@ -6557,7 +6557,7 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_acquire(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas2_acq -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i16_aligned_acquire: ; -O1: and w9, w1, #0xffff @@ -6575,7 +6575,7 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_release(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas2_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i16_aligned_release: ; -O1: and w9, w1, #0xffff @@ -6593,7 +6593,7 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_acq_rel(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i16_aligned_acq_rel: ; -O1: and w9, w1, #0xffff @@ -6611,7 +6611,7 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_seq_cst(ptr %ptr, i16 %value) { ; -O0: subs w9, w9, w8, uxth ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas2_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i16_aligned_seq_cst: ; -O1: and w9, w1, #0xffff @@ -6628,7 +6628,7 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_monotonic(ptr %ptr, i32 %value) ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas4_relax -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i32_aligned_monotonic: ; -O1: ldxr w8, [x0] @@ -6644,7 +6644,7 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_acquire(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas4_acq -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i32_aligned_acquire: ; -O1: ldaxr w8, [x0] @@ -6660,7 +6660,7 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_release(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas4_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i32_aligned_release: ; -O1: ldxr w8, [x0] @@ -6676,7 +6676,7 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_acq_rel(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i32_aligned_acq_rel: ; -O1: ldaxr w8, [x0] @@ -6692,7 +6692,7 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_seq_cst(ptr %ptr, i32 %value) { ; -O0: subs w9, w0, w8 ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas4_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i32_aligned_seq_cst: ; -O1: ldaxr w8, [x0] @@ -6708,7 +6708,7 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_monotonic(ptr %ptr, i64 %value) ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, ls ; -O0: bl __aarch64_cas8_relax -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_umin_i64_aligned_monotonic: ; -O1: ldxr x0, [x8] @@ -6724,7 +6724,7 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_acquire(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, ls ; -O0: bl __aarch64_cas8_acq -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_umin_i64_aligned_acquire: ; -O1: ldaxr x0, [x8] @@ -6740,7 +6740,7 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_release(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, ls ; -O0: bl __aarch64_cas8_rel -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_umin_i64_aligned_release: ; -O1: ldxr x0, [x8] @@ -6756,7 +6756,7 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_acq_rel(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, ls ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_umin_i64_aligned_acq_rel: ; -O1: ldaxr x0, [x8] @@ -6772,7 +6772,7 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_seq_cst(ptr %ptr, i64 %value) { ; -O0: subs x9, x0, x8 ; -O0: csel x1, x0, x8, ls ; -O0: bl __aarch64_cas8_acq_rel -; -O0: subs x8, x0, x8 +; -O0: subs x9, x8, x9 ; ; -O1-LABEL: atomicrmw_umin_i64_aligned_seq_cst: ; -O1: ldaxr x0, [x8] @@ -6789,8 +6789,8 @@ define dso_local i128 @atomicrmw_umin_i128_aligned_monotonic(ptr %ptr, i128 %val ; -O0: csel x2, x0, x9, hs ; -O0: csel x3, x1, x8, hs ; -O0: bl __aarch64_cas16_relax -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_umin_i128_aligned_monotonic: ; -O1: ldxp x0, x1, [x8] @@ -6808,8 +6808,8 @@ define dso_local i128 @atomicrmw_umin_i128_aligned_acquire(ptr %ptr, i128 %value ; -O0: csel x2, x0, x9, hs ; -O0: csel x3, x1, x8, hs ; -O0: bl __aarch64_cas16_acq -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_umin_i128_aligned_acquire: ; -O1: ldaxp x0, x1, [x8] @@ -6827,8 +6827,8 @@ define dso_local i128 @atomicrmw_umin_i128_aligned_release(ptr %ptr, i128 %value ; -O0: csel x2, x0, x9, hs ; -O0: csel x3, x1, x8, hs ; -O0: bl __aarch64_cas16_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_umin_i128_aligned_release: ; -O1: ldxp x0, x1, [x8] @@ -6846,8 +6846,8 @@ define dso_local i128 @atomicrmw_umin_i128_aligned_acq_rel(ptr %ptr, i128 %value ; -O0: csel x2, x0, x9, hs ; -O0: csel x3, x1, x8, hs ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_umin_i128_aligned_acq_rel: ; -O1: ldaxp x0, x1, [x8] @@ -6865,8 +6865,8 @@ define dso_local i128 @atomicrmw_umin_i128_aligned_seq_cst(ptr %ptr, i128 %value ; -O0: csel x2, x0, x9, hs ; -O0: csel x3, x1, x8, hs ; -O0: bl __aarch64_cas16_acq_rel -; -O0: subs x10, x10, x11 -; -O0: ccmp x8, x9, #0, eq +; -O0: subs x11, x0, x11 +; -O0: ccmp x1, x10, #0, eq ; ; -O1-LABEL: atomicrmw_umin_i128_aligned_seq_cst: ; -O1: ldaxp x0, x1, [x8] @@ -6884,7 +6884,7 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_monotonic(ptr %ptr, i8 %value) ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_relax -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i8_unaligned_monotonic: ; -O1: and w9, w1, #0xff @@ -6902,7 +6902,7 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_acquire(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_acq -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i8_unaligned_acquire: ; -O1: and w9, w1, #0xff @@ -6920,7 +6920,7 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_release(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i8_unaligned_release: ; -O1: and w9, w1, #0xff @@ -6938,7 +6938,7 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_acq_rel(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i8_unaligned_acq_rel: ; -O1: and w9, w1, #0xff @@ -6956,7 +6956,7 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_seq_cst(ptr %ptr, i8 %value) { ; -O0: subs w9, w9, w8, uxtb ; -O0: csel w1, w0, w8, ls ; -O0: bl __aarch64_cas1_acq_rel -; -O0: subs w8, w0, w8 +; -O0: subs w9, w8, w9 ; ; -O1-LABEL: atomicrmw_umin_i8_unaligned_seq_cst: ; -O1: and w9, w1, #0xff diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll index 6da7795f8b5e8..87ae4da8a0edf 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll @@ -528,23 +528,21 @@ define i32 @fetch_and_nand(ptr %p) #0 { ; CHECK-OUTLINE-O0-NEXT: sub sp, sp, #48 ; CHECK-OUTLINE-O0-NEXT: stp x29, x30, [sp, #32] ; 16-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill -; CHECK-OUTLINE-O0-NEXT: ldr w0, [x0] -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr w8, [x0] +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b LBB6_1 ; CHECK-OUTLINE-O0-NEXT: LBB6_1: ; %atomicrmw.start ; CHECK-OUTLINE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-OUTLINE-O0-NEXT: ldr w0, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #16] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #8] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: and w8, w0, #0x7 ; CHECK-OUTLINE-O0-NEXT: mvn w1, w8 ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas4_rel -; CHECK-OUTLINE-O0-NEXT: ldr w9, [sp, #8] ; 4-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr w9, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: mov w8, w0 -; CHECK-OUTLINE-O0-NEXT: mov w0, w8 -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: subs w8, w8, w9 -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #12] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: subs w9, w0, w9 +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB6_2 ; CHECK-OUTLINE-O0-NEXT: b LBB6_1 ; CHECK-OUTLINE-O0-NEXT: LBB6_2: ; %atomicrmw.end @@ -666,23 +664,21 @@ define i64 @fetch_and_nand_64(ptr %p) #0 { ; CHECK-OUTLINE-O0-NEXT: sub sp, sp, #48 ; CHECK-OUTLINE-O0-NEXT: stp x29, x30, [sp, #32] ; 16-byte Folded Spill ; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill -; CHECK-OUTLINE-O0-NEXT: ldr x0, [x0] -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #24] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr x8, [x0] +; CHECK-OUTLINE-O0-NEXT: str x8, [sp, #24] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: b LBB7_1 ; CHECK-OUTLINE-O0-NEXT: LBB7_1: ; %atomicrmw.start ; CHECK-OUTLINE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp, #24] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #16] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: str x0, [sp] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: and x8, x0, #0x7 ; CHECK-OUTLINE-O0-NEXT: mvn x1, x8 ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas8_acq_rel -; CHECK-OUTLINE-O0-NEXT: ldr x9, [sp] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr x9, [sp, #24] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: mov x8, x0 -; CHECK-OUTLINE-O0-NEXT: mov x0, x8 -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #8] ; 8-byte Spill -; CHECK-OUTLINE-O0-NEXT: subs x8, x8, x9 -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #24] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: str x8, [sp, #8] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: subs x9, x0, x9 +; CHECK-OUTLINE-O0-NEXT: str x8, [sp, #24] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB7_2 ; CHECK-OUTLINE-O0-NEXT: b LBB7_1 ; CHECK-OUTLINE-O0-NEXT: LBB7_2: ; %atomicrmw.end @@ -2195,8 +2191,8 @@ define i8 @atomicrmw_add_i8(ptr %ptr, i8 %rhs) { ; CHECK-NOLSE-O0-NEXT: .cfi_def_cfa_offset 32 ; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-NOLSE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-NOLSE-O0-NEXT: ldrb w8, [x0] -; CHECK-NOLSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill +; CHECK-NOLSE-O0-NEXT: ldrb w0, [x0] +; CHECK-NOLSE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill ; CHECK-NOLSE-O0-NEXT: b LBB30_1 ; CHECK-NOLSE-O0-NEXT: LBB30_1: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1 @@ -2287,8 +2283,8 @@ define i8 @atomicrmw_xchg_i8(ptr %ptr, i8 %rhs) { ; CHECK-NOLSE-O0-NEXT: .cfi_def_cfa_offset 32 ; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-NOLSE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-NOLSE-O0-NEXT: ldrb w8, [x0] -; CHECK-NOLSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill +; CHECK-NOLSE-O0-NEXT: ldrb w0, [x0] +; CHECK-NOLSE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill ; CHECK-NOLSE-O0-NEXT: b LBB31_1 ; CHECK-NOLSE-O0-NEXT: LBB31_1: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1 @@ -2379,8 +2375,8 @@ define i8 @atomicrmw_sub_i8(ptr %ptr, i8 %rhs) { ; CHECK-NOLSE-O0-NEXT: .cfi_def_cfa_offset 32 ; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-NOLSE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-NOLSE-O0-NEXT: ldrb w8, [x0] -; CHECK-NOLSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill +; CHECK-NOLSE-O0-NEXT: ldrb w0, [x0] +; CHECK-NOLSE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill ; CHECK-NOLSE-O0-NEXT: b LBB32_1 ; CHECK-NOLSE-O0-NEXT: LBB32_1: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1 @@ -2476,8 +2472,8 @@ define i8 @atomicrmw_and_i8(ptr %ptr, i8 %rhs) { ; CHECK-NOLSE-O0-NEXT: .cfi_def_cfa_offset 32 ; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-NOLSE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-NOLSE-O0-NEXT: ldrb w8, [x0] -; CHECK-NOLSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill +; CHECK-NOLSE-O0-NEXT: ldrb w0, [x0] +; CHECK-NOLSE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill ; CHECK-NOLSE-O0-NEXT: b LBB33_1 ; CHECK-NOLSE-O0-NEXT: LBB33_1: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1 @@ -2573,8 +2569,8 @@ define i8 @atomicrmw_or_i8(ptr %ptr, i8 %rhs) { ; CHECK-NOLSE-O0-NEXT: .cfi_def_cfa_offset 32 ; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-NOLSE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-NOLSE-O0-NEXT: ldrb w8, [x0] -; CHECK-NOLSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill +; CHECK-NOLSE-O0-NEXT: ldrb w0, [x0] +; CHECK-NOLSE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill ; CHECK-NOLSE-O0-NEXT: b LBB34_1 ; CHECK-NOLSE-O0-NEXT: LBB34_1: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1 @@ -2666,8 +2662,8 @@ define i8 @atomicrmw_xor_i8(ptr %ptr, i8 %rhs) { ; CHECK-NOLSE-O0-NEXT: .cfi_def_cfa_offset 32 ; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-NOLSE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-NOLSE-O0-NEXT: ldrb w8, [x0] -; CHECK-NOLSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill +; CHECK-NOLSE-O0-NEXT: ldrb w0, [x0] +; CHECK-NOLSE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill ; CHECK-NOLSE-O0-NEXT: b LBB35_1 ; CHECK-NOLSE-O0-NEXT: LBB35_1: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1 @@ -2762,8 +2758,8 @@ define i8 @atomicrmw_min_i8(ptr %ptr, i8 %rhs) { ; CHECK-NOLSE-O0-NEXT: .cfi_def_cfa_offset 32 ; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-NOLSE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-NOLSE-O0-NEXT: ldrb w8, [x0] -; CHECK-NOLSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill +; CHECK-NOLSE-O0-NEXT: ldrb w0, [x0] +; CHECK-NOLSE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill ; CHECK-NOLSE-O0-NEXT: b LBB36_1 ; CHECK-NOLSE-O0-NEXT: LBB36_1: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1 @@ -2814,16 +2810,16 @@ define i8 @atomicrmw_min_i8(ptr %ptr, i8 %rhs) { ; CHECK-OUTLINE-O0-NEXT: ldr w0, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #24] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #8] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: sxtb w9, w0 ; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w8, sxtb ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, le ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas1_acq -; CHECK-OUTLINE-O0-NEXT: ldr w9, [sp, #8] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: and w8, w0, #0xff -; CHECK-OUTLINE-O0-NEXT: subs w8, w8, w9, uxtb -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr w10, [sp, #28] ; 4-byte Reload +; CHECK-OUTLINE-O0-NEXT: mov w8, w0 +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #12] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: and w9, w8, #0xff +; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w10, uxtb +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB36_2 ; CHECK-OUTLINE-O0-NEXT: b LBB36_1 ; CHECK-OUTLINE-O0-NEXT: LBB36_2: ; %atomicrmw.end @@ -2880,8 +2876,8 @@ define i8 @atomicrmw_max_i8(ptr %ptr, i8 %rhs) { ; CHECK-NOLSE-O0-NEXT: .cfi_def_cfa_offset 32 ; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-NOLSE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-NOLSE-O0-NEXT: ldrb w8, [x0] -; CHECK-NOLSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill +; CHECK-NOLSE-O0-NEXT: ldrb w0, [x0] +; CHECK-NOLSE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill ; CHECK-NOLSE-O0-NEXT: b LBB37_1 ; CHECK-NOLSE-O0-NEXT: LBB37_1: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1 @@ -2932,16 +2928,16 @@ define i8 @atomicrmw_max_i8(ptr %ptr, i8 %rhs) { ; CHECK-OUTLINE-O0-NEXT: ldr w0, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #24] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #8] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: sxtb w9, w0 ; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w8, sxtb ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, gt ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas1_rel -; CHECK-OUTLINE-O0-NEXT: ldr w9, [sp, #8] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: and w8, w0, #0xff -; CHECK-OUTLINE-O0-NEXT: subs w8, w8, w9, uxtb -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr w10, [sp, #28] ; 4-byte Reload +; CHECK-OUTLINE-O0-NEXT: mov w8, w0 +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #12] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: and w9, w8, #0xff +; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w10, uxtb +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB37_2 ; CHECK-OUTLINE-O0-NEXT: b LBB37_1 ; CHECK-OUTLINE-O0-NEXT: LBB37_2: ; %atomicrmw.end @@ -3000,8 +2996,8 @@ define i8 @atomicrmw_umin_i8(ptr %ptr, i8 %rhs) { ; CHECK-NOLSE-O0-NEXT: .cfi_def_cfa_offset 32 ; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-NOLSE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-NOLSE-O0-NEXT: ldrb w8, [x0] -; CHECK-NOLSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill +; CHECK-NOLSE-O0-NEXT: ldrb w0, [x0] +; CHECK-NOLSE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill ; CHECK-NOLSE-O0-NEXT: b LBB38_1 ; CHECK-NOLSE-O0-NEXT: LBB38_1: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1 @@ -3052,16 +3048,16 @@ define i8 @atomicrmw_umin_i8(ptr %ptr, i8 %rhs) { ; CHECK-OUTLINE-O0-NEXT: ldr w0, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #24] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #8] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: and w9, w0, #0xff ; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w8, uxtb ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, ls ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas1_acq_rel -; CHECK-OUTLINE-O0-NEXT: ldr w9, [sp, #8] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: and w8, w0, #0xff -; CHECK-OUTLINE-O0-NEXT: subs w8, w8, w9, uxtb -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr w10, [sp, #28] ; 4-byte Reload +; CHECK-OUTLINE-O0-NEXT: mov w8, w0 +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #12] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: and w9, w8, #0xff +; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w10, uxtb +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB38_2 ; CHECK-OUTLINE-O0-NEXT: b LBB38_1 ; CHECK-OUTLINE-O0-NEXT: LBB38_2: ; %atomicrmw.end @@ -3120,8 +3116,8 @@ define i8 @atomicrmw_umax_i8(ptr %ptr, i8 %rhs) { ; CHECK-NOLSE-O0-NEXT: .cfi_def_cfa_offset 32 ; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-NOLSE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-NOLSE-O0-NEXT: ldrb w8, [x0] -; CHECK-NOLSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill +; CHECK-NOLSE-O0-NEXT: ldrb w0, [x0] +; CHECK-NOLSE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill ; CHECK-NOLSE-O0-NEXT: b LBB39_1 ; CHECK-NOLSE-O0-NEXT: LBB39_1: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1 @@ -3172,16 +3168,16 @@ define i8 @atomicrmw_umax_i8(ptr %ptr, i8 %rhs) { ; CHECK-OUTLINE-O0-NEXT: ldr w0, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #24] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #8] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: and w9, w0, #0xff ; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w8, uxtb ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, hi ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas1_relax -; CHECK-OUTLINE-O0-NEXT: ldr w9, [sp, #8] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: and w8, w0, #0xff -; CHECK-OUTLINE-O0-NEXT: subs w8, w8, w9, uxtb -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr w10, [sp, #28] ; 4-byte Reload +; CHECK-OUTLINE-O0-NEXT: mov w8, w0 +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #12] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: and w9, w8, #0xff +; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w10, uxtb +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB39_2 ; CHECK-OUTLINE-O0-NEXT: b LBB39_1 ; CHECK-OUTLINE-O0-NEXT: LBB39_2: ; %atomicrmw.end @@ -3853,26 +3849,24 @@ define i16 @atomicrmw_min_i16(ptr %ptr, i16 %rhs) { ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w29, -16 ; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: ldrh w0, [x0] -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldrh w8, [x0] +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b LBB46_1 ; CHECK-OUTLINE-O0-NEXT: LBB46_1: ; %atomicrmw.start ; CHECK-OUTLINE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-OUTLINE-O0-NEXT: ldr w0, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #24] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #8] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: sxth w9, w0 ; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w8, sxth ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, le ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas2_acq -; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #8] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: mov w9, w0 -; CHECK-OUTLINE-O0-NEXT: str w9, [sp, #12] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: uxth w8, w8 -; CHECK-OUTLINE-O0-NEXT: mov w0, w9 -; CHECK-OUTLINE-O0-NEXT: subs w8, w8, w9, uxth -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #28] ; 4-byte Reload +; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: uxth w9, w8 +; CHECK-OUTLINE-O0-NEXT: mov w8, w0 +; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w0, uxth +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB46_2 ; CHECK-OUTLINE-O0-NEXT: b LBB46_1 ; CHECK-OUTLINE-O0-NEXT: LBB46_2: ; %atomicrmw.end @@ -3974,26 +3968,24 @@ define i16 @atomicrmw_max_i16(ptr %ptr, i16 %rhs) { ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w29, -16 ; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: ldrh w0, [x0] -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldrh w8, [x0] +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b LBB47_1 ; CHECK-OUTLINE-O0-NEXT: LBB47_1: ; %atomicrmw.start ; CHECK-OUTLINE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-OUTLINE-O0-NEXT: ldr w0, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #24] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #8] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: sxth w9, w0 ; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w8, sxth ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, gt ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas2_rel -; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #8] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: mov w9, w0 -; CHECK-OUTLINE-O0-NEXT: str w9, [sp, #12] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: uxth w8, w8 -; CHECK-OUTLINE-O0-NEXT: mov w0, w9 -; CHECK-OUTLINE-O0-NEXT: subs w8, w8, w9, uxth -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #28] ; 4-byte Reload +; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: uxth w9, w8 +; CHECK-OUTLINE-O0-NEXT: mov w8, w0 +; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w0, uxth +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB47_2 ; CHECK-OUTLINE-O0-NEXT: b LBB47_1 ; CHECK-OUTLINE-O0-NEXT: LBB47_2: ; %atomicrmw.end @@ -4097,26 +4089,24 @@ define i16 @atomicrmw_umin_i16(ptr %ptr, i16 %rhs) { ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w29, -16 ; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: ldrh w0, [x0] -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldrh w8, [x0] +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b LBB48_1 ; CHECK-OUTLINE-O0-NEXT: LBB48_1: ; %atomicrmw.start ; CHECK-OUTLINE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-OUTLINE-O0-NEXT: ldr w0, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #24] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #8] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: uxth w9, w0 ; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w8, uxth ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, ls ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas2_acq_rel -; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #8] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: mov w9, w0 -; CHECK-OUTLINE-O0-NEXT: str w9, [sp, #12] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: uxth w8, w8 -; CHECK-OUTLINE-O0-NEXT: mov w0, w9 -; CHECK-OUTLINE-O0-NEXT: subs w8, w8, w9, uxth -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #28] ; 4-byte Reload +; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: uxth w9, w8 +; CHECK-OUTLINE-O0-NEXT: mov w8, w0 +; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w0, uxth +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB48_2 ; CHECK-OUTLINE-O0-NEXT: b LBB48_1 ; CHECK-OUTLINE-O0-NEXT: LBB48_2: ; %atomicrmw.end @@ -4220,26 +4210,24 @@ define i16 @atomicrmw_umax_i16(ptr %ptr, i16 %rhs) { ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w29, -16 ; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: ldrh w0, [x0] -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldrh w8, [x0] +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b LBB49_1 ; CHECK-OUTLINE-O0-NEXT: LBB49_1: ; %atomicrmw.start ; CHECK-OUTLINE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-OUTLINE-O0-NEXT: ldr w0, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #24] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #8] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: uxth w9, w0 ; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w8, uxth ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, hi ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas2_relax -; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #8] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: mov w9, w0 -; CHECK-OUTLINE-O0-NEXT: str w9, [sp, #12] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: uxth w8, w8 -; CHECK-OUTLINE-O0-NEXT: mov w0, w9 -; CHECK-OUTLINE-O0-NEXT: subs w8, w8, w9, uxth -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #28] ; 4-byte Reload +; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: uxth w9, w8 +; CHECK-OUTLINE-O0-NEXT: mov w8, w0 +; CHECK-OUTLINE-O0-NEXT: subs w9, w9, w0, uxth +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB49_2 ; CHECK-OUTLINE-O0-NEXT: b LBB49_1 ; CHECK-OUTLINE-O0-NEXT: LBB49_2: ; %atomicrmw.end @@ -4902,24 +4890,22 @@ define i32 @atomicrmw_min_i32(ptr %ptr, i32 %rhs) { ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w29, -16 ; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: ldr w0, [x0] -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr w8, [x0] +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b LBB56_1 ; CHECK-OUTLINE-O0-NEXT: LBB56_1: ; %atomicrmw.start ; CHECK-OUTLINE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-OUTLINE-O0-NEXT: ldr w0, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #24] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #8] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: subs w9, w0, w8 ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, le ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas4_acq -; CHECK-OUTLINE-O0-NEXT: ldr w9, [sp, #8] ; 4-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr w9, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: mov w8, w0 -; CHECK-OUTLINE-O0-NEXT: mov w0, w8 -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: subs w8, w8, w9 -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #12] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: subs w9, w0, w9 +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB56_2 ; CHECK-OUTLINE-O0-NEXT: b LBB56_1 ; CHECK-OUTLINE-O0-NEXT: LBB56_2: ; %atomicrmw.end @@ -5017,24 +5003,22 @@ define i32 @atomicrmw_max_i32(ptr %ptr, i32 %rhs) { ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w29, -16 ; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: ldr w0, [x0] -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr w8, [x0] +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b LBB57_1 ; CHECK-OUTLINE-O0-NEXT: LBB57_1: ; %atomicrmw.start ; CHECK-OUTLINE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-OUTLINE-O0-NEXT: ldr w0, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #24] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #8] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: subs w9, w0, w8 ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, gt ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas4_rel -; CHECK-OUTLINE-O0-NEXT: ldr w9, [sp, #8] ; 4-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr w9, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: mov w8, w0 -; CHECK-OUTLINE-O0-NEXT: mov w0, w8 -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: subs w8, w8, w9 -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #12] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: subs w9, w0, w9 +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB57_2 ; CHECK-OUTLINE-O0-NEXT: b LBB57_1 ; CHECK-OUTLINE-O0-NEXT: LBB57_2: ; %atomicrmw.end @@ -5132,24 +5116,22 @@ define i32 @atomicrmw_umin_i32(ptr %ptr, i32 %rhs) { ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w29, -16 ; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: ldr w0, [x0] -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr w8, [x0] +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b LBB58_1 ; CHECK-OUTLINE-O0-NEXT: LBB58_1: ; %atomicrmw.start ; CHECK-OUTLINE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-OUTLINE-O0-NEXT: ldr w0, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #24] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #8] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: subs w9, w0, w8 ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, ls ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas4_acq_rel -; CHECK-OUTLINE-O0-NEXT: ldr w9, [sp, #8] ; 4-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr w9, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: mov w8, w0 -; CHECK-OUTLINE-O0-NEXT: mov w0, w8 -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: subs w8, w8, w9 -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #12] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: subs w9, w0, w9 +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB58_2 ; CHECK-OUTLINE-O0-NEXT: b LBB58_1 ; CHECK-OUTLINE-O0-NEXT: LBB58_2: ; %atomicrmw.end @@ -5247,24 +5229,22 @@ define i32 @atomicrmw_umax_i32(ptr %ptr, i32 %rhs) { ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w29, -16 ; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: str w1, [sp, #24] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: ldr w0, [x0] -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr w8, [x0] +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b LBB59_1 ; CHECK-OUTLINE-O0-NEXT: LBB59_1: ; %atomicrmw.start ; CHECK-OUTLINE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-OUTLINE-O0-NEXT: ldr w0, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: ldr w8, [sp, #24] ; 4-byte Reload -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #8] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: subs w9, w0, w8 ; CHECK-OUTLINE-O0-NEXT: csel w1, w0, w8, hi ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas4_relax -; CHECK-OUTLINE-O0-NEXT: ldr w9, [sp, #8] ; 4-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr w9, [sp, #28] ; 4-byte Reload ; CHECK-OUTLINE-O0-NEXT: mov w8, w0 -; CHECK-OUTLINE-O0-NEXT: mov w0, w8 -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #12] ; 4-byte Spill -; CHECK-OUTLINE-O0-NEXT: subs w8, w8, w9 -; CHECK-OUTLINE-O0-NEXT: str w0, [sp, #28] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #12] ; 4-byte Spill +; CHECK-OUTLINE-O0-NEXT: subs w9, w0, w9 +; CHECK-OUTLINE-O0-NEXT: str w8, [sp, #28] ; 4-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB59_2 ; CHECK-OUTLINE-O0-NEXT: b LBB59_1 ; CHECK-OUTLINE-O0-NEXT: LBB59_2: ; %atomicrmw.end @@ -5919,37 +5899,35 @@ define i64 @atomicrmw_min_i64(ptr %ptr, i64 %rhs) { ; ; CHECK-OUTLINE-O0-LABEL: atomicrmw_min_i64: ; CHECK-OUTLINE-O0: ; %bb.0: -; CHECK-OUTLINE-O0-NEXT: sub sp, sp, #64 -; CHECK-OUTLINE-O0-NEXT: stp x29, x30, [sp, #48] ; 16-byte Folded Spill -; CHECK-OUTLINE-O0-NEXT: .cfi_def_cfa_offset 64 +; CHECK-OUTLINE-O0-NEXT: sub sp, sp, #48 +; CHECK-OUTLINE-O0-NEXT: stp x29, x30, [sp, #32] ; 16-byte Folded Spill +; CHECK-OUTLINE-O0-NEXT: .cfi_def_cfa_offset 48 ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w30, -8 ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w29, -16 -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #24] ; 8-byte Spill -; CHECK-OUTLINE-O0-NEXT: str x1, [sp, #32] ; 8-byte Spill -; CHECK-OUTLINE-O0-NEXT: ldr x0, [x0] -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #40] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #8] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: str x1, [sp, #16] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr x8, [x0] +; CHECK-OUTLINE-O0-NEXT: str x8, [sp, #24] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: b LBB66_1 ; CHECK-OUTLINE-O0-NEXT: LBB66_1: ; %atomicrmw.start ; CHECK-OUTLINE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp, #40] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #24] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: ldr x8, [sp, #32] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #8] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp, #24] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #8] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr x8, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: subs x9, x0, x8 ; CHECK-OUTLINE-O0-NEXT: csel x1, x0, x8, le ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas8_acq -; CHECK-OUTLINE-O0-NEXT: ldr x9, [sp, #8] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr x9, [sp, #24] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: mov x8, x0 -; CHECK-OUTLINE-O0-NEXT: mov x0, x8 -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill -; CHECK-OUTLINE-O0-NEXT: subs x8, x8, x9 -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #40] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: str x8, [sp] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: subs x9, x0, x9 +; CHECK-OUTLINE-O0-NEXT: str x8, [sp, #24] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB66_2 ; CHECK-OUTLINE-O0-NEXT: b LBB66_1 ; CHECK-OUTLINE-O0-NEXT: LBB66_2: ; %atomicrmw.end -; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp, #16] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: ldp x29, x30, [sp, #48] ; 16-byte Folded Reload -; CHECK-OUTLINE-O0-NEXT: add sp, sp, #64 +; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldp x29, x30, [sp, #32] ; 16-byte Folded Reload +; CHECK-OUTLINE-O0-NEXT: add sp, sp, #48 ; CHECK-OUTLINE-O0-NEXT: ret ; ; CHECK-LSE-O1-LABEL: atomicrmw_min_i64: @@ -6034,37 +6012,35 @@ define i64 @atomicrmw_max_i64(ptr %ptr, i64 %rhs) { ; ; CHECK-OUTLINE-O0-LABEL: atomicrmw_max_i64: ; CHECK-OUTLINE-O0: ; %bb.0: -; CHECK-OUTLINE-O0-NEXT: sub sp, sp, #64 -; CHECK-OUTLINE-O0-NEXT: stp x29, x30, [sp, #48] ; 16-byte Folded Spill -; CHECK-OUTLINE-O0-NEXT: .cfi_def_cfa_offset 64 +; CHECK-OUTLINE-O0-NEXT: sub sp, sp, #48 +; CHECK-OUTLINE-O0-NEXT: stp x29, x30, [sp, #32] ; 16-byte Folded Spill +; CHECK-OUTLINE-O0-NEXT: .cfi_def_cfa_offset 48 ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w30, -8 ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w29, -16 -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #24] ; 8-byte Spill -; CHECK-OUTLINE-O0-NEXT: str x1, [sp, #32] ; 8-byte Spill -; CHECK-OUTLINE-O0-NEXT: ldr x0, [x0] -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #40] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #8] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: str x1, [sp, #16] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr x8, [x0] +; CHECK-OUTLINE-O0-NEXT: str x8, [sp, #24] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: b LBB67_1 ; CHECK-OUTLINE-O0-NEXT: LBB67_1: ; %atomicrmw.start ; CHECK-OUTLINE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp, #40] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #24] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: ldr x8, [sp, #32] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #8] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp, #24] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #8] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr x8, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: subs x9, x0, x8 ; CHECK-OUTLINE-O0-NEXT: csel x1, x0, x8, gt ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas8_rel -; CHECK-OUTLINE-O0-NEXT: ldr x9, [sp, #8] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr x9, [sp, #24] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: mov x8, x0 -; CHECK-OUTLINE-O0-NEXT: mov x0, x8 -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill -; CHECK-OUTLINE-O0-NEXT: subs x8, x8, x9 -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #40] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: str x8, [sp] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: subs x9, x0, x9 +; CHECK-OUTLINE-O0-NEXT: str x8, [sp, #24] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB67_2 ; CHECK-OUTLINE-O0-NEXT: b LBB67_1 ; CHECK-OUTLINE-O0-NEXT: LBB67_2: ; %atomicrmw.end -; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp, #16] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: ldp x29, x30, [sp, #48] ; 16-byte Folded Reload -; CHECK-OUTLINE-O0-NEXT: add sp, sp, #64 +; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldp x29, x30, [sp, #32] ; 16-byte Folded Reload +; CHECK-OUTLINE-O0-NEXT: add sp, sp, #48 ; CHECK-OUTLINE-O0-NEXT: ret ; ; CHECK-LSE-O1-LABEL: atomicrmw_max_i64: @@ -6149,37 +6125,35 @@ define i64 @atomicrmw_umin_i64(ptr %ptr, i64 %rhs) { ; ; CHECK-OUTLINE-O0-LABEL: atomicrmw_umin_i64: ; CHECK-OUTLINE-O0: ; %bb.0: -; CHECK-OUTLINE-O0-NEXT: sub sp, sp, #64 -; CHECK-OUTLINE-O0-NEXT: stp x29, x30, [sp, #48] ; 16-byte Folded Spill -; CHECK-OUTLINE-O0-NEXT: .cfi_def_cfa_offset 64 +; CHECK-OUTLINE-O0-NEXT: sub sp, sp, #48 +; CHECK-OUTLINE-O0-NEXT: stp x29, x30, [sp, #32] ; 16-byte Folded Spill +; CHECK-OUTLINE-O0-NEXT: .cfi_def_cfa_offset 48 ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w30, -8 ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w29, -16 -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #24] ; 8-byte Spill -; CHECK-OUTLINE-O0-NEXT: str x1, [sp, #32] ; 8-byte Spill -; CHECK-OUTLINE-O0-NEXT: ldr x0, [x0] -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #40] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #8] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: str x1, [sp, #16] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr x8, [x0] +; CHECK-OUTLINE-O0-NEXT: str x8, [sp, #24] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: b LBB68_1 ; CHECK-OUTLINE-O0-NEXT: LBB68_1: ; %atomicrmw.start ; CHECK-OUTLINE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp, #40] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #24] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: ldr x8, [sp, #32] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #8] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp, #24] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #8] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr x8, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: subs x9, x0, x8 ; CHECK-OUTLINE-O0-NEXT: csel x1, x0, x8, ls ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas8_acq_rel -; CHECK-OUTLINE-O0-NEXT: ldr x9, [sp, #8] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr x9, [sp, #24] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: mov x8, x0 -; CHECK-OUTLINE-O0-NEXT: mov x0, x8 -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill -; CHECK-OUTLINE-O0-NEXT: subs x8, x8, x9 -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #40] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: str x8, [sp] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: subs x9, x0, x9 +; CHECK-OUTLINE-O0-NEXT: str x8, [sp, #24] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB68_2 ; CHECK-OUTLINE-O0-NEXT: b LBB68_1 ; CHECK-OUTLINE-O0-NEXT: LBB68_2: ; %atomicrmw.end -; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp, #16] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: ldp x29, x30, [sp, #48] ; 16-byte Folded Reload -; CHECK-OUTLINE-O0-NEXT: add sp, sp, #64 +; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldp x29, x30, [sp, #32] ; 16-byte Folded Reload +; CHECK-OUTLINE-O0-NEXT: add sp, sp, #48 ; CHECK-OUTLINE-O0-NEXT: ret ; ; CHECK-LSE-O1-LABEL: atomicrmw_umin_i64: @@ -6264,37 +6238,35 @@ define i64 @atomicrmw_umax_i64(ptr %ptr, i64 %rhs) { ; ; CHECK-OUTLINE-O0-LABEL: atomicrmw_umax_i64: ; CHECK-OUTLINE-O0: ; %bb.0: -; CHECK-OUTLINE-O0-NEXT: sub sp, sp, #64 -; CHECK-OUTLINE-O0-NEXT: stp x29, x30, [sp, #48] ; 16-byte Folded Spill -; CHECK-OUTLINE-O0-NEXT: .cfi_def_cfa_offset 64 +; CHECK-OUTLINE-O0-NEXT: sub sp, sp, #48 +; CHECK-OUTLINE-O0-NEXT: stp x29, x30, [sp, #32] ; 16-byte Folded Spill +; CHECK-OUTLINE-O0-NEXT: .cfi_def_cfa_offset 48 ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w30, -8 ; CHECK-OUTLINE-O0-NEXT: .cfi_offset w29, -16 -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #24] ; 8-byte Spill -; CHECK-OUTLINE-O0-NEXT: str x1, [sp, #32] ; 8-byte Spill -; CHECK-OUTLINE-O0-NEXT: ldr x0, [x0] -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #40] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #8] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: str x1, [sp, #16] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr x8, [x0] +; CHECK-OUTLINE-O0-NEXT: str x8, [sp, #24] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: b LBB69_1 ; CHECK-OUTLINE-O0-NEXT: LBB69_1: ; %atomicrmw.start ; CHECK-OUTLINE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp, #40] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #24] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: ldr x8, [sp, #32] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #8] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp, #24] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr x2, [sp, #8] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr x8, [sp, #16] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: subs x9, x0, x8 ; CHECK-OUTLINE-O0-NEXT: csel x1, x0, x8, hi ; CHECK-OUTLINE-O0-NEXT: bl ___aarch64_cas8_relax -; CHECK-OUTLINE-O0-NEXT: ldr x9, [sp, #8] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldr x9, [sp, #24] ; 8-byte Reload ; CHECK-OUTLINE-O0-NEXT: mov x8, x0 -; CHECK-OUTLINE-O0-NEXT: mov x0, x8 -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #16] ; 8-byte Spill -; CHECK-OUTLINE-O0-NEXT: subs x8, x8, x9 -; CHECK-OUTLINE-O0-NEXT: str x0, [sp, #40] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: str x8, [sp] ; 8-byte Spill +; CHECK-OUTLINE-O0-NEXT: subs x9, x0, x9 +; CHECK-OUTLINE-O0-NEXT: str x8, [sp, #24] ; 8-byte Spill ; CHECK-OUTLINE-O0-NEXT: b.eq LBB69_2 ; CHECK-OUTLINE-O0-NEXT: b LBB69_1 ; CHECK-OUTLINE-O0-NEXT: LBB69_2: ; %atomicrmw.end -; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp, #16] ; 8-byte Reload -; CHECK-OUTLINE-O0-NEXT: ldp x29, x30, [sp, #48] ; 16-byte Folded Reload -; CHECK-OUTLINE-O0-NEXT: add sp, sp, #64 +; CHECK-OUTLINE-O0-NEXT: ldr x0, [sp] ; 8-byte Reload +; CHECK-OUTLINE-O0-NEXT: ldp x29, x30, [sp, #32] ; 16-byte Folded Reload +; CHECK-OUTLINE-O0-NEXT: add sp, sp, #48 ; CHECK-OUTLINE-O0-NEXT: ret ; ; CHECK-LSE-O1-LABEL: atomicrmw_umax_i64: diff --git a/llvm/test/CodeGen/AArch64/O0-pipeline.ll b/llvm/test/CodeGen/AArch64/O0-pipeline.ll index b363e11f62811..8815a9f2b3b55 100644 --- a/llvm/test/CodeGen/AArch64/O0-pipeline.ll +++ b/llvm/test/CodeGen/AArch64/O0-pipeline.ll @@ -52,13 +52,9 @@ ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Machine Optimization Remark Emitter ; CHECK-NEXT: Machine SME ABI pass -; CHECK-NEXT: Eliminate PHI nodes for register allocation -; CHECK-NEXT: Two-Address instruction pass ; CHECK-NEXT: Fast Register Allocator ; CHECK-NEXT: Remove Redundant DEBUG_VALUE analysis ; CHECK-NEXT: Fixup Statepoint Caller Saved -; CHECK-NEXT: Lazy Machine Block Frequency Analysis -; CHECK-NEXT: Machine Optimization Remark Emitter ; CHECK-NEXT: Prologue/Epilogue Insertion & Frame Finalization ; CHECK-NEXT: Post-RA pseudo instruction expansion pass ; CHECK-NEXT: AArch64 pseudo instruction expansion pass diff --git a/llvm/test/CodeGen/AArch64/aarch64-mops-mte.ll b/llvm/test/CodeGen/AArch64/aarch64-mops-mte.ll index bccb70ccd5c03..a3d188bf9d9e5 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-mops-mte.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-mops-mte.ll @@ -38,7 +38,7 @@ entry: define ptr @memset_tagged_1_zeroval(ptr %dst, i64 %size) { ; GISel-O0-LABEL: memset_tagged_1_zeroval: ; GISel-O0: // %bb.0: // %entry -; GISel-O0-NEXT: mov w8, #1 +; GISel-O0-NEXT: mov w8, #1 // =0x1 ; GISel-O0-NEXT: // kill: def $x8 killed $w8 ; GISel-O0-NEXT: mov x9, xzr ; GISel-O0-NEXT: setgp [x0]!, x8!, x9 @@ -48,7 +48,7 @@ define ptr @memset_tagged_1_zeroval(ptr %dst, i64 %size) { ; ; GISel-LABEL: memset_tagged_1_zeroval: ; GISel: // %bb.0: // %entry -; GISel-NEXT: mov w8, #1 +; GISel-NEXT: mov w8, #1 // =0x1 ; GISel-NEXT: setgp [x0]!, x8!, xzr ; GISel-NEXT: setgm [x0]!, x8!, xzr ; GISel-NEXT: setge [x0]!, x8!, xzr @@ -56,7 +56,7 @@ define ptr @memset_tagged_1_zeroval(ptr %dst, i64 %size) { ; ; SDAG-LABEL: memset_tagged_1_zeroval: ; SDAG: // %bb.0: // %entry -; SDAG-NEXT: mov w8, #1 +; SDAG-NEXT: mov w8, #1 // =0x1 ; SDAG-NEXT: setgp [x0]!, x8!, xzr ; SDAG-NEXT: setgm [x0]!, x8!, xzr ; SDAG-NEXT: setge [x0]!, x8!, xzr @@ -69,7 +69,7 @@ entry: define ptr @memset_tagged_10_zeroval(ptr %dst, i64 %size) { ; GISel-O0-LABEL: memset_tagged_10_zeroval: ; GISel-O0: // %bb.0: // %entry -; GISel-O0-NEXT: mov w8, #10 +; GISel-O0-NEXT: mov w8, #10 // =0xa ; GISel-O0-NEXT: // kill: def $x8 killed $w8 ; GISel-O0-NEXT: mov x9, xzr ; GISel-O0-NEXT: setgp [x0]!, x8!, x9 @@ -79,7 +79,7 @@ define ptr @memset_tagged_10_zeroval(ptr %dst, i64 %size) { ; ; GISel-LABEL: memset_tagged_10_zeroval: ; GISel: // %bb.0: // %entry -; GISel-NEXT: mov w8, #10 +; GISel-NEXT: mov w8, #10 // =0xa ; GISel-NEXT: setgp [x0]!, x8!, xzr ; GISel-NEXT: setgm [x0]!, x8!, xzr ; GISel-NEXT: setge [x0]!, x8!, xzr @@ -87,7 +87,7 @@ define ptr @memset_tagged_10_zeroval(ptr %dst, i64 %size) { ; ; SDAG-LABEL: memset_tagged_10_zeroval: ; SDAG: // %bb.0: // %entry -; SDAG-NEXT: mov w8, #10 +; SDAG-NEXT: mov w8, #10 // =0xa ; SDAG-NEXT: setgp [x0]!, x8!, xzr ; SDAG-NEXT: setgm [x0]!, x8!, xzr ; SDAG-NEXT: setge [x0]!, x8!, xzr @@ -100,7 +100,7 @@ entry: define ptr @memset_tagged_10000_zeroval(ptr %dst, i64 %size) { ; GISel-O0-LABEL: memset_tagged_10000_zeroval: ; GISel-O0: // %bb.0: // %entry -; GISel-O0-NEXT: mov w8, #10000 +; GISel-O0-NEXT: mov w8, #10000 // =0x2710 ; GISel-O0-NEXT: // kill: def $x8 killed $w8 ; GISel-O0-NEXT: mov x9, xzr ; GISel-O0-NEXT: setgp [x0]!, x8!, x9 @@ -110,7 +110,7 @@ define ptr @memset_tagged_10000_zeroval(ptr %dst, i64 %size) { ; ; GISel-LABEL: memset_tagged_10000_zeroval: ; GISel: // %bb.0: // %entry -; GISel-NEXT: mov w8, #10000 +; GISel-NEXT: mov w8, #10000 // =0x2710 ; GISel-NEXT: setgp [x0]!, x8!, xzr ; GISel-NEXT: setgm [x0]!, x8!, xzr ; GISel-NEXT: setge [x0]!, x8!, xzr @@ -118,7 +118,7 @@ define ptr @memset_tagged_10000_zeroval(ptr %dst, i64 %size) { ; ; SDAG-LABEL: memset_tagged_10000_zeroval: ; SDAG: // %bb.0: // %entry -; SDAG-NEXT: mov w8, #10000 +; SDAG-NEXT: mov w8, #10000 // =0x2710 ; SDAG-NEXT: setgp [x0]!, x8!, xzr ; SDAG-NEXT: setgm [x0]!, x8!, xzr ; SDAG-NEXT: setge [x0]!, x8!, xzr @@ -192,7 +192,7 @@ entry: define ptr @memset_tagged_1(ptr %dst, i64 %size, i32 %value) { ; GISel-O0-LABEL: memset_tagged_1: ; GISel-O0: // %bb.0: // %entry -; GISel-O0-NEXT: mov w8, #1 +; GISel-O0-NEXT: mov w8, #1 // =0x1 ; GISel-O0-NEXT: // kill: def $x8 killed $w8 ; GISel-O0-NEXT: // implicit-def: $x9 ; GISel-O0-NEXT: mov w9, w2 @@ -203,7 +203,7 @@ define ptr @memset_tagged_1(ptr %dst, i64 %size, i32 %value) { ; ; GISel-LABEL: memset_tagged_1: ; GISel: // %bb.0: // %entry -; GISel-NEXT: mov w8, #1 +; GISel-NEXT: mov w8, #1 // =0x1 ; GISel-NEXT: // kill: def $w2 killed $w2 def $x2 ; GISel-NEXT: setgp [x0]!, x8!, x2 ; GISel-NEXT: setgm [x0]!, x8!, x2 @@ -212,7 +212,7 @@ define ptr @memset_tagged_1(ptr %dst, i64 %size, i32 %value) { ; ; SDAG-LABEL: memset_tagged_1: ; SDAG: // %bb.0: // %entry -; SDAG-NEXT: mov w8, #1 +; SDAG-NEXT: mov w8, #1 // =0x1 ; SDAG-NEXT: // kill: def $w2 killed $w2 def $x2 ; SDAG-NEXT: setgp [x0]!, x8!, x2 ; SDAG-NEXT: setgm [x0]!, x8!, x2 @@ -227,7 +227,7 @@ entry: define ptr @memset_tagged_10(ptr %dst, i64 %size, i32 %value) { ; GISel-O0-LABEL: memset_tagged_10: ; GISel-O0: // %bb.0: // %entry -; GISel-O0-NEXT: mov w8, #10 +; GISel-O0-NEXT: mov w8, #10 // =0xa ; GISel-O0-NEXT: // kill: def $x8 killed $w8 ; GISel-O0-NEXT: // implicit-def: $x9 ; GISel-O0-NEXT: mov w9, w2 @@ -238,7 +238,7 @@ define ptr @memset_tagged_10(ptr %dst, i64 %size, i32 %value) { ; ; GISel-LABEL: memset_tagged_10: ; GISel: // %bb.0: // %entry -; GISel-NEXT: mov w8, #10 +; GISel-NEXT: mov w8, #10 // =0xa ; GISel-NEXT: // kill: def $w2 killed $w2 def $x2 ; GISel-NEXT: setgp [x0]!, x8!, x2 ; GISel-NEXT: setgm [x0]!, x8!, x2 @@ -247,7 +247,7 @@ define ptr @memset_tagged_10(ptr %dst, i64 %size, i32 %value) { ; ; SDAG-LABEL: memset_tagged_10: ; SDAG: // %bb.0: // %entry -; SDAG-NEXT: mov w8, #10 +; SDAG-NEXT: mov w8, #10 // =0xa ; SDAG-NEXT: // kill: def $w2 killed $w2 def $x2 ; SDAG-NEXT: setgp [x0]!, x8!, x2 ; SDAG-NEXT: setgm [x0]!, x8!, x2 @@ -262,7 +262,7 @@ entry: define ptr @memset_tagged_10000(ptr %dst, i64 %size, i32 %value) { ; GISel-O0-LABEL: memset_tagged_10000: ; GISel-O0: // %bb.0: // %entry -; GISel-O0-NEXT: mov w8, #10000 +; GISel-O0-NEXT: mov w8, #10000 // =0x2710 ; GISel-O0-NEXT: // kill: def $x8 killed $w8 ; GISel-O0-NEXT: // implicit-def: $x9 ; GISel-O0-NEXT: mov w9, w2 @@ -273,7 +273,7 @@ define ptr @memset_tagged_10000(ptr %dst, i64 %size, i32 %value) { ; ; GISel-LABEL: memset_tagged_10000: ; GISel: // %bb.0: // %entry -; GISel-NEXT: mov w8, #10000 +; GISel-NEXT: mov w8, #10000 // =0x2710 ; GISel-NEXT: // kill: def $w2 killed $w2 def $x2 ; GISel-NEXT: setgp [x0]!, x8!, x2 ; GISel-NEXT: setgm [x0]!, x8!, x2 @@ -282,7 +282,7 @@ define ptr @memset_tagged_10000(ptr %dst, i64 %size, i32 %value) { ; ; SDAG-LABEL: memset_tagged_10000: ; SDAG: // %bb.0: // %entry -; SDAG-NEXT: mov w8, #10000 +; SDAG-NEXT: mov w8, #10000 // =0x2710 ; SDAG-NEXT: // kill: def $w2 killed $w2 def $x2 ; SDAG-NEXT: setgp [x0]!, x8!, x2 ; SDAG-NEXT: setgm [x0]!, x8!, x2 diff --git a/llvm/test/CodeGen/AArch64/arm64_32-null.ll b/llvm/test/CodeGen/AArch64/arm64_32-null.ll index b4d6581a3ac53..7bed6d6893c6f 100644 --- a/llvm/test/CodeGen/AArch64/arm64_32-null.ll +++ b/llvm/test/CodeGen/AArch64/arm64_32-null.ll @@ -13,10 +13,10 @@ define void @test_store(ptr %p) { define void @test_phi(ptr %p) { ; CHECK-LABEL: test_phi: ; CHECK: mov [[R1:x[0-9]+]], xzr -; CHECK: str [[R1]], [sp, #8] +; CHECK: str [[R1]], [sp, #{{[0-9]+}}] ; CHECK: b [[BB:LBB[0-9_]+]] ; CHECK: [[BB]]: -; CHECK: ldr x0, [sp, #8] +; CHECK: ldr x0, [sp, #{{[0-9]+}}] ; CHECK: mov w8, w0 ; CHECK: str w8, [x{{.*}}] diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-O0.ll b/llvm/test/CodeGen/AArch64/atomicrmw-O0.ll index f7f6a723fd4f6..a1d1e1823ac43 100644 --- a/llvm/test/CodeGen/AArch64/atomicrmw-O0.ll +++ b/llvm/test/CodeGen/AArch64/atomicrmw-O0.ll @@ -10,8 +10,8 @@ define i8 @test_rmw_add_8(ptr %dst) { ; NOLSE-NEXT: sub sp, sp, #32 ; NOLSE-NEXT: .cfi_def_cfa_offset 32 ; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill -; NOLSE-NEXT: ldrb w8, [x0] -; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill +; NOLSE-NEXT: ldrb w0, [x0] +; NOLSE-NEXT: str w0, [sp, #28] // 4-byte Spill ; NOLSE-NEXT: b .LBB0_1 ; NOLSE-NEXT: .LBB0_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Loop Header: Depth=1 @@ -57,8 +57,8 @@ define i16 @test_rmw_add_16(ptr %dst) { ; NOLSE-NEXT: sub sp, sp, #32 ; NOLSE-NEXT: .cfi_def_cfa_offset 32 ; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill -; NOLSE-NEXT: ldrh w8, [x0] -; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill +; NOLSE-NEXT: ldrh w0, [x0] +; NOLSE-NEXT: str w0, [sp, #28] // 4-byte Spill ; NOLSE-NEXT: b .LBB1_1 ; NOLSE-NEXT: .LBB1_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Loop Header: Depth=1 @@ -104,8 +104,8 @@ define i32 @test_rmw_add_32(ptr %dst) { ; NOLSE-NEXT: sub sp, sp, #32 ; NOLSE-NEXT: .cfi_def_cfa_offset 32 ; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill -; NOLSE-NEXT: ldr w8, [x0] -; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill +; NOLSE-NEXT: ldr w0, [x0] +; NOLSE-NEXT: str w0, [sp, #28] // 4-byte Spill ; NOLSE-NEXT: b .LBB2_1 ; NOLSE-NEXT: .LBB2_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Loop Header: Depth=1 @@ -151,8 +151,8 @@ define i64 @test_rmw_add_64(ptr %dst) { ; NOLSE-NEXT: sub sp, sp, #32 ; NOLSE-NEXT: .cfi_def_cfa_offset 32 ; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill -; NOLSE-NEXT: ldr x8, [x0] -; NOLSE-NEXT: str x8, [sp, #24] // 8-byte Spill +; NOLSE-NEXT: ldr x0, [x0] +; NOLSE-NEXT: str x0, [sp, #24] // 8-byte Spill ; NOLSE-NEXT: b .LBB3_1 ; NOLSE-NEXT: .LBB3_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Loop Header: Depth=1 @@ -199,16 +199,16 @@ define i128 @test_rmw_add_128(ptr %dst) { ; NOLSE-NEXT: sub sp, sp, #48 ; NOLSE-NEXT: .cfi_def_cfa_offset 48 ; NOLSE-NEXT: str x0, [sp, #24] // 8-byte Spill -; NOLSE-NEXT: ldr x8, [x0, #8] -; NOLSE-NEXT: ldr x9, [x0] -; NOLSE-NEXT: str x9, [sp, #32] // 8-byte Spill -; NOLSE-NEXT: str x8, [sp, #40] // 8-byte Spill +; NOLSE-NEXT: ldr x1, [x0, #8] +; NOLSE-NEXT: ldr x0, [x0] +; NOLSE-NEXT: str x1, [sp, #32] // 8-byte Spill +; NOLSE-NEXT: str x0, [sp, #40] // 8-byte Spill ; NOLSE-NEXT: b .LBB4_1 ; NOLSE-NEXT: .LBB4_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Loop Header: Depth=1 ; NOLSE-NEXT: // Child Loop BB4_2 Depth 2 -; NOLSE-NEXT: ldr x13, [sp, #40] // 8-byte Reload -; NOLSE-NEXT: ldr x11, [sp, #32] // 8-byte Reload +; NOLSE-NEXT: ldr x13, [sp, #32] // 8-byte Reload +; NOLSE-NEXT: ldr x11, [sp, #40] // 8-byte Reload ; NOLSE-NEXT: ldr x9, [sp, #24] // 8-byte Reload ; NOLSE-NEXT: adds x14, x11, #1 ; NOLSE-NEXT: cinc x15, x13, hs @@ -238,8 +238,8 @@ define i128 @test_rmw_add_128(ptr %dst) { ; NOLSE-NEXT: str x9, [sp, #16] // 8-byte Spill ; NOLSE-NEXT: subs x12, x12, x13 ; NOLSE-NEXT: ccmp x10, x11, #0, eq -; NOLSE-NEXT: str x9, [sp, #32] // 8-byte Spill -; NOLSE-NEXT: str x8, [sp, #40] // 8-byte Spill +; NOLSE-NEXT: str x9, [sp, #40] // 8-byte Spill +; NOLSE-NEXT: str x8, [sp, #32] // 8-byte Spill ; NOLSE-NEXT: b.ne .LBB4_1 ; NOLSE-NEXT: b .LBB4_6 ; NOLSE-NEXT: .LBB4_6: // %atomicrmw.end @@ -253,15 +253,15 @@ define i128 @test_rmw_add_128(ptr %dst) { ; LSE-NEXT: sub sp, sp, #48 ; LSE-NEXT: .cfi_def_cfa_offset 48 ; LSE-NEXT: str x0, [sp, #24] // 8-byte Spill -; LSE-NEXT: ldr x8, [x0, #8] -; LSE-NEXT: ldr x9, [x0] -; LSE-NEXT: str x9, [sp, #32] // 8-byte Spill -; LSE-NEXT: str x8, [sp, #40] // 8-byte Spill +; LSE-NEXT: ldr x1, [x0, #8] +; LSE-NEXT: ldr x0, [x0] +; LSE-NEXT: str x1, [sp, #32] // 8-byte Spill +; LSE-NEXT: str x0, [sp, #40] // 8-byte Spill ; LSE-NEXT: b .LBB4_1 ; LSE-NEXT: .LBB4_1: // %atomicrmw.start ; LSE-NEXT: // =>This Inner Loop Header: Depth=1 -; LSE-NEXT: ldr x11, [sp, #40] // 8-byte Reload -; LSE-NEXT: ldr x10, [sp, #32] // 8-byte Reload +; LSE-NEXT: ldr x11, [sp, #32] // 8-byte Reload +; LSE-NEXT: ldr x10, [sp, #40] // 8-byte Reload ; LSE-NEXT: ldr x8, [sp, #24] // 8-byte Reload ; LSE-NEXT: mov x0, x10 ; LSE-NEXT: mov x1, x11 @@ -276,8 +276,8 @@ define i128 @test_rmw_add_128(ptr %dst) { ; LSE-NEXT: str x8, [sp, #16] // 8-byte Spill ; LSE-NEXT: subs x11, x8, x11 ; LSE-NEXT: ccmp x9, x10, #0, eq -; LSE-NEXT: str x9, [sp, #32] // 8-byte Spill -; LSE-NEXT: str x8, [sp, #40] // 8-byte Spill +; LSE-NEXT: str x9, [sp, #40] // 8-byte Spill +; LSE-NEXT: str x8, [sp, #32] // 8-byte Spill ; LSE-NEXT: b.ne .LBB4_1 ; LSE-NEXT: b .LBB4_2 ; LSE-NEXT: .LBB4_2: // %atomicrmw.end @@ -295,8 +295,8 @@ define i8 @test_rmw_nand_8(ptr %dst) { ; NOLSE-NEXT: sub sp, sp, #32 ; NOLSE-NEXT: .cfi_def_cfa_offset 32 ; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill -; NOLSE-NEXT: ldrb w8, [x0] -; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill +; NOLSE-NEXT: ldrb w0, [x0] +; NOLSE-NEXT: str w0, [sp, #28] // 4-byte Spill ; NOLSE-NEXT: b .LBB5_1 ; NOLSE-NEXT: .LBB5_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Loop Header: Depth=1 @@ -332,8 +332,8 @@ define i8 @test_rmw_nand_8(ptr %dst) { ; LSE-NEXT: sub sp, sp, #32 ; LSE-NEXT: .cfi_def_cfa_offset 32 ; LSE-NEXT: str x0, [sp, #16] // 8-byte Spill -; LSE-NEXT: ldrb w8, [x0] -; LSE-NEXT: str w8, [sp, #28] // 4-byte Spill +; LSE-NEXT: ldrb w0, [x0] +; LSE-NEXT: str w0, [sp, #28] // 4-byte Spill ; LSE-NEXT: b .LBB5_1 ; LSE-NEXT: .LBB5_1: // %atomicrmw.start ; LSE-NEXT: // =>This Inner Loop Header: Depth=1 @@ -363,8 +363,8 @@ define i16 @test_rmw_nand_16(ptr %dst) { ; NOLSE-NEXT: sub sp, sp, #32 ; NOLSE-NEXT: .cfi_def_cfa_offset 32 ; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill -; NOLSE-NEXT: ldrh w8, [x0] -; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill +; NOLSE-NEXT: ldrh w0, [x0] +; NOLSE-NEXT: str w0, [sp, #28] // 4-byte Spill ; NOLSE-NEXT: b .LBB6_1 ; NOLSE-NEXT: .LBB6_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Loop Header: Depth=1 @@ -400,8 +400,8 @@ define i16 @test_rmw_nand_16(ptr %dst) { ; LSE-NEXT: sub sp, sp, #32 ; LSE-NEXT: .cfi_def_cfa_offset 32 ; LSE-NEXT: str x0, [sp, #16] // 8-byte Spill -; LSE-NEXT: ldrh w8, [x0] -; LSE-NEXT: str w8, [sp, #28] // 4-byte Spill +; LSE-NEXT: ldrh w0, [x0] +; LSE-NEXT: str w0, [sp, #28] // 4-byte Spill ; LSE-NEXT: b .LBB6_1 ; LSE-NEXT: .LBB6_1: // %atomicrmw.start ; LSE-NEXT: // =>This Inner Loop Header: Depth=1 @@ -431,8 +431,8 @@ define i32 @test_rmw_nand_32(ptr %dst) { ; NOLSE-NEXT: sub sp, sp, #32 ; NOLSE-NEXT: .cfi_def_cfa_offset 32 ; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill -; NOLSE-NEXT: ldr w8, [x0] -; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill +; NOLSE-NEXT: ldr w0, [x0] +; NOLSE-NEXT: str w0, [sp, #28] // 4-byte Spill ; NOLSE-NEXT: b .LBB7_1 ; NOLSE-NEXT: .LBB7_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Loop Header: Depth=1 @@ -468,8 +468,8 @@ define i32 @test_rmw_nand_32(ptr %dst) { ; LSE-NEXT: sub sp, sp, #32 ; LSE-NEXT: .cfi_def_cfa_offset 32 ; LSE-NEXT: str x0, [sp, #16] // 8-byte Spill -; LSE-NEXT: ldr w8, [x0] -; LSE-NEXT: str w8, [sp, #28] // 4-byte Spill +; LSE-NEXT: ldr w0, [x0] +; LSE-NEXT: str w0, [sp, #28] // 4-byte Spill ; LSE-NEXT: b .LBB7_1 ; LSE-NEXT: .LBB7_1: // %atomicrmw.start ; LSE-NEXT: // =>This Inner Loop Header: Depth=1 @@ -499,8 +499,8 @@ define i64 @test_rmw_nand_64(ptr %dst) { ; NOLSE-NEXT: sub sp, sp, #32 ; NOLSE-NEXT: .cfi_def_cfa_offset 32 ; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill -; NOLSE-NEXT: ldr x8, [x0] -; NOLSE-NEXT: str x8, [sp, #24] // 8-byte Spill +; NOLSE-NEXT: ldr x0, [x0] +; NOLSE-NEXT: str x0, [sp, #24] // 8-byte Spill ; NOLSE-NEXT: b .LBB8_1 ; NOLSE-NEXT: .LBB8_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Loop Header: Depth=1 @@ -539,8 +539,8 @@ define i64 @test_rmw_nand_64(ptr %dst) { ; LSE-NEXT: sub sp, sp, #32 ; LSE-NEXT: .cfi_def_cfa_offset 32 ; LSE-NEXT: str x0, [sp, #16] // 8-byte Spill -; LSE-NEXT: ldr x8, [x0] -; LSE-NEXT: str x8, [sp, #24] // 8-byte Spill +; LSE-NEXT: ldr x0, [x0] +; LSE-NEXT: str x0, [sp, #24] // 8-byte Spill ; LSE-NEXT: b .LBB8_1 ; LSE-NEXT: .LBB8_1: // %atomicrmw.start ; LSE-NEXT: // =>This Inner Loop Header: Depth=1 @@ -573,16 +573,16 @@ define i128 @test_rmw_nand_128(ptr %dst) { ; NOLSE-NEXT: sub sp, sp, #48 ; NOLSE-NEXT: .cfi_def_cfa_offset 48 ; NOLSE-NEXT: str x0, [sp, #24] // 8-byte Spill -; NOLSE-NEXT: ldr x8, [x0, #8] -; NOLSE-NEXT: ldr x9, [x0] -; NOLSE-NEXT: str x9, [sp, #32] // 8-byte Spill -; NOLSE-NEXT: str x8, [sp, #40] // 8-byte Spill +; NOLSE-NEXT: ldr x1, [x0, #8] +; NOLSE-NEXT: ldr x0, [x0] +; NOLSE-NEXT: str x1, [sp, #32] // 8-byte Spill +; NOLSE-NEXT: str x0, [sp, #40] // 8-byte Spill ; NOLSE-NEXT: b .LBB9_1 ; NOLSE-NEXT: .LBB9_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Loop Header: Depth=1 ; NOLSE-NEXT: // Child Loop BB9_2 Depth 2 -; NOLSE-NEXT: ldr x13, [sp, #40] // 8-byte Reload -; NOLSE-NEXT: ldr x11, [sp, #32] // 8-byte Reload +; NOLSE-NEXT: ldr x13, [sp, #32] // 8-byte Reload +; NOLSE-NEXT: ldr x11, [sp, #40] // 8-byte Reload ; NOLSE-NEXT: ldr x9, [sp, #24] // 8-byte Reload ; NOLSE-NEXT: mov w8, w11 ; NOLSE-NEXT: mvn w10, w8 @@ -616,8 +616,8 @@ define i128 @test_rmw_nand_128(ptr %dst) { ; NOLSE-NEXT: str x9, [sp, #16] // 8-byte Spill ; NOLSE-NEXT: subs x12, x12, x13 ; NOLSE-NEXT: ccmp x10, x11, #0, eq -; NOLSE-NEXT: str x9, [sp, #32] // 8-byte Spill -; NOLSE-NEXT: str x8, [sp, #40] // 8-byte Spill +; NOLSE-NEXT: str x9, [sp, #40] // 8-byte Spill +; NOLSE-NEXT: str x8, [sp, #32] // 8-byte Spill ; NOLSE-NEXT: b.ne .LBB9_1 ; NOLSE-NEXT: b .LBB9_6 ; NOLSE-NEXT: .LBB9_6: // %atomicrmw.end @@ -631,15 +631,15 @@ define i128 @test_rmw_nand_128(ptr %dst) { ; LSE-NEXT: sub sp, sp, #48 ; LSE-NEXT: .cfi_def_cfa_offset 48 ; LSE-NEXT: str x0, [sp, #24] // 8-byte Spill -; LSE-NEXT: ldr x8, [x0, #8] -; LSE-NEXT: ldr x9, [x0] -; LSE-NEXT: str x9, [sp, #32] // 8-byte Spill -; LSE-NEXT: str x8, [sp, #40] // 8-byte Spill +; LSE-NEXT: ldr x1, [x0, #8] +; LSE-NEXT: ldr x0, [x0] +; LSE-NEXT: str x1, [sp, #32] // 8-byte Spill +; LSE-NEXT: str x0, [sp, #40] // 8-byte Spill ; LSE-NEXT: b .LBB9_1 ; LSE-NEXT: .LBB9_1: // %atomicrmw.start ; LSE-NEXT: // =>This Inner Loop Header: Depth=1 -; LSE-NEXT: ldr x11, [sp, #40] // 8-byte Reload -; LSE-NEXT: ldr x10, [sp, #32] // 8-byte Reload +; LSE-NEXT: ldr x11, [sp, #32] // 8-byte Reload +; LSE-NEXT: ldr x10, [sp, #40] // 8-byte Reload ; LSE-NEXT: ldr x8, [sp, #24] // 8-byte Reload ; LSE-NEXT: mov x0, x10 ; LSE-NEXT: mov x1, x11 @@ -658,8 +658,8 @@ define i128 @test_rmw_nand_128(ptr %dst) { ; LSE-NEXT: str x8, [sp, #16] // 8-byte Spill ; LSE-NEXT: subs x11, x8, x11 ; LSE-NEXT: ccmp x9, x10, #0, eq -; LSE-NEXT: str x9, [sp, #32] // 8-byte Spill -; LSE-NEXT: str x8, [sp, #40] // 8-byte Spill +; LSE-NEXT: str x9, [sp, #40] // 8-byte Spill +; LSE-NEXT: str x8, [sp, #32] // 8-byte Spill ; LSE-NEXT: b.ne .LBB9_1 ; LSE-NEXT: b .LBB9_2 ; LSE-NEXT: .LBB9_2: // %atomicrmw.end diff --git a/llvm/test/CodeGen/AArch64/phi-dbg.ll b/llvm/test/CodeGen/AArch64/phi-dbg.ll index 4f7c005f8026f..32e1ebc81adc4 100644 --- a/llvm/test/CodeGen/AArch64/phi-dbg.ll +++ b/llvm/test/CodeGen/AArch64/phi-dbg.ll @@ -1,4 +1,5 @@ -; RUN: llc -O0 %s -mtriple=aarch64 -stop-after=phi-node-elimination -o - | FileCheck %s +; RUN: llc -O0 %s -mtriple=aarch64 -regalloc-fast-ssa=0 -stop-after=phi-node-elimination -o - | FileCheck %s +; -regalloc-fast-ssa=0 restores phi-node-elimination, which -stop-after names. ; Test that a DEBUG_VALUE node is create for variable c after the phi has been ; converted to a ldr. The DEBUG_VALUE must be *after* the ldr and not before it. diff --git a/llvm/test/CodeGen/AArch64/pr48188.ll b/llvm/test/CodeGen/AArch64/pr48188.ll index 836690c1e7f0d..645ebe87542a0 100644 --- a/llvm/test/CodeGen/AArch64/pr48188.ll +++ b/llvm/test/CodeGen/AArch64/pr48188.ll @@ -23,17 +23,13 @@ define void @test() nounwind { ; SDAG-LABEL: test: ; SDAG: // %bb.0: // %entry ; SDAG-NEXT: sub sp, sp, #16 -; SDAG-NEXT: mov x1, xzr -; SDAG-NEXT: mov x0, x1 +; SDAG-NEXT: mov x0, xzr +; SDAG-NEXT: mov x1, x0 ; SDAG-NEXT: str x1, [sp] // 8-byte Spill ; SDAG-NEXT: str x0, [sp, #8] // 8-byte Spill ; SDAG-NEXT: b .LBB0_1 ; SDAG-NEXT: .LBB0_1: // %loop ; SDAG-NEXT: // =>This Inner Loop Header: Depth=1 -; SDAG-NEXT: ldr x0, [sp, #8] // 8-byte Reload -; SDAG-NEXT: ldr x1, [sp] // 8-byte Reload -; SDAG-NEXT: str x1, [sp] // 8-byte Spill -; SDAG-NEXT: str x0, [sp, #8] // 8-byte Spill ; SDAG-NEXT: b .LBB0_1 entry: br label %loop diff --git a/llvm/test/CodeGen/AArch64/regallocfast-ssa-phi-cycle.ll b/llvm/test/CodeGen/AArch64/regallocfast-ssa-phi-cycle.ll new file mode 100644 index 0000000000000..af5ff7bc684ac --- /dev/null +++ b/llvm/test/CodeGen/AArch64/regallocfast-ssa-phi-cycle.ll @@ -0,0 +1,153 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -mtriple=aarch64-linux -O0 -regalloc-fast-ssa -verify-machineinstrs < %s | FileCheck %s + +; Loop-carried PHI webs whose edge moves form permutation cycles: the +; parallel-move resolver must park one value in a held register per cycle. + +; Three values rotated across the back edge. +define i64 @rotate3(i64 %n, i64 %a0, i64 %b0, i64 %c0) nounwind { +; CHECK-LABEL: rotate3: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sub sp, sp, #64 +; CHECK-NEXT: str x0, [sp, #24] // 8-byte Spill +; CHECK-NEXT: mov x10, x1 +; CHECK-NEXT: mov x9, x2 +; CHECK-NEXT: mov x8, x3 +; CHECK-NEXT: mov x11, xzr +; CHECK-NEXT: str x11, [sp, #56] // 8-byte Spill +; CHECK-NEXT: str x10, [sp, #32] // 8-byte Spill +; CHECK-NEXT: str x9, [sp, #40] // 8-byte Spill +; CHECK-NEXT: str x8, [sp, #48] // 8-byte Spill +; CHECK-NEXT: b .LBB0_1 +; CHECK-NEXT: .LBB0_1: // %loop +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldr x8, [sp, #56] // 8-byte Reload +; CHECK-NEXT: ldr x10, [sp, #48] // 8-byte Reload +; CHECK-NEXT: ldr x11, [sp, #40] // 8-byte Reload +; CHECK-NEXT: ldr x9, [sp, #32] // 8-byte Reload +; CHECK-NEXT: ldr x12, [sp, #24] // 8-byte Reload +; CHECK-NEXT: str x10, [sp] // 8-byte Spill +; CHECK-NEXT: str x11, [sp, #8] // 8-byte Spill +; CHECK-NEXT: str x9, [sp, #16] // 8-byte Spill +; CHECK-NEXT: add x8, x8, #1 +; CHECK-NEXT: subs x12, x8, x12 +; CHECK-NEXT: str x11, [sp, #32] // 8-byte Spill +; CHECK-NEXT: str x10, [sp, #40] // 8-byte Spill +; CHECK-NEXT: str x9, [sp, #48] // 8-byte Spill +; CHECK-NEXT: str x8, [sp, #56] // 8-byte Spill +; CHECK-NEXT: b.lo .LBB0_1 +; CHECK-NEXT: b .LBB0_2 +; CHECK-NEXT: .LBB0_2: // %exit +; CHECK-NEXT: ldr x9, [sp] // 8-byte Reload +; CHECK-NEXT: ldr x8, [sp, #16] // 8-byte Reload +; CHECK-NEXT: ldr x10, [sp, #8] // 8-byte Reload +; CHECK-NEXT: add x8, x8, x10 +; CHECK-NEXT: add x0, x8, x9 +; CHECK-NEXT: add sp, sp, #64 +; CHECK-NEXT: ret +entry: + br label %loop + +loop: + %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] + %a = phi i64 [ %a0, %entry ], [ %b, %loop ] + %b = phi i64 [ %b0, %entry ], [ %c, %loop ] + %c = phi i64 [ %c0, %entry ], [ %a, %loop ] + %i.next = add i64 %i, 1 + %cond = icmp ult i64 %i.next, %n + br i1 %cond, label %loop, label %exit + +exit: + %s1 = add i64 %a, %b + %s2 = add i64 %s1, %c + ret i64 %s2 +} + +; Two values swapped across the back edge. +define i64 @swap2(i64 %n, i64 %a0, i64 %b0) nounwind { +; CHECK-LABEL: swap2: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sub sp, sp, #48 +; CHECK-NEXT: str x0, [sp, #16] // 8-byte Spill +; CHECK-NEXT: mov x9, x1 +; CHECK-NEXT: mov x8, x2 +; CHECK-NEXT: mov x10, xzr +; CHECK-NEXT: str x10, [sp, #40] // 8-byte Spill +; CHECK-NEXT: str x9, [sp, #24] // 8-byte Spill +; CHECK-NEXT: str x8, [sp, #32] // 8-byte Spill +; CHECK-NEXT: b .LBB1_1 +; CHECK-NEXT: .LBB1_1: // %loop +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldr x8, [sp, #40] // 8-byte Reload +; CHECK-NEXT: ldr x10, [sp, #32] // 8-byte Reload +; CHECK-NEXT: ldr x9, [sp, #24] // 8-byte Reload +; CHECK-NEXT: ldr x11, [sp, #16] // 8-byte Reload +; CHECK-NEXT: str x10, [sp] // 8-byte Spill +; CHECK-NEXT: str x9, [sp, #8] // 8-byte Spill +; CHECK-NEXT: add x8, x8, #1 +; CHECK-NEXT: subs x11, x8, x11 +; CHECK-NEXT: str x10, [sp, #24] // 8-byte Spill +; CHECK-NEXT: str x9, [sp, #32] // 8-byte Spill +; CHECK-NEXT: str x8, [sp, #40] // 8-byte Spill +; CHECK-NEXT: b.lo .LBB1_1 +; CHECK-NEXT: b .LBB1_2 +; CHECK-NEXT: .LBB1_2: // %exit +; CHECK-NEXT: ldr x8, [sp, #8] // 8-byte Reload +; CHECK-NEXT: ldr x9, [sp] // 8-byte Reload +; CHECK-NEXT: subs x0, x8, x9 +; CHECK-NEXT: add sp, sp, #48 +; CHECK-NEXT: ret +entry: + br label %loop + +loop: + %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] + %a = phi i64 [ %a0, %entry ], [ %b, %loop ] + %b = phi i64 [ %b0, %entry ], [ %a, %loop ] + %i.next = add i64 %i, 1 + %cond = icmp ult i64 %i.next, %n + br i1 %cond, label %loop, label %exit + +exit: + %s = sub i64 %a, %b + ret i64 %s +} + +; PHI receiving a value over a callbr indirect edge: the moves are inserted +; before the INLINEASM_BR. +define i32 @callbr_phi(i32 %x) nounwind { +; CHECK-LABEL: callbr_phi: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sub sp, sp, #16 +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: str w8, [sp, #4] // 4-byte Spill +; CHECK-NEXT: add w0, w8, #1 +; CHECK-NEXT: str w0, [sp, #8] // 4-byte Spill +; CHECK-NEXT: str w8, [sp, #12] // 4-byte Spill +; CHECK-NEXT: //APP +; CHECK-NEXT: //NO_APP +; CHECK-NEXT: b .LBB2_1 +; CHECK-NEXT: .LBB2_1: // %fall +; CHECK-NEXT: ldr w8, [sp, #8] // 4-byte Reload +; CHECK-NEXT: str w8, [sp, #12] // 4-byte Spill +; CHECK-NEXT: b .LBB2_2 +; CHECK-NEXT: .LBB2_2: // Inline asm indirect target +; CHECK-NEXT: // %ind +; CHECK-NEXT: // Label of block must be emitted +; CHECK-NEXT: ldr w8, [sp, #12] // 4-byte Reload +; CHECK-NEXT: ldr w9, [sp, #4] // 4-byte Reload +; CHECK-NEXT: add w0, w8, w9 +; CHECK-NEXT: add sp, sp, #16 +; CHECK-NEXT: ret +entry: + %y = add i32 %x, 1 + callbr void asm sideeffect "", "!i"() to label %fall [label %ind] + +fall: + br label %ind + +ind: + %p = phi i32 [ %x, %entry ], [ %y, %fall ] + %q = add i32 %p, %x + ret i32 %q +} diff --git a/llvm/test/CodeGen/AArch64/regallocfast-ssa-reg-sequence.ll b/llvm/test/CodeGen/AArch64/regallocfast-ssa-reg-sequence.ll new file mode 100644 index 0000000000000..031f9d194f126 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/regallocfast-ssa-reg-sequence.ll @@ -0,0 +1,39 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=aarch64 -O0 -regalloc-fast-ssa -verify-machineinstrs < %s | FileCheck %s + +;; REG_SEQUENCE reaches the allocator only on the SSA path; TwoAddressInstruction +;; pass lowers it otherwise. Left in place it survives to the AsmPrinter, which +;; cannot print it. NEON tuple intrinsics are the reachable source at -O0, the +;; 128-bit atomics that cover this elsewhere doing so incidentally. + +declare void @llvm.aarch64.neon.st2.v4i32.p0(<4 x i32>, <4 x i32>, ptr) +declare void @llvm.aarch64.neon.st4.v4i32.p0(<4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, ptr) + +define void @reg_sequence_pair(<4 x i32> %a, <4 x i32> %b, ptr %p) nounwind { +; CHECK-LABEL: reg_sequence_pair: +; CHECK: // %bb.0: +; CHECK-NEXT: mov v2.16b, v1.16b +; CHECK-NEXT: // kill: def $q0 killed $q0 def $q0_q1 +; CHECK-NEXT: mov v1.16b, v2.16b +; CHECK-NEXT: st2 { v0.4s, v1.4s }, [x0] +; CHECK-NEXT: ret + call void @llvm.aarch64.neon.st2.v4i32.p0(<4 x i32> %a, <4 x i32> %b, ptr %p) + ret void +} + +;; Four sources: only the first subregister COPY may be marked undef. +define void @reg_sequence_quad(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d, ptr %p) nounwind { +; CHECK-LABEL: reg_sequence_quad: +; CHECK: // %bb.0: +; CHECK-NEXT: mov v6.16b, v1.16b +; CHECK-NEXT: mov v5.16b, v2.16b +; CHECK-NEXT: mov v4.16b, v3.16b +; CHECK-NEXT: // kill: def $q0 killed $q0 def $q0_q1_q2_q3 +; CHECK-NEXT: mov v1.16b, v6.16b +; CHECK-NEXT: mov v2.16b, v5.16b +; CHECK-NEXT: mov v3.16b, v4.16b +; CHECK-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0] +; CHECK-NEXT: ret + call void @llvm.aarch64.neon.st4.v4i32.p0(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d, ptr %p) + ret void +} diff --git a/llvm/test/CodeGen/AArch64/swifterror.ll b/llvm/test/CodeGen/AArch64/swifterror.ll index 1ef88cfc16069..18487e80b1e2f 100644 --- a/llvm/test/CodeGen/AArch64/swifterror.ll +++ b/llvm/test/CodeGen/AArch64/swifterror.ll @@ -549,7 +549,8 @@ define float @foo_loop(ptr swifterror %error_ptr_ref, i32 %cc, float %cc2) { ; CHECK-O0-AARCH64-NEXT: .cfi_offset w29, -16 ; CHECK-O0-AARCH64-NEXT: str s0, [sp, #16] ; 4-byte Spill ; CHECK-O0-AARCH64-NEXT: stur w0, [x29, #-12] ; 4-byte Folded Spill -; CHECK-O0-AARCH64-NEXT: stur x21, [x29, #-8] ; 8-byte Folded Spill +; CHECK-O0-AARCH64-NEXT: mov x0, x21 +; CHECK-O0-AARCH64-NEXT: stur x0, [x29, #-8] ; 8-byte Folded Spill ; CHECK-O0-AARCH64-NEXT: b LBB4_1 ; CHECK-O0-AARCH64-NEXT: LBB4_1: ; %bb_loop ; CHECK-O0-AARCH64-NEXT: ; =>This Inner Loop Header: Depth=1 @@ -590,7 +591,8 @@ define float @foo_loop(ptr swifterror %error_ptr_ref, i32 %cc, float %cc2) { ; CHECK-O0-ARM64_32-NEXT: .cfi_offset w30, -16 ; CHECK-O0-ARM64_32-NEXT: str s0, [sp, #16] ; 4-byte Spill ; CHECK-O0-ARM64_32-NEXT: str w0, [sp, #20] ; 4-byte Spill -; CHECK-O0-ARM64_32-NEXT: str x21, [sp, #24] ; 8-byte Spill +; CHECK-O0-ARM64_32-NEXT: mov x0, x21 +; CHECK-O0-ARM64_32-NEXT: str x0, [sp, #24] ; 8-byte Spill ; CHECK-O0-ARM64_32-NEXT: b LBB4_1 ; CHECK-O0-ARM64_32-NEXT: LBB4_1: ; %bb_loop ; CHECK-O0-ARM64_32-NEXT: ; =>This Inner Loop Header: Depth=1 diff --git a/llvm/test/CodeGen/X86/2011-06-12-FastAllocSpill.ll b/llvm/test/CodeGen/X86/2011-06-12-FastAllocSpill.ll index 650cc0a77c7ae..5edbd55680c55 100644 --- a/llvm/test/CodeGen/X86/2011-06-12-FastAllocSpill.ll +++ b/llvm/test/CodeGen/X86/2011-06-12-FastAllocSpill.ll @@ -3,7 +3,7 @@ ; ; This test should not cause any spilling with RAFast. ; -; CHECK: Number of copies coalesced +; CHECK: Number of blocks selected entirely by fast isel ; CHECK-NOT: Number of stores added ; target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64" diff --git a/llvm/test/CodeGen/X86/AMX/amx-across-func.ll b/llvm/test/CodeGen/X86/AMX/amx-across-func.ll index cbd4ee0370642..d13c385129a53 100644 --- a/llvm/test/CodeGen/X86/AMX/amx-across-func.ll +++ b/llvm/test/CodeGen/X86/AMX/amx-across-func.ll @@ -346,10 +346,10 @@ define dso_local i32 @test_loop(i32 %0) nounwind { ; O0-NEXT: vzeroupper ; O0-NEXT: callq foo ; O0-NEXT: # %bb.1: -; O0-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload -; O0-NEXT: xorl %eax, %eax -; O0-NEXT: cmpl $0, %ecx -; O0-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; O0-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; O0-NEXT: xorl %ecx, %ecx +; O0-NEXT: movl %ecx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; O0-NEXT: cmpl $0, %eax ; O0-NEXT: jg .LBB2_4 ; O0-NEXT: jmp .LBB2_3 ; O0-NEXT: .LBB2_2: @@ -358,8 +358,6 @@ define dso_local i32 @test_loop(i32 %0) nounwind { ; O0-NEXT: je .LBB2_5 ; O0-NEXT: jmp .LBB2_4 ; O0-NEXT: .LBB2_3: # =>This Inner Loop Header: Depth=1 -; O0-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload -; O0-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill ; O0-NEXT: movl $buf, %ecx ; O0-NEXT: movl $32, %edx ; O0-NEXT: movw $8, %ax @@ -566,8 +564,6 @@ define dso_local void @test_loop2(i32 %0) nounwind { ; O0-NEXT: xorl %eax, %eax ; O0-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill ; O0-NEXT: .LBB3_1: # =>This Inner Loop Header: Depth=1 -; O0-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload -; O0-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill ; O0-NEXT: vzeroupper ; O0-NEXT: callq foo ; O0-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll index e8a3084563573..f2a847960a1ab 100644 --- a/llvm/test/CodeGen/X86/O0-pipeline.ll +++ b/llvm/test/CodeGen/X86/O0-pipeline.ll @@ -47,8 +47,6 @@ ; CHECK-NEXT: X86 EFLAGS copy lowering ; CHECK-NEXT: X86 DynAlloca Expander ; CHECK-NEXT: Fast Tile Register Preconfigure -; CHECK-NEXT: Eliminate PHI nodes for register allocation -; CHECK-NEXT: Two-Address instruction pass ; CHECK-NEXT: Fast Register Allocator ; CHECK-NEXT: Fast Tile Register Configure ; CHECK-NEXT: X86 Lower Tile Copy diff --git a/llvm/test/CodeGen/X86/atomic-load-store.ll b/llvm/test/CodeGen/X86/atomic-load-store.ll index 7cfe7af47748a..b998492e4cbfd 100644 --- a/llvm/test/CodeGen/X86/atomic-load-store.ll +++ b/llvm/test/CodeGen/X86/atomic-load-store.ll @@ -980,7 +980,7 @@ define void @store_atomic_vec4_half(ptr %x, <4 x half> %v) nounwind { ; CHECK-SSE2-O0-NEXT: movaps %xmm3, %xmm2 ; CHECK-SSE2-O0-NEXT: psrlq $48, %xmm2 ; CHECK-SSE2-O0-NEXT: movaps %xmm3, %xmm1 -; CHECK-SSE2-O0-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1,1,1] +; CHECK-SSE2-O0-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1],xmm3[1,1] ; CHECK-SSE2-O0-NEXT: psrld $16, %xmm3 ; CHECK-SSE2-O0-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3] ; CHECK-SSE2-O0-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] diff --git a/llvm/test/CodeGen/X86/atomic-unordered.ll b/llvm/test/CodeGen/X86/atomic-unordered.ll index edfdec37150c2..79478972ab1b6 100644 --- a/llvm/test/CodeGen/X86/atomic-unordered.ll +++ b/llvm/test/CodeGen/X86/atomic-unordered.ll @@ -1428,10 +1428,10 @@ define void @rmw_fold_sdiv1(ptr %p, i64 %v) { ; CHECK-O0-NEXT: imulq %rdx ; CHECK-O0-NEXT: movq %rdx, %rax ; CHECK-O0-NEXT: addq %rcx, %rax -; CHECK-O0-NEXT: movq %rax, %rcx -; CHECK-O0-NEXT: shrq $63, %rcx +; CHECK-O0-NEXT: movq %rax, %rdx +; CHECK-O0-NEXT: shrq $63, %rdx ; CHECK-O0-NEXT: sarq $3, %rax -; CHECK-O0-NEXT: addq %rcx, %rax +; CHECK-O0-NEXT: addq %rdx, %rax ; CHECK-O0-NEXT: movq %rax, (%rdi) ; CHECK-O0-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/atomic32.ll b/llvm/test/CodeGen/X86/atomic32.ll index f4666738db7d2..4c17c3efad18c 100644 --- a/llvm/test/CodeGen/X86/atomic32.ll +++ b/llvm/test/CodeGen/X86/atomic32.ll @@ -331,8 +331,8 @@ define void @atomic_fetch_max32(i32 %x) nounwind { ; X86-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NOCMOV-NEXT: .LBB6_4: # %atomicrmw.start ; X86-NOCMOV-NEXT: # in Loop: Header=BB6_1 Depth=1 -; X86-NOCMOV-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NOCMOV-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOCMOV-NEXT: lock cmpxchgl %ecx, sc32 ; X86-NOCMOV-NEXT: sete %cl ; X86-NOCMOV-NEXT: testb $1, %cl @@ -365,8 +365,8 @@ define void @atomic_fetch_max32(i32 %x) nounwind { ; X86-NOX87-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NOX87-NEXT: .LBB6_4: # %atomicrmw.start ; X86-NOX87-NEXT: # in Loop: Header=BB6_1 Depth=1 -; X86-NOX87-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOX87-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NOX87-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOX87-NEXT: lock cmpxchgl %ecx, sc32 ; X86-NOX87-NEXT: sete %cl ; X86-NOX87-NEXT: testb $1, %cl @@ -448,8 +448,8 @@ define void @atomic_fetch_min32(i32 %x) nounwind { ; X86-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NOCMOV-NEXT: .LBB7_4: # %atomicrmw.start ; X86-NOCMOV-NEXT: # in Loop: Header=BB7_1 Depth=1 -; X86-NOCMOV-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NOCMOV-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOCMOV-NEXT: lock cmpxchgl %ecx, sc32 ; X86-NOCMOV-NEXT: sete %cl ; X86-NOCMOV-NEXT: testb $1, %cl @@ -482,8 +482,8 @@ define void @atomic_fetch_min32(i32 %x) nounwind { ; X86-NOX87-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NOX87-NEXT: .LBB7_4: # %atomicrmw.start ; X86-NOX87-NEXT: # in Loop: Header=BB7_1 Depth=1 -; X86-NOX87-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOX87-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NOX87-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOX87-NEXT: lock cmpxchgl %ecx, sc32 ; X86-NOX87-NEXT: sete %cl ; X86-NOX87-NEXT: testb $1, %cl @@ -565,8 +565,8 @@ define void @atomic_fetch_umax32(i32 %x) nounwind { ; X86-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NOCMOV-NEXT: .LBB8_4: # %atomicrmw.start ; X86-NOCMOV-NEXT: # in Loop: Header=BB8_1 Depth=1 -; X86-NOCMOV-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NOCMOV-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOCMOV-NEXT: lock cmpxchgl %ecx, sc32 ; X86-NOCMOV-NEXT: sete %cl ; X86-NOCMOV-NEXT: testb $1, %cl @@ -599,8 +599,8 @@ define void @atomic_fetch_umax32(i32 %x) nounwind { ; X86-NOX87-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NOX87-NEXT: .LBB8_4: # %atomicrmw.start ; X86-NOX87-NEXT: # in Loop: Header=BB8_1 Depth=1 -; X86-NOX87-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOX87-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NOX87-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOX87-NEXT: lock cmpxchgl %ecx, sc32 ; X86-NOX87-NEXT: sete %cl ; X86-NOX87-NEXT: testb $1, %cl @@ -682,8 +682,8 @@ define void @atomic_fetch_umin32(i32 %x) nounwind { ; X86-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NOCMOV-NEXT: .LBB9_4: # %atomicrmw.start ; X86-NOCMOV-NEXT: # in Loop: Header=BB9_1 Depth=1 -; X86-NOCMOV-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NOCMOV-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOCMOV-NEXT: lock cmpxchgl %ecx, sc32 ; X86-NOCMOV-NEXT: sete %cl ; X86-NOCMOV-NEXT: testb $1, %cl @@ -716,8 +716,8 @@ define void @atomic_fetch_umin32(i32 %x) nounwind { ; X86-NOX87-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NOX87-NEXT: .LBB9_4: # %atomicrmw.start ; X86-NOX87-NEXT: # in Loop: Header=BB9_1 Depth=1 -; X86-NOX87-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOX87-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NOX87-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-NOX87-NEXT: lock cmpxchgl %ecx, sc32 ; X86-NOX87-NEXT: sete %cl ; X86-NOX87-NEXT: testb $1, %cl diff --git a/llvm/test/CodeGen/X86/atomic64.ll b/llvm/test/CodeGen/X86/atomic64.ll index 8f4da356e06cb..5568de25dee0c 100644 --- a/llvm/test/CodeGen/X86/atomic64.ll +++ b/llvm/test/CodeGen/X86/atomic64.ll @@ -359,10 +359,10 @@ define void @atomic_fetch_max64(i64 %x) nounwind { ; I486-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; I486-NEXT: .LBB6_4: # %atomicrmw.start ; I486-NEXT: # in Loop: Header=BB6_1 Depth=1 -; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload +; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload +; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; I486-NEXT: movl %esi, {{[0-9]+}}(%esp) ; I486-NEXT: movl %eax, {{[0-9]+}}(%esp) ; I486-NEXT: movl %esp, %eax @@ -451,10 +451,10 @@ define void @atomic_fetch_min64(i64 %x) nounwind { ; I486-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; I486-NEXT: .LBB7_4: # %atomicrmw.start ; I486-NEXT: # in Loop: Header=BB7_1 Depth=1 -; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload +; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload +; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; I486-NEXT: movl %esi, {{[0-9]+}}(%esp) ; I486-NEXT: movl %eax, {{[0-9]+}}(%esp) ; I486-NEXT: movl %esp, %eax @@ -543,10 +543,10 @@ define void @atomic_fetch_umax64(i64 %x) nounwind { ; I486-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; I486-NEXT: .LBB8_4: # %atomicrmw.start ; I486-NEXT: # in Loop: Header=BB8_1 Depth=1 -; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload +; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload +; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; I486-NEXT: movl %esi, {{[0-9]+}}(%esp) ; I486-NEXT: movl %eax, {{[0-9]+}}(%esp) ; I486-NEXT: movl %esp, %eax @@ -635,10 +635,10 @@ define void @atomic_fetch_umin64(i64 %x) nounwind { ; I486-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; I486-NEXT: .LBB9_4: # %atomicrmw.start ; I486-NEXT: # in Loop: Header=BB9_1 Depth=1 -; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload +; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload +; I486-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; I486-NEXT: movl %esi, {{[0-9]+}}(%esp) ; I486-NEXT: movl %eax, {{[0-9]+}}(%esp) ; I486-NEXT: movl %esp, %eax diff --git a/llvm/test/CodeGen/X86/atomic6432.ll b/llvm/test/CodeGen/X86/atomic6432.ll index 8ff5f338e1482..af2754f0bf95b 100644 --- a/llvm/test/CodeGen/X86/atomic6432.ll +++ b/llvm/test/CodeGen/X86/atomic6432.ll @@ -9,10 +9,10 @@ define void @atomic_fetch_add64() nounwind { ; X32-NEXT: pushl %ebx ; X32-NEXT: pushl %esi ; X32-NEXT: subl $40, %esp -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB0_1 ; X32-NEXT: .LBB0_1: # %atomicrmw.start14 ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -23,15 +23,17 @@ define void @atomic_fetch_add64() nounwind { ; X32-NEXT: movl %edx, %ecx ; X32-NEXT: adcl $0, %ecx ; X32-NEXT: lock cmpxchg8b sc64 +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB0_1 ; X32-NEXT: jmp .LBB0_2 ; X32-NEXT: .LBB0_2: # %atomicrmw.end13 -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB0_3 ; X32-NEXT: .LBB0_3: # %atomicrmw.start8 ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -42,15 +44,17 @@ define void @atomic_fetch_add64() nounwind { ; X32-NEXT: movl %edx, %ecx ; X32-NEXT: adcl $0, %ecx ; X32-NEXT: lock cmpxchg8b sc64 +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB0_3 ; X32-NEXT: jmp .LBB0_4 ; X32-NEXT: .LBB0_4: # %atomicrmw.end7 -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB0_5 ; X32-NEXT: .LBB0_5: # %atomicrmw.start2 ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -61,31 +65,35 @@ define void @atomic_fetch_add64() nounwind { ; X32-NEXT: movl %edx, %ecx ; X32-NEXT: adcl $0, %ecx ; X32-NEXT: lock cmpxchg8b sc64 -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %eax, (%esp) # 4-byte Spill +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB0_5 ; X32-NEXT: jmp .LBB0_6 ; X32-NEXT: .LBB0_6: # %atomicrmw.end1 -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax -; X32-NEXT: movl %eax, (%esp) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB0_7 ; X32-NEXT: .LBB0_7: # %atomicrmw.start ; X32-NEXT: # =>This Inner Loop Header: Depth=1 ; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload -; X32-NEXT: movl (%esp), %eax # 4-byte Reload -; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload +; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload +; X32-NEXT: movl (%esp), %esi # 4-byte Reload ; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X32-NEXT: movl %eax, %ebx ; X32-NEXT: addl %ecx, %ebx ; X32-NEXT: movl %edx, %ecx ; X32-NEXT: adcl %esi, %ecx ; X32-NEXT: lock cmpxchg8b sc64 -; X32-NEXT: movl %eax, (%esp) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB0_7 ; X32-NEXT: jmp .LBB0_8 ; X32-NEXT: .LBB0_8: # %atomicrmw.end @@ -107,10 +115,10 @@ define void @atomic_fetch_sub64() nounwind { ; X32-NEXT: pushl %ebx ; X32-NEXT: pushl %esi ; X32-NEXT: subl $40, %esp -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB1_1 ; X32-NEXT: .LBB1_1: # %atomicrmw.start14 ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -121,15 +129,17 @@ define void @atomic_fetch_sub64() nounwind { ; X32-NEXT: movl %edx, %ecx ; X32-NEXT: adcl $-1, %ecx ; X32-NEXT: lock cmpxchg8b sc64 +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB1_1 ; X32-NEXT: jmp .LBB1_2 ; X32-NEXT: .LBB1_2: # %atomicrmw.end13 -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB1_3 ; X32-NEXT: .LBB1_3: # %atomicrmw.start8 ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -140,15 +150,17 @@ define void @atomic_fetch_sub64() nounwind { ; X32-NEXT: movl %edx, %ecx ; X32-NEXT: adcl $-1, %ecx ; X32-NEXT: lock cmpxchg8b sc64 +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB1_3 ; X32-NEXT: jmp .LBB1_4 ; X32-NEXT: .LBB1_4: # %atomicrmw.end7 -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB1_5 ; X32-NEXT: .LBB1_5: # %atomicrmw.start2 ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -159,31 +171,35 @@ define void @atomic_fetch_sub64() nounwind { ; X32-NEXT: movl %edx, %ecx ; X32-NEXT: adcl $-1, %ecx ; X32-NEXT: lock cmpxchg8b sc64 -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %eax, (%esp) # 4-byte Spill +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB1_5 ; X32-NEXT: jmp .LBB1_6 ; X32-NEXT: .LBB1_6: # %atomicrmw.end1 -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax -; X32-NEXT: movl %eax, (%esp) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB1_7 ; X32-NEXT: .LBB1_7: # %atomicrmw.start ; X32-NEXT: # =>This Inner Loop Header: Depth=1 ; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload -; X32-NEXT: movl (%esp), %eax # 4-byte Reload -; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload +; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload +; X32-NEXT: movl (%esp), %esi # 4-byte Reload ; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X32-NEXT: movl %eax, %ebx ; X32-NEXT: subl %ecx, %ebx ; X32-NEXT: movl %edx, %ecx ; X32-NEXT: sbbl %esi, %ecx ; X32-NEXT: lock cmpxchg8b sc64 -; X32-NEXT: movl %eax, (%esp) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB1_7 ; X32-NEXT: jmp .LBB1_8 ; X32-NEXT: .LBB1_8: # %atomicrmw.end @@ -204,10 +220,10 @@ define void @atomic_fetch_and64() nounwind { ; X32-NEXT: pushl %ebx ; X32-NEXT: pushl %esi ; X32-NEXT: subl $32, %esp -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB2_1 ; X32-NEXT: .LBB2_1: # %atomicrmw.start8 ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -217,15 +233,17 @@ define void @atomic_fetch_and64() nounwind { ; X32-NEXT: andl $3, %ebx ; X32-NEXT: xorl %ecx, %ecx ; X32-NEXT: lock cmpxchg8b sc64 +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB2_1 ; X32-NEXT: jmp .LBB2_2 ; X32-NEXT: .LBB2_2: # %atomicrmw.end7 -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB2_3 ; X32-NEXT: .LBB2_3: # %atomicrmw.start2 ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -236,31 +254,35 @@ define void @atomic_fetch_and64() nounwind { ; X32-NEXT: movl %edx, %ecx ; X32-NEXT: andl $1, %ecx ; X32-NEXT: lock cmpxchg8b sc64 -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %eax, (%esp) # 4-byte Spill +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB2_3 ; X32-NEXT: jmp .LBB2_4 ; X32-NEXT: .LBB2_4: # %atomicrmw.end1 -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax -; X32-NEXT: movl %eax, (%esp) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB2_5 ; X32-NEXT: .LBB2_5: # %atomicrmw.start ; X32-NEXT: # =>This Inner Loop Header: Depth=1 ; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload -; X32-NEXT: movl (%esp), %eax # 4-byte Reload -; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload +; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload +; X32-NEXT: movl (%esp), %esi # 4-byte Reload ; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X32-NEXT: movl %eax, %ebx ; X32-NEXT: andl %ecx, %ebx ; X32-NEXT: movl %edx, %ecx ; X32-NEXT: andl %esi, %ecx ; X32-NEXT: lock cmpxchg8b sc64 -; X32-NEXT: movl %eax, (%esp) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB2_5 ; X32-NEXT: jmp .LBB2_6 ; X32-NEXT: .LBB2_6: # %atomicrmw.end @@ -280,10 +302,10 @@ define void @atomic_fetch_or64() nounwind { ; X32-NEXT: pushl %ebx ; X32-NEXT: pushl %esi ; X32-NEXT: subl $32, %esp -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB3_1 ; X32-NEXT: .LBB3_1: # %atomicrmw.start8 ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -293,15 +315,17 @@ define void @atomic_fetch_or64() nounwind { ; X32-NEXT: orl $3, %ebx ; X32-NEXT: movl %ecx, %edx ; X32-NEXT: lock cmpxchg8b sc64 +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB3_1 ; X32-NEXT: jmp .LBB3_2 ; X32-NEXT: .LBB3_2: # %atomicrmw.end7 -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB3_3 ; X32-NEXT: .LBB3_3: # %atomicrmw.start2 ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -312,31 +336,35 @@ define void @atomic_fetch_or64() nounwind { ; X32-NEXT: movl %edx, %ecx ; X32-NEXT: orl $1, %ecx ; X32-NEXT: lock cmpxchg8b sc64 -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %eax, (%esp) # 4-byte Spill +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB3_3 ; X32-NEXT: jmp .LBB3_4 ; X32-NEXT: .LBB3_4: # %atomicrmw.end1 -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax -; X32-NEXT: movl %eax, (%esp) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB3_5 ; X32-NEXT: .LBB3_5: # %atomicrmw.start ; X32-NEXT: # =>This Inner Loop Header: Depth=1 ; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload -; X32-NEXT: movl (%esp), %eax # 4-byte Reload -; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload +; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload +; X32-NEXT: movl (%esp), %esi # 4-byte Reload ; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X32-NEXT: movl %eax, %ebx ; X32-NEXT: orl %ecx, %ebx ; X32-NEXT: movl %edx, %ecx ; X32-NEXT: orl %esi, %ecx ; X32-NEXT: lock cmpxchg8b sc64 -; X32-NEXT: movl %eax, (%esp) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB3_5 ; X32-NEXT: jmp .LBB3_6 ; X32-NEXT: .LBB3_6: # %atomicrmw.end @@ -356,10 +384,10 @@ define void @atomic_fetch_xor64() nounwind { ; X32-NEXT: pushl %ebx ; X32-NEXT: pushl %esi ; X32-NEXT: subl $32, %esp -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB4_1 ; X32-NEXT: .LBB4_1: # %atomicrmw.start8 ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -369,15 +397,17 @@ define void @atomic_fetch_xor64() nounwind { ; X32-NEXT: xorl $3, %ebx ; X32-NEXT: movl %ecx, %edx ; X32-NEXT: lock cmpxchg8b sc64 +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB4_1 ; X32-NEXT: jmp .LBB4_2 ; X32-NEXT: .LBB4_2: # %atomicrmw.end7 -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB4_3 ; X32-NEXT: .LBB4_3: # %atomicrmw.start2 ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -388,31 +418,35 @@ define void @atomic_fetch_xor64() nounwind { ; X32-NEXT: movl %edx, %ecx ; X32-NEXT: xorl $1, %ecx ; X32-NEXT: lock cmpxchg8b sc64 -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %eax, (%esp) # 4-byte Spill +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB4_3 ; X32-NEXT: jmp .LBB4_4 ; X32-NEXT: .LBB4_4: # %atomicrmw.end1 -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax -; X32-NEXT: movl %eax, (%esp) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB4_5 ; X32-NEXT: .LBB4_5: # %atomicrmw.start ; X32-NEXT: # =>This Inner Loop Header: Depth=1 ; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload -; X32-NEXT: movl (%esp), %eax # 4-byte Reload -; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload +; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload +; X32-NEXT: movl (%esp), %esi # 4-byte Reload ; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X32-NEXT: movl %eax, %ebx ; X32-NEXT: xorl %ecx, %ebx ; X32-NEXT: movl %edx, %ecx ; X32-NEXT: xorl %esi, %ecx ; X32-NEXT: lock cmpxchg8b sc64 -; X32-NEXT: movl %eax, (%esp) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB4_5 ; X32-NEXT: jmp .LBB4_6 ; X32-NEXT: .LBB4_6: # %atomicrmw.end @@ -437,10 +471,10 @@ define void @atomic_fetch_nand64(i64 %x) nounwind { ; X32-NEXT: movl %eax, (%esp) # 4-byte Spill ; X32-NEXT: movl {{[0-9]+}}(%esp), %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB5_1 ; X32-NEXT: .LBB5_1: # %atomicrmw.start ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -455,8 +489,10 @@ define void @atomic_fetch_nand64(i64 %x) nounwind { ; X32-NEXT: notl %ebx ; X32-NEXT: notl %ecx ; X32-NEXT: lock cmpxchg8b sc64 +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB5_1 ; X32-NEXT: jmp .LBB5_2 ; X32-NEXT: .LBB5_2: # %atomicrmw.end @@ -479,10 +515,10 @@ define void @atomic_fetch_max64(i64 %x) nounwind { ; X32-NEXT: movl %eax, (%esp) # 4-byte Spill ; X32-NEXT: movl {{[0-9]+}}(%esp), %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB6_1 ; X32-NEXT: .LBB6_1: # %atomicrmw.start ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -497,8 +533,10 @@ define void @atomic_fetch_max64(i64 %x) nounwind { ; X32-NEXT: cmovll %edx, %ecx ; X32-NEXT: cmovll %eax, %ebx ; X32-NEXT: lock cmpxchg8b sc64 +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB6_1 ; X32-NEXT: jmp .LBB6_2 ; X32-NEXT: .LBB6_2: # %atomicrmw.end @@ -520,10 +558,10 @@ define void @atomic_fetch_min64(i64 %x) nounwind { ; X32-NEXT: movl %eax, (%esp) # 4-byte Spill ; X32-NEXT: movl {{[0-9]+}}(%esp), %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB7_1 ; X32-NEXT: .LBB7_1: # %atomicrmw.start ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -538,8 +576,10 @@ define void @atomic_fetch_min64(i64 %x) nounwind { ; X32-NEXT: cmovgel %edx, %ecx ; X32-NEXT: cmovgel %eax, %ebx ; X32-NEXT: lock cmpxchg8b sc64 +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB7_1 ; X32-NEXT: jmp .LBB7_2 ; X32-NEXT: .LBB7_2: # %atomicrmw.end @@ -561,10 +601,10 @@ define void @atomic_fetch_umax64(i64 %x) nounwind { ; X32-NEXT: movl %eax, (%esp) # 4-byte Spill ; X32-NEXT: movl {{[0-9]+}}(%esp), %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB8_1 ; X32-NEXT: .LBB8_1: # %atomicrmw.start ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -579,8 +619,10 @@ define void @atomic_fetch_umax64(i64 %x) nounwind { ; X32-NEXT: cmovbl %edx, %ecx ; X32-NEXT: cmovbl %eax, %ebx ; X32-NEXT: lock cmpxchg8b sc64 +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB8_1 ; X32-NEXT: jmp .LBB8_2 ; X32-NEXT: .LBB8_2: # %atomicrmw.end @@ -602,10 +644,10 @@ define void @atomic_fetch_umin64(i64 %x) nounwind { ; X32-NEXT: movl %eax, (%esp) # 4-byte Spill ; X32-NEXT: movl {{[0-9]+}}(%esp), %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB9_1 ; X32-NEXT: .LBB9_1: # %atomicrmw.start ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -620,8 +662,10 @@ define void @atomic_fetch_umin64(i64 %x) nounwind { ; X32-NEXT: cmovael %edx, %ecx ; X32-NEXT: cmovael %eax, %ebx ; X32-NEXT: lock cmpxchg8b sc64 +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB9_1 ; X32-NEXT: jmp .LBB9_2 ; X32-NEXT: .LBB9_2: # %atomicrmw.end @@ -670,10 +714,10 @@ define void @atomic_fetch_swap64(i64 %x) nounwind { ; X32-NEXT: movl %eax, (%esp) # 4-byte Spill ; X32-NEXT: movl {{[0-9]+}}(%esp), %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl sc64+4, %edx +; X32-NEXT: movl sc64+4, %eax +; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl sc64, %eax ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jmp .LBB12_1 ; X32-NEXT: .LBB12_1: # %atomicrmw.start ; X32-NEXT: # =>This Inner Loop Header: Depth=1 @@ -682,8 +726,10 @@ define void @atomic_fetch_swap64(i64 %x) nounwind { ; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload ; X32-NEXT: movl (%esp), %ecx # 4-byte Reload ; X32-NEXT: lock cmpxchg8b sc64 +; X32-NEXT: movl %eax, %ecx +; X32-NEXT: movl %edx, %eax +; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X32-NEXT: jne .LBB12_1 ; X32-NEXT: jmp .LBB12_2 ; X32-NEXT: .LBB12_2: # %atomicrmw.end diff --git a/llvm/test/CodeGen/X86/clobber_base_ptr.ll b/llvm/test/CodeGen/X86/clobber_base_ptr.ll index 2c39560f02d16..65e808191037d 100644 --- a/llvm/test/CodeGen/X86/clobber_base_ptr.ll +++ b/llvm/test/CodeGen/X86/clobber_base_ptr.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc < %s | FileCheck %s +; RUN: llc -regalloc=fast -regalloc-fast-ssa < %s | FileCheck %s target datalayout = "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:32-n8:16:32-a:0:32-S32" target triple = "i386-pc-windows-gnu" diff --git a/llvm/test/CodeGen/X86/fast-isel-gep.ll b/llvm/test/CodeGen/X86/fast-isel-gep.ll index 847559443e86d..5db231d3ad016 100644 --- a/llvm/test/CodeGen/X86/fast-isel-gep.ll +++ b/llvm/test/CodeGen/X86/fast-isel-gep.ll @@ -113,8 +113,8 @@ declare ptr @_ZNK18G__FastAllocString4dataEv() nounwind ; happen before the store. define i32 @test7(ptr %tmp1, i32 %tmp71, i32 %tmp63) nounwind { ; X64-LABEL: test7: -; X64: movl 8({{%rdi|%rcx}}), %eax -; X64: movl $4, 8({{%rdi|%rcx}}) +; X64: movl 8([[BASE:%r[a-z0-9]+]]), {{%e[a-z0-9]+}} +; X64: movl $4, 8([[BASE]]) %tmp29 = getelementptr inbounds {i32,i32,i32}, ptr %tmp1, i32 0, i32 2 diff --git a/llvm/test/CodeGen/X86/fast-isel-x86-64.ll b/llvm/test/CodeGen/X86/fast-isel-x86-64.ll index 4db7f0ccb4eae..fc5e08c8cd3b0 100644 --- a/llvm/test/CodeGen/X86/fast-isel-x86-64.ll +++ b/llvm/test/CodeGen/X86/fast-isel-x86-64.ll @@ -73,7 +73,7 @@ define void @test5(i32 %x, ptr %p) nounwind { ; CHECK-LABEL: test5: ; CHECK: movl $50000, %ecx -; CHECK: sarl %cl, %edi +; CHECK: sarl %cl, {{%e[a-z0-9]+}} ; CHECK: ret } diff --git a/llvm/test/CodeGen/X86/llc-pipeline-npm.ll b/llvm/test/CodeGen/X86/llc-pipeline-npm.ll index 114d1e0c1b050..bcd2bccc258c4 100644 --- a/llvm/test/CodeGen/X86/llc-pipeline-npm.ll +++ b/llvm/test/CodeGen/X86/llc-pipeline-npm.ll @@ -48,8 +48,6 @@ ; O0-NEXT: x86-flags-copy-lowering ; O0-NEXT: x86-dyn-alloca-expander ; O0-NEXT: x86-fast-pre-tile-config -; O0-NEXT: phi-node-elimination -; O0-NEXT: two-address-instruction ; O0-NEXT: regallocfast ; O0-NEXT: x86-lower-tile-copy ; O0-NEXT: x86-fp-stackifier @@ -245,8 +243,6 @@ ; O0-WINDOWS-NEXT: x86-flags-copy-lowering ; O0-WINDOWS-NEXT: x86-dyn-alloca-expander ; O0-WINDOWS-NEXT: x86-fast-pre-tile-config -; O0-WINDOWS-NEXT: phi-node-elimination -; O0-WINDOWS-NEXT: two-address-instruction ; O0-WINDOWS-NEXT: regallocfast ; O0-WINDOWS-NEXT: x86-lower-tile-copy ; O0-WINDOWS-NEXT: x86-fp-stackifier diff --git a/llvm/test/CodeGen/X86/pcsections-atomics.ll b/llvm/test/CodeGen/X86/pcsections-atomics.ll index 3b813d2e8a674..084d9d3cc6900 100644 --- a/llvm/test/CodeGen/X86/pcsections-atomics.ll +++ b/llvm/test/CodeGen/X86/pcsections-atomics.ll @@ -12597,10 +12597,10 @@ define void @atomic128_store_unordered(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection383: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection384: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection385: ; O0-NEXT: jmp .LBB203_1 ; O0-NEXT: .LBB203_1: # %atomicrmw.start @@ -12616,8 +12616,10 @@ define void @atomic128_store_unordered(ptr %a) { ; O0-NEXT: movl $42, %ebx ; O0-NEXT: .Lpcsection389: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection390: ; O0-NEXT: jne .LBB203_1 ; O0-NEXT: jmp .LBB203_2 @@ -12734,10 +12736,10 @@ define void @atomic128_store_monotonic(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection391: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection392: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection393: ; O0-NEXT: jmp .LBB204_1 ; O0-NEXT: .LBB204_1: # %atomicrmw.start @@ -12753,8 +12755,10 @@ define void @atomic128_store_monotonic(ptr %a) { ; O0-NEXT: movl $42, %ebx ; O0-NEXT: .Lpcsection397: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection398: ; O0-NEXT: jne .LBB204_1 ; O0-NEXT: jmp .LBB204_2 @@ -12871,10 +12875,10 @@ define void @atomic128_store_release(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection399: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection400: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection401: ; O0-NEXT: jmp .LBB205_1 ; O0-NEXT: .LBB205_1: # %atomicrmw.start @@ -12890,8 +12894,10 @@ define void @atomic128_store_release(ptr %a) { ; O0-NEXT: movl $42, %ebx ; O0-NEXT: .Lpcsection405: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection406: ; O0-NEXT: jne .LBB205_1 ; O0-NEXT: jmp .LBB205_2 @@ -13008,10 +13014,10 @@ define void @atomic128_store_seq_cst(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection407: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection408: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection409: ; O0-NEXT: jmp .LBB206_1 ; O0-NEXT: .LBB206_1: # %atomicrmw.start @@ -13027,8 +13033,10 @@ define void @atomic128_store_seq_cst(ptr %a) { ; O0-NEXT: movl $42, %ebx ; O0-NEXT: .Lpcsection413: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection414: ; O0-NEXT: jne .LBB206_1 ; O0-NEXT: jmp .LBB206_2 @@ -13194,10 +13202,10 @@ define void @atomic128_xchg_monotonic(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection416: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection417: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection418: ; O0-NEXT: jmp .LBB208_1 ; O0-NEXT: .LBB208_1: # %atomicrmw.start @@ -13213,8 +13221,10 @@ define void @atomic128_xchg_monotonic(ptr %a) { ; O0-NEXT: movl $42, %ebx ; O0-NEXT: .Lpcsection422: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection423: ; O0-NEXT: jne .LBB208_1 ; O0-NEXT: jmp .LBB208_2 @@ -13348,10 +13358,10 @@ define void @atomic128_add_monotonic(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection424: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection425: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection426: ; O0-NEXT: jmp .LBB209_1 ; O0-NEXT: .LBB209_1: # %atomicrmw.start @@ -13367,8 +13377,10 @@ define void @atomic128_add_monotonic(ptr %a) { ; O0-NEXT: adcq $0, %rcx ; O0-NEXT: .Lpcsection429: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection430: ; O0-NEXT: jne .LBB209_1 ; O0-NEXT: jmp .LBB209_2 @@ -13510,10 +13522,10 @@ define void @atomic128_sub_monotonic(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection431: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection432: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection433: ; O0-NEXT: jmp .LBB210_1 ; O0-NEXT: .LBB210_1: # %atomicrmw.start @@ -13529,8 +13541,10 @@ define void @atomic128_sub_monotonic(ptr %a) { ; O0-NEXT: adcq $-1, %rcx ; O0-NEXT: .Lpcsection436: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection437: ; O0-NEXT: jne .LBB210_1 ; O0-NEXT: jmp .LBB210_2 @@ -13672,10 +13686,10 @@ define void @atomic128_and_monotonic(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection438: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection439: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection440: ; O0-NEXT: jmp .LBB211_1 ; O0-NEXT: .LBB211_1: # %atomicrmw.start @@ -13693,8 +13707,10 @@ define void @atomic128_and_monotonic(ptr %a) { ; O0-NEXT: # kill: def $rcx killed $ecx ; O0-NEXT: .Lpcsection444: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection445: ; O0-NEXT: jne .LBB211_1 ; O0-NEXT: jmp .LBB211_2 @@ -13832,10 +13848,10 @@ define void @atomic128_or_monotonic(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection446: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection447: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection448: ; O0-NEXT: jmp .LBB212_1 ; O0-NEXT: .LBB212_1: # %atomicrmw.start @@ -13849,8 +13865,10 @@ define void @atomic128_or_monotonic(ptr %a) { ; O0-NEXT: movq %rcx, %rdx ; O0-NEXT: .Lpcsection450: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection451: ; O0-NEXT: jne .LBB212_1 ; O0-NEXT: jmp .LBB212_2 @@ -13984,10 +14002,10 @@ define void @atomic128_xor_monotonic(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection452: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection453: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection454: ; O0-NEXT: jmp .LBB213_1 ; O0-NEXT: .LBB213_1: # %atomicrmw.start @@ -14001,8 +14019,10 @@ define void @atomic128_xor_monotonic(ptr %a) { ; O0-NEXT: movq %rcx, %rdx ; O0-NEXT: .Lpcsection456: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection457: ; O0-NEXT: jne .LBB213_1 ; O0-NEXT: jmp .LBB213_2 @@ -14136,10 +14156,10 @@ define void @atomic128_nand_monotonic(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection458: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection459: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection460: ; O0-NEXT: jmp .LBB214_1 ; O0-NEXT: .LBB214_1: # %atomicrmw.start @@ -14159,8 +14179,10 @@ define void @atomic128_nand_monotonic(ptr %a) { ; O0-NEXT: movq $-1, %rcx ; O0-NEXT: .Lpcsection465: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection466: ; O0-NEXT: jne .LBB214_1 ; O0-NEXT: jmp .LBB214_2 @@ -14306,10 +14328,10 @@ define void @atomic128_xchg_acquire(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection467: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection468: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection469: ; O0-NEXT: jmp .LBB215_1 ; O0-NEXT: .LBB215_1: # %atomicrmw.start @@ -14325,8 +14347,10 @@ define void @atomic128_xchg_acquire(ptr %a) { ; O0-NEXT: movl $42, %ebx ; O0-NEXT: .Lpcsection473: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection474: ; O0-NEXT: jne .LBB215_1 ; O0-NEXT: jmp .LBB215_2 @@ -14460,10 +14484,10 @@ define void @atomic128_add_acquire(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection475: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection476: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection477: ; O0-NEXT: jmp .LBB216_1 ; O0-NEXT: .LBB216_1: # %atomicrmw.start @@ -14479,8 +14503,10 @@ define void @atomic128_add_acquire(ptr %a) { ; O0-NEXT: adcq $0, %rcx ; O0-NEXT: .Lpcsection480: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection481: ; O0-NEXT: jne .LBB216_1 ; O0-NEXT: jmp .LBB216_2 @@ -14622,10 +14648,10 @@ define void @atomic128_sub_acquire(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection482: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection483: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection484: ; O0-NEXT: jmp .LBB217_1 ; O0-NEXT: .LBB217_1: # %atomicrmw.start @@ -14641,8 +14667,10 @@ define void @atomic128_sub_acquire(ptr %a) { ; O0-NEXT: adcq $-1, %rcx ; O0-NEXT: .Lpcsection487: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection488: ; O0-NEXT: jne .LBB217_1 ; O0-NEXT: jmp .LBB217_2 @@ -14784,10 +14812,10 @@ define void @atomic128_and_acquire(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection489: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection490: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection491: ; O0-NEXT: jmp .LBB218_1 ; O0-NEXT: .LBB218_1: # %atomicrmw.start @@ -14805,8 +14833,10 @@ define void @atomic128_and_acquire(ptr %a) { ; O0-NEXT: # kill: def $rcx killed $ecx ; O0-NEXT: .Lpcsection495: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection496: ; O0-NEXT: jne .LBB218_1 ; O0-NEXT: jmp .LBB218_2 @@ -14944,10 +14974,10 @@ define void @atomic128_or_acquire(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection497: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection498: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection499: ; O0-NEXT: jmp .LBB219_1 ; O0-NEXT: .LBB219_1: # %atomicrmw.start @@ -14961,8 +14991,10 @@ define void @atomic128_or_acquire(ptr %a) { ; O0-NEXT: movq %rcx, %rdx ; O0-NEXT: .Lpcsection501: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection502: ; O0-NEXT: jne .LBB219_1 ; O0-NEXT: jmp .LBB219_2 @@ -15096,10 +15128,10 @@ define void @atomic128_xor_acquire(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection503: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection504: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection505: ; O0-NEXT: jmp .LBB220_1 ; O0-NEXT: .LBB220_1: # %atomicrmw.start @@ -15113,8 +15145,10 @@ define void @atomic128_xor_acquire(ptr %a) { ; O0-NEXT: movq %rcx, %rdx ; O0-NEXT: .Lpcsection507: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection508: ; O0-NEXT: jne .LBB220_1 ; O0-NEXT: jmp .LBB220_2 @@ -15248,10 +15282,10 @@ define void @atomic128_nand_acquire(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection509: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection510: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection511: ; O0-NEXT: jmp .LBB221_1 ; O0-NEXT: .LBB221_1: # %atomicrmw.start @@ -15271,8 +15305,10 @@ define void @atomic128_nand_acquire(ptr %a) { ; O0-NEXT: movq $-1, %rcx ; O0-NEXT: .Lpcsection516: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection517: ; O0-NEXT: jne .LBB221_1 ; O0-NEXT: jmp .LBB221_2 @@ -15418,10 +15454,10 @@ define void @atomic128_xchg_release(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection518: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection519: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection520: ; O0-NEXT: jmp .LBB222_1 ; O0-NEXT: .LBB222_1: # %atomicrmw.start @@ -15437,8 +15473,10 @@ define void @atomic128_xchg_release(ptr %a) { ; O0-NEXT: movl $42, %ebx ; O0-NEXT: .Lpcsection524: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection525: ; O0-NEXT: jne .LBB222_1 ; O0-NEXT: jmp .LBB222_2 @@ -15571,10 +15609,10 @@ define void @atomic128_add_release(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection526: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection527: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection528: ; O0-NEXT: jmp .LBB223_1 ; O0-NEXT: .LBB223_1: # %atomicrmw.start @@ -15590,8 +15628,10 @@ define void @atomic128_add_release(ptr %a) { ; O0-NEXT: adcq $0, %rcx ; O0-NEXT: .Lpcsection531: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection532: ; O0-NEXT: jne .LBB223_1 ; O0-NEXT: jmp .LBB223_2 @@ -15733,10 +15773,10 @@ define void @atomic128_sub_release(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection533: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection534: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection535: ; O0-NEXT: jmp .LBB224_1 ; O0-NEXT: .LBB224_1: # %atomicrmw.start @@ -15752,8 +15792,10 @@ define void @atomic128_sub_release(ptr %a) { ; O0-NEXT: adcq $-1, %rcx ; O0-NEXT: .Lpcsection538: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection539: ; O0-NEXT: jne .LBB224_1 ; O0-NEXT: jmp .LBB224_2 @@ -15895,10 +15937,10 @@ define void @atomic128_and_release(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection540: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection541: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection542: ; O0-NEXT: jmp .LBB225_1 ; O0-NEXT: .LBB225_1: # %atomicrmw.start @@ -15916,8 +15958,10 @@ define void @atomic128_and_release(ptr %a) { ; O0-NEXT: # kill: def $rcx killed $ecx ; O0-NEXT: .Lpcsection546: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection547: ; O0-NEXT: jne .LBB225_1 ; O0-NEXT: jmp .LBB225_2 @@ -16055,10 +16099,10 @@ define void @atomic128_or_release(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection548: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection549: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection550: ; O0-NEXT: jmp .LBB226_1 ; O0-NEXT: .LBB226_1: # %atomicrmw.start @@ -16072,8 +16116,10 @@ define void @atomic128_or_release(ptr %a) { ; O0-NEXT: movq %rcx, %rdx ; O0-NEXT: .Lpcsection552: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection553: ; O0-NEXT: jne .LBB226_1 ; O0-NEXT: jmp .LBB226_2 @@ -16207,10 +16253,10 @@ define void @atomic128_xor_release(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection554: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection555: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection556: ; O0-NEXT: jmp .LBB227_1 ; O0-NEXT: .LBB227_1: # %atomicrmw.start @@ -16224,8 +16270,10 @@ define void @atomic128_xor_release(ptr %a) { ; O0-NEXT: movq %rcx, %rdx ; O0-NEXT: .Lpcsection558: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection559: ; O0-NEXT: jne .LBB227_1 ; O0-NEXT: jmp .LBB227_2 @@ -16359,10 +16407,10 @@ define void @atomic128_nand_release(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection560: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection561: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection562: ; O0-NEXT: jmp .LBB228_1 ; O0-NEXT: .LBB228_1: # %atomicrmw.start @@ -16382,8 +16430,10 @@ define void @atomic128_nand_release(ptr %a) { ; O0-NEXT: movq $-1, %rcx ; O0-NEXT: .Lpcsection567: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection568: ; O0-NEXT: jne .LBB228_1 ; O0-NEXT: jmp .LBB228_2 @@ -16529,10 +16579,10 @@ define void @atomic128_xchg_acq_rel(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection569: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection570: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection571: ; O0-NEXT: jmp .LBB229_1 ; O0-NEXT: .LBB229_1: # %atomicrmw.start @@ -16548,8 +16598,10 @@ define void @atomic128_xchg_acq_rel(ptr %a) { ; O0-NEXT: movl $42, %ebx ; O0-NEXT: .Lpcsection575: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection576: ; O0-NEXT: jne .LBB229_1 ; O0-NEXT: jmp .LBB229_2 @@ -16683,10 +16735,10 @@ define void @atomic128_add_acq_rel(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection577: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection578: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection579: ; O0-NEXT: jmp .LBB230_1 ; O0-NEXT: .LBB230_1: # %atomicrmw.start @@ -16702,8 +16754,10 @@ define void @atomic128_add_acq_rel(ptr %a) { ; O0-NEXT: adcq $0, %rcx ; O0-NEXT: .Lpcsection582: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection583: ; O0-NEXT: jne .LBB230_1 ; O0-NEXT: jmp .LBB230_2 @@ -16845,10 +16899,10 @@ define void @atomic128_sub_acq_rel(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection584: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection585: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection586: ; O0-NEXT: jmp .LBB231_1 ; O0-NEXT: .LBB231_1: # %atomicrmw.start @@ -16864,8 +16918,10 @@ define void @atomic128_sub_acq_rel(ptr %a) { ; O0-NEXT: adcq $-1, %rcx ; O0-NEXT: .Lpcsection589: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection590: ; O0-NEXT: jne .LBB231_1 ; O0-NEXT: jmp .LBB231_2 @@ -17007,10 +17063,10 @@ define void @atomic128_and_acq_rel(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection591: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection592: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection593: ; O0-NEXT: jmp .LBB232_1 ; O0-NEXT: .LBB232_1: # %atomicrmw.start @@ -17028,8 +17084,10 @@ define void @atomic128_and_acq_rel(ptr %a) { ; O0-NEXT: # kill: def $rcx killed $ecx ; O0-NEXT: .Lpcsection597: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection598: ; O0-NEXT: jne .LBB232_1 ; O0-NEXT: jmp .LBB232_2 @@ -17167,10 +17225,10 @@ define void @atomic128_or_acq_rel(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection599: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection600: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection601: ; O0-NEXT: jmp .LBB233_1 ; O0-NEXT: .LBB233_1: # %atomicrmw.start @@ -17184,8 +17242,10 @@ define void @atomic128_or_acq_rel(ptr %a) { ; O0-NEXT: movq %rcx, %rdx ; O0-NEXT: .Lpcsection603: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection604: ; O0-NEXT: jne .LBB233_1 ; O0-NEXT: jmp .LBB233_2 @@ -17319,10 +17379,10 @@ define void @atomic128_xor_acq_rel(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection605: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection606: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection607: ; O0-NEXT: jmp .LBB234_1 ; O0-NEXT: .LBB234_1: # %atomicrmw.start @@ -17336,8 +17396,10 @@ define void @atomic128_xor_acq_rel(ptr %a) { ; O0-NEXT: movq %rcx, %rdx ; O0-NEXT: .Lpcsection609: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection610: ; O0-NEXT: jne .LBB234_1 ; O0-NEXT: jmp .LBB234_2 @@ -17471,10 +17533,10 @@ define void @atomic128_nand_acq_rel(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection611: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection612: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection613: ; O0-NEXT: jmp .LBB235_1 ; O0-NEXT: .LBB235_1: # %atomicrmw.start @@ -17494,8 +17556,10 @@ define void @atomic128_nand_acq_rel(ptr %a) { ; O0-NEXT: movq $-1, %rcx ; O0-NEXT: .Lpcsection618: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection619: ; O0-NEXT: jne .LBB235_1 ; O0-NEXT: jmp .LBB235_2 @@ -17641,10 +17705,10 @@ define void @atomic128_xchg_seq_cst(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection620: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection621: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection622: ; O0-NEXT: jmp .LBB236_1 ; O0-NEXT: .LBB236_1: # %atomicrmw.start @@ -17660,8 +17724,10 @@ define void @atomic128_xchg_seq_cst(ptr %a) { ; O0-NEXT: movl $42, %ebx ; O0-NEXT: .Lpcsection626: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection627: ; O0-NEXT: jne .LBB236_1 ; O0-NEXT: jmp .LBB236_2 @@ -17795,10 +17861,10 @@ define void @atomic128_add_seq_cst(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection628: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection629: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection630: ; O0-NEXT: jmp .LBB237_1 ; O0-NEXT: .LBB237_1: # %atomicrmw.start @@ -17814,8 +17880,10 @@ define void @atomic128_add_seq_cst(ptr %a) { ; O0-NEXT: adcq $0, %rcx ; O0-NEXT: .Lpcsection633: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection634: ; O0-NEXT: jne .LBB237_1 ; O0-NEXT: jmp .LBB237_2 @@ -17957,10 +18025,10 @@ define void @atomic128_sub_seq_cst(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection635: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection636: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection637: ; O0-NEXT: jmp .LBB238_1 ; O0-NEXT: .LBB238_1: # %atomicrmw.start @@ -17976,8 +18044,10 @@ define void @atomic128_sub_seq_cst(ptr %a) { ; O0-NEXT: adcq $-1, %rcx ; O0-NEXT: .Lpcsection640: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection641: ; O0-NEXT: jne .LBB238_1 ; O0-NEXT: jmp .LBB238_2 @@ -18119,10 +18189,10 @@ define void @atomic128_and_seq_cst(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection642: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection643: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection644: ; O0-NEXT: jmp .LBB239_1 ; O0-NEXT: .LBB239_1: # %atomicrmw.start @@ -18140,8 +18210,10 @@ define void @atomic128_and_seq_cst(ptr %a) { ; O0-NEXT: # kill: def $rcx killed $ecx ; O0-NEXT: .Lpcsection648: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection649: ; O0-NEXT: jne .LBB239_1 ; O0-NEXT: jmp .LBB239_2 @@ -18279,10 +18351,10 @@ define void @atomic128_or_seq_cst(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection650: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection651: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection652: ; O0-NEXT: jmp .LBB240_1 ; O0-NEXT: .LBB240_1: # %atomicrmw.start @@ -18296,8 +18368,10 @@ define void @atomic128_or_seq_cst(ptr %a) { ; O0-NEXT: movq %rcx, %rdx ; O0-NEXT: .Lpcsection654: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection655: ; O0-NEXT: jne .LBB240_1 ; O0-NEXT: jmp .LBB240_2 @@ -18431,10 +18505,10 @@ define void @atomic128_xor_seq_cst(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection656: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection657: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection658: ; O0-NEXT: jmp .LBB241_1 ; O0-NEXT: .LBB241_1: # %atomicrmw.start @@ -18448,8 +18522,10 @@ define void @atomic128_xor_seq_cst(ptr %a) { ; O0-NEXT: movq %rcx, %rdx ; O0-NEXT: .Lpcsection660: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection661: ; O0-NEXT: jne .LBB241_1 ; O0-NEXT: jmp .LBB241_2 @@ -18583,10 +18659,10 @@ define void @atomic128_nand_seq_cst(ptr %a) { ; O0-NEXT: movq foo(%rip), %rax ; O0-NEXT: .Lpcsection662: ; O0-NEXT: movq (%rdi), %rax +; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection663: -; O0-NEXT: movq 8(%rdi), %rdx +; O0-NEXT: movq 8(%rdi), %rax ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection664: ; O0-NEXT: jmp .LBB242_1 ; O0-NEXT: .LBB242_1: # %atomicrmw.start @@ -18606,8 +18682,10 @@ define void @atomic128_nand_seq_cst(ptr %a) { ; O0-NEXT: movq $-1, %rcx ; O0-NEXT: .Lpcsection669: ; O0-NEXT: lock cmpxchg16b (%rsi) +; O0-NEXT: movq %rax, %rcx +; O0-NEXT: movq %rdx, %rax +; O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; O0-NEXT: .Lpcsection670: ; O0-NEXT: jne .LBB242_1 ; O0-NEXT: jmp .LBB242_2 diff --git a/llvm/test/CodeGen/X86/pr11415.ll b/llvm/test/CodeGen/X86/pr11415.ll index c8c902c4e05bd..1e9ece4ed5d9e 100644 --- a/llvm/test/CodeGen/X86/pr11415.ll +++ b/llvm/test/CodeGen/X86/pr11415.ll @@ -1,4 +1,5 @@ ; RUN: llc -mtriple=x86_64-pc-linux %s -o - -regalloc=fast | FileCheck %s +; RUN: llc -mtriple=x86_64-pc-linux -O0 -regalloc-fast-ssa %s -o - | FileCheck --check-prefix=O0 %s ; We used to consider the early clobber in the second asm statement as ; defining %0 before it was read. This caused us to omit the @@ -13,6 +14,19 @@ ; CHECK-NEXT: movq -8(%rsp), %rax ; CHECK-NEXT: ret +; The early-clobber output is tied to the first input while the second input +; is the same value: the tied operand gets a copy of the value in the +; early-clobber register, and the other input keeps a distinct register. +; O0: #APP +; O0-NEXT: #NO_APP +; O0-NEXT: movq %rcx, %rdx +; O0-NEXT: movq %rdx, %rcx +; O0-NEXT: #APP +; O0-NEXT: #NO_APP +; O0-NEXT: movq %rcx, -8(%rsp) +; O0-NEXT: movq -8(%rsp), %rax +; O0-NEXT: retq + define i64 @foo() { entry: %0 = tail call i64 asm "", "={cx}"() nounwind diff --git a/llvm/test/CodeGen/X86/pr32241.ll b/llvm/test/CodeGen/X86/pr32241.ll index 30cd55e6774ee..d24f9b447b759 100644 --- a/llvm/test/CodeGen/X86/pr32241.ll +++ b/llvm/test/CodeGen/X86/pr32241.ll @@ -4,16 +4,16 @@ define i32 @_Z3foov() { ; CHECK-LABEL: _Z3foov: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: subl $16, %esp -; CHECK-NEXT: .cfi_def_cfa_offset 20 +; CHECK-NEXT: subl $12, %esp +; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: movw $10959, {{[0-9]+}}(%esp) # imm = 0x2ACF ; CHECK-NEXT: movw $-15498, {{[0-9]+}}(%esp) # imm = 0xC376 ; CHECK-NEXT: movw $19417, {{[0-9]+}}(%esp) # imm = 0x4BD9 ; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %eax -; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: movl %eax, (%esp) # 4-byte Spill ; CHECK-NEXT: movb $1, %al -; CHECK-NEXT: cmpw $0, {{[0-9]+}}(%esp) ; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill +; CHECK-NEXT: cmpw $0, {{[0-9]+}}(%esp) ; CHECK-NEXT: jne .LBB0_2 ; CHECK-NEXT: # %bb.1: # %lor.rhs ; CHECK-NEXT: xorl %eax, %eax @@ -21,18 +21,18 @@ define i32 @_Z3foov() { ; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill ; CHECK-NEXT: jmp .LBB0_2 ; CHECK-NEXT: .LBB0_2: # %lor.end -; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: movl (%esp), %eax # 4-byte Reload ; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload ; CHECK-NEXT: andb $1, %cl ; CHECK-NEXT: movzbl %cl, %ecx ; CHECK-NEXT: cmpl %ecx, %eax ; CHECK-NEXT: setl %al ; CHECK-NEXT: andb $1, %al -; CHECK-NEXT: movzbl %al, %ecx -; CHECK-NEXT: xorl $-1, %ecx -; CHECK-NEXT: movb $1, %al -; CHECK-NEXT: cmpl $0, %ecx -; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill +; CHECK-NEXT: movzbl %al, %eax +; CHECK-NEXT: xorl $-1, %eax +; CHECK-NEXT: movb $1, %cl +; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill +; CHECK-NEXT: cmpl $0, %eax ; CHECK-NEXT: jne .LBB0_4 ; CHECK-NEXT: # %bb.3: # %lor.rhs4 ; CHECK-NEXT: xorl %eax, %eax @@ -46,7 +46,7 @@ define i32 @_Z3foov() { ; CHECK-NEXT: # kill: def $ax killed $ax killed $eax ; CHECK-NEXT: movw %ax, {{[0-9]+}}(%esp) ; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %eax -; CHECK-NEXT: addl $16, %esp +; CHECK-NEXT: addl $12, %esp ; CHECK-NEXT: .cfi_def_cfa_offset 4 ; CHECK-NEXT: retl entry: diff --git a/llvm/test/CodeGen/X86/pr32256.ll b/llvm/test/CodeGen/X86/pr32256.ll index 225a3af551a2c..09f7d92c1db2e 100644 --- a/llvm/test/CodeGen/X86/pr32256.ll +++ b/llvm/test/CodeGen/X86/pr32256.ll @@ -9,12 +9,12 @@ define void @_Z1av() { ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: subl $2, %esp ; CHECK-NEXT: .cfi_def_cfa_offset 6 -; CHECK-NEXT: movb c, %cl -; CHECK-NEXT: xorb $-1, %cl -; CHECK-NEXT: xorl %eax, %eax -; CHECK-NEXT: # kill: def $al killed $al killed $eax -; CHECK-NEXT: testb $1, %cl -; CHECK-NEXT: movb %al, (%esp) # 1-byte Spill +; CHECK-NEXT: movb c, %al +; CHECK-NEXT: xorb $-1, %al +; CHECK-NEXT: xorl %ecx, %ecx +; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx +; CHECK-NEXT: movb %cl, (%esp) # 1-byte Spill +; CHECK-NEXT: testb $1, %al ; CHECK-NEXT: jne .LBB0_1 ; CHECK-NEXT: jmp .LBB0_2 ; CHECK-NEXT: .LBB0_1: # %land.rhs diff --git a/llvm/test/CodeGen/X86/pr32345.ll b/llvm/test/CodeGen/X86/pr32345.ll index c7405e982660c..c638196035ea5 100644 --- a/llvm/test/CodeGen/X86/pr32345.ll +++ b/llvm/test/CodeGen/X86/pr32345.ll @@ -41,31 +41,33 @@ define void @foo() { ; X86-O0-NEXT: .cfi_offset %ebp, -8 ; X86-O0-NEXT: movl %esp, %ebp ; X86-O0-NEXT: .cfi_def_cfa_register %ebp +; X86-O0-NEXT: pushl %esi ; X86-O0-NEXT: andl $-8, %esp ; X86-O0-NEXT: subl $24, %esp +; X86-O0-NEXT: .cfi_offset %esi, -12 ; X86-O0-NEXT: movzwl var_22, %eax ; X86-O0-NEXT: movl %eax, {{[0-9]+}}(%esp) ; X86-O0-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-O0-NEXT: movzwl var_22, %edx -; X86-O0-NEXT: movb var_27, %cl -; X86-O0-NEXT: addb $30, %cl -; X86-O0-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill -; X86-O0-NEXT: xorl %eax, %eax -; X86-O0-NEXT: shrdl %cl, %eax, %edx -; X86-O0-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload -; X86-O0-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-O0-NEXT: testb $32, %cl -; X86-O0-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-O0-NEXT: movb var_27, %al +; X86-O0-NEXT: addb $30, %al +; X86-O0-NEXT: xorl %esi, %esi +; X86-O0-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-O0-NEXT: movb %al, %cl +; X86-O0-NEXT: shrdl %cl, %esi, %edx +; X86-O0-NEXT: movl %edx, (%esp) # 4-byte Spill +; X86-O0-NEXT: testb $32, %al ; X86-O0-NEXT: jne .LBB0_2 ; X86-O0-NEXT: # %bb.1: # %bb -; X86-O0-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload +; X86-O0-NEXT: movl (%esp), %eax # 4-byte Reload ; X86-O0-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-O0-NEXT: .LBB0_2: # %bb ; X86-O0-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; X86-O0-NEXT: movb %al, %cl ; X86-O0-NEXT: # implicit-def: $eax ; X86-O0-NEXT: movb %cl, (%eax) -; X86-O0-NEXT: movl %ebp, %esp +; X86-O0-NEXT: leal -4(%ebp), %esp +; X86-O0-NEXT: popl %esi ; X86-O0-NEXT: popl %ebp ; X86-O0-NEXT: .cfi_def_cfa %esp, 4 ; X86-O0-NEXT: retl diff --git a/llvm/test/CodeGen/X86/pr47000.ll b/llvm/test/CodeGen/X86/pr47000.ll index c0cd5a8d7f50c..d4254322c81ee 100644 --- a/llvm/test/CodeGen/X86/pr47000.ll +++ b/llvm/test/CodeGen/X86/pr47000.ll @@ -15,13 +15,13 @@ define <4 x half> @doTheTestMod(<4 x half> %0, <4 x half> %1) nounwind { ; CHECK-NEXT: movaps %xmm0, %xmm3 ; CHECK-NEXT: psrlq $48, %xmm3 ; CHECK-NEXT: movaps %xmm0, %xmm2 -; CHECK-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1,1,1] +; CHECK-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1] ; CHECK-NEXT: psrld $16, %xmm0 ; CHECK-NEXT: movaps %xmm6, %xmm7 ; CHECK-NEXT: movaps %xmm6, %xmm4 ; CHECK-NEXT: psrlq $48, %xmm4 ; CHECK-NEXT: movaps %xmm6, %xmm5 -; CHECK-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,1,1,1] +; CHECK-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,1],xmm6[1,1] ; CHECK-NEXT: psrld $16, %xmm6 ; CHECK-NEXT: pextrw $0, %xmm7, %eax ; CHECK-NEXT: # kill: def $ax killed $ax killed $eax diff --git a/llvm/test/CodeGen/X86/pr49587.ll b/llvm/test/CodeGen/X86/pr49587.ll index 7dc54a526608c..962d8dd3547c2 100644 --- a/llvm/test/CodeGen/X86/pr49587.ll +++ b/llvm/test/CodeGen/X86/pr49587.ll @@ -5,10 +5,10 @@ define i32 @test(i64 %arg) nounwind { ; CHECK-LABEL: test: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: subq $1, %rdi -; CHECK-NEXT: setb %cl -; CHECK-NEXT: xorl %eax, %eax -; CHECK-NEXT: testb $1, %cl -; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: setb %al +; CHECK-NEXT: xorl %ecx, %ecx +; CHECK-NEXT: movl %ecx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: testb $1, %al ; CHECK-NEXT: jne .LBB0_2 ; CHECK-NEXT: # %bb.1: # %no_overflow ; CHECK-NEXT: movl $1, %eax diff --git a/llvm/test/CodeGen/X86/regallocfast-ssa-phi-cycle.ll b/llvm/test/CodeGen/X86/regallocfast-ssa-phi-cycle.ll new file mode 100644 index 0000000000000..f956658fb9bd5 --- /dev/null +++ b/llvm/test/CodeGen/X86/regallocfast-ssa-phi-cycle.ll @@ -0,0 +1,149 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -mtriple=x86_64-linux -O0 -regalloc-fast-ssa -verify-machineinstrs < %s | FileCheck %s + +; Loop-carried PHI webs whose edge moves form permutation cycles: the +; parallel-move resolver must park one value in a held register per cycle. And a +; PHI on an INLINEASM_BR edge, whose copy has to precede the asm rather than sit +; before the terminators. + +; Three values rotated across the back edge. +define i64 @rotate3(i64 %n, i64 %a0, i64 %b0, i64 %c0) nounwind { +; CHECK-LABEL: rotate3: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movq %rcx, %rax +; CHECK-NEXT: movq %rdx, %rcx +; CHECK-NEXT: movq %rsi, %rdx +; CHECK-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: xorl %esi, %esi +; CHECK-NEXT: # kill: def $rsi killed $esi +; CHECK-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: jmp .LBB0_1 +; CHECK-NEXT: .LBB0_1: # %loop +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload +; CHECK-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: addq $1, %rax +; CHECK-NEXT: cmpq %rdi, %rax +; CHECK-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: jb .LBB0_1 +; CHECK-NEXT: # %bb.2: # %exit +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload +; CHECK-NEXT: addq %rdx, %rax +; CHECK-NEXT: addq %rcx, %rax +; CHECK-NEXT: retq +entry: + br label %loop + +loop: + %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] + %a = phi i64 [ %a0, %entry ], [ %b, %loop ] + %b = phi i64 [ %b0, %entry ], [ %c, %loop ] + %c = phi i64 [ %c0, %entry ], [ %a, %loop ] + %i.next = add i64 %i, 1 + %cond = icmp ult i64 %i.next, %n + br i1 %cond, label %loop, label %exit + +exit: + %s1 = add i64 %a, %b + %s2 = add i64 %s1, %c + ret i64 %s2 +} + +; Two values swapped across the back edge. +define i64 @swap2(i64 %n, i64 %a0, i64 %b0) nounwind { +; CHECK-LABEL: swap2: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movq %rdx, %rax +; CHECK-NEXT: movq %rsi, %rcx +; CHECK-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: xorl %edx, %edx +; CHECK-NEXT: # kill: def $rdx killed $edx +; CHECK-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: jmp .LBB1_1 +; CHECK-NEXT: .LBB1_1: # %loop +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload +; CHECK-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: addq $1, %rax +; CHECK-NEXT: cmpq %rsi, %rax +; CHECK-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: jb .LBB1_1 +; CHECK-NEXT: # %bb.2: # %exit +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload +; CHECK-NEXT: subq %rcx, %rax +; CHECK-NEXT: retq +entry: + br label %loop + +loop: + %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] + %a = phi i64 [ %a0, %entry ], [ %b, %loop ] + %b = phi i64 [ %b0, %entry ], [ %a, %loop ] + %i.next = add i64 %i, 1 + %cond = icmp ult i64 %i.next, %n + br i1 %cond, label %loop, label %exit + +exit: + %s = sub i64 %a, %b + ret i64 %s +} + +; PHI receiving a value over a callbr indirect edge. +define i32 @callbr_phi(i32 %x) nounwind { +; CHECK-LABEL: callbr_phi: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %edi, %eax +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: movl %eax, %edi +; CHECK-NEXT: incl %edi +; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: #APP +; CHECK-NEXT: #NO_APP +; CHECK-NEXT: jmp .LBB2_1 +; CHECK-NEXT: .LBB2_1: # %fall +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: jmp .LBB2_2 +; CHECK-NEXT: .LBB2_2: # Inline asm indirect target +; CHECK-NEXT: # %ind +; CHECK-NEXT: # Label of block must be emitted +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload +; CHECK-NEXT: addl %ecx, %eax +; CHECK-NEXT: retq +entry: + %y = add i32 %x, 1 + callbr void asm sideeffect "", "!i"() to label %fall [label %ind] + +fall: + br label %ind + +ind: + %p = phi i32 [ %x, %entry ], [ %y, %fall ] + %q = add i32 %p, %x + ret i32 %q +} diff --git a/llvm/test/CodeGen/X86/regallocfast-ssa-phi-eh.ll b/llvm/test/CodeGen/X86/regallocfast-ssa-phi-eh.ll new file mode 100644 index 0000000000000..ae107a2af4315 --- /dev/null +++ b/llvm/test/CodeGen/X86/regallocfast-ssa-phi-eh.ll @@ -0,0 +1,113 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -mtriple=x86_64-linux -O0 -regalloc-fast-ssa -verify-machineinstrs < %s | FileCheck %s + +; PHIs reached over an exceptional edge. findPHICopyInsertPoint places the +; transfer before the call that may throw rather than before the terminators, +; so it also runs when the call returns normally. + +declare void @may_throw(i32) +declare i32 @__gxx_personality_v0(...) + +; The landing pad and the normal destination merge the same two values. The +; landing pad's transfers precede the calls, the normal destination's follow +; them. +define i32 @phi_on_eh_edge(i32 %a, i32 %b, i1 %c) personality ptr @__gxx_personality_v0 { +; CHECK-LABEL: phi_on_eh_edge: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: subq $24, %rsp +; CHECK-NEXT: .cfi_def_cfa_offset 32 +; CHECK-NEXT: movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: movb %dl, %al +; CHECK-NEXT: testb $1, %al +; CHECK-NEXT: jne .LBB0_1 +; CHECK-NEXT: jmp .LBB0_2 +; CHECK-NEXT: .LBB0_1: # %try1 +; CHECK-NEXT: .Ltmp2: # EH_LABEL +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: movl %eax, %edi +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: callq may_throw@PLT +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: .Ltmp3: # EH_LABEL +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: jmp .LBB0_3 +; CHECK-NEXT: .LBB0_2: # %try2 +; CHECK-NEXT: .Ltmp0: # EH_LABEL +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: movl %eax, %edi +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: callq may_throw@PLT +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: .Ltmp1: # EH_LABEL +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: jmp .LBB0_3 +; CHECK-NEXT: .LBB0_3: # %cont +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: addq $24, %rsp +; CHECK-NEXT: .cfi_def_cfa_offset 8 +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB0_4: # %lpad +; CHECK-NEXT: .cfi_def_cfa_offset 32 +; CHECK-NEXT: .Ltmp4: # EH_LABEL +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: addq $24, %rsp +; CHECK-NEXT: .cfi_def_cfa_offset 8 +; CHECK-NEXT: retq +entry: + br i1 %c, label %try1, label %try2 + +try1: + invoke void @may_throw(i32 %a) to label %cont unwind label %lpad + +try2: + invoke void @may_throw(i32 %b) to label %cont unwind label %lpad + +cont: + %n = phi i32 [ %a, %try1 ], [ %b, %try2 ] + ret i32 %n + +lpad: + %e = phi i32 [ %a, %try1 ], [ %b, %try2 ] + %pad = landingpad { ptr, i32 } cleanup + ret i32 %e +} + +; A landing pad whose PHI has one predecessor, with the incoming value defined +; there and dying at the PHI. The transfer still precedes the call. +define i32 @eh_phi_source_dies(i32 %a, i32 %b) personality ptr @__gxx_personality_v0 { +; CHECK-LABEL: eh_phi_source_dies: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: pushq %rax +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: movl %edi, %eax +; CHECK-NEXT: addl %esi, %eax +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: .Ltmp5: # EH_LABEL +; CHECK-NEXT: callq may_throw@PLT +; CHECK-NEXT: .Ltmp6: # EH_LABEL +; CHECK-NEXT: jmp .LBB1_1 +; CHECK-NEXT: .LBB1_1: # %cont +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: popq %rcx +; CHECK-NEXT: .cfi_def_cfa_offset 8 +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB1_2: # %lpad +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .Ltmp7: # EH_LABEL +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: popq %rcx +; CHECK-NEXT: .cfi_def_cfa_offset 8 +; CHECK-NEXT: retq +entry: + %s = add i32 %a, %b + invoke void @may_throw(i32 %a) to label %cont unwind label %lpad + +cont: + ret i32 0 + +lpad: + %e = phi i32 [ %s, %entry ] + %pad = landingpad { ptr, i32 } cleanup + ret i32 %e +} diff --git a/llvm/test/CodeGen/X86/regallocfast-ssa-phi-live-out.ll b/llvm/test/CodeGen/X86/regallocfast-ssa-phi-live-out.ll new file mode 100644 index 0000000000000..681c164ece612 --- /dev/null +++ b/llvm/test/CodeGen/X86/regallocfast-ssa-phi-live-out.ll @@ -0,0 +1,182 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -mtriple=x86_64-linux -O0 -regalloc-fast-ssa -verify-machineinstrs < %s | FileCheck %s + +; A PHI is lowered by writing the incoming value into the destination's stack +; slot in the predecessor. That write also executes when control leaves the +; predecessor by another edge, and lands ahead of the end of the predecessor +; when the source's slot is shared, so it is only sound while nothing can read +; the destination's slot in between. +; +; The generated checks wildcard stack offsets, so they do not show which slot +; an access uses: a regression appears here as a missing copy, not as an +; obviously wrong reload. Read the assembly before regenerating them. + +; %dst escapes the loop, so the back edge must not write its slot. Returns n-1. +define i32 @live_out_of_loop(i32 %n) nounwind { +; CHECK-LABEL: live_out_of_loop: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: jmp .LBB0_1 +; CHECK-NEXT: .LBB0_1: # %loop +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: addl $1, %eax +; CHECK-NEXT: cmpl %ecx, %eax +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: jl .LBB0_1 +; CHECK-NEXT: # %bb.2: # %exit +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: retq +entry: + br label %loop + +loop: ; preds = %entry, %loop + %dst = phi i32 [ 0, %entry ], [ %next, %loop ] + %next = add i32 %dst, 1 + %cmp = icmp slt i32 %next, %n + br i1 %cmp, label %loop, label %exit + +exit: ; preds = %loop + ret i32 %dst +} + +; Same CFG, but every use of %dst is in the PHI's own block, so the slot is +; read only after an edge that writes it: the copy-free lowering applies even +; over this critical edge. +define i32 @uses_in_phi_block(i32 %n) nounwind { +; CHECK-LABEL: uses_in_phi_block: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: jmp .LBB1_1 +; CHECK-NEXT: .LBB1_1: # %loop +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload +; CHECK-NEXT: addl $1, %eax +; CHECK-NEXT: cmpl %ecx, %eax +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: jl .LBB1_1 +; CHECK-NEXT: # %bb.2: # %exit +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: retq +entry: + br label %loop + +loop: ; preds = %entry, %loop + %dst = phi i32 [ 0, %entry ], [ %next, %loop ] + %next = add i32 %dst, 1 + %cmp = icmp slt i32 %next, %n + br i1 %cmp, label %loop, label %exit + +exit: ; preds = %loop + ret i32 %n +} + +; %x escapes the loop, but the latch's only successor is the header, so the +; write has no other edge to escape on: also copy-free. +define i32 @single_successor_latch(i32 %n) nounwind { +; CHECK-LABEL: single_successor_latch: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: jmp .LBB2_1 +; CHECK-NEXT: .LBB2_1: # %header +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload +; CHECK-NEXT: cmpl %ecx, %eax +; CHECK-NEXT: jge .LBB2_3 +; CHECK-NEXT: # %bb.2: # %latch +; CHECK-NEXT: # in Loop: Header=BB2_1 Depth=1 +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: addl $1, %eax +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: jmp .LBB2_1 +; CHECK-NEXT: .LBB2_3: # %exit +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: retq +entry: + br label %header + +header: ; preds = %entry, %latch + %x = phi i32 [ 0, %entry ], [ %next, %latch ] + %cmp = icmp slt i32 %x, %n + br i1 %cmp, label %latch, label %exit + +latch: ; preds = %header + %next = add i32 %x, 1 + br label %header + +exit: ; preds = %header + ret i32 %x +} + +; The edge itself is safe here, so the PHI keeps the slot lowering, but %next +; must not take over %dst's slot: that store lands at %next's definition, and +; the clobbers force %dst to be reloaded from the slot after it. %use2 is +; %dst + 7, not %next + 7. +define i32 @shared_slot_dst_used_in_pred(i32 %n, ptr %p) nounwind { +; CHECK-LABEL: shared_slot_dst_used_in_pred: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: pushq %r15 +; CHECK-NEXT: pushq %r14 +; CHECK-NEXT: pushq %r13 +; CHECK-NEXT: pushq %r12 +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: jmp .LBB3_2 +; CHECK-NEXT: .LBB3_1: # %latch +; CHECK-NEXT: # in Loop: Header=BB3_2 Depth=1 +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: addl $1, %eax +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: #APP +; CHECK-NEXT: #NO_APP +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 4-byte Reload +; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: addl $7, %edx +; CHECK-NEXT: movl %edx, (%rcx) +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: .LBB3_2: # %header +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload +; CHECK-NEXT: cmpl %ecx, %eax +; CHECK-NEXT: jl .LBB3_1 +; CHECK-NEXT: # %bb.3: # %exit +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: popq %r12 +; CHECK-NEXT: popq %r13 +; CHECK-NEXT: popq %r14 +; CHECK-NEXT: popq %r15 +; CHECK-NEXT: retq +entry: + br label %header + +latch: ; preds = %header + %next = add i32 %dst, 1 + call void asm sideeffect "", "~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"() + %use2 = add i32 %dst, 7 + store volatile i32 %use2, ptr %p + br label %header + +header: ; preds = %entry, %latch + %dst = phi i32 [ 0, %entry ], [ %next, %latch ] + %cmp = icmp slt i32 %dst, %n + br i1 %cmp, label %latch, label %exit + +exit: ; preds = %header + ret i32 %dst +} diff --git a/llvm/test/CodeGen/X86/regallocfast-ssa-phi-regclass.ll b/llvm/test/CodeGen/X86/regallocfast-ssa-phi-regclass.ll new file mode 100644 index 0000000000000..486ba89dc862d --- /dev/null +++ b/llvm/test/CodeGen/X86/regallocfast-ssa-phi-regclass.ll @@ -0,0 +1,61 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=x86_64 -O2 -regalloc=fast -regalloc-fast-ssa -verify-machineinstrs < %s | FileCheck %s + +;; A PHI whose operand has a different register class than its destination +;; cannot use the stack-slot lowering: the operand's slot is sized for its own +;; class, while an edge move reads it using the destination's. Here optimized +;; instruction selection gives the zero arm fr32 (FsFLD0SS) and the PHI vr128, +;; because the result feeds a bitmask; reading a 4-byte slot as vr128 is a +;; 16-byte load out of it. Such PHIs go through a virtual register instead. + +;; -O2 is deliberate: unoptimized selection does not produce this shape, so the +;; SSA path has to be forced past the optimization-level restriction. + +declare float @llvm.fabs.f32(float) + +define float @phi_regclass_mismatch(i32 %n, float %x) nounwind { +; CHECK-LABEL: phi_regclass_mismatch: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: xorps %xmm0, %xmm0 +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: movaps {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN] +; CHECK-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: .p2align 4 +; CHECK-NEXT: .LBB0_1: # %loop +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload +; CHECK-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 4-byte Reload +; CHECK-NEXT: # xmm1 = mem[0],zero,zero,zero +; CHECK-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; CHECK-NEXT: andps %xmm2, %xmm0 +; CHECK-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: addss %xmm1, %xmm0 +; CHECK-NEXT: incl %eax +; CHECK-NEXT: cmpl %ecx, %eax +; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: jl .LBB0_1 +; CHECK-NEXT: # %bb.2: # %exit +; CHECK-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-NEXT: retq +entry: + br label %loop + +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %acc = phi float [ 0.000000e+00, %entry ], [ %next, %loop ] + %r = call float @llvm.fabs.f32(float %acc) + %next = fadd float %r, %x + %i.next = add i32 %i, 1 + %c = icmp slt i32 %i.next, %n + br i1 %c, label %loop, label %exit + +exit: + ret float %r +} diff --git a/llvm/test/CodeGen/X86/regallocfast-ssa-phi-slot-share.ll b/llvm/test/CodeGen/X86/regallocfast-ssa-phi-slot-share.ll new file mode 100644 index 0000000000000..37fc92424b18f --- /dev/null +++ b/llvm/test/CodeGen/X86/regallocfast-ssa-phi-slot-share.ll @@ -0,0 +1,88 @@ +; RUN: llc -mtriple=x86_64-linux -O0 -regalloc-fast-ssa -verify-machineinstrs < %s | FileCheck %s + +;; A PHI source defined in the incoming predecessor and dying there takes the +;; destination's stack slot, so its spill is the edge transfer and no copy is +;; needed. That spill lands before the end of the predecessor, so nothing may +;; read the destination's slot in between -- a PHI in a successor does, on the +;; edge. +;; +;; Offsets are spelled out in the checks on purpose: which slot each store +;; targets is the property under test, and update_llc_test_checks.py wildcards +;; spill offsets unconditionally, so do not regenerate this file with it. + +;; Three edge copies (no sharing): +;; %k1 into %k: %k is read in %latch, by the add. +;; %s into %d: %o reads %d on the %latch -> %head edge, which happens in +;; %latch, after a shared spill of %s would have overwritten %d's slot. +;; %d into %o: %d is not defined in %latch. +;; The copy reading %d is ordered ahead of the one writing it. +define i32 @phi_reads_dst_on_edge(i32 %n) nounwind { +; CHECK-LABEL: phi_reads_dst_on_edge: +; CHECK: movl $100, %eax +; CHECK-NEXT: movl %eax, -8(%rsp) +; CHECK: # %latch +; CHECK: movl -8(%rsp), %ecx +; CHECK-NEXT: movl -4(%rsp), %edx +; CHECK-NEXT: addl $1, %edx +; CHECK-NEXT: movl %edx, %eax +; CHECK-NEXT: addl $200, %eax +; CHECK-NEXT: movl %edx, -4(%rsp) +; CHECK-NEXT: movl %ecx, -12(%rsp) +; CHECK-NEXT: movl %eax, -8(%rsp) +; CHECK: # %exit +; CHECK-NEXT: movl -12(%rsp), %eax +; CHECK-NEXT: retq +entry: + br label %head + +head: + %k = phi i32 [ 0, %entry ], [ %k1, %latch ] + %d = phi i32 [ 100, %entry ], [ %s, %latch ] + %o = phi i32 [ -1, %entry ], [ %d, %latch ] + %c = icmp slt i32 %k, %n + br i1 %c, label %latch, label %exit + +latch: + %k1 = add i32 %k, 1 + %s = add i32 %k1, 200 + br label %head + +exit: + ret i32 %o +} + +;; Same loop without %o. Nothing reads %d in %latch or on the edge out of it, so +;; %s takes %d's slot and the add of $200 stores straight into it -- no edge +;; copy for %d. %k1 still cannot take %k's slot, %k being read in %latch, so a +;; single copy remains. +define i32 @share_across_back_edge(i32 %n) nounwind { +; CHECK-LABEL: share_across_back_edge: +; CHECK: movl $100, %eax +; CHECK-NEXT: movl %eax, -8(%rsp) +; CHECK: # %latch +; CHECK: movl -4(%rsp), %eax +; CHECK-NEXT: addl $1, %eax +; CHECK-NEXT: movl %eax, %ecx +; CHECK-NEXT: addl $200, %ecx +; CHECK-NEXT: movl %ecx, -8(%rsp) +; CHECK-NEXT: movl %eax, -4(%rsp) +; CHECK: # %exit +; CHECK-NEXT: movl -8(%rsp), %eax +; CHECK-NEXT: retq +entry: + br label %head + +head: + %k = phi i32 [ 0, %entry ], [ %k1, %latch ] + %d = phi i32 [ 100, %entry ], [ %s, %latch ] + %c = icmp slt i32 %k, %n + br i1 %c, label %latch, label %exit + +latch: + %k1 = add i32 %k, 1 + %s = add i32 %k1, 200 + br label %head + +exit: + ret i32 %d +} diff --git a/llvm/test/CodeGen/X86/regallocfast-ssa-tied.ll b/llvm/test/CodeGen/X86/regallocfast-ssa-tied.ll new file mode 100644 index 0000000000000..27c067d6a547f --- /dev/null +++ b/llvm/test/CodeGen/X86/regallocfast-ssa-tied.ll @@ -0,0 +1,91 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -mtriple=x86_64-linux -O0 -regalloc-fast-ssa -verify-machineinstrs < %s | FileCheck %s + +;; Tied operands, which the SSA path rewrites itself instead of leaving to +;; TwoAddressInstructionPass. + +;; The tied use dies at the instruction, so it takes over the tied def's +;; register and no copy is needed. +define i32 @tied_use_dies(i32 %a, i32 %b) nounwind { +; CHECK-LABEL: tied_use_dies: +; CHECK: # %bb.0: +; CHECK-NEXT: movl %edi, %eax +; CHECK-NEXT: addl %esi, %eax +; CHECK-NEXT: retq + %c = add i32 %a, %b + ret i32 %c +} + +;; The tied use is read again afterwards, so it has to be copied into the def's +;; register first, as TwoAddressInstructionPass would have done. +define i32 @tied_use_lives_on(i32 %a, i32 %b) nounwind { +; CHECK-LABEL: tied_use_lives_on: +; CHECK: # %bb.0: +; CHECK-NEXT: movl %edi, %eax +; CHECK-NEXT: addl %esi, %eax +; CHECK-NEXT: addl %edi, %eax +; CHECK-NEXT: retq + %c = add i32 %a, %b + %d = add i32 %c, %a + ret i32 %d +} + +;; The result is allocated first and has no copy to take a hint from; tracing +;; the hint through the tied def to the tied use reaches the argument's +;; register, so the argument copy coalesces away and the add runs in %edi. +define void @tied_hint_follows_tie(i32 %a, i32 %b, ptr %p) nounwind { +; CHECK-LABEL: tied_hint_follows_tie: +; CHECK: # %bb.0: +; CHECK-NEXT: addl %esi, %edi +; CHECK-NEXT: movl %edi, (%rdx) +; CHECK-NEXT: retq + %r = add i32 %a, %b + store volatile i32 %r, ptr %p + ret void +} + +;; The high-byte read needs GR32_ABCD, which does not contain the register the +;; shift's def was assigned, so the tied use cannot take it over. The i128 store +;; is what FastISel gives up on, handing the block to SelectionDAG, which is the +;; selector that produces the high-byte read. +define void @tied_use_class_mismatch(ptr %p, i32 %x, i128 %fallback) nounwind { +; CHECK-LABEL: tied_use_class_mismatch: +; CHECK: # %bb.0: +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: movl (%rdi), %ebx +; CHECK-NEXT: movb %bh, %al +; CHECK-NEXT: movzbl %al, %eax +; CHECK-NEXT: movl %ebx, %r8d +; CHECK-NEXT: shrl %r8d +; CHECK-NEXT: orl %r8d, %esi +; CHECK-NEXT: xorl %eax, %esi +; CHECK-NEXT: movl %esi, (%rdi) +; CHECK-NEXT: movq %rcx, 8(%rdi) +; CHECK-NEXT: movq %rdx, (%rdi) +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: retq + %v = load i32, ptr %p + %hi = lshr i32 %v, 8 + %byte = and i32 %hi, 255 + %shr = lshr i32 %v, 1 + %or = or i32 %x, %shr + %xor = xor i32 %or, %byte + store i32 %xor, ptr %p + store i128 %fallback, ptr %p + ret void +} + +;; Instruction selection gives the tied and the untied read separate registers, +;; so the early-clobber rule is exercised from MIR instead; see +;; regallocfast-ssa.mir. +define i32 @tied_early_clobber(i32 %a) nounwind { +; CHECK-LABEL: tied_early_clobber: +; CHECK: # %bb.0: +; CHECK-NEXT: movl %edi, %eax +; CHECK-NEXT: #APP +; CHECK-NEXT: # %eax %eax %edi +; CHECK-NEXT: #NO_APP +; CHECK-NEXT: retq + %r = call i32 asm "// $0 $1 $2", "=&r,0,r"(i32 %a, i32 %a) + ret i32 %r +} diff --git a/llvm/test/CodeGen/X86/regallocfast-ssa.mir b/llvm/test/CodeGen/X86/regallocfast-ssa.mir new file mode 100644 index 0000000000000..bce78eff66b9d --- /dev/null +++ b/llvm/test/CodeGen/X86/regallocfast-ssa.mir @@ -0,0 +1,326 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 +# RUN: llc -mtriple=x86_64-linux -run-pass=regallocfast -regalloc-fast-ssa -verify-machineinstrs %s -o - | FileCheck %s + +# Shapes the SSA path has to handle that instruction selection does not +# produce reliably: PHI operands carrying a subregister index, an undef flag or +# a repeated predecessor, a PHI source reaching its destination from two +# predecessors or defined by another PHI, a tied use whose value the +# early-clobber def also reads through another operand, a tied use of a +# subregister, a tied use whose register class excludes the def's register, and +# INSERT_SUBREG on an undef base. + +--- | + define i32 @phi_subreg_and_undef_operands(i64 %a, i32 %c) { ret i32 0 } + define i32 @phi_repeated_predecessor(i32 %a, i32 %b) { ret i32 0 } + define i32 @phi_source_on_two_edges(i32 %a, i32 %b) { ret i32 0 } + define i32 @phi_source_defined_by_phi(i32 %a, i32 %b) { ret i32 0 } + define i32 @tied_early_clobber_shared_use(i32 %a) { ret i32 0 } + define i16 @tied_use_subreg(i32 %a, i16 %b) { ret i16 0 } + define void @tied_use_class_mismatch(i32 %a) { ret void } + define i64 @insert_subreg_undef_base(i32 %a) { ret i64 0 } +... +--- +name: phi_subreg_and_undef_operands +tracksRegLiveness: true +body: | + ; CHECK-LABEL: name: phi_subreg_and_undef_operands + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: liveins: $rdi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: TEST32rr killed renamable $esi, renamable $esi, implicit-def $eflags + ; CHECK-NEXT: renamable $eax = COPY renamable $edi, implicit killed $rdi + ; CHECK-NEXT: MOV32mr %stack.0, 1, $noreg, 0, $noreg, killed $eax :: (store (s32) into %stack.0) + ; CHECK-NEXT: JCC_1 %bb.2, 4, implicit killed $eflags + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.2(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $eax = IMPLICIT_DEF + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: $eax = MOV32rm %stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %stack.0) + ; CHECK-NEXT: RET64 implicit killed $eax + bb.0: + successors: %bb.1, %bb.2 + liveins: $rdi, $esi + %0:gr64 = COPY $rdi + %3:gr32 = COPY $esi + TEST32rr %3, %3, implicit-def $eflags + JCC_1 %bb.2, 4, implicit $eflags + + bb.1: + successors: %bb.2 + %1:gr32 = IMPLICIT_DEF + + bb.2: + %2:gr32 = PHI %0.sub_32bit, %bb.0, undef %1, %bb.1 + $eax = COPY %2 + RET64 implicit $eax +... +--- +# A PHI naming the same predecessor twice, which the machine verifier permits +# while a repeated successor edge is invalid. One transfer per edge suffices, +# for a destination read outside the loop (%2) as for one that is not (%3). +name: phi_repeated_predecessor +tracksRegLiveness: true +body: | + ; CHECK-LABEL: name: phi_repeated_predecessor + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $eax = COPY killed $edi + ; CHECK-NEXT: MOV32mr %stack.0, 1, $noreg, 0, $noreg, killed $esi :: (store (s32) into %stack.0) + ; CHECK-NEXT: MOV32mr %stack.1, 1, $noreg, 0, $noreg, killed $eax :: (store (s32) into %stack.1) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.2(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: $eax = MOV32rm %stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %stack.0) + ; CHECK-NEXT: $ecx = MOV32rm %stack.1, 1, $noreg, 0, $noreg :: (load (s32) from %stack.1) + ; CHECK-NEXT: MOV32mr %stack.4, 1, $noreg, 0, $noreg, $ecx :: (store (s32) into %stack.4) + ; CHECK-NEXT: renamable $ecx = ADD32rr renamable $ecx, renamable $eax, implicit-def dead $eflags + ; CHECK-NEXT: MOV32mr %stack.3, 1, $noreg, 0, $noreg, killed $ecx :: (store (s32) into %stack.3) + ; CHECK-NEXT: renamable $eax = ADD32ri renamable $eax, 1, implicit-def dead $eflags + ; CHECK-NEXT: MOV32mr %stack.2, 1, $noreg, 0, $noreg, killed $eax :: (store (s32) into %stack.2) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: $eax = MOV32rm %stack.2, 1, $noreg, 0, $noreg :: (load (s32) from %stack.2) + ; CHECK-NEXT: $ecx = MOV32rm %stack.3, 1, $noreg, 0, $noreg :: (load (s32) from %stack.3) + ; CHECK-NEXT: TEST32rr renamable $eax, renamable $eax, implicit-def $eflags + ; CHECK-NEXT: MOV32mr %stack.1, 1, $noreg, 0, $noreg, killed $ecx :: (store (s32) into %stack.1) + ; CHECK-NEXT: MOV32mr %stack.0, 1, $noreg, 0, $noreg, killed $eax :: (store (s32) into %stack.0) + ; CHECK-NEXT: JCC_1 %bb.1, 4, implicit killed $eflags + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.3: + ; CHECK-NEXT: $eax = MOV32rm %stack.4, 1, $noreg, 0, $noreg :: (load (s32) from %stack.4) + ; CHECK-NEXT: RET64 implicit killed $eax + bb.0: + successors: %bb.1 + liveins: $edi, $esi + %0:gr32 = COPY $edi + %1:gr32 = COPY $esi + + bb.1: + successors: %bb.2 + %2:gr32 = PHI %0, %bb.0, %4, %bb.2, %4, %bb.2 + %3:gr32 = PHI %1, %bb.0, %5, %bb.2, %5, %bb.2 + %4:gr32 = ADD32rr %2, %3, implicit-def dead $eflags + %5:gr32 = ADD32ri %3, 1, implicit-def dead $eflags + + bb.2: + successors: %bb.1, %bb.3 + TEST32rr %5, %5, implicit-def $eflags + JCC_1 %bb.1, 4, implicit $eflags + + bb.3: + $eax = COPY %2 + RET64 implicit $eax +... +--- +# %0 reaches %3 from two predecessors, so it cannot take %3's slot: only the +# edge leaving %0's own predecessor writes that slot, and bb.2 would fall +# through with the 7 bb.1 spilled there. %2 still shares despite naming bb.1 +# twice -- one predecessor, one transfer. +name: phi_source_on_two_edges +tracksRegLiveness: true +body: | + ; CHECK-LABEL: name: phi_source_on_two_edges + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000) + ; CHECK-NEXT: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $eax = COPY killed $edi + ; CHECK-NEXT: MOV32mr %stack.2, 1, $noreg, 0, $noreg, $eax :: (store (s32) into %stack.2) + ; CHECK-NEXT: MOV32mr %stack.1, 1, $noreg, 0, $noreg, killed $esi :: (store (s32) into %stack.1) + ; CHECK-NEXT: TEST32rr renamable $eax, renamable $eax, implicit-def $eflags + ; CHECK-NEXT: MOV32mr %stack.0, 1, $noreg, 0, $noreg, killed $eax :: (store (s32) into %stack.0) + ; CHECK-NEXT: JCC_1 %bb.3, 4, implicit killed $eflags + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: $eax = MOV32rm %stack.1, 1, $noreg, 0, $noreg :: (load (s32) from %stack.1) + ; CHECK-NEXT: renamable $ecx = MOV32ri 7 + ; CHECK-NEXT: MOV32mr %stack.0, 1, $noreg, 0, $noreg, killed $ecx :: (store (s32) into %stack.0) + ; CHECK-NEXT: TEST32rr renamable $eax, renamable $eax, implicit-def $eflags + ; CHECK-NEXT: JCC_1 %bb.3, 5, implicit killed $eflags + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: successors: %bb.3(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: $eax = MOV32rm %stack.2, 1, $noreg, 0, $noreg :: (load (s32) from %stack.2) + ; CHECK-NEXT: MOV32mr %stack.0, 1, $noreg, 0, $noreg, killed $eax :: (store (s32) into %stack.0) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.3: + ; CHECK-NEXT: $eax = MOV32rm %stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %stack.0) + ; CHECK-NEXT: RET64 implicit killed $eax + bb.0: + successors: %bb.1, %bb.3 + liveins: $edi, $esi + %0:gr32 = COPY $edi + %1:gr32 = COPY $esi + TEST32rr %0, %0, implicit-def $eflags + JCC_1 %bb.3, 4, implicit $eflags + + bb.1: + successors: %bb.2, %bb.3 + %2:gr32 = MOV32ri 7 + TEST32rr %1, %1, implicit-def $eflags + JCC_1 %bb.3, 5, implicit $eflags + + bb.2: + successors: %bb.3 + + bb.3: + %3:gr32 = PHI %0, %bb.0, %2, %bb.1, %2, %bb.1, %0, %bb.2 + $eax = COPY %3 + RET64 implicit $eax +... +--- +# %2 is defined by a PHI, so its own lowering writes it at the ends of bb.0 and +# bb.1 rather than in bb.2. Sharing %3's slot would put that write in the same +# block as the copy bb.1 makes for %3, and one would overwrite the other. +name: phi_source_defined_by_phi +tracksRegLiveness: true +body: | + ; CHECK-LABEL: name: phi_source_defined_by_phi + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000) + ; CHECK-NEXT: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $eax = COPY killed $edi + ; CHECK-NEXT: MOV32mr %stack.2, 1, $noreg, 0, $noreg, $eax :: (store (s32) into %stack.2) + ; CHECK-NEXT: MOV32mr %stack.1, 1, $noreg, 0, $noreg, $esi :: (store (s32) into %stack.1) + ; CHECK-NEXT: TEST32rr renamable $esi, renamable $esi, implicit-def $eflags + ; CHECK-NEXT: MOV32mr %stack.0, 1, $noreg, 0, $noreg, killed $eax :: (store (s32) into %stack.0) + ; CHECK-NEXT: JCC_1 %bb.2, 4, implicit killed $eflags + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: $eax = MOV32rm %stack.2, 1, $noreg, 0, $noreg :: (load (s32) from %stack.2) + ; CHECK-NEXT: $ecx = MOV32rm %stack.1, 1, $noreg, 0, $noreg :: (load (s32) from %stack.1) + ; CHECK-NEXT: TEST32rr renamable $eax, renamable $eax, implicit-def $eflags + ; CHECK-NEXT: MOV32mr %stack.0, 1, $noreg, 0, $noreg, killed $ecx :: (store (s32) into %stack.0) + ; CHECK-NEXT: MOV32mr %stack.3, 1, $noreg, 0, $noreg, killed $eax :: (store (s32) into %stack.3) + ; CHECK-NEXT: JCC_1 %bb.2, 4, implicit killed $eflags + ; CHECK-NEXT: JMP_1 %bb.3 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: successors: %bb.3(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: $eax = MOV32rm %stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %stack.0) + ; CHECK-NEXT: MOV32mr %stack.3, 1, $noreg, 0, $noreg, killed $eax :: (store (s32) into %stack.3) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.3: + ; CHECK-NEXT: $eax = MOV32rm %stack.3, 1, $noreg, 0, $noreg :: (load (s32) from %stack.3) + ; CHECK-NEXT: RET64 implicit killed $eax + bb.0: + successors: %bb.2, %bb.1 + liveins: $edi, $esi + %0:gr32 = COPY $edi + %1:gr32 = COPY $esi + TEST32rr %1, %1, implicit-def $eflags + JCC_1 %bb.2, 4, implicit $eflags + + bb.1: + successors: %bb.2, %bb.3 + TEST32rr %0, %0, implicit-def $eflags + JCC_1 %bb.2, 4, implicit $eflags + JMP_1 %bb.3 + + bb.2: + successors: %bb.3 + %2:gr32 = PHI %0, %bb.0, %1, %bb.1 + + bb.3: + %3:gr32 = PHI %0, %bb.1, %2, %bb.2 + $eax = COPY %3 + RET64 implicit $eax +... +--- +name: tied_early_clobber_shared_use +tracksRegLiveness: true +body: | + bb.0: + liveins: $edi + ; CHECK-LABEL: name: tied_early_clobber_shared_use + ; CHECK: liveins: $edi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: $eax = COPY $edi + ; CHECK-NEXT: INLINEASM &"// $0 $1 $2", attdialect, regdef-ec:GR32, def early-clobber renamable $eax, reguse tiedto:$0, killed renamable $eax(tied-def 3), reguse:GR32, renamable $edi + ; CHECK-NEXT: RET64 implicit killed $eax + %0:gr32 = COPY $edi + INLINEASM &"// $0 $1 $2", attdialect, regdef-ec:GR32, def early-clobber %1:gr32, reguse tiedto:$0, %0(tied-def 3), reguse:GR32, %0 + $eax = COPY %1 + RET64 implicit $eax +... +--- +# A tied use reading a subregister is a truncation, so it cannot take over the +# def's register: the copy performs the truncation and the operand reads the +# def's register whole, as TwoAddressInstructionPass does. %0 is read again +# afterwards, so it keeps its own register. +name: tied_use_subreg +tracksRegLiveness: true +body: | + bb.0: + liveins: $edi, $esi + ; CHECK-LABEL: name: tied_use_subreg + ; CHECK: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $ecx = COPY killed $edi + ; CHECK-NEXT: $ax = COPY $cx + ; CHECK-NEXT: renamable $ax = ADD16rr renamable $ax, killed renamable $si, implicit-def dead $eflags + ; CHECK-NEXT: RET64 implicit killed $ax, implicit killed $ecx + %0:gr32 = COPY $edi + %1:gr16 = COPY $si + %2:gr16 = ADD16rr %0.sub_16bit, %1, implicit-def dead $eflags + $ecx = COPY %0 + $ax = COPY %2 + RET64 implicit $ax, implicit $ecx +... +--- +# The high-byte read forces GR32_ABCD on %0, which does not contain the $esi +# the tied def was assigned, so %0 cannot take that register over and is copied +# into it instead. +name: tied_use_class_mismatch +tracksRegLiveness: true +body: | + bb.0: + liveins: $edi + ; CHECK-LABEL: name: tied_use_class_mismatch + ; CHECK: liveins: $edi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $eax = COPY killed $edi + ; CHECK-NEXT: renamable $cl = COPY renamable $ah + ; CHECK-NEXT: $esi = COPY $eax + ; CHECK-NEXT: renamable $esi = SHR32ri killed renamable $esi, 1, implicit-def dead $eflags + ; CHECK-NEXT: RET64 implicit killed $esi, implicit killed $cl + %0:gr32_abcd = COPY $edi + %1:gr8_norex = COPY %0.sub_8bit_hi + %2:gr32 = SHR32ri %0, 1, implicit-def dead $eflags + $esi = COPY %2 + $cl = COPY %1 + RET64 implicit $esi, implicit $cl +... +--- +name: insert_subreg_undef_base +tracksRegLiveness: true +body: | + bb.0: + liveins: $edi + ; CHECK-LABEL: name: insert_subreg_undef_base + ; CHECK: liveins: $edi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $eax = COPY killed $edi + ; CHECK-NEXT: undef renamable $eax = COPY killed renamable $eax, implicit-def $rax + ; CHECK-NEXT: RET64 implicit killed $rax + %0:gr32 = COPY $edi + %1:gr64 = INSERT_SUBREG undef %2:gr64, %0, %subreg.sub_32bit + $rax = COPY %1 + RET64 implicit $rax +... diff --git a/llvm/test/CodeGen/X86/sink-local-value.ll b/llvm/test/CodeGen/X86/sink-local-value.ll index 4732fb48a922d..7660178bf206f 100644 --- a/llvm/test/CodeGen/X86/sink-local-value.ll +++ b/llvm/test/CodeGen/X86/sink-local-value.ll @@ -99,12 +99,12 @@ if.end: ; preds = %if.else, %if.then ; CHECK: # %entry ; CHECK: cmpl $0, ; CHECK: # %if.then -; CHECK: movl $42, %[[REG:[a-z]+]] -; CHECK: #DEBUG_VALUE: phi_const_cast:4 <- $[[REG]] +; CHECK: movl $42, %{{[a-z]+}} +; CHECK: #DEBUG_VALUE: phi_const_cast:4 <- [$esp+{{[0-9]+}}] ; CHECK: jmp ; CHECK: # %if.else -; CHECK: movl $1, %[[REG:[a-z]+]] -; CHECK: #DEBUG_VALUE: phi_const_cast:5 <- $[[REG]] +; CHECK: movl $1, %{{[a-z]+}} +; CHECK: #DEBUG_VALUE: phi_const_cast:5 <- [$esp+{{[0-9]+}}] ; CHECK: # %if.end declare void @may_throw() local_unnamed_addr #1 diff --git a/llvm/test/CodeGen/X86/swifterror.ll b/llvm/test/CodeGen/X86/swifterror.ll index 5699c447baf41..3b6d4cd2e5847 100644 --- a/llvm/test/CodeGen/X86/swifterror.ll +++ b/llvm/test/CodeGen/X86/swifterror.ll @@ -422,7 +422,8 @@ define float @foo_loop(ptr swifterror %error_ptr_ref, i32 %cc, float %cc2) { ; CHECK-O0-NEXT: .cfi_def_cfa_offset 48 ; CHECK-O0-NEXT: movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill ; CHECK-O0-NEXT: movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill -; CHECK-O0-NEXT: movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill +; CHECK-O0-NEXT: movq %r12, %rax +; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill ; CHECK-O0-NEXT: jmp LBB4_1 ; CHECK-O0-NEXT: LBB4_1: ## %bb_loop ; CHECK-O0-NEXT: ## =>This Inner Loop Header: Depth=1 @@ -957,11 +958,11 @@ define void @swifterror_isel(ptr) { ; CHECK-O0-NEXT: .cfi_def_cfa_offset 64 ; CHECK-O0-NEXT: .cfi_offset %r12, -24 ; CHECK-O0-NEXT: .cfi_offset %r13, -16 +; CHECK-O0-NEXT: ## implicit-def: $rax ; CHECK-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill +; CHECK-O0-NEXT: ## implicit-def: $ax ; CHECK-O0-NEXT: ## implicit-def: $al ; CHECK-O0-NEXT: testb $1, %al -; CHECK-O0-NEXT: ## implicit-def: $ax -; CHECK-O0-NEXT: ## implicit-def: $r12 ; CHECK-O0-NEXT: jne LBB8_2 ; CHECK-O0-NEXT: LBB8_1: ## =>This Inner Loop Header: Depth=1 ; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 ## 8-byte Reload @@ -971,10 +972,11 @@ define void @swifterror_isel(ptr) { ; CHECK-O0-NEXT: movw %ax, %di ; CHECK-O0-NEXT: ## implicit-def: $rax ; CHECK-O0-NEXT: callq *%rax -; CHECK-O0-NEXT: ## implicit-def: $rax -; CHECK-O0-NEXT: movw (%rax), %ax -; CHECK-O0-NEXT: movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill -; CHECK-O0-NEXT: movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill +; CHECK-O0-NEXT: movq %r12, %rax +; CHECK-O0-NEXT: ## implicit-def: $rcx +; CHECK-O0-NEXT: movw (%rcx), %cx +; CHECK-O0-NEXT: movw %cx, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill +; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill ; CHECK-O0-NEXT: jmp LBB8_1 ; CHECK-O0-NEXT: LBB8_2: ; CHECK-O0-NEXT: addq $40, %rsp diff --git a/llvm/test/CodeGen/X86/switch.ll b/llvm/test/CodeGen/X86/switch.ll index c75819c2fd2c5..19c179985ec71 100644 --- a/llvm/test/CodeGen/X86/switch.ll +++ b/llvm/test/CodeGen/X86/switch.ll @@ -1327,7 +1327,8 @@ define void @phi_node_trouble(ptr %s) { ; ; NOOPT-LABEL: phi_node_trouble: ; NOOPT: # %bb.0: # %entry -; NOOPT-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; NOOPT-NEXT: movq %rdi, %rax +; NOOPT-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; NOOPT-NEXT: jmp .LBB13_1 ; NOOPT-NEXT: .LBB13_1: # %header ; NOOPT-NEXT: # =>This Inner Loop Header: Depth=1 @@ -2526,9 +2527,9 @@ define i32 @pr27135(i32 %i) { ; NOOPT: # %bb.0: # %entry ; NOOPT-NEXT: movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill ; NOOPT-NEXT: xorl %eax, %eax -; NOOPT-NEXT: # implicit-def: $cl -; NOOPT-NEXT: testb $1, %cl ; NOOPT-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; NOOPT-NEXT: # implicit-def: $al +; NOOPT-NEXT: testb $1, %al ; NOOPT-NEXT: jne .LBB24_1 ; NOOPT-NEXT: jmp .LBB24_4 ; NOOPT-NEXT: .LBB24_1: # %sw diff --git a/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir b/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir index 6dbd2cd4faa71..48e9471d0bab3 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir @@ -1,5 +1,6 @@ # RUN: llc -start-after=phi-node-elimination -stop-after=virtregrewriter %s -mtriple=x86_64-unknown-unknown -o - -experimental-debug-variable-locations | FileCheck %s -# RUN: llc -O0 -start-after=phi-node-elimination -stop-after=regallocfast %s -mtriple=x86_64-unknown-unknown -o - -experimental-debug-variable-locations | FileCheck %s --check-prefix=FASTREG +# RUN: llc -O0 -regalloc-fast-ssa=0 -start-after=phi-node-elimination -stop-after=regallocfast %s -mtriple=x86_64-unknown-unknown -o - -experimental-debug-variable-locations | FileCheck %s --check-prefix=FASTREG +# -regalloc-fast-ssa=0 restores phi-node-elimination, which -start-after names. # # Test that DBG_INSTR_REFs can pass through livedebugvariables to the end of # regalloc without problem. Program body copied from diff --git a/llvm/test/DebugInfo/X86/fission-ranges.ll b/llvm/test/DebugInfo/X86/fission-ranges.ll index be0a3759973c3..80c707a3a731b 100644 --- a/llvm/test/DebugInfo/X86/fission-ranges.ll +++ b/llvm/test/DebugInfo/X86/fission-ranges.ll @@ -10,11 +10,11 @@ ; LiveDebugValues should produce DBG_VALUEs for variable "b" in successive ; blocks once we recognize that it is spilled. ; CHECK-MIR: ![[BDIVAR:[0-9]+]] = !DILocalVariable(name: "b" -; CHECK-MIR: DBG_VALUE $rsp, 0, ![[BDIVAR]], !DIExpression(DW_OP_constu, 24, DW_OP_minus) +; CHECK-MIR: DBG_VALUE $rsp, 0, ![[BDIVAR]], !DIExpression(DW_OP_constu, 32, DW_OP_minus) ; CHECK-MIR-LABEL: bb.6.for.inc13: -; CHECK-MIR: DBG_VALUE $rsp, 0, ![[BDIVAR]], !DIExpression(DW_OP_constu, 24, DW_OP_minus) +; CHECK-MIR: DBG_VALUE $rsp, 0, ![[BDIVAR]], !DIExpression(DW_OP_constu, 32, DW_OP_minus) ; CHECK-MIR-LABEL: bb.7.for.inc16: -; CHECK-MIR: DBG_VALUE $rsp, 0, ![[BDIVAR]], !DIExpression(DW_OP_constu, 24, DW_OP_minus) +; CHECK-MIR: DBG_VALUE $rsp, 0, ![[BDIVAR]], !DIExpression(DW_OP_constu, 32, DW_OP_minus) ; CHECK: .debug_info contents: @@ -47,19 +47,19 @@ ; CHECK: [[A]]: ; CHECK-NEXT: DW_LLE_startx_length (0x00000001, 0x00000011): DW_OP_consts +0, DW_OP_stack_value ; CHECK-NEXT: DW_LLE_startx_length (0x00000002, 0x0000000b): DW_OP_reg0 RAX -; CHECK-NEXT: DW_LLE_startx_length (0x00000003, 0x00000012): DW_OP_breg7 RSP-4 +; CHECK-NEXT: DW_LLE_startx_length (0x00000003, 0x00000012): DW_OP_breg7 RSP-12 ; CHECK-NEXT: DW_LLE_end_of_list () ; CHECK: [[E]]: ; CHECK-NEXT: DW_LLE_startx_length (0x00000004, 0x0000000b): DW_OP_reg0 RAX -; CHECK-NEXT: DW_LLE_startx_length (0x00000005, 0x0000005a): DW_OP_breg7 RSP-36 +; CHECK-NEXT: DW_LLE_startx_length (0x00000005, 0x0000005a): DW_OP_breg7 RSP-8 ; CHECK-NEXT: DW_LLE_end_of_list () ; CHECK: [[B]]: ; CHECK-NEXT: DW_LLE_startx_length (0x00000006, 0x0000000b): DW_OP_reg0 RAX -; CHECK-NEXT: DW_LLE_startx_length (0x00000007, 0x00000042): DW_OP_breg7 RSP-24 +; CHECK-NEXT: DW_LLE_startx_length (0x00000007, 0x00000042): DW_OP_breg7 RSP-32 ; CHECK-NEXT: DW_LLE_end_of_list () ; CHECK: [[D]]: ; CHECK-NEXT: DW_LLE_startx_length (0x00000008, 0x0000000b): DW_OP_reg0 RAX -; CHECK-NEXT: DW_LLE_startx_length (0x00000009, 0x0000002a): DW_OP_breg7 RSP-12 +; CHECK-NEXT: DW_LLE_startx_length (0x00000009, 0x0000002a): DW_OP_breg7 RSP-20 ; CHECK-NEXT: DW_LLE_end_of_list () ; Make sure we don't produce any relocations in any .dwo section (though in particular, debug_info.dwo)