1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
| //===- InstCombineAtomicRMW.cpp -------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the visit functions for atomic rmw instructions.
//
//===----------------------------------------------------------------------===//
#include "InstCombineInternal.h"
#include "llvm/IR/Instructions.h"
using namespace llvm;
namespace {
/// Return true if and only if the given instruction does not modify the memory
/// location referenced. Note that an idemptent atomicrmw may still have
/// ordering effects on nearby instructions, or be volatile.
/// TODO: Common w/ the version in AtomicExpandPass, and change the term used.
/// Idemptotent is confusing in this context.
bool isIdempotentRMW(AtomicRMWInst& RMWI) {
if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
switch(RMWI.getOperation()) {
case AtomicRMWInst::FAdd: // -0.0
return CF->isZero() && CF->isNegative();
case AtomicRMWInst::FSub: // +0.0
return CF->isZero() && !CF->isNegative();
default:
return false;
};
auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
if(!C)
return false;
switch(RMWI.getOperation()) {
case AtomicRMWInst::Add:
case AtomicRMWInst::Sub:
case AtomicRMWInst::Or:
case AtomicRMWInst::Xor:
return C->isZero();
case AtomicRMWInst::And:
return C->isMinusOne();
case AtomicRMWInst::Min:
return C->isMaxValue(true);
case AtomicRMWInst::Max:
return C->isMinValue(true);
case AtomicRMWInst::UMin:
return C->isMaxValue(false);
case AtomicRMWInst::UMax:
return C->isMinValue(false);
default:
return false;
}
}
/// Return true if the given instruction always produces a value in memory
/// equivalent to its value operand.
bool isSaturating(AtomicRMWInst& RMWI) {
if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
switch(RMWI.getOperation()) {
case AtomicRMWInst::FAdd:
case AtomicRMWInst::FSub:
return CF->isNaN();
default:
return false;
};
auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
if(!C)
return false;
switch(RMWI.getOperation()) {
default:
return false;
case AtomicRMWInst::Xchg:
return true;
case AtomicRMWInst::Or:
return C->isAllOnesValue();
case AtomicRMWInst::And:
return C->isZero();
case AtomicRMWInst::Min:
return C->isMinValue(true);
case AtomicRMWInst::Max:
return C->isMaxValue(true);
case AtomicRMWInst::UMin:
return C->isMinValue(false);
case AtomicRMWInst::UMax:
return C->isMaxValue(false);
};
}
}
Instruction *InstCombiner::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
// Volatile RMWs perform a load and a store, we cannot replace this by just a
// load or just a store. We chose not to canonicalize out of general paranoia
// about user expectations around volatile.
if (RMWI.isVolatile())
return nullptr;
// Any atomicrmw op which produces a known result in memory can be
// replaced w/an atomicrmw xchg.
if (isSaturating(RMWI) &&
RMWI.getOperation() != AtomicRMWInst::Xchg) {
RMWI.setOperation(AtomicRMWInst::Xchg);
return &RMWI;
}
AtomicOrdering Ordering = RMWI.getOrdering();
assert(Ordering != AtomicOrdering::NotAtomic &&
Ordering != AtomicOrdering::Unordered &&
"AtomicRMWs don't make sense with Unordered or NotAtomic");
// Any atomicrmw xchg with no uses can be converted to a atomic store if the
// ordering is compatible.
if (RMWI.getOperation() == AtomicRMWInst::Xchg &&
RMWI.use_empty()) {
if (Ordering != AtomicOrdering::Release &&
Ordering != AtomicOrdering::Monotonic)
return nullptr;
auto *SI = new StoreInst(RMWI.getValOperand(),
RMWI.getPointerOperand(), &RMWI);
SI->setAtomic(Ordering, RMWI.getSyncScopeID());
SI->setAlignment(MaybeAlign(DL.getABITypeAlignment(RMWI.getType())));
return eraseInstFromFunction(RMWI);
}
if (!isIdempotentRMW(RMWI))
return nullptr;
// We chose to canonicalize all idempotent operations to an single
// operation code and constant. This makes it easier for the rest of the
// optimizer to match easily. The choices of or w/0 and fadd w/-0.0 are
// arbitrary.
if (RMWI.getType()->isIntegerTy() &&
RMWI.getOperation() != AtomicRMWInst::Or) {
RMWI.setOperation(AtomicRMWInst::Or);
RMWI.setOperand(1, ConstantInt::get(RMWI.getType(), 0));
return &RMWI;
} else if (RMWI.getType()->isFloatingPointTy() &&
RMWI.getOperation() != AtomicRMWInst::FAdd) {
RMWI.setOperation(AtomicRMWInst::FAdd);
RMWI.setOperand(1, ConstantFP::getNegativeZero(RMWI.getType()));
return &RMWI;
}
// Check if the required ordering is compatible with an atomic load.
if (Ordering != AtomicOrdering::Acquire &&
Ordering != AtomicOrdering::Monotonic)
return nullptr;
LoadInst *Load = new LoadInst(RMWI.getType(), RMWI.getPointerOperand());
Load->setAtomic(Ordering, RMWI.getSyncScopeID());
Load->setAlignment(MaybeAlign(DL.getABITypeAlignment(RMWI.getType())));
return Load;
}
|