1 /*
2  * Copyright (C) 2016 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM_VIXL_H_
18 #define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM_VIXL_H_
19 
20 #include "base/enums.h"
21 #include "code_generator.h"
22 #include "common_arm.h"
23 #include "dex/string_reference.h"
24 #include "dex/type_reference.h"
25 #include "driver/compiler_options.h"
26 #include "nodes.h"
27 #include "parallel_move_resolver.h"
28 #include "utils/arm/assembler_arm_vixl.h"
29 
30 // TODO(VIXL): make vixl clean wrt -Wshadow.
31 #pragma GCC diagnostic push
32 #pragma GCC diagnostic ignored "-Wshadow"
33 #include "aarch32/constants-aarch32.h"
34 #include "aarch32/instructions-aarch32.h"
35 #include "aarch32/macro-assembler-aarch32.h"
36 #pragma GCC diagnostic pop
37 
38 namespace art {
39 
40 namespace linker {
41 class Thumb2RelativePatcherTest;
42 }  // namespace linker
43 
44 namespace arm {
45 
46 // This constant is used as an approximate margin when emission of veneer and literal pools
47 // must be blocked.
48 static constexpr int kMaxMacroInstructionSizeInBytes =
49     15 * vixl::aarch32::kMaxInstructionSizeInBytes;
50 
51 static const vixl::aarch32::Register kParameterCoreRegistersVIXL[] = {
52     vixl::aarch32::r1,
53     vixl::aarch32::r2,
54     vixl::aarch32::r3
55 };
56 static const size_t kParameterCoreRegistersLengthVIXL = arraysize(kParameterCoreRegistersVIXL);
57 static const vixl::aarch32::SRegister kParameterFpuRegistersVIXL[] = {
58     vixl::aarch32::s0,
59     vixl::aarch32::s1,
60     vixl::aarch32::s2,
61     vixl::aarch32::s3,
62     vixl::aarch32::s4,
63     vixl::aarch32::s5,
64     vixl::aarch32::s6,
65     vixl::aarch32::s7,
66     vixl::aarch32::s8,
67     vixl::aarch32::s9,
68     vixl::aarch32::s10,
69     vixl::aarch32::s11,
70     vixl::aarch32::s12,
71     vixl::aarch32::s13,
72     vixl::aarch32::s14,
73     vixl::aarch32::s15
74 };
75 static const size_t kParameterFpuRegistersLengthVIXL = arraysize(kParameterFpuRegistersVIXL);
76 
77 static const vixl::aarch32::Register kMethodRegister = vixl::aarch32::r0;
78 
79 // Callee saves core registers r5, r6, r7, r8 (except when emitting Baker
80 // read barriers, where it is used as Marking Register), r10, r11, and lr.
81 static const vixl::aarch32::RegisterList kCoreCalleeSaves = vixl::aarch32::RegisterList::Union(
82     vixl::aarch32::RegisterList(vixl::aarch32::r5,
83                                 vixl::aarch32::r6,
84                                 vixl::aarch32::r7),
85     // Do not consider r8 as a callee-save register with Baker read barriers.
86     ((kEmitCompilerReadBarrier && kUseBakerReadBarrier)
87          ? vixl::aarch32::RegisterList()
88          : vixl::aarch32::RegisterList(vixl::aarch32::r8)),
89     vixl::aarch32::RegisterList(vixl::aarch32::r10,
90                                 vixl::aarch32::r11,
91                                 vixl::aarch32::lr));
92 
93 // Callee saves FP registers s16 to s31 inclusive.
94 static const vixl::aarch32::SRegisterList kFpuCalleeSaves =
95     vixl::aarch32::SRegisterList(vixl::aarch32::s16, 16);
96 
97 static const vixl::aarch32::Register kRuntimeParameterCoreRegistersVIXL[] = {
98     vixl::aarch32::r0,
99     vixl::aarch32::r1,
100     vixl::aarch32::r2,
101     vixl::aarch32::r3
102 };
103 static const size_t kRuntimeParameterCoreRegistersLengthVIXL =
104     arraysize(kRuntimeParameterCoreRegistersVIXL);
105 static const vixl::aarch32::SRegister kRuntimeParameterFpuRegistersVIXL[] = {
106     vixl::aarch32::s0,
107     vixl::aarch32::s1,
108     vixl::aarch32::s2,
109     vixl::aarch32::s3
110 };
111 static const size_t kRuntimeParameterFpuRegistersLengthVIXL =
112     arraysize(kRuntimeParameterFpuRegistersVIXL);
113 
114 class LoadClassSlowPathARMVIXL;
115 class CodeGeneratorARMVIXL;
116 
117 using VIXLInt32Literal = vixl::aarch32::Literal<int32_t>;
118 using VIXLUInt32Literal = vixl::aarch32::Literal<uint32_t>;
119 
120 class JumpTableARMVIXL : public DeletableArenaObject<kArenaAllocSwitchTable> {
121  public:
JumpTableARMVIXL(HPackedSwitch * switch_instr)122   explicit JumpTableARMVIXL(HPackedSwitch* switch_instr)
123       : switch_instr_(switch_instr),
124         table_start_(),
125         bb_addresses_(switch_instr->GetAllocator()->Adapter(kArenaAllocCodeGenerator)) {
126     uint32_t num_entries = switch_instr_->GetNumEntries();
127     for (uint32_t i = 0; i < num_entries; i++) {
128       VIXLInt32Literal *lit = new VIXLInt32Literal(0, vixl32::RawLiteral::kManuallyPlaced);
129       bb_addresses_.emplace_back(lit);
130     }
131   }
132 
GetTableStartLabel()133   vixl::aarch32::Label* GetTableStartLabel() { return &table_start_; }
134 
135   void EmitTable(CodeGeneratorARMVIXL* codegen);
136   void FixTable(CodeGeneratorARMVIXL* codegen);
137 
138  private:
139   HPackedSwitch* const switch_instr_;
140   vixl::aarch32::Label table_start_;
141   ArenaVector<std::unique_ptr<VIXLInt32Literal>> bb_addresses_;
142 
143   DISALLOW_COPY_AND_ASSIGN(JumpTableARMVIXL);
144 };
145 
146 class InvokeRuntimeCallingConventionARMVIXL
147     : public CallingConvention<vixl::aarch32::Register, vixl::aarch32::SRegister> {
148  public:
InvokeRuntimeCallingConventionARMVIXL()149   InvokeRuntimeCallingConventionARMVIXL()
150       : CallingConvention(kRuntimeParameterCoreRegistersVIXL,
151                           kRuntimeParameterCoreRegistersLengthVIXL,
152                           kRuntimeParameterFpuRegistersVIXL,
153                           kRuntimeParameterFpuRegistersLengthVIXL,
154                           kArmPointerSize) {}
155 
156  private:
157   DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConventionARMVIXL);
158 };
159 
160 class InvokeDexCallingConventionARMVIXL
161     : public CallingConvention<vixl::aarch32::Register, vixl::aarch32::SRegister> {
162  public:
InvokeDexCallingConventionARMVIXL()163   InvokeDexCallingConventionARMVIXL()
164       : CallingConvention(kParameterCoreRegistersVIXL,
165                           kParameterCoreRegistersLengthVIXL,
166                           kParameterFpuRegistersVIXL,
167                           kParameterFpuRegistersLengthVIXL,
168                           kArmPointerSize) {}
169 
170  private:
171   DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConventionARMVIXL);
172 };
173 
174 class InvokeDexCallingConventionVisitorARMVIXL : public InvokeDexCallingConventionVisitor {
175  public:
InvokeDexCallingConventionVisitorARMVIXL()176   InvokeDexCallingConventionVisitorARMVIXL() {}
~InvokeDexCallingConventionVisitorARMVIXL()177   virtual ~InvokeDexCallingConventionVisitorARMVIXL() {}
178 
179   Location GetNextLocation(DataType::Type type) override;
180   Location GetReturnLocation(DataType::Type type) const override;
181   Location GetMethodLocation() const override;
182 
183  private:
184   InvokeDexCallingConventionARMVIXL calling_convention;
185   uint32_t double_index_ = 0;
186 
187   DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConventionVisitorARMVIXL);
188 };
189 
190 class FieldAccessCallingConventionARMVIXL : public FieldAccessCallingConvention {
191  public:
FieldAccessCallingConventionARMVIXL()192   FieldAccessCallingConventionARMVIXL() {}
193 
GetObjectLocation()194   Location GetObjectLocation() const override {
195     return helpers::LocationFrom(vixl::aarch32::r1);
196   }
GetFieldIndexLocation()197   Location GetFieldIndexLocation() const override {
198     return helpers::LocationFrom(vixl::aarch32::r0);
199   }
GetReturnLocation(DataType::Type type)200   Location GetReturnLocation(DataType::Type type) const override {
201     return DataType::Is64BitType(type)
202         ? helpers::LocationFrom(vixl::aarch32::r0, vixl::aarch32::r1)
203         : helpers::LocationFrom(vixl::aarch32::r0);
204   }
GetSetValueLocation(DataType::Type type,bool is_instance)205   Location GetSetValueLocation(DataType::Type type, bool is_instance) const override {
206     return DataType::Is64BitType(type)
207         ? helpers::LocationFrom(vixl::aarch32::r2, vixl::aarch32::r3)
208         : (is_instance
209             ? helpers::LocationFrom(vixl::aarch32::r2)
210             : helpers::LocationFrom(vixl::aarch32::r1));
211   }
GetFpuLocation(DataType::Type type)212   Location GetFpuLocation(DataType::Type type) const override {
213     return DataType::Is64BitType(type)
214         ? helpers::LocationFrom(vixl::aarch32::s0, vixl::aarch32::s1)
215         : helpers::LocationFrom(vixl::aarch32::s0);
216   }
217 
218  private:
219   DISALLOW_COPY_AND_ASSIGN(FieldAccessCallingConventionARMVIXL);
220 };
221 
222 class SlowPathCodeARMVIXL : public SlowPathCode {
223  public:
SlowPathCodeARMVIXL(HInstruction * instruction)224   explicit SlowPathCodeARMVIXL(HInstruction* instruction)
225       : SlowPathCode(instruction), entry_label_(), exit_label_() {}
226 
GetEntryLabel()227   vixl::aarch32::Label* GetEntryLabel() { return &entry_label_; }
GetExitLabel()228   vixl::aarch32::Label* GetExitLabel() { return &exit_label_; }
229 
230   void SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) override;
231   void RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) override;
232 
233  private:
234   vixl::aarch32::Label entry_label_;
235   vixl::aarch32::Label exit_label_;
236 
237   DISALLOW_COPY_AND_ASSIGN(SlowPathCodeARMVIXL);
238 };
239 
240 class ParallelMoveResolverARMVIXL : public ParallelMoveResolverWithSwap {
241  public:
ParallelMoveResolverARMVIXL(ArenaAllocator * allocator,CodeGeneratorARMVIXL * codegen)242   ParallelMoveResolverARMVIXL(ArenaAllocator* allocator, CodeGeneratorARMVIXL* codegen)
243       : ParallelMoveResolverWithSwap(allocator), codegen_(codegen) {}
244 
245   void EmitMove(size_t index) override;
246   void EmitSwap(size_t index) override;
247   void SpillScratch(int reg) override;
248   void RestoreScratch(int reg) override;
249 
250   ArmVIXLAssembler* GetAssembler() const;
251 
252  private:
253   void Exchange(vixl32::Register reg, int mem);
254   void Exchange(int mem1, int mem2);
255 
256   CodeGeneratorARMVIXL* const codegen_;
257 
258   DISALLOW_COPY_AND_ASSIGN(ParallelMoveResolverARMVIXL);
259 };
260 
261 class LocationsBuilderARMVIXL : public HGraphVisitor {
262  public:
LocationsBuilderARMVIXL(HGraph * graph,CodeGeneratorARMVIXL * codegen)263   LocationsBuilderARMVIXL(HGraph* graph, CodeGeneratorARMVIXL* codegen)
264       : HGraphVisitor(graph), codegen_(codegen) {}
265 
266 #define DECLARE_VISIT_INSTRUCTION(name, super)     \
267   void Visit##name(H##name* instr) override;
268 
269   FOR_EACH_CONCRETE_INSTRUCTION_COMMON(DECLARE_VISIT_INSTRUCTION)
FOR_EACH_CONCRETE_INSTRUCTION_ARM(DECLARE_VISIT_INSTRUCTION)270   FOR_EACH_CONCRETE_INSTRUCTION_ARM(DECLARE_VISIT_INSTRUCTION)
271   FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION)
272 
273 #undef DECLARE_VISIT_INSTRUCTION
274 
275   void VisitInstruction(HInstruction* instruction) override {
276     LOG(FATAL) << "Unreachable instruction " << instruction->DebugName()
277                << " (id " << instruction->GetId() << ")";
278   }
279 
280  private:
281   void HandleInvoke(HInvoke* invoke);
282   void HandleBitwiseOperation(HBinaryOperation* operation, Opcode opcode);
283   void HandleCondition(HCondition* condition);
284   void HandleIntegerRotate(LocationSummary* locations);
285   void HandleLongRotate(LocationSummary* locations);
286   void HandleShift(HBinaryOperation* operation);
287   void HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info);
288   void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info);
289 
290   Location ArithmeticZeroOrFpuRegister(HInstruction* input);
291   Location ArmEncodableConstantOrRegister(HInstruction* constant, Opcode opcode);
292   bool CanEncodeConstantAsImmediate(HConstant* input_cst, Opcode opcode);
293 
294   CodeGeneratorARMVIXL* const codegen_;
295   InvokeDexCallingConventionVisitorARMVIXL parameter_visitor_;
296 
297   DISALLOW_COPY_AND_ASSIGN(LocationsBuilderARMVIXL);
298 };
299 
300 class InstructionCodeGeneratorARMVIXL : public InstructionCodeGenerator {
301  public:
302   InstructionCodeGeneratorARMVIXL(HGraph* graph, CodeGeneratorARMVIXL* codegen);
303 
304 #define DECLARE_VISIT_INSTRUCTION(name, super)     \
305   void Visit##name(H##name* instr) override;
306 
307   FOR_EACH_CONCRETE_INSTRUCTION_COMMON(DECLARE_VISIT_INSTRUCTION)
FOR_EACH_CONCRETE_INSTRUCTION_ARM(DECLARE_VISIT_INSTRUCTION)308   FOR_EACH_CONCRETE_INSTRUCTION_ARM(DECLARE_VISIT_INSTRUCTION)
309   FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION)
310 
311 #undef DECLARE_VISIT_INSTRUCTION
312 
313   void VisitInstruction(HInstruction* instruction) override {
314     LOG(FATAL) << "Unreachable instruction " << instruction->DebugName()
315                << " (id " << instruction->GetId() << ")";
316   }
317 
GetAssembler()318   ArmVIXLAssembler* GetAssembler() const { return assembler_; }
GetVIXLAssembler()319   ArmVIXLMacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); }
320 
321  private:
322   // Generate code for the given suspend check. If not null, `successor`
323   // is the block to branch to if the suspend check is not needed, and after
324   // the suspend call.
325   void GenerateSuspendCheck(HSuspendCheck* instruction, HBasicBlock* successor);
326   void GenerateClassInitializationCheck(LoadClassSlowPathARMVIXL* slow_path,
327                                         vixl32::Register class_reg);
328   void GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check,
329                                          vixl::aarch32::Register temp,
330                                          vixl::aarch32::FlagsUpdate flags_update);
331   void GenerateAndConst(vixl::aarch32::Register out, vixl::aarch32::Register first, uint32_t value);
332   void GenerateOrrConst(vixl::aarch32::Register out, vixl::aarch32::Register first, uint32_t value);
333   void GenerateEorConst(vixl::aarch32::Register out, vixl::aarch32::Register first, uint32_t value);
334   void GenerateAddLongConst(Location out, Location first, uint64_t value);
335   void HandleBitwiseOperation(HBinaryOperation* operation);
336   void HandleCondition(HCondition* condition);
337   void HandleIntegerRotate(HRor* ror);
338   void HandleLongRotate(HRor* ror);
339   void HandleShift(HBinaryOperation* operation);
340 
341   void GenerateWideAtomicStore(vixl::aarch32::Register addr,
342                                uint32_t offset,
343                                vixl::aarch32::Register value_lo,
344                                vixl::aarch32::Register value_hi,
345                                vixl::aarch32::Register temp1,
346                                vixl::aarch32::Register temp2,
347                                HInstruction* instruction);
348   void GenerateWideAtomicLoad(vixl::aarch32::Register addr,
349                               uint32_t offset,
350                               vixl::aarch32::Register out_lo,
351                               vixl::aarch32::Register out_hi);
352 
353   void HandleFieldSet(HInstruction* instruction,
354                       const FieldInfo& field_info,
355                       bool value_can_be_null);
356   void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info);
357 
358   void GenerateMinMaxInt(LocationSummary* locations, bool is_min);
359   void GenerateMinMaxLong(LocationSummary* locations, bool is_min);
360   void GenerateMinMaxFloat(HInstruction* minmax, bool is_min);
361   void GenerateMinMaxDouble(HInstruction* minmax, bool is_min);
362   void GenerateMinMax(HBinaryOperation* minmax, bool is_min);
363 
364   // Generate a heap reference load using one register `out`:
365   //
366   //   out <- *(out + offset)
367   //
368   // while honoring heap poisoning and/or read barriers (if any).
369   //
370   // Location `maybe_temp` is used when generating a read barrier and
371   // shall be a register in that case; it may be an invalid location
372   // otherwise.
373   void GenerateReferenceLoadOneRegister(HInstruction* instruction,
374                                         Location out,
375                                         uint32_t offset,
376                                         Location maybe_temp,
377                                         ReadBarrierOption read_barrier_option);
378   // Generate a heap reference load using two different registers
379   // `out` and `obj`:
380   //
381   //   out <- *(obj + offset)
382   //
383   // while honoring heap poisoning and/or read barriers (if any).
384   //
385   // Location `maybe_temp` is used when generating a Baker's (fast
386   // path) read barrier and shall be a register in that case; it may
387   // be an invalid location otherwise.
388   void GenerateReferenceLoadTwoRegisters(HInstruction* instruction,
389                                          Location out,
390                                          Location obj,
391                                          uint32_t offset,
392                                          Location maybe_temp,
393                                          ReadBarrierOption read_barrier_option);
394   void GenerateTestAndBranch(HInstruction* instruction,
395                              size_t condition_input_index,
396                              vixl::aarch32::Label* true_target,
397                              vixl::aarch32::Label* false_target,
398                              bool far_target = true);
399   void GenerateCompareTestAndBranch(HCondition* condition,
400                                     vixl::aarch32::Label* true_target,
401                                     vixl::aarch32::Label* false_target,
402                                     bool is_far_target = true);
403   void DivRemOneOrMinusOne(HBinaryOperation* instruction);
404   void DivRemByPowerOfTwo(HBinaryOperation* instruction);
405   void GenerateDivRemWithAnyConstant(HBinaryOperation* instruction);
406   void GenerateDivRemConstantIntegral(HBinaryOperation* instruction);
407   void HandleGoto(HInstruction* got, HBasicBlock* successor);
408 
409   vixl::aarch32::MemOperand VecAddress(
410       HVecMemoryOperation* instruction,
411       // This function may acquire a scratch register.
412       vixl::aarch32::UseScratchRegisterScope* temps_scope,
413       /*out*/ vixl32::Register* scratch);
414   vixl::aarch32::AlignedMemOperand VecAddressUnaligned(
415       HVecMemoryOperation* instruction,
416       // This function may acquire a scratch register.
417       vixl::aarch32::UseScratchRegisterScope* temps_scope,
418       /*out*/ vixl32::Register* scratch);
419 
420   ArmVIXLAssembler* const assembler_;
421   CodeGeneratorARMVIXL* const codegen_;
422 
423   DISALLOW_COPY_AND_ASSIGN(InstructionCodeGeneratorARMVIXL);
424 };
425 
426 class CodeGeneratorARMVIXL : public CodeGenerator {
427  public:
428   CodeGeneratorARMVIXL(HGraph* graph,
429                        const CompilerOptions& compiler_options,
430                        OptimizingCompilerStats* stats = nullptr);
~CodeGeneratorARMVIXL()431   virtual ~CodeGeneratorARMVIXL() {}
432 
433   void GenerateFrameEntry() override;
434   void GenerateFrameExit() override;
435   void Bind(HBasicBlock* block) override;
436   void MoveConstant(Location destination, int32_t value) override;
437   void MoveLocation(Location dst, Location src, DataType::Type dst_type) override;
438   void AddLocationAsTemp(Location location, LocationSummary* locations) override;
439 
440   size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) override;
441   size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) override;
442   size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) override;
443   size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) override;
444 
GetWordSize()445   size_t GetWordSize() const override {
446     return static_cast<size_t>(kArmPointerSize);
447   }
448 
GetCalleePreservedFPWidth()449   size_t GetCalleePreservedFPWidth() const override {
450     return vixl::aarch32::kSRegSizeInBytes;
451   }
452 
GetLocationBuilder()453   HGraphVisitor* GetLocationBuilder() override { return &location_builder_; }
454 
GetInstructionVisitor()455   HGraphVisitor* GetInstructionVisitor() override { return &instruction_visitor_; }
456 
GetAssembler()457   ArmVIXLAssembler* GetAssembler() override { return &assembler_; }
458 
GetAssembler()459   const ArmVIXLAssembler& GetAssembler() const override { return assembler_; }
460 
GetVIXLAssembler()461   ArmVIXLMacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); }
462 
GetAddressOf(HBasicBlock * block)463   uintptr_t GetAddressOf(HBasicBlock* block) override {
464     vixl::aarch32::Label* block_entry_label = GetLabelOf(block);
465     DCHECK(block_entry_label->IsBound());
466     return block_entry_label->GetLocation();
467   }
468 
469   void FixJumpTables();
470   void SetupBlockedRegisters() const override;
471 
472   void DumpCoreRegister(std::ostream& stream, int reg) const override;
473   void DumpFloatingPointRegister(std::ostream& stream, int reg) const override;
474 
GetMoveResolver()475   ParallelMoveResolver* GetMoveResolver() override { return &move_resolver_; }
GetInstructionSet()476   InstructionSet GetInstructionSet() const override { return InstructionSet::kThumb2; }
477 
478   const ArmInstructionSetFeatures& GetInstructionSetFeatures() const;
479 
480   // Helper method to move a 32-bit value between two locations.
481   void Move32(Location destination, Location source);
482 
483   void LoadFromShiftedRegOffset(DataType::Type type,
484                                 Location out_loc,
485                                 vixl::aarch32::Register base,
486                                 vixl::aarch32::Register reg_index,
487                                 vixl::aarch32::Condition cond = vixl::aarch32::al);
488   void StoreToShiftedRegOffset(DataType::Type type,
489                                Location out_loc,
490                                vixl::aarch32::Register base,
491                                vixl::aarch32::Register reg_index,
492                                vixl::aarch32::Condition cond = vixl::aarch32::al);
493 
494   // Generate code to invoke a runtime entry point.
495   void InvokeRuntime(QuickEntrypointEnum entrypoint,
496                      HInstruction* instruction,
497                      uint32_t dex_pc,
498                      SlowPathCode* slow_path = nullptr) override;
499 
500   // Generate code to invoke a runtime entry point, but do not record
501   // PC-related information in a stack map.
502   void InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset,
503                                            HInstruction* instruction,
504                                            SlowPathCode* slow_path);
505 
506   // Emit a write barrier.
507   void MarkGCCard(vixl::aarch32::Register temp,
508                   vixl::aarch32::Register card,
509                   vixl::aarch32::Register object,
510                   vixl::aarch32::Register value,
511                   bool can_be_null);
512 
513   void GenerateMemoryBarrier(MemBarrierKind kind);
514 
GetLabelOf(HBasicBlock * block)515   vixl::aarch32::Label* GetLabelOf(HBasicBlock* block) {
516     block = FirstNonEmptyBlock(block);
517     return &(block_labels_[block->GetBlockId()]);
518   }
519 
520   vixl32::Label* GetFinalLabel(HInstruction* instruction, vixl32::Label* final_label);
521 
Initialize()522   void Initialize() override {
523     block_labels_.resize(GetGraph()->GetBlocks().size());
524   }
525 
526   void Finalize(CodeAllocator* allocator) override;
527 
NeedsTwoRegisters(DataType::Type type)528   bool NeedsTwoRegisters(DataType::Type type) const override {
529     return type == DataType::Type::kFloat64 || type == DataType::Type::kInt64;
530   }
531 
532   void ComputeSpillMask() override;
533 
GetFrameEntryLabel()534   vixl::aarch32::Label* GetFrameEntryLabel() { return &frame_entry_label_; }
535 
536   // Check if the desired_string_load_kind is supported. If it is, return it,
537   // otherwise return a fall-back kind that should be used instead.
538   HLoadString::LoadKind GetSupportedLoadStringKind(
539       HLoadString::LoadKind desired_string_load_kind) override;
540 
541   // Check if the desired_class_load_kind is supported. If it is, return it,
542   // otherwise return a fall-back kind that should be used instead.
543   HLoadClass::LoadKind GetSupportedLoadClassKind(
544       HLoadClass::LoadKind desired_class_load_kind) override;
545 
546   // Check if the desired_dispatch_info is supported. If it is, return it,
547   // otherwise return a fall-back info that should be used instead.
548   HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
549       const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
550       ArtMethod* method) override;
551 
552   void GenerateStaticOrDirectCall(
553       HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
554   void GenerateVirtualCall(
555       HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
556 
557   void MoveFromReturnRegister(Location trg, DataType::Type type) override;
558 
559   // The PcRelativePatchInfo is used for PC-relative addressing of methods/strings/types,
560   // whether through .data.bimg.rel.ro, .bss, or directly in the boot image.
561   //
562   // The PC-relative address is loaded with three instructions,
563   // MOVW+MOVT to load the offset to base_reg and then ADD base_reg, PC. The offset
564   // is calculated from the ADD's effective PC, i.e. PC+4 on Thumb2. Though we
565   // currently emit these 3 instructions together, instruction scheduling could
566   // split this sequence apart, so we keep separate labels for each of them.
567   struct PcRelativePatchInfo {
PcRelativePatchInfoPcRelativePatchInfo568     PcRelativePatchInfo(const DexFile* dex_file, uint32_t off_or_idx)
569         : target_dex_file(dex_file), offset_or_index(off_or_idx) { }
570     PcRelativePatchInfo(PcRelativePatchInfo&& other) = default;
571 
572     // Target dex file or null for .data.bmig.rel.ro patches.
573     const DexFile* target_dex_file;
574     // Either the boot image offset (to write to .data.bmig.rel.ro) or string/type/method index.
575     uint32_t offset_or_index;
576     vixl::aarch32::Label movw_label;
577     vixl::aarch32::Label movt_label;
578     vixl::aarch32::Label add_pc_label;
579   };
580 
581   PcRelativePatchInfo* NewBootImageIntrinsicPatch(uint32_t intrinsic_data);
582   PcRelativePatchInfo* NewBootImageRelRoPatch(uint32_t boot_image_offset);
583   PcRelativePatchInfo* NewBootImageMethodPatch(MethodReference target_method);
584   PcRelativePatchInfo* NewMethodBssEntryPatch(MethodReference target_method);
585   PcRelativePatchInfo* NewBootImageTypePatch(const DexFile& dex_file, dex::TypeIndex type_index);
586   PcRelativePatchInfo* NewTypeBssEntryPatch(const DexFile& dex_file, dex::TypeIndex type_index);
587   PcRelativePatchInfo* NewBootImageStringPatch(const DexFile& dex_file,
588                                                dex::StringIndex string_index);
589   PcRelativePatchInfo* NewStringBssEntryPatch(const DexFile& dex_file,
590                                               dex::StringIndex string_index);
591 
592   // Emit the BL instruction for entrypoint thunk call and record the associated patch for AOT.
593   void EmitEntrypointThunkCall(ThreadOffset32 entrypoint_offset);
594 
595   // Emit the BNE instruction for baker read barrier and record
596   // the associated patch for AOT or slow path for JIT.
597   void EmitBakerReadBarrierBne(uint32_t custom_data);
598 
599   VIXLUInt32Literal* DeduplicateBootImageAddressLiteral(uint32_t address);
600   VIXLUInt32Literal* DeduplicateJitStringLiteral(const DexFile& dex_file,
601                                                  dex::StringIndex string_index,
602                                                  Handle<mirror::String> handle);
603   VIXLUInt32Literal* DeduplicateJitClassLiteral(const DexFile& dex_file,
604                                                 dex::TypeIndex type_index,
605                                                 Handle<mirror::Class> handle);
606 
607   void LoadBootImageAddress(vixl::aarch32::Register reg, uint32_t boot_image_reference);
608   void AllocateInstanceForIntrinsic(HInvokeStaticOrDirect* invoke, uint32_t boot_image_offset);
609 
610   void EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) override;
611   bool NeedsThunkCode(const linker::LinkerPatch& patch) const override;
612   void EmitThunkCode(const linker::LinkerPatch& patch,
613                      /*out*/ ArenaVector<uint8_t>* code,
614                      /*out*/ std::string* debug_name) override;
615 
616   void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) override;
617 
618   // Generate a GC root reference load:
619   //
620   //   root <- *(obj + offset)
621   //
622   // while honoring read barriers based on read_barrier_option.
623   void GenerateGcRootFieldLoad(HInstruction* instruction,
624                                Location root,
625                                vixl::aarch32::Register obj,
626                                uint32_t offset,
627                                ReadBarrierOption read_barrier_option);
628   // Generate ADD for UnsafeCASObject to reconstruct the old value from
629   // `old_value - expected` and mark it with Baker read barrier.
630   void GenerateUnsafeCasOldValueAddWithBakerReadBarrier(vixl::aarch32::Register old_value,
631                                                         vixl::aarch32::Register adjusted_old_value,
632                                                         vixl::aarch32::Register expected);
633   // Fast path implementation of ReadBarrier::Barrier for a heap
634   // reference field load when Baker's read barriers are used.
635   // Overload suitable for Unsafe.getObject/-Volatile() intrinsic.
636   void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
637                                              Location ref,
638                                              vixl::aarch32::Register obj,
639                                              const vixl::aarch32::MemOperand& src,
640                                              bool needs_null_check);
641   // Fast path implementation of ReadBarrier::Barrier for a heap
642   // reference field load when Baker's read barriers are used.
643   void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
644                                              Location ref,
645                                              vixl::aarch32::Register obj,
646                                              uint32_t offset,
647                                              Location temp,
648                                              bool needs_null_check);
649   // Fast path implementation of ReadBarrier::Barrier for a heap
650   // reference array load when Baker's read barriers are used.
651   void GenerateArrayLoadWithBakerReadBarrier(Location ref,
652                                              vixl::aarch32::Register obj,
653                                              uint32_t data_offset,
654                                              Location index,
655                                              Location temp,
656                                              bool needs_null_check);
657 
658   // Emit code checking the status of the Marking Register, and
659   // aborting the program if MR does not match the value stored in the
660   // art::Thread object. Code is only emitted in debug mode and if
661   // CompilerOptions::EmitRunTimeChecksInDebugMode returns true.
662   //
663   // Argument `code` is used to identify the different occurrences of
664   // MaybeGenerateMarkingRegisterCheck in the code generator, and is
665   // used together with kMarkingRegisterCheckBreakCodeBaseCode to
666   // create the value passed to the BKPT instruction. Note that unlike
667   // in the ARM64 code generator, where `__LINE__` is passed as `code`
668   // argument to
669   // CodeGeneratorARM64::MaybeGenerateMarkingRegisterCheck, we cannot
670   // realistically do that here, as Encoding T1 for the BKPT
671   // instruction only accepts 8-bit immediate values.
672   //
673   // If `temp_loc` is a valid location, it is expected to be a
674   // register and will be used as a temporary to generate code;
675   // otherwise, a temporary will be fetched from the core register
676   // scratch pool.
677   virtual void MaybeGenerateMarkingRegisterCheck(int code,
678                                                  Location temp_loc = Location::NoLocation());
679 
680   // Generate a read barrier for a heap reference within `instruction`
681   // using a slow path.
682   //
683   // A read barrier for an object reference read from the heap is
684   // implemented as a call to the artReadBarrierSlow runtime entry
685   // point, which is passed the values in locations `ref`, `obj`, and
686   // `offset`:
687   //
688   //   mirror::Object* artReadBarrierSlow(mirror::Object* ref,
689   //                                      mirror::Object* obj,
690   //                                      uint32_t offset);
691   //
692   // The `out` location contains the value returned by
693   // artReadBarrierSlow.
694   //
695   // When `index` is provided (i.e. for array accesses), the offset
696   // value passed to artReadBarrierSlow is adjusted to take `index`
697   // into account.
698   void GenerateReadBarrierSlow(HInstruction* instruction,
699                                Location out,
700                                Location ref,
701                                Location obj,
702                                uint32_t offset,
703                                Location index = Location::NoLocation());
704 
705   // If read barriers are enabled, generate a read barrier for a heap
706   // reference using a slow path. If heap poisoning is enabled, also
707   // unpoison the reference in `out`.
708   void MaybeGenerateReadBarrierSlow(HInstruction* instruction,
709                                     Location out,
710                                     Location ref,
711                                     Location obj,
712                                     uint32_t offset,
713                                     Location index = Location::NoLocation());
714 
715   // Generate a read barrier for a GC root within `instruction` using
716   // a slow path.
717   //
718   // A read barrier for an object reference GC root is implemented as
719   // a call to the artReadBarrierForRootSlow runtime entry point,
720   // which is passed the value in location `root`:
721   //
722   //   mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root);
723   //
724   // The `out` location contains the value returned by
725   // artReadBarrierForRootSlow.
726   void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root);
727 
728   void GenerateNop() override;
729 
730   void GenerateImplicitNullCheck(HNullCheck* instruction) override;
731   void GenerateExplicitNullCheck(HNullCheck* instruction) override;
732 
CreateJumpTable(HPackedSwitch * switch_instr)733   JumpTableARMVIXL* CreateJumpTable(HPackedSwitch* switch_instr) {
734     jump_tables_.emplace_back(new (GetGraph()->GetAllocator()) JumpTableARMVIXL(switch_instr));
735     return jump_tables_.back().get();
736   }
737   void EmitJumpTables();
738 
739   void EmitMovwMovtPlaceholder(CodeGeneratorARMVIXL::PcRelativePatchInfo* labels,
740                                vixl::aarch32::Register out);
741 
742   // `temp` is an extra temporary register that is used for some conditions;
743   // callers may not specify it, in which case the method will use a scratch
744   // register instead.
745   void GenerateConditionWithZero(IfCondition condition,
746                                  vixl::aarch32::Register out,
747                                  vixl::aarch32::Register in,
748                                  vixl::aarch32::Register temp = vixl32::Register());
749 
MaybeRecordImplicitNullCheck(HInstruction * instr)750   void MaybeRecordImplicitNullCheck(HInstruction* instr) final {
751     // The function must be only be called within special scopes
752     // (EmissionCheckScope, ExactAssemblyScope) which prevent generation of
753     // veneer/literal pools by VIXL assembler.
754     CHECK_EQ(GetVIXLAssembler()->ArePoolsBlocked(), true)
755         << "The function must only be called within EmissionCheckScope or ExactAssemblyScope";
756     CodeGenerator::MaybeRecordImplicitNullCheck(instr);
757   }
758 
759   void MaybeGenerateInlineCacheCheck(HInstruction* instruction, vixl32::Register klass);
760   void MaybeIncrementHotness(bool is_frame_entry);
761 
762  private:
763   // Encoding of thunk type and data for link-time generated thunks for Baker read barriers.
764 
765   enum class BakerReadBarrierKind : uint8_t {
766     kField,       // Field get or array get with constant offset (i.e. constant index).
767     kArray,       // Array get with index in register.
768     kGcRoot,      // GC root load.
769     kUnsafeCas,   // UnsafeCASObject intrinsic.
770     kLast = kUnsafeCas
771   };
772 
773   enum class BakerReadBarrierWidth : uint8_t {
774     kWide,          // 32-bit LDR (and 32-bit NEG if heap poisoning is enabled).
775     kNarrow,        // 16-bit LDR (and 16-bit NEG if heap poisoning is enabled).
776     kLast = kNarrow
777   };
778 
779   static constexpr uint32_t kBakerReadBarrierInvalidEncodedReg = /* pc is invalid */ 15u;
780 
781   static constexpr size_t kBitsForBakerReadBarrierKind =
782       MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast));
783   static constexpr size_t kBakerReadBarrierBitsForRegister =
784       MinimumBitsToStore(kBakerReadBarrierInvalidEncodedReg);
785   using BakerReadBarrierKindField =
786       BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>;
787   using BakerReadBarrierFirstRegField =
788       BitField<uint32_t, kBitsForBakerReadBarrierKind, kBakerReadBarrierBitsForRegister>;
789   using BakerReadBarrierSecondRegField =
790       BitField<uint32_t,
791                kBitsForBakerReadBarrierKind + kBakerReadBarrierBitsForRegister,
792                kBakerReadBarrierBitsForRegister>;
793   static constexpr size_t kBitsForBakerReadBarrierWidth =
794       MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierWidth::kLast));
795   using BakerReadBarrierWidthField =
796       BitField<BakerReadBarrierWidth,
797                kBitsForBakerReadBarrierKind + 2 * kBakerReadBarrierBitsForRegister,
798                kBitsForBakerReadBarrierWidth>;
799 
CheckValidReg(uint32_t reg)800   static void CheckValidReg(uint32_t reg) {
801     DCHECK(reg < vixl::aarch32::ip.GetCode() && reg != mr.GetCode()) << reg;
802   }
803 
EncodeBakerReadBarrierFieldData(uint32_t base_reg,uint32_t holder_reg,bool narrow)804   static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg,
805                                                   uint32_t holder_reg,
806                                                   bool narrow) {
807     CheckValidReg(base_reg);
808     CheckValidReg(holder_reg);
809     DCHECK(!narrow || base_reg < 8u) << base_reg;
810     BakerReadBarrierWidth width =
811         narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide;
812     return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) |
813            BakerReadBarrierFirstRegField::Encode(base_reg) |
814            BakerReadBarrierSecondRegField::Encode(holder_reg) |
815            BakerReadBarrierWidthField::Encode(width);
816   }
817 
EncodeBakerReadBarrierArrayData(uint32_t base_reg)818   static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
819     CheckValidReg(base_reg);
820     return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) |
821            BakerReadBarrierFirstRegField::Encode(base_reg) |
822            BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg) |
823            BakerReadBarrierWidthField::Encode(BakerReadBarrierWidth::kWide);
824   }
825 
EncodeBakerReadBarrierGcRootData(uint32_t root_reg,bool narrow)826   static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) {
827     CheckValidReg(root_reg);
828     DCHECK(!narrow || root_reg < 8u) << root_reg;
829     BakerReadBarrierWidth width =
830         narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide;
831     return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) |
832            BakerReadBarrierFirstRegField::Encode(root_reg) |
833            BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg) |
834            BakerReadBarrierWidthField::Encode(width);
835   }
836 
EncodeBakerReadBarrierUnsafeCasData(uint32_t root_reg)837   static uint32_t EncodeBakerReadBarrierUnsafeCasData(uint32_t root_reg) {
838     CheckValidReg(root_reg);
839     return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kUnsafeCas) |
840            BakerReadBarrierFirstRegField::Encode(root_reg) |
841            BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg) |
842            BakerReadBarrierWidthField::Encode(BakerReadBarrierWidth::kWide);
843   }
844 
845   void CompileBakerReadBarrierThunk(ArmVIXLAssembler& assembler,
846                                     uint32_t encoded_data,
847                                     /*out*/ std::string* debug_name);
848 
849   vixl::aarch32::Register GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke,
850                                                                 vixl::aarch32::Register temp);
851 
852   using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, VIXLUInt32Literal*>;
853   using StringToLiteralMap = ArenaSafeMap<StringReference,
854                                           VIXLUInt32Literal*,
855                                           StringReferenceValueComparator>;
856   using TypeToLiteralMap = ArenaSafeMap<TypeReference,
857                                         VIXLUInt32Literal*,
858                                         TypeReferenceValueComparator>;
859 
860   struct BakerReadBarrierPatchInfo {
BakerReadBarrierPatchInfoBakerReadBarrierPatchInfo861     explicit BakerReadBarrierPatchInfo(uint32_t data) : label(), custom_data(data) { }
862 
863     vixl::aarch32::Label label;
864     uint32_t custom_data;
865   };
866 
867   VIXLUInt32Literal* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map);
868   PcRelativePatchInfo* NewPcRelativePatch(const DexFile* dex_file,
869                                           uint32_t offset_or_index,
870                                           ArenaDeque<PcRelativePatchInfo>* patches);
871   template <linker::LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)>
872   static void EmitPcRelativeLinkerPatches(const ArenaDeque<PcRelativePatchInfo>& infos,
873                                           ArenaVector<linker::LinkerPatch>* linker_patches);
874 
875   // Labels for each block that will be compiled.
876   // We use a deque so that the `vixl::aarch32::Label` objects do not move in memory.
877   ArenaDeque<vixl::aarch32::Label> block_labels_;  // Indexed by block id.
878   vixl::aarch32::Label frame_entry_label_;
879 
880   ArenaVector<std::unique_ptr<JumpTableARMVIXL>> jump_tables_;
881   LocationsBuilderARMVIXL location_builder_;
882   InstructionCodeGeneratorARMVIXL instruction_visitor_;
883   ParallelMoveResolverARMVIXL move_resolver_;
884 
885   ArmVIXLAssembler assembler_;
886 
887   // PC-relative method patch info for kBootImageLinkTimePcRelative.
888   ArenaDeque<PcRelativePatchInfo> boot_image_method_patches_;
889   // PC-relative method patch info for kBssEntry.
890   ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_;
891   // PC-relative type patch info for kBootImageLinkTimePcRelative.
892   ArenaDeque<PcRelativePatchInfo> boot_image_type_patches_;
893   // PC-relative type patch info for kBssEntry.
894   ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
895   // PC-relative String patch info for kBootImageLinkTimePcRelative.
896   ArenaDeque<PcRelativePatchInfo> boot_image_string_patches_;
897   // PC-relative String patch info for kBssEntry.
898   ArenaDeque<PcRelativePatchInfo> string_bss_entry_patches_;
899   // PC-relative patch info for IntrinsicObjects for the boot image,
900   // and for method/type/string patches for kBootImageRelRo otherwise.
901   ArenaDeque<PcRelativePatchInfo> boot_image_other_patches_;
902   // Patch info for calls to entrypoint dispatch thunks. Used for slow paths.
903   ArenaDeque<PatchInfo<vixl::aarch32::Label>> call_entrypoint_patches_;
904   // Baker read barrier patch info.
905   ArenaDeque<BakerReadBarrierPatchInfo> baker_read_barrier_patches_;
906 
907   // Deduplication map for 32-bit literals, used for JIT for boot image addresses.
908   Uint32ToLiteralMap uint32_literals_;
909   // Patches for string literals in JIT compiled code.
910   StringToLiteralMap jit_string_patches_;
911   // Patches for class literals in JIT compiled code.
912   TypeToLiteralMap jit_class_patches_;
913 
914   // Baker read barrier slow paths, mapping custom data (uint32_t) to label.
915   // Wrap the label to work around vixl::aarch32::Label being non-copyable
916   // and non-moveable and as such unusable in ArenaSafeMap<>.
917   struct LabelWrapper {
LabelWrapperLabelWrapper918     LabelWrapper(const LabelWrapper& src)
919         : label() {
920       DCHECK(!src.label.IsReferenced() && !src.label.IsBound());
921     }
922     LabelWrapper() = default;
923     vixl::aarch32::Label label;
924   };
925   ArenaSafeMap<uint32_t, LabelWrapper> jit_baker_read_barrier_slow_paths_;
926 
927   friend class linker::Thumb2RelativePatcherTest;
928   DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARMVIXL);
929 };
930 
931 }  // namespace arm
932 }  // namespace art
933 
934 #endif  // ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM_VIXL_H_
935