1 /*
2  * Copyright (C) 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_X86_64_H_
18 #define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_X86_64_H_
19 
20 #include "arch/x86_64/instruction_set_features_x86_64.h"
21 #include "code_generator.h"
22 #include "driver/compiler_options.h"
23 #include "nodes.h"
24 #include "parallel_move_resolver.h"
25 #include "utils/x86_64/assembler_x86_64.h"
26 
27 namespace art {
28 namespace x86_64 {
29 
30 // Use a local definition to prevent copying mistakes.
31 static constexpr size_t kX86_64WordSize = static_cast<size_t>(kX86_64PointerSize);
32 
33 // Some x86_64 instructions require a register to be available as temp.
34 static constexpr Register TMP = R11;
35 
36 static constexpr Register kParameterCoreRegisters[] = { RSI, RDX, RCX, R8, R9 };
37 static constexpr FloatRegister kParameterFloatRegisters[] =
38     { XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7 };
39 
40 static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters);
41 static constexpr size_t kParameterFloatRegistersLength = arraysize(kParameterFloatRegisters);
42 
43 static constexpr Register kRuntimeParameterCoreRegisters[] = { RDI, RSI, RDX, RCX };
44 static constexpr size_t kRuntimeParameterCoreRegistersLength =
45     arraysize(kRuntimeParameterCoreRegisters);
46 static constexpr FloatRegister kRuntimeParameterFpuRegisters[] = { XMM0, XMM1 };
47 static constexpr size_t kRuntimeParameterFpuRegistersLength =
48     arraysize(kRuntimeParameterFpuRegisters);
49 
50 // These XMM registers are non-volatile in ART ABI, but volatile in native ABI.
51 // If the ART ABI changes, this list must be updated.  It is used to ensure that
52 // these are not clobbered by any direct call to native code (such as math intrinsics).
53 static constexpr FloatRegister non_volatile_xmm_regs[] = { XMM12, XMM13, XMM14, XMM15 };
54 
55 
56 class InvokeRuntimeCallingConvention : public CallingConvention<Register, FloatRegister> {
57  public:
InvokeRuntimeCallingConvention()58   InvokeRuntimeCallingConvention()
59       : CallingConvention(kRuntimeParameterCoreRegisters,
60                           kRuntimeParameterCoreRegistersLength,
61                           kRuntimeParameterFpuRegisters,
62                           kRuntimeParameterFpuRegistersLength,
63                           kX86_64PointerSize) {}
64 
65  private:
66   DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention);
67 };
68 
69 class InvokeDexCallingConvention : public CallingConvention<Register, FloatRegister> {
70  public:
InvokeDexCallingConvention()71   InvokeDexCallingConvention() : CallingConvention(
72       kParameterCoreRegisters,
73       kParameterCoreRegistersLength,
74       kParameterFloatRegisters,
75       kParameterFloatRegistersLength,
76       kX86_64PointerSize) {}
77 
78  private:
79   DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConvention);
80 };
81 
82 class FieldAccessCallingConventionX86_64 : public FieldAccessCallingConvention {
83  public:
FieldAccessCallingConventionX86_64()84   FieldAccessCallingConventionX86_64() {}
85 
GetObjectLocation()86   Location GetObjectLocation() const override {
87     return Location::RegisterLocation(RSI);
88   }
GetFieldIndexLocation()89   Location GetFieldIndexLocation() const override {
90     return Location::RegisterLocation(RDI);
91   }
GetReturnLocation(DataType::Type type ATTRIBUTE_UNUSED)92   Location GetReturnLocation(DataType::Type type ATTRIBUTE_UNUSED) const override {
93     return Location::RegisterLocation(RAX);
94   }
GetSetValueLocation(DataType::Type type ATTRIBUTE_UNUSED,bool is_instance)95   Location GetSetValueLocation(DataType::Type type ATTRIBUTE_UNUSED, bool is_instance)
96       const override {
97     return is_instance
98         ? Location::RegisterLocation(RDX)
99         : Location::RegisterLocation(RSI);
100   }
GetFpuLocation(DataType::Type type ATTRIBUTE_UNUSED)101   Location GetFpuLocation(DataType::Type type ATTRIBUTE_UNUSED) const override {
102     return Location::FpuRegisterLocation(XMM0);
103   }
104 
105  private:
106   DISALLOW_COPY_AND_ASSIGN(FieldAccessCallingConventionX86_64);
107 };
108 
109 
110 class InvokeDexCallingConventionVisitorX86_64 : public InvokeDexCallingConventionVisitor {
111  public:
InvokeDexCallingConventionVisitorX86_64()112   InvokeDexCallingConventionVisitorX86_64() {}
~InvokeDexCallingConventionVisitorX86_64()113   virtual ~InvokeDexCallingConventionVisitorX86_64() {}
114 
115   Location GetNextLocation(DataType::Type type) override;
116   Location GetReturnLocation(DataType::Type type) const override;
117   Location GetMethodLocation() const override;
118 
119  private:
120   InvokeDexCallingConvention calling_convention;
121 
122   DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConventionVisitorX86_64);
123 };
124 
125 class CodeGeneratorX86_64;
126 
127 class ParallelMoveResolverX86_64 : public ParallelMoveResolverWithSwap {
128  public:
ParallelMoveResolverX86_64(ArenaAllocator * allocator,CodeGeneratorX86_64 * codegen)129   ParallelMoveResolverX86_64(ArenaAllocator* allocator, CodeGeneratorX86_64* codegen)
130       : ParallelMoveResolverWithSwap(allocator), codegen_(codegen) {}
131 
132   void EmitMove(size_t index) override;
133   void EmitSwap(size_t index) override;
134   void SpillScratch(int reg) override;
135   void RestoreScratch(int reg) override;
136 
137   X86_64Assembler* GetAssembler() const;
138 
139  private:
140   void Exchange32(CpuRegister reg, int mem);
141   void Exchange32(XmmRegister reg, int mem);
142   void Exchange64(CpuRegister reg1, CpuRegister reg2);
143   void Exchange64(CpuRegister reg, int mem);
144   void Exchange64(XmmRegister reg, int mem);
145   void Exchange128(XmmRegister reg, int mem);
146   void ExchangeMemory32(int mem1, int mem2);
147   void ExchangeMemory64(int mem1, int mem2, int num_of_qwords);
148 
149   CodeGeneratorX86_64* const codegen_;
150 
151   DISALLOW_COPY_AND_ASSIGN(ParallelMoveResolverX86_64);
152 };
153 
154 class LocationsBuilderX86_64 : public HGraphVisitor {
155  public:
LocationsBuilderX86_64(HGraph * graph,CodeGeneratorX86_64 * codegen)156   LocationsBuilderX86_64(HGraph* graph, CodeGeneratorX86_64* codegen)
157       : HGraphVisitor(graph), codegen_(codegen) {}
158 
159 #define DECLARE_VISIT_INSTRUCTION(name, super)     \
160   void Visit##name(H##name* instr) override;
161 
162   FOR_EACH_CONCRETE_INSTRUCTION_COMMON(DECLARE_VISIT_INSTRUCTION)
FOR_EACH_CONCRETE_INSTRUCTION_X86_64(DECLARE_VISIT_INSTRUCTION)163   FOR_EACH_CONCRETE_INSTRUCTION_X86_64(DECLARE_VISIT_INSTRUCTION)
164   FOR_EACH_CONCRETE_INSTRUCTION_X86_COMMON(DECLARE_VISIT_INSTRUCTION)
165 
166 #undef DECLARE_VISIT_INSTRUCTION
167 
168   void VisitInstruction(HInstruction* instruction) override {
169     LOG(FATAL) << "Unreachable instruction " << instruction->DebugName()
170                << " (id " << instruction->GetId() << ")";
171   }
172 
173  private:
174   void HandleInvoke(HInvoke* invoke);
175   void HandleBitwiseOperation(HBinaryOperation* operation);
176   void HandleCondition(HCondition* condition);
177   void HandleShift(HBinaryOperation* operation);
178   void HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info);
179   void HandleFieldGet(HInstruction* instruction);
180   bool CpuHasAvxFeatureFlag();
181   bool CpuHasAvx2FeatureFlag();
182 
183   CodeGeneratorX86_64* const codegen_;
184   InvokeDexCallingConventionVisitorX86_64 parameter_visitor_;
185 
186   DISALLOW_COPY_AND_ASSIGN(LocationsBuilderX86_64);
187 };
188 
189 class InstructionCodeGeneratorX86_64 : public InstructionCodeGenerator {
190  public:
191   InstructionCodeGeneratorX86_64(HGraph* graph, CodeGeneratorX86_64* codegen);
192 
193 #define DECLARE_VISIT_INSTRUCTION(name, super)     \
194   void Visit##name(H##name* instr) override;
195 
196   FOR_EACH_CONCRETE_INSTRUCTION_COMMON(DECLARE_VISIT_INSTRUCTION)
FOR_EACH_CONCRETE_INSTRUCTION_X86_64(DECLARE_VISIT_INSTRUCTION)197   FOR_EACH_CONCRETE_INSTRUCTION_X86_64(DECLARE_VISIT_INSTRUCTION)
198   FOR_EACH_CONCRETE_INSTRUCTION_X86_COMMON(DECLARE_VISIT_INSTRUCTION)
199 
200 #undef DECLARE_VISIT_INSTRUCTION
201 
202   void VisitInstruction(HInstruction* instruction) override {
203     LOG(FATAL) << "Unreachable instruction " << instruction->DebugName()
204                << " (id " << instruction->GetId() << ")";
205   }
206 
GetAssembler()207   X86_64Assembler* GetAssembler() const { return assembler_; }
208 
209  private:
210   // Generate code for the given suspend check. If not null, `successor`
211   // is the block to branch to if the suspend check is not needed, and after
212   // the suspend call.
213   void GenerateSuspendCheck(HSuspendCheck* instruction, HBasicBlock* successor);
214   void GenerateClassInitializationCheck(SlowPathCode* slow_path, CpuRegister class_reg);
215   void GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check, CpuRegister temp);
216   void HandleBitwiseOperation(HBinaryOperation* operation);
217   void GenerateRemFP(HRem* rem);
218   void DivRemOneOrMinusOne(HBinaryOperation* instruction);
219   void DivByPowerOfTwo(HDiv* instruction);
220   void RemByPowerOfTwo(HRem* instruction);
221   void GenerateDivRemWithAnyConstant(HBinaryOperation* instruction);
222   void GenerateDivRemIntegral(HBinaryOperation* instruction);
223   void HandleCondition(HCondition* condition);
224   void HandleShift(HBinaryOperation* operation);
225 
226   void HandleFieldSet(HInstruction* instruction,
227                       const FieldInfo& field_info,
228                       bool value_can_be_null);
229   void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info);
230 
231   void GenerateMinMaxInt(LocationSummary* locations, bool is_min, DataType::Type type);
232   void GenerateMinMaxFP(LocationSummary* locations, bool is_min, DataType::Type type);
233   void GenerateMinMax(HBinaryOperation* minmax, bool is_min);
234 
235   // Generate a heap reference load using one register `out`:
236   //
237   //   out <- *(out + offset)
238   //
239   // while honoring heap poisoning and/or read barriers (if any).
240   //
241   // Location `maybe_temp` is used when generating a read barrier and
242   // shall be a register in that case; it may be an invalid location
243   // otherwise.
244   void GenerateReferenceLoadOneRegister(HInstruction* instruction,
245                                         Location out,
246                                         uint32_t offset,
247                                         Location maybe_temp,
248                                         ReadBarrierOption read_barrier_option);
249   // Generate a heap reference load using two different registers
250   // `out` and `obj`:
251   //
252   //   out <- *(obj + offset)
253   //
254   // while honoring heap poisoning and/or read barriers (if any).
255   //
256   // Location `maybe_temp` is used when generating a Baker's (fast
257   // path) read barrier and shall be a register in that case; it may
258   // be an invalid location otherwise.
259   void GenerateReferenceLoadTwoRegisters(HInstruction* instruction,
260                                          Location out,
261                                          Location obj,
262                                          uint32_t offset,
263                                          ReadBarrierOption read_barrier_option);
264   // Generate a GC root reference load:
265   //
266   //   root <- *address
267   //
268   // while honoring read barriers based on read_barrier_option.
269   void GenerateGcRootFieldLoad(HInstruction* instruction,
270                                Location root,
271                                const Address& address,
272                                Label* fixup_label,
273                                ReadBarrierOption read_barrier_option);
274 
275   void PushOntoFPStack(Location source, uint32_t temp_offset,
276                        uint32_t stack_adjustment, bool is_float);
277   void GenerateCompareTest(HCondition* condition);
278   template<class LabelType>
279   void GenerateTestAndBranch(HInstruction* instruction,
280                              size_t condition_input_index,
281                              LabelType* true_target,
282                              LabelType* false_target);
283   template<class LabelType>
284   void GenerateCompareTestAndBranch(HCondition* condition,
285                                     LabelType* true_target,
286                                     LabelType* false_target);
287   template<class LabelType>
288   void GenerateFPJumps(HCondition* cond, LabelType* true_label, LabelType* false_label);
289 
290   void HandleGoto(HInstruction* got, HBasicBlock* successor);
291 
292   bool CpuHasAvxFeatureFlag();
293   bool CpuHasAvx2FeatureFlag();
294 
295   X86_64Assembler* const assembler_;
296   CodeGeneratorX86_64* const codegen_;
297 
298   DISALLOW_COPY_AND_ASSIGN(InstructionCodeGeneratorX86_64);
299 };
300 
301 // Class for fixups to jump tables.
302 class JumpTableRIPFixup;
303 
304 class CodeGeneratorX86_64 : public CodeGenerator {
305  public:
306   CodeGeneratorX86_64(HGraph* graph,
307                   const CompilerOptions& compiler_options,
308                   OptimizingCompilerStats* stats = nullptr);
~CodeGeneratorX86_64()309   virtual ~CodeGeneratorX86_64() {}
310 
311   void GenerateFrameEntry() override;
312   void GenerateFrameExit() override;
313   void Bind(HBasicBlock* block) override;
314   void MoveConstant(Location destination, int32_t value) override;
315   void MoveLocation(Location dst, Location src, DataType::Type dst_type) override;
316   void AddLocationAsTemp(Location location, LocationSummary* locations) override;
317 
318   size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) override;
319   size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) override;
320   size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) override;
321   size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) override;
322 
323   // Generate code to invoke a runtime entry point.
324   void InvokeRuntime(QuickEntrypointEnum entrypoint,
325                      HInstruction* instruction,
326                      uint32_t dex_pc,
327                      SlowPathCode* slow_path = nullptr) override;
328 
329   // Generate code to invoke a runtime entry point, but do not record
330   // PC-related information in a stack map.
331   void InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset,
332                                            HInstruction* instruction,
333                                            SlowPathCode* slow_path);
334 
335   void GenerateInvokeRuntime(int32_t entry_point_offset);
336 
GetWordSize()337   size_t GetWordSize() const override {
338     return kX86_64WordSize;
339   }
340 
GetSlowPathFPWidth()341   size_t GetSlowPathFPWidth() const override {
342     return GetGraph()->HasSIMD()
343         ? 2 * kX86_64WordSize   // 16 bytes == 2 x86_64 words for each spill
344         : 1 * kX86_64WordSize;  //  8 bytes == 1 x86_64 words for each spill
345   }
346 
GetCalleePreservedFPWidth()347   size_t GetCalleePreservedFPWidth() const override {
348     return 1 * kX86_64WordSize;
349   }
350 
GetLocationBuilder()351   HGraphVisitor* GetLocationBuilder() override {
352     return &location_builder_;
353   }
354 
GetInstructionVisitor()355   HGraphVisitor* GetInstructionVisitor() override {
356     return &instruction_visitor_;
357   }
358 
GetAssembler()359   X86_64Assembler* GetAssembler() override {
360     return &assembler_;
361   }
362 
GetAssembler()363   const X86_64Assembler& GetAssembler() const override {
364     return assembler_;
365   }
366 
GetMoveResolver()367   ParallelMoveResolverX86_64* GetMoveResolver() override {
368     return &move_resolver_;
369   }
370 
GetAddressOf(HBasicBlock * block)371   uintptr_t GetAddressOf(HBasicBlock* block) override {
372     return GetLabelOf(block)->Position();
373   }
374 
375   void SetupBlockedRegisters() const override;
376   void DumpCoreRegister(std::ostream& stream, int reg) const override;
377   void DumpFloatingPointRegister(std::ostream& stream, int reg) const override;
378   void Finalize(CodeAllocator* allocator) override;
379 
GetInstructionSet()380   InstructionSet GetInstructionSet() const override {
381     return InstructionSet::kX86_64;
382   }
383 
384   const X86_64InstructionSetFeatures& GetInstructionSetFeatures() const;
385 
386   // Emit a write barrier.
387   void MarkGCCard(CpuRegister temp,
388                   CpuRegister card,
389                   CpuRegister object,
390                   CpuRegister value,
391                   bool value_can_be_null);
392 
393   void GenerateMemoryBarrier(MemBarrierKind kind);
394 
395   // Helper method to move a value between two locations.
396   void Move(Location destination, Location source);
397 
GetLabelOf(HBasicBlock * block)398   Label* GetLabelOf(HBasicBlock* block) const {
399     return CommonGetLabelOf<Label>(block_labels_, block);
400   }
401 
Initialize()402   void Initialize() override {
403     block_labels_ = CommonInitializeLabels<Label>();
404   }
405 
NeedsTwoRegisters(DataType::Type type ATTRIBUTE_UNUSED)406   bool NeedsTwoRegisters(DataType::Type type ATTRIBUTE_UNUSED) const override {
407     return false;
408   }
409 
410   // Check if the desired_string_load_kind is supported. If it is, return it,
411   // otherwise return a fall-back kind that should be used instead.
412   HLoadString::LoadKind GetSupportedLoadStringKind(
413       HLoadString::LoadKind desired_string_load_kind) override;
414 
415   // Check if the desired_class_load_kind is supported. If it is, return it,
416   // otherwise return a fall-back kind that should be used instead.
417   HLoadClass::LoadKind GetSupportedLoadClassKind(
418       HLoadClass::LoadKind desired_class_load_kind) override;
419 
420   // Check if the desired_dispatch_info is supported. If it is, return it,
421   // otherwise return a fall-back info that should be used instead.
422   HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
423       const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
424       ArtMethod* method) override;
425 
426   void GenerateStaticOrDirectCall(
427       HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
428   void GenerateVirtualCall(
429       HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
430 
431   void RecordBootImageIntrinsicPatch(uint32_t intrinsic_data);
432   void RecordBootImageRelRoPatch(uint32_t boot_image_offset);
433   void RecordBootImageMethodPatch(HInvokeStaticOrDirect* invoke);
434   void RecordMethodBssEntryPatch(HInvokeStaticOrDirect* invoke);
435   void RecordBootImageTypePatch(HLoadClass* load_class);
436   Label* NewTypeBssEntryPatch(HLoadClass* load_class);
437   void RecordBootImageStringPatch(HLoadString* load_string);
438   Label* NewStringBssEntryPatch(HLoadString* load_string);
439   Label* NewJitRootStringPatch(const DexFile& dex_file,
440                                dex::StringIndex string_index,
441                                Handle<mirror::String> handle);
442   Label* NewJitRootClassPatch(const DexFile& dex_file,
443                               dex::TypeIndex type_index,
444                               Handle<mirror::Class> handle);
445 
446   void LoadBootImageAddress(CpuRegister reg, uint32_t boot_image_reference);
447   void AllocateInstanceForIntrinsic(HInvokeStaticOrDirect* invoke, uint32_t boot_image_offset);
448 
449   void EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) override;
450 
451   void PatchJitRootUse(uint8_t* code,
452                        const uint8_t* roots_data,
453                        const PatchInfo<Label>& info,
454                        uint64_t index_in_table) const;
455 
456   void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) override;
457 
458   // Fast path implementation of ReadBarrier::Barrier for a heap
459   // reference field load when Baker's read barriers are used.
460   void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
461                                              Location ref,
462                                              CpuRegister obj,
463                                              uint32_t offset,
464                                              bool needs_null_check);
465   // Fast path implementation of ReadBarrier::Barrier for a heap
466   // reference array load when Baker's read barriers are used.
467   void GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction,
468                                              Location ref,
469                                              CpuRegister obj,
470                                              uint32_t data_offset,
471                                              Location index,
472                                              bool needs_null_check);
473   // Factored implementation, used by GenerateFieldLoadWithBakerReadBarrier,
474   // GenerateArrayLoadWithBakerReadBarrier and some intrinsics.
475   //
476   // Load the object reference located at address `src`, held by
477   // object `obj`, into `ref`, and mark it if needed.  The base of
478   // address `src` must be `obj`.
479   //
480   // If `always_update_field` is true, the value of the reference is
481   // atomically updated in the holder (`obj`).  This operation
482   // requires two temporary registers, which must be provided as
483   // non-null pointers (`temp1` and `temp2`).
484   void GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction,
485                                                  Location ref,
486                                                  CpuRegister obj,
487                                                  const Address& src,
488                                                  bool needs_null_check,
489                                                  bool always_update_field = false,
490                                                  CpuRegister* temp1 = nullptr,
491                                                  CpuRegister* temp2 = nullptr);
492 
493   // Generate a read barrier for a heap reference within `instruction`
494   // using a slow path.
495   //
496   // A read barrier for an object reference read from the heap is
497   // implemented as a call to the artReadBarrierSlow runtime entry
498   // point, which is passed the values in locations `ref`, `obj`, and
499   // `offset`:
500   //
501   //   mirror::Object* artReadBarrierSlow(mirror::Object* ref,
502   //                                      mirror::Object* obj,
503   //                                      uint32_t offset);
504   //
505   // The `out` location contains the value returned by
506   // artReadBarrierSlow.
507   //
508   // When `index` provided (i.e., when it is different from
509   // Location::NoLocation()), the offset value passed to
510   // artReadBarrierSlow is adjusted to take `index` into account.
511   void GenerateReadBarrierSlow(HInstruction* instruction,
512                                Location out,
513                                Location ref,
514                                Location obj,
515                                uint32_t offset,
516                                Location index = Location::NoLocation());
517 
518   // If read barriers are enabled, generate a read barrier for a heap
519   // reference using a slow path. If heap poisoning is enabled, also
520   // unpoison the reference in `out`.
521   void MaybeGenerateReadBarrierSlow(HInstruction* instruction,
522                                     Location out,
523                                     Location ref,
524                                     Location obj,
525                                     uint32_t offset,
526                                     Location index = Location::NoLocation());
527 
528   // Generate a read barrier for a GC root within `instruction` using
529   // a slow path.
530   //
531   // A read barrier for an object reference GC root is implemented as
532   // a call to the artReadBarrierForRootSlow runtime entry point,
533   // which is passed the value in location `root`:
534   //
535   //   mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root);
536   //
537   // The `out` location contains the value returned by
538   // artReadBarrierForRootSlow.
539   void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root);
540 
ConstantAreaStart()541   int ConstantAreaStart() const {
542     return constant_area_start_;
543   }
544 
545   Address LiteralDoubleAddress(double v);
546   Address LiteralFloatAddress(float v);
547   Address LiteralInt32Address(int32_t v);
548   Address LiteralInt64Address(int64_t v);
549 
550   // Load a 32/64-bit value into a register in the most efficient manner.
551   void Load32BitValue(CpuRegister dest, int32_t value);
552   void Load64BitValue(CpuRegister dest, int64_t value);
553   void Load32BitValue(XmmRegister dest, int32_t value);
554   void Load64BitValue(XmmRegister dest, int64_t value);
555   void Load32BitValue(XmmRegister dest, float value);
556   void Load64BitValue(XmmRegister dest, double value);
557 
558   // Compare a register with a 32/64-bit value in the most efficient manner.
559   void Compare32BitValue(CpuRegister dest, int32_t value);
560   void Compare64BitValue(CpuRegister dest, int64_t value);
561 
562   // Compare int values. Supports register locations for `lhs`.
563   void GenerateIntCompare(Location lhs, Location rhs);
564   void GenerateIntCompare(CpuRegister lhs, Location rhs);
565 
566   // Compare long values. Supports only register locations for `lhs`.
567   void GenerateLongCompare(Location lhs, Location rhs);
568 
569   // Construct address for array access.
570   static Address ArrayAddress(CpuRegister obj,
571                               Location index,
572                               ScaleFactor scale,
573                               uint32_t data_offset);
574 
575   Address LiteralCaseTable(HPackedSwitch* switch_instr);
576 
577   // Store a 64 bit value into a DoubleStackSlot in the most efficient manner.
578   void Store64BitValueToStack(Location dest, int64_t value);
579 
580   void MoveFromReturnRegister(Location trg, DataType::Type type) override;
581 
582   // Assign a 64 bit constant to an address.
583   void MoveInt64ToAddress(const Address& addr_low,
584                           const Address& addr_high,
585                           int64_t v,
586                           HInstruction* instruction);
587 
588   // Ensure that prior stores complete to memory before subsequent loads.
589   // The locked add implementation will avoid serializing device memory, but will
590   // touch (but not change) the top of the stack.
591   // The 'non_temporal' parameter should be used to ensure ordering of non-temporal stores.
592   void MemoryFence(bool force_mfence = false) {
593     if (!force_mfence) {
594       assembler_.lock()->addl(Address(CpuRegister(RSP), 0), Immediate(0));
595     } else {
596       assembler_.mfence();
597     }
598   }
599 
600   void GenerateNop() override;
601   void GenerateImplicitNullCheck(HNullCheck* instruction) override;
602   void GenerateExplicitNullCheck(HNullCheck* instruction) override;
603   void MaybeGenerateInlineCacheCheck(HInstruction* instruction, CpuRegister cls);
604 
605 
606   void MaybeIncrementHotness(bool is_frame_entry);
607 
608   // When we don't know the proper offset for the value, we use kDummy32BitOffset.
609   // We will fix this up in the linker later to have the right value.
610   static constexpr int32_t kDummy32BitOffset = 256;
611 
612  private:
613   template <linker::LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)>
614   static void EmitPcRelativeLinkerPatches(const ArenaDeque<PatchInfo<Label>>& infos,
615                                           ArenaVector<linker::LinkerPatch>* linker_patches);
616 
617   // Labels for each block that will be compiled.
618   Label* block_labels_;  // Indexed by block id.
619   Label frame_entry_label_;
620   LocationsBuilderX86_64 location_builder_;
621   InstructionCodeGeneratorX86_64 instruction_visitor_;
622   ParallelMoveResolverX86_64 move_resolver_;
623   X86_64Assembler assembler_;
624 
625   // Offset to the start of the constant area in the assembled code.
626   // Used for fixups to the constant area.
627   int constant_area_start_;
628 
629   // PC-relative method patch info for kBootImageLinkTimePcRelative.
630   ArenaDeque<PatchInfo<Label>> boot_image_method_patches_;
631   // PC-relative method patch info for kBssEntry.
632   ArenaDeque<PatchInfo<Label>> method_bss_entry_patches_;
633   // PC-relative type patch info for kBootImageLinkTimePcRelative.
634   ArenaDeque<PatchInfo<Label>> boot_image_type_patches_;
635   // PC-relative type patch info for kBssEntry.
636   ArenaDeque<PatchInfo<Label>> type_bss_entry_patches_;
637   // PC-relative String patch info for kBootImageLinkTimePcRelative.
638   ArenaDeque<PatchInfo<Label>> boot_image_string_patches_;
639   // PC-relative String patch info for kBssEntry.
640   ArenaDeque<PatchInfo<Label>> string_bss_entry_patches_;
641   // PC-relative patch info for IntrinsicObjects for the boot image,
642   // and for method/type/string patches for kBootImageRelRo otherwise.
643   ArenaDeque<PatchInfo<Label>> boot_image_other_patches_;
644 
645   // Patches for string literals in JIT compiled code.
646   ArenaDeque<PatchInfo<Label>> jit_string_patches_;
647   // Patches for class literals in JIT compiled code.
648   ArenaDeque<PatchInfo<Label>> jit_class_patches_;
649 
650   // Fixups for jump tables need to be handled specially.
651   ArenaVector<JumpTableRIPFixup*> fixups_to_jump_tables_;
652 
653   DISALLOW_COPY_AND_ASSIGN(CodeGeneratorX86_64);
654 };
655 
656 }  // namespace x86_64
657 }  // namespace art
658 
659 #endif  // ART_COMPILER_OPTIMIZING_CODE_GENERATOR_X86_64_H_
660