1 /*
2  * Copyright (C) 2016 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "jni_macro_assembler_arm64.h"
18 
19 #include "entrypoints/quick/quick_entrypoints.h"
20 #include "managed_register_arm64.h"
21 #include "offsets.h"
22 #include "thread.h"
23 
24 using namespace vixl::aarch64;  // NOLINT(build/namespaces)
25 
26 namespace art {
27 namespace arm64 {
28 
29 #ifdef ___
30 #error "ARM64 Assembler macro already defined."
31 #else
32 #define ___   asm_.GetVIXLAssembler()->
33 #endif
34 
35 #define reg_x(X) Arm64Assembler::reg_x(X)
36 #define reg_w(W) Arm64Assembler::reg_w(W)
37 #define reg_d(D) Arm64Assembler::reg_d(D)
38 #define reg_s(S) Arm64Assembler::reg_s(S)
39 
40 // The AAPCS64 requires 16-byte alignement. This is the same as the Managed ABI stack alignment.
41 static constexpr size_t kAapcs64StackAlignment = 16u;
42 static_assert(kAapcs64StackAlignment == kStackAlignment);
43 
~Arm64JNIMacroAssembler()44 Arm64JNIMacroAssembler::~Arm64JNIMacroAssembler() {
45 }
46 
FinalizeCode()47 void Arm64JNIMacroAssembler::FinalizeCode() {
48   for (const std::unique_ptr<Arm64Exception>& exception : exception_blocks_) {
49     EmitExceptionPoll(exception.get());
50   }
51   ___ FinalizeCode();
52 }
53 
GetCurrentThread(ManagedRegister tr)54 void Arm64JNIMacroAssembler::GetCurrentThread(ManagedRegister tr) {
55   ___ Mov(reg_x(tr.AsArm64().AsXRegister()), reg_x(TR));
56 }
57 
GetCurrentThread(FrameOffset offset,ManagedRegister)58 void Arm64JNIMacroAssembler::GetCurrentThread(FrameOffset offset, ManagedRegister /* scratch */) {
59   StoreToOffset(TR, SP, offset.Int32Value());
60 }
61 
62 // See Arm64 PCS Section 5.2.2.1.
IncreaseFrameSize(size_t adjust)63 void Arm64JNIMacroAssembler::IncreaseFrameSize(size_t adjust) {
64   if (adjust != 0u) {
65     CHECK_ALIGNED(adjust, kStackAlignment);
66     AddConstant(SP, -adjust);
67     cfi().AdjustCFAOffset(adjust);
68   }
69 }
70 
71 // See Arm64 PCS Section 5.2.2.1.
DecreaseFrameSize(size_t adjust)72 void Arm64JNIMacroAssembler::DecreaseFrameSize(size_t adjust) {
73   if (adjust != 0u) {
74     CHECK_ALIGNED(adjust, kStackAlignment);
75     AddConstant(SP, adjust);
76     cfi().AdjustCFAOffset(-adjust);
77   }
78 }
79 
AddConstant(XRegister rd,int32_t value,Condition cond)80 void Arm64JNIMacroAssembler::AddConstant(XRegister rd, int32_t value, Condition cond) {
81   AddConstant(rd, rd, value, cond);
82 }
83 
AddConstant(XRegister rd,XRegister rn,int32_t value,Condition cond)84 void Arm64JNIMacroAssembler::AddConstant(XRegister rd,
85                                          XRegister rn,
86                                          int32_t value,
87                                          Condition cond) {
88   if ((cond == al) || (cond == nv)) {
89     // VIXL macro-assembler handles all variants.
90     ___ Add(reg_x(rd), reg_x(rn), value);
91   } else {
92     // temp = rd + value
93     // rd = cond ? temp : rn
94     UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
95     temps.Exclude(reg_x(rd), reg_x(rn));
96     Register temp = temps.AcquireX();
97     ___ Add(temp, reg_x(rn), value);
98     ___ Csel(reg_x(rd), temp, reg_x(rd), cond);
99   }
100 }
101 
StoreWToOffset(StoreOperandType type,WRegister source,XRegister base,int32_t offset)102 void Arm64JNIMacroAssembler::StoreWToOffset(StoreOperandType type,
103                                             WRegister source,
104                                             XRegister base,
105                                             int32_t offset) {
106   switch (type) {
107     case kStoreByte:
108       ___ Strb(reg_w(source), MEM_OP(reg_x(base), offset));
109       break;
110     case kStoreHalfword:
111       ___ Strh(reg_w(source), MEM_OP(reg_x(base), offset));
112       break;
113     case kStoreWord:
114       ___ Str(reg_w(source), MEM_OP(reg_x(base), offset));
115       break;
116     default:
117       LOG(FATAL) << "UNREACHABLE";
118   }
119 }
120 
StoreToOffset(XRegister source,XRegister base,int32_t offset)121 void Arm64JNIMacroAssembler::StoreToOffset(XRegister source, XRegister base, int32_t offset) {
122   CHECK_NE(source, SP);
123   ___ Str(reg_x(source), MEM_OP(reg_x(base), offset));
124 }
125 
StoreSToOffset(SRegister source,XRegister base,int32_t offset)126 void Arm64JNIMacroAssembler::StoreSToOffset(SRegister source, XRegister base, int32_t offset) {
127   ___ Str(reg_s(source), MEM_OP(reg_x(base), offset));
128 }
129 
StoreDToOffset(DRegister source,XRegister base,int32_t offset)130 void Arm64JNIMacroAssembler::StoreDToOffset(DRegister source, XRegister base, int32_t offset) {
131   ___ Str(reg_d(source), MEM_OP(reg_x(base), offset));
132 }
133 
Store(FrameOffset offs,ManagedRegister m_src,size_t size)134 void Arm64JNIMacroAssembler::Store(FrameOffset offs, ManagedRegister m_src, size_t size) {
135   Arm64ManagedRegister src = m_src.AsArm64();
136   if (src.IsNoRegister()) {
137     CHECK_EQ(0u, size);
138   } else if (src.IsWRegister()) {
139     CHECK_EQ(4u, size);
140     StoreWToOffset(kStoreWord, src.AsWRegister(), SP, offs.Int32Value());
141   } else if (src.IsXRegister()) {
142     CHECK_EQ(8u, size);
143     StoreToOffset(src.AsXRegister(), SP, offs.Int32Value());
144   } else if (src.IsSRegister()) {
145     StoreSToOffset(src.AsSRegister(), SP, offs.Int32Value());
146   } else {
147     CHECK(src.IsDRegister()) << src;
148     StoreDToOffset(src.AsDRegister(), SP, offs.Int32Value());
149   }
150 }
151 
StoreRef(FrameOffset offs,ManagedRegister m_src)152 void Arm64JNIMacroAssembler::StoreRef(FrameOffset offs, ManagedRegister m_src) {
153   Arm64ManagedRegister src = m_src.AsArm64();
154   CHECK(src.IsXRegister()) << src;
155   StoreWToOffset(kStoreWord, src.AsOverlappingWRegister(), SP,
156                  offs.Int32Value());
157 }
158 
StoreRawPtr(FrameOffset offs,ManagedRegister m_src)159 void Arm64JNIMacroAssembler::StoreRawPtr(FrameOffset offs, ManagedRegister m_src) {
160   Arm64ManagedRegister src = m_src.AsArm64();
161   CHECK(src.IsXRegister()) << src;
162   StoreToOffset(src.AsXRegister(), SP, offs.Int32Value());
163 }
164 
StoreImmediateToFrame(FrameOffset offs,uint32_t imm,ManagedRegister m_scratch)165 void Arm64JNIMacroAssembler::StoreImmediateToFrame(FrameOffset offs,
166                                                    uint32_t imm,
167                                                    ManagedRegister m_scratch) {
168   Arm64ManagedRegister scratch = m_scratch.AsArm64();
169   CHECK(scratch.IsXRegister()) << scratch;
170   LoadImmediate(scratch.AsXRegister(), imm);
171   StoreWToOffset(kStoreWord, scratch.AsOverlappingWRegister(), SP,
172                  offs.Int32Value());
173 }
174 
StoreStackOffsetToThread(ThreadOffset64 tr_offs,FrameOffset fr_offs,ManagedRegister m_scratch)175 void Arm64JNIMacroAssembler::StoreStackOffsetToThread(ThreadOffset64 tr_offs,
176                                                       FrameOffset fr_offs,
177                                                       ManagedRegister m_scratch) {
178   Arm64ManagedRegister scratch = m_scratch.AsArm64();
179   CHECK(scratch.IsXRegister()) << scratch;
180   AddConstant(scratch.AsXRegister(), SP, fr_offs.Int32Value());
181   StoreToOffset(scratch.AsXRegister(), TR, tr_offs.Int32Value());
182 }
183 
StoreStackPointerToThread(ThreadOffset64 tr_offs)184 void Arm64JNIMacroAssembler::StoreStackPointerToThread(ThreadOffset64 tr_offs) {
185   UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
186   Register temp = temps.AcquireX();
187   ___ Mov(temp, reg_x(SP));
188   ___ Str(temp, MEM_OP(reg_x(TR), tr_offs.Int32Value()));
189 }
190 
StoreSpanning(FrameOffset dest_off,ManagedRegister m_source,FrameOffset in_off,ManagedRegister m_scratch)191 void Arm64JNIMacroAssembler::StoreSpanning(FrameOffset dest_off,
192                                            ManagedRegister m_source,
193                                            FrameOffset in_off,
194                                            ManagedRegister m_scratch) {
195   Arm64ManagedRegister source = m_source.AsArm64();
196   Arm64ManagedRegister scratch = m_scratch.AsArm64();
197   StoreToOffset(source.AsXRegister(), SP, dest_off.Int32Value());
198   LoadFromOffset(scratch.AsXRegister(), SP, in_off.Int32Value());
199   StoreToOffset(scratch.AsXRegister(), SP, dest_off.Int32Value() + 8);
200 }
201 
202 // Load routines.
LoadImmediate(XRegister dest,int32_t value,Condition cond)203 void Arm64JNIMacroAssembler::LoadImmediate(XRegister dest, int32_t value, Condition cond) {
204   if ((cond == al) || (cond == nv)) {
205     ___ Mov(reg_x(dest), value);
206   } else {
207     // temp = value
208     // rd = cond ? temp : rd
209     if (value != 0) {
210       UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
211       temps.Exclude(reg_x(dest));
212       Register temp = temps.AcquireX();
213       ___ Mov(temp, value);
214       ___ Csel(reg_x(dest), temp, reg_x(dest), cond);
215     } else {
216       ___ Csel(reg_x(dest), reg_x(XZR), reg_x(dest), cond);
217     }
218   }
219 }
220 
LoadWFromOffset(LoadOperandType type,WRegister dest,XRegister base,int32_t offset)221 void Arm64JNIMacroAssembler::LoadWFromOffset(LoadOperandType type,
222                                              WRegister dest,
223                                              XRegister base,
224                                              int32_t offset) {
225   switch (type) {
226     case kLoadSignedByte:
227       ___ Ldrsb(reg_w(dest), MEM_OP(reg_x(base), offset));
228       break;
229     case kLoadSignedHalfword:
230       ___ Ldrsh(reg_w(dest), MEM_OP(reg_x(base), offset));
231       break;
232     case kLoadUnsignedByte:
233       ___ Ldrb(reg_w(dest), MEM_OP(reg_x(base), offset));
234       break;
235     case kLoadUnsignedHalfword:
236       ___ Ldrh(reg_w(dest), MEM_OP(reg_x(base), offset));
237       break;
238     case kLoadWord:
239       ___ Ldr(reg_w(dest), MEM_OP(reg_x(base), offset));
240       break;
241     default:
242         LOG(FATAL) << "UNREACHABLE";
243   }
244 }
245 
246 // Note: We can extend this member by adding load type info - see
247 // sign extended A64 load variants.
LoadFromOffset(XRegister dest,XRegister base,int32_t offset)248 void Arm64JNIMacroAssembler::LoadFromOffset(XRegister dest, XRegister base, int32_t offset) {
249   CHECK_NE(dest, SP);
250   ___ Ldr(reg_x(dest), MEM_OP(reg_x(base), offset));
251 }
252 
LoadSFromOffset(SRegister dest,XRegister base,int32_t offset)253 void Arm64JNIMacroAssembler::LoadSFromOffset(SRegister dest, XRegister base, int32_t offset) {
254   ___ Ldr(reg_s(dest), MEM_OP(reg_x(base), offset));
255 }
256 
LoadDFromOffset(DRegister dest,XRegister base,int32_t offset)257 void Arm64JNIMacroAssembler::LoadDFromOffset(DRegister dest, XRegister base, int32_t offset) {
258   ___ Ldr(reg_d(dest), MEM_OP(reg_x(base), offset));
259 }
260 
Load(Arm64ManagedRegister dest,XRegister base,int32_t offset,size_t size)261 void Arm64JNIMacroAssembler::Load(Arm64ManagedRegister dest,
262                                   XRegister base,
263                                   int32_t offset,
264                                   size_t size) {
265   if (dest.IsNoRegister()) {
266     CHECK_EQ(0u, size) << dest;
267   } else if (dest.IsWRegister()) {
268     CHECK_EQ(4u, size) << dest;
269     ___ Ldr(reg_w(dest.AsWRegister()), MEM_OP(reg_x(base), offset));
270   } else if (dest.IsXRegister()) {
271     CHECK_NE(dest.AsXRegister(), SP) << dest;
272 
273     if (size == 1u) {
274       ___ Ldrb(reg_w(dest.AsOverlappingWRegister()), MEM_OP(reg_x(base), offset));
275     } else if (size == 4u) {
276       ___ Ldr(reg_w(dest.AsOverlappingWRegister()), MEM_OP(reg_x(base), offset));
277     }  else {
278       CHECK_EQ(8u, size) << dest;
279       ___ Ldr(reg_x(dest.AsXRegister()), MEM_OP(reg_x(base), offset));
280     }
281   } else if (dest.IsSRegister()) {
282     ___ Ldr(reg_s(dest.AsSRegister()), MEM_OP(reg_x(base), offset));
283   } else {
284     CHECK(dest.IsDRegister()) << dest;
285     ___ Ldr(reg_d(dest.AsDRegister()), MEM_OP(reg_x(base), offset));
286   }
287 }
288 
Load(ManagedRegister m_dst,FrameOffset src,size_t size)289 void Arm64JNIMacroAssembler::Load(ManagedRegister m_dst, FrameOffset src, size_t size) {
290   return Load(m_dst.AsArm64(), SP, src.Int32Value(), size);
291 }
292 
LoadFromThread(ManagedRegister m_dst,ThreadOffset64 src,size_t size)293 void Arm64JNIMacroAssembler::LoadFromThread(ManagedRegister m_dst,
294                                             ThreadOffset64 src,
295                                             size_t size) {
296   return Load(m_dst.AsArm64(), TR, src.Int32Value(), size);
297 }
298 
LoadRef(ManagedRegister m_dst,FrameOffset offs)299 void Arm64JNIMacroAssembler::LoadRef(ManagedRegister m_dst, FrameOffset offs) {
300   Arm64ManagedRegister dst = m_dst.AsArm64();
301   CHECK(dst.IsXRegister()) << dst;
302   LoadWFromOffset(kLoadWord, dst.AsOverlappingWRegister(), SP, offs.Int32Value());
303 }
304 
LoadRef(ManagedRegister m_dst,ManagedRegister m_base,MemberOffset offs,bool unpoison_reference)305 void Arm64JNIMacroAssembler::LoadRef(ManagedRegister m_dst,
306                                      ManagedRegister m_base,
307                                      MemberOffset offs,
308                                      bool unpoison_reference) {
309   Arm64ManagedRegister dst = m_dst.AsArm64();
310   Arm64ManagedRegister base = m_base.AsArm64();
311   CHECK(dst.IsXRegister() && base.IsXRegister());
312   LoadWFromOffset(kLoadWord, dst.AsOverlappingWRegister(), base.AsXRegister(),
313                   offs.Int32Value());
314   if (unpoison_reference) {
315     WRegister ref_reg = dst.AsOverlappingWRegister();
316     asm_.MaybeUnpoisonHeapReference(reg_w(ref_reg));
317   }
318 }
319 
LoadRawPtr(ManagedRegister m_dst,ManagedRegister m_base,Offset offs)320 void Arm64JNIMacroAssembler::LoadRawPtr(ManagedRegister m_dst,
321                                         ManagedRegister m_base,
322                                         Offset offs) {
323   Arm64ManagedRegister dst = m_dst.AsArm64();
324   Arm64ManagedRegister base = m_base.AsArm64();
325   CHECK(dst.IsXRegister() && base.IsXRegister());
326   // Remove dst and base form the temp list - higher level API uses IP1, IP0.
327   UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
328   temps.Exclude(reg_x(dst.AsXRegister()), reg_x(base.AsXRegister()));
329   ___ Ldr(reg_x(dst.AsXRegister()), MEM_OP(reg_x(base.AsXRegister()), offs.Int32Value()));
330 }
331 
LoadRawPtrFromThread(ManagedRegister m_dst,ThreadOffset64 offs)332 void Arm64JNIMacroAssembler::LoadRawPtrFromThread(ManagedRegister m_dst, ThreadOffset64 offs) {
333   Arm64ManagedRegister dst = m_dst.AsArm64();
334   CHECK(dst.IsXRegister()) << dst;
335   LoadFromOffset(dst.AsXRegister(), TR, offs.Int32Value());
336 }
337 
338 // Copying routines.
Move(ManagedRegister m_dst,ManagedRegister m_src,size_t size)339 void Arm64JNIMacroAssembler::Move(ManagedRegister m_dst, ManagedRegister m_src, size_t size) {
340   Arm64ManagedRegister dst = m_dst.AsArm64();
341   Arm64ManagedRegister src = m_src.AsArm64();
342   if (!dst.Equals(src)) {
343     if (dst.IsXRegister()) {
344       if (size == 4) {
345         CHECK(src.IsWRegister());
346         ___ Mov(reg_w(dst.AsOverlappingWRegister()), reg_w(src.AsWRegister()));
347       } else {
348         if (src.IsXRegister()) {
349           ___ Mov(reg_x(dst.AsXRegister()), reg_x(src.AsXRegister()));
350         } else {
351           ___ Mov(reg_x(dst.AsXRegister()), reg_x(src.AsOverlappingXRegister()));
352         }
353       }
354     } else if (dst.IsWRegister()) {
355       CHECK(src.IsWRegister()) << src;
356       ___ Mov(reg_w(dst.AsWRegister()), reg_w(src.AsWRegister()));
357     } else if (dst.IsSRegister()) {
358       CHECK(src.IsSRegister()) << src;
359       ___ Fmov(reg_s(dst.AsSRegister()), reg_s(src.AsSRegister()));
360     } else {
361       CHECK(dst.IsDRegister()) << dst;
362       CHECK(src.IsDRegister()) << src;
363       ___ Fmov(reg_d(dst.AsDRegister()), reg_d(src.AsDRegister()));
364     }
365   }
366 }
367 
CopyRawPtrFromThread(FrameOffset fr_offs,ThreadOffset64 tr_offs,ManagedRegister m_scratch)368 void Arm64JNIMacroAssembler::CopyRawPtrFromThread(FrameOffset fr_offs,
369                                                   ThreadOffset64 tr_offs,
370                                                   ManagedRegister m_scratch) {
371   Arm64ManagedRegister scratch = m_scratch.AsArm64();
372   CHECK(scratch.IsXRegister()) << scratch;
373   LoadFromOffset(scratch.AsXRegister(), TR, tr_offs.Int32Value());
374   StoreToOffset(scratch.AsXRegister(), SP, fr_offs.Int32Value());
375 }
376 
CopyRawPtrToThread(ThreadOffset64 tr_offs,FrameOffset fr_offs,ManagedRegister m_scratch)377 void Arm64JNIMacroAssembler::CopyRawPtrToThread(ThreadOffset64 tr_offs,
378                                                 FrameOffset fr_offs,
379                                                 ManagedRegister m_scratch) {
380   Arm64ManagedRegister scratch = m_scratch.AsArm64();
381   CHECK(scratch.IsXRegister()) << scratch;
382   LoadFromOffset(scratch.AsXRegister(), SP, fr_offs.Int32Value());
383   StoreToOffset(scratch.AsXRegister(), TR, tr_offs.Int32Value());
384 }
385 
CopyRef(FrameOffset dest,FrameOffset src,ManagedRegister m_scratch)386 void Arm64JNIMacroAssembler::CopyRef(FrameOffset dest, FrameOffset src, ManagedRegister m_scratch) {
387   Arm64ManagedRegister scratch = m_scratch.AsArm64();
388   CHECK(scratch.IsXRegister()) << scratch;
389   LoadWFromOffset(kLoadWord, scratch.AsOverlappingWRegister(),
390                   SP, src.Int32Value());
391   StoreWToOffset(kStoreWord, scratch.AsOverlappingWRegister(),
392                  SP, dest.Int32Value());
393 }
394 
Copy(FrameOffset dest,FrameOffset src,ManagedRegister m_scratch,size_t size)395 void Arm64JNIMacroAssembler::Copy(FrameOffset dest,
396                                   FrameOffset src,
397                                   ManagedRegister m_scratch,
398                                   size_t size) {
399   Arm64ManagedRegister scratch = m_scratch.AsArm64();
400   CHECK(scratch.IsXRegister()) << scratch;
401   CHECK(size == 4 || size == 8) << size;
402   if (size == 4) {
403     LoadWFromOffset(kLoadWord, scratch.AsOverlappingWRegister(), SP, src.Int32Value());
404     StoreWToOffset(kStoreWord, scratch.AsOverlappingWRegister(), SP, dest.Int32Value());
405   } else if (size == 8) {
406     LoadFromOffset(scratch.AsXRegister(), SP, src.Int32Value());
407     StoreToOffset(scratch.AsXRegister(), SP, dest.Int32Value());
408   } else {
409     UNIMPLEMENTED(FATAL) << "We only support Copy() of size 4 and 8";
410   }
411 }
412 
Copy(FrameOffset dest,ManagedRegister src_base,Offset src_offset,ManagedRegister m_scratch,size_t size)413 void Arm64JNIMacroAssembler::Copy(FrameOffset dest,
414                                   ManagedRegister src_base,
415                                   Offset src_offset,
416                                   ManagedRegister m_scratch,
417                                   size_t size) {
418   Arm64ManagedRegister scratch = m_scratch.AsArm64();
419   Arm64ManagedRegister base = src_base.AsArm64();
420   CHECK(base.IsXRegister()) << base;
421   CHECK(scratch.IsXRegister() || scratch.IsWRegister()) << scratch;
422   CHECK(size == 4 || size == 8) << size;
423   if (size == 4) {
424     LoadWFromOffset(kLoadWord, scratch.AsWRegister(), base.AsXRegister(),
425                    src_offset.Int32Value());
426     StoreWToOffset(kStoreWord, scratch.AsWRegister(), SP, dest.Int32Value());
427   } else if (size == 8) {
428     LoadFromOffset(scratch.AsXRegister(), base.AsXRegister(), src_offset.Int32Value());
429     StoreToOffset(scratch.AsXRegister(), SP, dest.Int32Value());
430   } else {
431     UNIMPLEMENTED(FATAL) << "We only support Copy() of size 4 and 8";
432   }
433 }
434 
Copy(ManagedRegister m_dest_base,Offset dest_offs,FrameOffset src,ManagedRegister m_scratch,size_t size)435 void Arm64JNIMacroAssembler::Copy(ManagedRegister m_dest_base,
436                                   Offset dest_offs,
437                                   FrameOffset src,
438                                   ManagedRegister m_scratch,
439                                   size_t size) {
440   Arm64ManagedRegister scratch = m_scratch.AsArm64();
441   Arm64ManagedRegister base = m_dest_base.AsArm64();
442   CHECK(base.IsXRegister()) << base;
443   CHECK(scratch.IsXRegister() || scratch.IsWRegister()) << scratch;
444   CHECK(size == 4 || size == 8) << size;
445   if (size == 4) {
446     LoadWFromOffset(kLoadWord, scratch.AsWRegister(), SP, src.Int32Value());
447     StoreWToOffset(kStoreWord, scratch.AsWRegister(), base.AsXRegister(),
448                    dest_offs.Int32Value());
449   } else if (size == 8) {
450     LoadFromOffset(scratch.AsXRegister(), SP, src.Int32Value());
451     StoreToOffset(scratch.AsXRegister(), base.AsXRegister(), dest_offs.Int32Value());
452   } else {
453     UNIMPLEMENTED(FATAL) << "We only support Copy() of size 4 and 8";
454   }
455 }
456 
Copy(FrameOffset,FrameOffset,Offset,ManagedRegister,size_t)457 void Arm64JNIMacroAssembler::Copy(FrameOffset /*dst*/,
458                                   FrameOffset /*src_base*/,
459                                   Offset /*src_offset*/,
460                                   ManagedRegister /*mscratch*/,
461                                   size_t /*size*/) {
462   UNIMPLEMENTED(FATAL) << "Unimplemented Copy() variant";
463 }
464 
Copy(ManagedRegister m_dest,Offset dest_offset,ManagedRegister m_src,Offset src_offset,ManagedRegister m_scratch,size_t size)465 void Arm64JNIMacroAssembler::Copy(ManagedRegister m_dest,
466                                   Offset dest_offset,
467                                   ManagedRegister m_src,
468                                   Offset src_offset,
469                                   ManagedRegister m_scratch,
470                                   size_t size) {
471   Arm64ManagedRegister scratch = m_scratch.AsArm64();
472   Arm64ManagedRegister src = m_src.AsArm64();
473   Arm64ManagedRegister dest = m_dest.AsArm64();
474   CHECK(dest.IsXRegister()) << dest;
475   CHECK(src.IsXRegister()) << src;
476   CHECK(scratch.IsXRegister() || scratch.IsWRegister()) << scratch;
477   CHECK(size == 4 || size == 8) << size;
478   if (size == 4) {
479     if (scratch.IsWRegister()) {
480       LoadWFromOffset(kLoadWord, scratch.AsWRegister(), src.AsXRegister(),
481                     src_offset.Int32Value());
482       StoreWToOffset(kStoreWord, scratch.AsWRegister(), dest.AsXRegister(),
483                    dest_offset.Int32Value());
484     } else {
485       LoadWFromOffset(kLoadWord, scratch.AsOverlappingWRegister(), src.AsXRegister(),
486                     src_offset.Int32Value());
487       StoreWToOffset(kStoreWord, scratch.AsOverlappingWRegister(), dest.AsXRegister(),
488                    dest_offset.Int32Value());
489     }
490   } else if (size == 8) {
491     LoadFromOffset(scratch.AsXRegister(), src.AsXRegister(), src_offset.Int32Value());
492     StoreToOffset(scratch.AsXRegister(), dest.AsXRegister(), dest_offset.Int32Value());
493   } else {
494     UNIMPLEMENTED(FATAL) << "We only support Copy() of size 4 and 8";
495   }
496 }
497 
Copy(FrameOffset,Offset,FrameOffset,Offset,ManagedRegister,size_t)498 void Arm64JNIMacroAssembler::Copy(FrameOffset /*dst*/,
499                                   Offset /*dest_offset*/,
500                                   FrameOffset /*src*/,
501                                   Offset /*src_offset*/,
502                                   ManagedRegister /*scratch*/,
503                                   size_t /*size*/) {
504   UNIMPLEMENTED(FATAL) << "Unimplemented Copy() variant";
505 }
506 
MemoryBarrier(ManagedRegister m_scratch ATTRIBUTE_UNUSED)507 void Arm64JNIMacroAssembler::MemoryBarrier(ManagedRegister m_scratch ATTRIBUTE_UNUSED) {
508   // TODO: Should we check that m_scratch is IP? - see arm.
509   ___ Dmb(InnerShareable, BarrierAll);
510 }
511 
SignExtend(ManagedRegister mreg,size_t size)512 void Arm64JNIMacroAssembler::SignExtend(ManagedRegister mreg, size_t size) {
513   Arm64ManagedRegister reg = mreg.AsArm64();
514   CHECK(size == 1 || size == 2) << size;
515   CHECK(reg.IsWRegister()) << reg;
516   if (size == 1) {
517     ___ Sxtb(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
518   } else {
519     ___ Sxth(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
520   }
521 }
522 
ZeroExtend(ManagedRegister mreg,size_t size)523 void Arm64JNIMacroAssembler::ZeroExtend(ManagedRegister mreg, size_t size) {
524   Arm64ManagedRegister reg = mreg.AsArm64();
525   CHECK(size == 1 || size == 2) << size;
526   CHECK(reg.IsWRegister()) << reg;
527   if (size == 1) {
528     ___ Uxtb(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
529   } else {
530     ___ Uxth(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
531   }
532 }
533 
VerifyObject(ManagedRegister,bool)534 void Arm64JNIMacroAssembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) {
535   // TODO: not validating references.
536 }
537 
VerifyObject(FrameOffset,bool)538 void Arm64JNIMacroAssembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) {
539   // TODO: not validating references.
540 }
541 
Jump(ManagedRegister m_base,Offset offs,ManagedRegister m_scratch)542 void Arm64JNIMacroAssembler::Jump(ManagedRegister m_base, Offset offs, ManagedRegister m_scratch) {
543   Arm64ManagedRegister base = m_base.AsArm64();
544   Arm64ManagedRegister scratch = m_scratch.AsArm64();
545   CHECK(base.IsXRegister()) << base;
546   CHECK(scratch.IsXRegister()) << scratch;
547   LoadFromOffset(scratch.AsXRegister(), base.AsXRegister(), offs.Int32Value());
548   ___ Br(reg_x(scratch.AsXRegister()));
549 }
550 
Call(ManagedRegister m_base,Offset offs,ManagedRegister m_scratch)551 void Arm64JNIMacroAssembler::Call(ManagedRegister m_base, Offset offs, ManagedRegister m_scratch) {
552   Arm64ManagedRegister base = m_base.AsArm64();
553   Arm64ManagedRegister scratch = m_scratch.AsArm64();
554   CHECK(base.IsXRegister()) << base;
555   CHECK(scratch.IsXRegister()) << scratch;
556   LoadFromOffset(scratch.AsXRegister(), base.AsXRegister(), offs.Int32Value());
557   ___ Blr(reg_x(scratch.AsXRegister()));
558 }
559 
Call(FrameOffset base,Offset offs,ManagedRegister m_scratch)560 void Arm64JNIMacroAssembler::Call(FrameOffset base, Offset offs, ManagedRegister m_scratch) {
561   Arm64ManagedRegister scratch = m_scratch.AsArm64();
562   CHECK(scratch.IsXRegister()) << scratch;
563   // Call *(*(SP + base) + offset)
564   LoadFromOffset(scratch.AsXRegister(), SP, base.Int32Value());
565   LoadFromOffset(scratch.AsXRegister(), scratch.AsXRegister(), offs.Int32Value());
566   ___ Blr(reg_x(scratch.AsXRegister()));
567 }
568 
CallFromThread(ThreadOffset64 offset ATTRIBUTE_UNUSED,ManagedRegister scratch ATTRIBUTE_UNUSED)569 void Arm64JNIMacroAssembler::CallFromThread(ThreadOffset64 offset ATTRIBUTE_UNUSED,
570                                             ManagedRegister scratch ATTRIBUTE_UNUSED) {
571   UNIMPLEMENTED(FATAL) << "Unimplemented Call() variant";
572 }
573 
CreateHandleScopeEntry(ManagedRegister m_out_reg,FrameOffset handle_scope_offs,ManagedRegister m_in_reg,bool null_allowed)574 void Arm64JNIMacroAssembler::CreateHandleScopeEntry(ManagedRegister m_out_reg,
575                                                     FrameOffset handle_scope_offs,
576                                                     ManagedRegister m_in_reg,
577                                                     bool null_allowed) {
578   Arm64ManagedRegister out_reg = m_out_reg.AsArm64();
579   Arm64ManagedRegister in_reg = m_in_reg.AsArm64();
580   // For now we only hold stale handle scope entries in x registers.
581   CHECK(in_reg.IsNoRegister() || in_reg.IsXRegister()) << in_reg;
582   CHECK(out_reg.IsXRegister()) << out_reg;
583   if (null_allowed) {
584     // Null values get a handle scope entry value of 0.  Otherwise, the handle scope entry is
585     // the address in the handle scope holding the reference.
586     // e.g. out_reg = (handle == 0) ? 0 : (SP+handle_offset)
587     if (in_reg.IsNoRegister()) {
588       LoadWFromOffset(kLoadWord, out_reg.AsOverlappingWRegister(), SP,
589                       handle_scope_offs.Int32Value());
590       in_reg = out_reg;
591     }
592     ___ Cmp(reg_w(in_reg.AsOverlappingWRegister()), 0);
593     if (!out_reg.Equals(in_reg)) {
594       LoadImmediate(out_reg.AsXRegister(), 0, eq);
595     }
596     AddConstant(out_reg.AsXRegister(), SP, handle_scope_offs.Int32Value(), ne);
597   } else {
598     AddConstant(out_reg.AsXRegister(), SP, handle_scope_offs.Int32Value(), al);
599   }
600 }
601 
CreateHandleScopeEntry(FrameOffset out_off,FrameOffset handle_scope_offset,ManagedRegister m_scratch,bool null_allowed)602 void Arm64JNIMacroAssembler::CreateHandleScopeEntry(FrameOffset out_off,
603                                                     FrameOffset handle_scope_offset,
604                                                     ManagedRegister m_scratch,
605                                                     bool null_allowed) {
606   Arm64ManagedRegister scratch = m_scratch.AsArm64();
607   CHECK(scratch.IsXRegister()) << scratch;
608   if (null_allowed) {
609     LoadWFromOffset(kLoadWord, scratch.AsOverlappingWRegister(), SP,
610                     handle_scope_offset.Int32Value());
611     // Null values get a handle scope entry value of 0.  Otherwise, the handle scope entry is
612     // the address in the handle scope holding the reference.
613     // e.g. scratch = (scratch == 0) ? 0 : (SP+handle_scope_offset)
614     ___ Cmp(reg_w(scratch.AsOverlappingWRegister()), 0);
615     // Move this logic in add constants with flags.
616     AddConstant(scratch.AsXRegister(), SP, handle_scope_offset.Int32Value(), ne);
617   } else {
618     AddConstant(scratch.AsXRegister(), SP, handle_scope_offset.Int32Value(), al);
619   }
620   StoreToOffset(scratch.AsXRegister(), SP, out_off.Int32Value());
621 }
622 
LoadReferenceFromHandleScope(ManagedRegister m_out_reg,ManagedRegister m_in_reg)623 void Arm64JNIMacroAssembler::LoadReferenceFromHandleScope(ManagedRegister m_out_reg,
624                                                           ManagedRegister m_in_reg) {
625   Arm64ManagedRegister out_reg = m_out_reg.AsArm64();
626   Arm64ManagedRegister in_reg = m_in_reg.AsArm64();
627   CHECK(out_reg.IsXRegister()) << out_reg;
628   CHECK(in_reg.IsXRegister()) << in_reg;
629   vixl::aarch64::Label exit;
630   if (!out_reg.Equals(in_reg)) {
631     // FIXME: Who sets the flags here?
632     LoadImmediate(out_reg.AsXRegister(), 0, eq);
633   }
634   ___ Cbz(reg_x(in_reg.AsXRegister()), &exit);
635   LoadFromOffset(out_reg.AsXRegister(), in_reg.AsXRegister(), 0);
636   ___ Bind(&exit);
637 }
638 
ExceptionPoll(ManagedRegister m_scratch,size_t stack_adjust)639 void Arm64JNIMacroAssembler::ExceptionPoll(ManagedRegister m_scratch, size_t stack_adjust) {
640   CHECK_ALIGNED(stack_adjust, kStackAlignment);
641   Arm64ManagedRegister scratch = m_scratch.AsArm64();
642   exception_blocks_.emplace_back(new Arm64Exception(scratch, stack_adjust));
643   LoadFromOffset(scratch.AsXRegister(),
644                  TR,
645                  Thread::ExceptionOffset<kArm64PointerSize>().Int32Value());
646   ___ Cbnz(reg_x(scratch.AsXRegister()), exception_blocks_.back()->Entry());
647 }
648 
CreateLabel()649 std::unique_ptr<JNIMacroLabel> Arm64JNIMacroAssembler::CreateLabel() {
650   return std::unique_ptr<JNIMacroLabel>(new Arm64JNIMacroLabel());
651 }
652 
Jump(JNIMacroLabel * label)653 void Arm64JNIMacroAssembler::Jump(JNIMacroLabel* label) {
654   CHECK(label != nullptr);
655   ___ B(Arm64JNIMacroLabel::Cast(label)->AsArm64());
656 }
657 
Jump(JNIMacroLabel * label,JNIMacroUnaryCondition condition,ManagedRegister test)658 void Arm64JNIMacroAssembler::Jump(JNIMacroLabel* label,
659                                   JNIMacroUnaryCondition condition,
660                                   ManagedRegister test) {
661   CHECK(label != nullptr);
662 
663   switch (condition) {
664     case JNIMacroUnaryCondition::kZero:
665       ___ Cbz(reg_x(test.AsArm64().AsXRegister()), Arm64JNIMacroLabel::Cast(label)->AsArm64());
666       break;
667     case JNIMacroUnaryCondition::kNotZero:
668       ___ Cbnz(reg_x(test.AsArm64().AsXRegister()), Arm64JNIMacroLabel::Cast(label)->AsArm64());
669       break;
670     default:
671       LOG(FATAL) << "Not implemented unary condition: " << static_cast<int>(condition);
672       UNREACHABLE();
673   }
674 }
675 
Bind(JNIMacroLabel * label)676 void Arm64JNIMacroAssembler::Bind(JNIMacroLabel* label) {
677   CHECK(label != nullptr);
678   ___ Bind(Arm64JNIMacroLabel::Cast(label)->AsArm64());
679 }
680 
EmitExceptionPoll(Arm64Exception * exception)681 void Arm64JNIMacroAssembler::EmitExceptionPoll(Arm64Exception* exception) {
682   UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
683   temps.Exclude(reg_x(exception->scratch_.AsXRegister()));
684   Register temp = temps.AcquireX();
685 
686   // Bind exception poll entry.
687   ___ Bind(exception->Entry());
688   if (exception->stack_adjust_ != 0) {  // Fix up the frame.
689     DecreaseFrameSize(exception->stack_adjust_);
690   }
691   // Pass exception object as argument.
692   // Don't care about preserving X0 as this won't return.
693   ___ Mov(reg_x(X0), reg_x(exception->scratch_.AsXRegister()));
694   ___ Ldr(temp,
695           MEM_OP(reg_x(TR),
696                  QUICK_ENTRYPOINT_OFFSET(kArm64PointerSize, pDeliverException).Int32Value()));
697 
698   ___ Blr(temp);
699   // Call should never return.
700   ___ Brk();
701 }
702 
BuildFrame(size_t frame_size,ManagedRegister method_reg,ArrayRef<const ManagedRegister> callee_save_regs,const ManagedRegisterEntrySpills & entry_spills)703 void Arm64JNIMacroAssembler::BuildFrame(size_t frame_size,
704                                         ManagedRegister method_reg,
705                                         ArrayRef<const ManagedRegister> callee_save_regs,
706                                         const ManagedRegisterEntrySpills& entry_spills) {
707   // Setup VIXL CPURegList for callee-saves.
708   CPURegList core_reg_list(CPURegister::kRegister, kXRegSize, 0);
709   CPURegList fp_reg_list(CPURegister::kVRegister, kDRegSize, 0);
710   for (auto r : callee_save_regs) {
711     Arm64ManagedRegister reg = r.AsArm64();
712     if (reg.IsXRegister()) {
713       core_reg_list.Combine(reg_x(reg.AsXRegister()).GetCode());
714     } else {
715       DCHECK(reg.IsDRegister());
716       fp_reg_list.Combine(reg_d(reg.AsDRegister()).GetCode());
717     }
718   }
719   size_t core_reg_size = core_reg_list.GetTotalSizeInBytes();
720   size_t fp_reg_size = fp_reg_list.GetTotalSizeInBytes();
721 
722   // Increase frame to required size.
723   DCHECK_ALIGNED(frame_size, kStackAlignment);
724   // Must at least have space for Method* if we're going to spill it.
725   DCHECK_GE(frame_size,
726             core_reg_size + fp_reg_size + (method_reg.IsRegister() ? kXRegSizeInBytes : 0u));
727   IncreaseFrameSize(frame_size);
728 
729   // Save callee-saves.
730   asm_.SpillRegisters(core_reg_list, frame_size - core_reg_size);
731   asm_.SpillRegisters(fp_reg_list, frame_size - core_reg_size - fp_reg_size);
732 
733   if (method_reg.IsRegister()) {
734     // Write ArtMethod*
735     DCHECK(X0 == method_reg.AsArm64().AsXRegister());
736     StoreToOffset(X0, SP, 0);
737   }
738 
739   // Write out entry spills
740   int32_t offset = frame_size + static_cast<size_t>(kArm64PointerSize);
741   for (const ManagedRegisterSpill& spill : entry_spills) {
742     Arm64ManagedRegister reg = spill.AsArm64();
743     if (reg.IsNoRegister()) {
744       // only increment stack offset.
745       offset += spill.getSize();
746     } else if (reg.IsXRegister()) {
747       StoreToOffset(reg.AsXRegister(), SP, offset);
748       offset += 8;
749     } else if (reg.IsWRegister()) {
750       StoreWToOffset(kStoreWord, reg.AsWRegister(), SP, offset);
751       offset += 4;
752     } else if (reg.IsDRegister()) {
753       StoreDToOffset(reg.AsDRegister(), SP, offset);
754       offset += 8;
755     } else if (reg.IsSRegister()) {
756       StoreSToOffset(reg.AsSRegister(), SP, offset);
757       offset += 4;
758     }
759   }
760 }
761 
RemoveFrame(size_t frame_size,ArrayRef<const ManagedRegister> callee_save_regs,bool may_suspend)762 void Arm64JNIMacroAssembler::RemoveFrame(size_t frame_size,
763                                          ArrayRef<const ManagedRegister> callee_save_regs,
764                                          bool may_suspend) {
765   // Setup VIXL CPURegList for callee-saves.
766   CPURegList core_reg_list(CPURegister::kRegister, kXRegSize, 0);
767   CPURegList fp_reg_list(CPURegister::kVRegister, kDRegSize, 0);
768   for (auto r : callee_save_regs) {
769     Arm64ManagedRegister reg = r.AsArm64();
770     if (reg.IsXRegister()) {
771       core_reg_list.Combine(reg_x(reg.AsXRegister()).GetCode());
772     } else {
773       DCHECK(reg.IsDRegister());
774       fp_reg_list.Combine(reg_d(reg.AsDRegister()).GetCode());
775     }
776   }
777   size_t core_reg_size = core_reg_list.GetTotalSizeInBytes();
778   size_t fp_reg_size = fp_reg_list.GetTotalSizeInBytes();
779 
780   // For now we only check that the size of the frame is large enough to hold spills and method
781   // reference.
782   DCHECK_GE(frame_size, core_reg_size + fp_reg_size);
783   DCHECK_ALIGNED(frame_size, kAapcs64StackAlignment);
784 
785   cfi().RememberState();
786 
787   // Restore callee-saves.
788   asm_.UnspillRegisters(core_reg_list, frame_size - core_reg_size);
789   asm_.UnspillRegisters(fp_reg_list, frame_size - core_reg_size - fp_reg_size);
790 
791   if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
792     vixl::aarch64::Register mr = reg_x(MR);  // Marking Register.
793     vixl::aarch64::Register tr = reg_x(TR);  // Thread Register.
794 
795     if (may_suspend) {
796       // The method may be suspended; refresh the Marking Register.
797       ___ Ldr(mr.W(), MemOperand(tr, Thread::IsGcMarkingOffset<kArm64PointerSize>().Int32Value()));
798     } else {
799       // The method shall not be suspended; no need to refresh the Marking Register.
800 
801       // The Marking Register is a callee-save register and thus has been
802       // preserved by native code following the AAPCS64 calling convention.
803 
804       // The following condition is a compile-time one, so it does not have a run-time cost.
805       if (kIsDebugBuild) {
806         // The following condition is a run-time one; it is executed after the
807         // previous compile-time test, to avoid penalizing non-debug builds.
808         if (emit_run_time_checks_in_debug_mode_) {
809           // Emit a run-time check verifying that the Marking Register is up-to-date.
810           UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
811           Register temp = temps.AcquireW();
812           // Ensure we are not clobbering a callee-save register that was restored before.
813           DCHECK(!core_reg_list.IncludesAliasOf(temp.X()))
814               << "core_reg_list should not contain scratch register X" << temp.GetCode();
815           asm_.GenerateMarkingRegisterCheck(temp);
816         }
817       }
818     }
819   }
820 
821   // Decrease frame size to start of callee saved regs.
822   DecreaseFrameSize(frame_size);
823 
824   // Return to LR.
825   ___ Ret();
826 
827   // The CFI should be restored for any code that follows the exit block.
828   cfi().RestoreState();
829   cfi().DefCFAOffset(frame_size);
830 }
831 
832 #undef ___
833 
834 }  // namespace arm64
835 }  // namespace art
836