/* * Copyright (C) 2023 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "asm_support_riscv64.S" #include "interpreter/cfi_asm_support.h" #include "arch/quick_alloc_entrypoints.S" #include "arch/quick_field_entrypoints.S" .macro INVOKE_STUB_CREATE_FRAME // Save RA, FP, xSELF (current thread), A4, A5 (they will be needed in the invoke stub return). INCREASE_FRAME 48 // Slot (8*0) is used for `ArtMethod*` (if no args), args or padding, see below. SAVE_GPR xSELF, (8*1) SAVE_GPR a4, (8*2) SAVE_GPR a5, (8*3) SAVE_GPR fp, (8*4) // Store FP just under the return address. SAVE_GPR ra, (8*5) // Make the new FP point to the location where we stored the old FP. // Some stack-walking tools may rely on this simply-linked list of saved FPs. addi fp, sp, (8*4) // save frame pointer .cfi_def_cfa fp, 48 - (8*4) // We already have space for `ArtMethod*` on the stack but we need space for args above // the `ArtMethod*`, so add sufficient space now, pushing the `ArtMethod*` slot down. addi t0, a2, 0xf // Reserve space for arguments and andi t0, t0, ~0xf // round up for 16-byte stack alignment. sub sp, sp, t0 mv xSELF, a3 // Copy arguments on stack (4 bytes per slot): // A1: source address // A2: arguments length // T0: destination address if there are any args. beqz a2, 2f // loop through 4-byte arguments from the last to the first addi t0, sp, 8 // destination address is bottom of the stack + 8 bytes for ArtMethod* (null) 1: addi a2, a2, -4 add t1, a1, a2 // T1 is the source address of the next copied argument lw t2, (t1) // T2 is the 4 bytes at address T1 add t1, t0, a2 // T1 is the destination address of the next copied argument sw t2, (t1) // save T2 at the destination address T1 bnez a2, 1b 2: sd zero, (sp) // Store null into ArtMethod* at bottom of frame. .endm .macro INVOKE_STUB_CALL_AND_RETURN // Call the method. ld t0, ART_METHOD_QUICK_CODE_OFFSET_64(a0) jalr t0 addi sp, fp, -(8*4) // restore SP (see `INVOKE_STUB_CREATE_FRAME`) .cfi_def_cfa sp, 48 // Restore ra, fp, xSELF (current thread) a4 (shorty), a5 (result pointer) from stack. RESTORE_GPR xSELF, (8*1) RESTORE_GPR a4, (8*2) RESTORE_GPR a5, (8*3) RESTORE_GPR fp, (8*4) RESTORE_GPR ra, (8*5) DECREASE_FRAME 48 // Load result type (1-byte symbol) from a5. // Check result type and store the correct register into the jvalue in memory at a4 address. lbu t0, (a5) li t1, 'V' // void (do not store result at all) beq t1, t0, 1f li t1, 'D' // double beq t1, t0, 2f li t1, 'F' // float beq t1, t0, 3f // Otherwise, result is in a0 (either 8 or 4 bytes, but it is fine to store 8 bytes as the // upper bytes in a0 in that case are zero, and jvalue has enough space). sd a0, (a4) 1: ret 2: // double: result in fa0 (8 bytes) fsd fa0, (a4) ret 3: // float: result in fa0 (4 bytes) fsw fa0, (a4) ret .endm ENTRY art_deliver_pending_exception DELIVER_PENDING_EXCEPTION END art_deliver_pending_exception // The size of the handler emitted by `INVOKE_STUB_LOAD_REG` below. #define INVOKE_STUB_LOAD_REG_SIZE 8 // The offset within `INVOKE_STUB_LOAD_REG` for skipping arguments. #define INVOKE_STUB_LOAD_REG_SKIP_OFFSET 6 // Macro for loading an argument into a register. // load - instruction used for loading, // reg - the register to load, // args - pointer to next argument, // size - the size of the register - 4 or 8 bytes, used as an offset for the load, // handler_reg - the register with the address of the handler (points to this handler on entry), // handler_diff - the difference in bytes from the current to the next handler, // cont - the base name of the label for continuing the shorty processing loop, // sfx - suffix added to all labels to make labels unique for different users. .macro INVOKE_STUB_LOAD_REG load, reg, args, size, handler_reg, handler_diff, cont, sfx .Linvoke_stub_\load\reg\sfx: \load \reg, -\size(\args) c.addi \handler_reg, \handler_diff .org .Linvoke_stub_\load\reg\sfx + INVOKE_STUB_LOAD_REG_SKIP_OFFSET // Enforce skip offset. c.j \cont\sfx .org .Linvoke_stub_\load\reg\sfx + INVOKE_STUB_LOAD_REG_SIZE // Enforce handler size. .endm // Fill registers a1 to a7 and fa0 to fa7 with parameters. // Parse the passed shorty to determine which register to load. // a5 - shorty, // t0 - points to arguments on the stack if any (undefined for static method without args), // sfx - suffix added to all labels to make labels unique for different users. .macro INVOKE_STUB_LOAD_ALL_ARGS sfx addi t1, a5, 1 // Load shorty address, plus one to skip the return type. // Load this (if instance method) and record the number of GPRs to fill. .ifc \sfx, _instance lwu a1, (t0) // Load "this" parameter, addi t0, t0, 4 // and increment arg pointer. .equ NUM_GPRS_TO_FILL, 6 .else .equ NUM_GPRS_TO_FILL, 7 .endif .equ NUM_FPRS_TO_FILL, 8 // Load addresses for routines that load argument GPRs and FPRs. lla t4, .Lreg_handlers_start\sfx // First handler for non-FP args. addi t5, t4, (3 * NUM_GPRS_TO_FILL * INVOKE_STUB_LOAD_REG_SIZE) // First handler for FP args. // Loop to fill registers. .Lfill_regs\sfx: lb t2, (t1) // Load next character in signature, addi t1, t1, 1 // and increment. beqz t2, .Lcall_method\sfx // Exit at end of signature. Shorty 0 terminated. li t3, 'L' beq t2, t3, .Lload_reference\sfx // Is this a reference? li t3, 'J' beq t2, t3, .Lload_long\sfx // Is this a long? li t3, 'F' beq t2, t3, .Lload_float\sfx // Is this a float? li t3, 'D' beq t2, t3, .Lload_double\sfx // Is this a double? // Everything else uses a 4-byte value sign-extened to a 64 bit GPR. addi t0, t0, 4 jalr x0, 0(t4) .Lload_reference\sfx: addi t0, t0, 4 jalr x0, (NUM_GPRS_TO_FILL * INVOKE_STUB_LOAD_REG_SIZE)(t4) .Lload_long\sfx: addi t0, t0, 8 jalr x0, (2 * NUM_GPRS_TO_FILL * INVOKE_STUB_LOAD_REG_SIZE)(t4) .Lload_float\sfx: addi t0, t0, 4 jalr x0, 0(t5) .Lload_double\sfx: addi t0, t0, 8 jalr x0, (NUM_FPRS_TO_FILL * INVOKE_STUB_LOAD_REG_SIZE)(t5) .Lreg_handlers_start\sfx: // Handlers for loading other args (not reference/long/float/double) into GPRs. .ifnc \sfx, _instance INVOKE_STUB_LOAD_REG lw, a1, t0, 4, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx .endif INVOKE_STUB_LOAD_REG lw, a2, t0, 4, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG lw, a3, t0, 4, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG lw, a4, t0, 4, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG lw, a5, t0, 4, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG lw, a6, t0, 4, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG lw, a7, t0, 4, t4, INVOKE_STUB_LOAD_REG_SKIP_OFFSET, .Lfill_regs, \sfx // Handlers for loading reference args into GPRs. .ifnc \sfx, _instance INVOKE_STUB_LOAD_REG lwu, a1, t0, 4, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx .endif INVOKE_STUB_LOAD_REG lwu, a2, t0, 4, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG lwu, a3, t0, 4, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG lwu, a4, t0, 4, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG lwu, a5, t0, 4, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG lwu, a6, t0, 4, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG lwu, a7, t0, 4, t4, INVOKE_STUB_LOAD_REG_SKIP_OFFSET, .Lfill_regs, \sfx // Handlers for loading long args into GPRs. .ifnc \sfx, _instance INVOKE_STUB_LOAD_REG ld, a1, t0, 8, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx .endif INVOKE_STUB_LOAD_REG ld, a2, t0, 8, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG ld, a3, t0, 8, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG ld, a4, t0, 8, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG ld, a5, t0, 8, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG ld, a6, t0, 8, t4, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG ld, a7, t0, 8, t4, INVOKE_STUB_LOAD_REG_SKIP_OFFSET, .Lfill_regs, \sfx // Handlers for loading floats into FPRs. INVOKE_STUB_LOAD_REG flw, fa0, t0, 4, t5, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG flw, fa1, t0, 4, t5, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG flw, fa2, t0, 4, t5, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG flw, fa3, t0, 4, t5, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG flw, fa4, t0, 4, t5, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG flw, fa5, t0, 4, t5, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG flw, fa6, t0, 4, t5, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG flw, fa7, t0, 4, t5, INVOKE_STUB_LOAD_REG_SKIP_OFFSET, .Lfill_regs, \sfx // Handlers for loading doubles into FPRs. INVOKE_STUB_LOAD_REG fld, fa0, t0, 8, t5, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG fld, fa1, t0, 8, t5, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG fld, fa2, t0, 8, t5, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG fld, fa3, t0, 8, t5, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG fld, fa4, t0, 8, t5, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG fld, fa5, t0, 8, t5, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG fld, fa6, t0, 8, t5, INVOKE_STUB_LOAD_REG_SIZE, .Lfill_regs, \sfx INVOKE_STUB_LOAD_REG fld, fa7, t0, 8, t5, INVOKE_STUB_LOAD_REG_SKIP_OFFSET, .Lfill_regs, \sfx .Lcall_method\sfx: .endm // void art_quick_invoke_stub(ArtMethod* method, // a0 // uint32_t* args, // a1 // uint32_t argsize, // a2 // Thread* self, // a3 // JValue* result, // a4 // char* shorty) // a5 ENTRY art_quick_invoke_stub INVOKE_STUB_CREATE_FRAME // Load args into registers. INVOKE_STUB_LOAD_ALL_ARGS _instance // Call the method and return. INVOKE_STUB_CALL_AND_RETURN END art_quick_invoke_stub // void art_quick_invoke_static_stub(ArtMethod* method, // a0 // uint32_t* args, // a1 // uint32_t argsize, // a2 // Thread* self, // a3 // JValue* result, // a4 // char* shorty) // a5 ENTRY art_quick_invoke_static_stub INVOKE_STUB_CREATE_FRAME // Load args into registers. INVOKE_STUB_LOAD_ALL_ARGS _static // Call the method and return. INVOKE_STUB_CALL_AND_RETURN END art_quick_invoke_static_stub ENTRY art_quick_generic_jni_trampoline SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_A0 // Save sp, so we can have static CFI info. mv fp, sp .cfi_def_cfa_register fp li t0, GENERIC_JNI_TRAMPOLINE_RESERVED_AREA sub sp, sp, t0 mv a0, xSELF // Thread* mv a1, fp // SP for the managed frame. mv a2, sp // reserved area for arguments and other saved data (up to managed frame) call artQuickGenericJniTrampoline // Check for error (class init check or locking for synchronized native method can throw). beqz a0, .Lexception_in_native mv t2, a0 // save pointer to native method code into temporary // Load argument GPRs from stack (saved there by artQuickGenericJniTrampoline). ld a0, 8*0(sp) // JniEnv* for the native method ld a1, 8*1(sp) ld a2, 8*2(sp) ld a3, 8*3(sp) ld a4, 8*4(sp) ld a5, 8*5(sp) ld a6, 8*6(sp) ld a7, 8*7(sp) // Load argument FPRs from stack (saved there by artQuickGenericJniTrampoline). fld fa0, 8*8(sp) fld fa1, 8*9(sp) fld fa2, 8*10(sp) fld fa3, 8*11(sp) fld fa4, 8*12(sp) fld fa5, 8*13(sp) fld fa6, 8*14(sp) fld fa7, 8*15(sp) ld t0, 8*16(sp) // @CriticalNative arg, used by art_jni_dlsym_lookup_critical_stub ld t1, 8*17(sp) // restore stack mv sp, t1 jalr t2 // call native method // result sign extension is handled in C code, prepare for artQuickGenericJniEndTrampoline call: // uint64_t artQuickGenericJniEndTrampoline(Thread* self, // a0 // jvalue result, // a1 (need to move from a0) // uint64_t result_f) // a2 (need to move from fa0) mv a1, a0 mv a0, xSELF fmv.x.d a2, fa0 call artQuickGenericJniEndTrampoline // Pending exceptions possible. ld t0, THREAD_EXCEPTION_OFFSET(xSELF) bnez t0, .Lexception_in_native // Tear down the alloca. mv sp, fp CFI_REMEMBER_STATE .cfi_def_cfa_register sp LOAD_RUNTIME_INSTANCE a1 ld a1, RUNTIME_INSTRUMENTATION_OFFSET(a1) lb a1, INSTRUMENTATION_RUN_EXIT_HOOKS_OFFSET(a1) bnez a1, .Lcall_method_exit_hook .Lcall_method_exit_hook_done: // This does not clobber the result register a0. a1 is not used for result as the managed code // does not have a 128-bit type. Alternatively we could restore a subset of these registers. RESTORE_SAVE_REFS_AND_ARGS_FRAME fmv.d.x fa0, a0 ret CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_REFS_AND_ARGS .Lcall_method_exit_hook: fmv.d.x fa0, a0 li a4, FRAME_SIZE_SAVE_REFS_AND_ARGS call art_quick_method_exit_hook j .Lcall_method_exit_hook_done .Lexception_in_native: // Move to a1 then sp to please assembler. ld a1, THREAD_TOP_QUICK_FRAME_OFFSET(xSELF) addi sp, a1, -1 // Remove the GenericJNI tag. call art_deliver_pending_exception END art_quick_generic_jni_trampoline ENTRY art_quick_to_interpreter_bridge SETUP_SAVE_REFS_AND_ARGS_FRAME // uint64_t artQuickToInterpreterBridge(ArtMethod* method, Thread* self, ArtMethod** sp) // a0 will contain ArtMethod* mv a1, xSELF mv a2, sp call artQuickToInterpreterBridge // TODO: no need to restore arguments in this case. RESTORE_SAVE_REFS_AND_ARGS_FRAME fmv.d.x fa0, a0 // copy the result to FP result register RETURN_OR_DELIVER_PENDING_EXCEPTION_REG t0 END art_quick_to_interpreter_bridge .extern artMethodEntryHook ENTRY art_quick_method_entry_hook SETUP_SAVE_EVERYTHING_FRAME ld a0, FRAME_SIZE_SAVE_EVERYTHING(sp) // Pass ArtMethod*. mv a1, xSELF // Pass Thread::Current(). mv a2, sp // pass SP call artMethodEntryHook // (ArtMethod*, Thread*, SP) CFI_REMEMBER_STATE bnez a0, .Lentryhook_deopt RESTORE_SAVE_EVERYTHING_FRAME ret .Lentryhook_deopt: // Deoptimize. CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING call art_quick_do_long_jump // (Context*) unimp // Unreached END art_quick_method_entry_hook .extern artMethodExitHook ENTRY art_quick_method_exit_hook SETUP_SAVE_EVERYTHING_FRAME // frame_size is passed in A4 from JITed code and `art_quick_generic_jni_trampoline`. addi a3, sp, SAVE_EVERYTHING_FRAME_OFFSET_FA0 // FP result ptr in kSaveEverything frame addi a2, sp, SAVE_EVERYTHING_FRAME_OFFSET_A0 // integer result ptr in kSaveEverything frame addi a1, sp, FRAME_SIZE_SAVE_EVERYTHING // ArtMethod** mv a0, xSELF // Thread::Current call artMethodExitHook // (Thread*, ArtMethod**, gpr_res*, fpr_res*, // frame_size) CFI_REMEMBER_STATE bnez a0, .Lexithook_deopt_or_exception // Normal return. RESTORE_SAVE_EVERYTHING_FRAME ret .Lexithook_deopt_or_exception: // Deoptimize or exception thrown. CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING call art_quick_do_long_jump // (Context*) unimp // Unreached END art_quick_method_exit_hook // On entry a0 is the long jump context. This is expected to be returned from a previous entrypoint // call which threw an exception or deoptimized. ENTRY art_quick_do_long_jump // We don't use the RA, but it is needed for unwinding. INCREASE_FRAME 16 SAVE_GPR ra, 0*8 // Reserve space for the gprs + fprs; INCREASE_FRAME RISCV64_LONG_JUMP_CONTEXT_SIZE mv a1, sp add a2, sp, RISCV64_LONG_JUMP_GPRS_SIZE call artContextCopyForLongJump // Context* context, uintptr_t* gprs, uintptr_t* fprs add a0, sp, RISCV64_LONG_JUMP_GPRS_SIZE // Load FPRs fld ft0, 8*0(a0) // f0 fld ft1, 8*1(a0) // f1 fld ft2, 8*2(a0) // f2 fld ft3, 8*3(a0) // f3 fld ft4, 8*4(a0) // f4 fld ft5, 8*5(a0) // f5 fld ft6, 8*6(a0) // f6 fld ft7, 8*7(a0) // f7 fld fs0, 8*8(a0) // f8 fld fs1, 8*9(a0) // f9 fld fa0, 8*10(a0) // f10 fld fa1, 8*11(a0) // f11 fld fa2, 8*12(a0) // f12 fld fa3, 8*13(a0) // f13 fld fa4, 8*14(a0) // f14 fld fa5, 8*15(a0) // f15 fld fa6, 8*16(a0) // f16 fld fa7, 8*17(a0) // f17 fld fs2, 8*18(a0) // f18 fld fs3, 8*19(a0) // f19 fld fs4, 8*20(a0) // f20 fld fs5, 8*21(a0) // f21 fld fs6, 8*22(a0) // f22 fld fs7, 8*23(a0) // f23 fld fs8, 8*24(a0) // f24 fld fs9, 8*25(a0) // f25 fld fs10, 8*26(a0) // f26 fld fs11, 8*27(a0) // f27 fld ft8, 8*28(a0) // f28 fld ft9, 8*29(a0) // f29 fld ft10, 8*30(a0) // f30 fld ft11, 8*31(a0) // f31 // Load GPRs. // Skip slot 8*0(sp) for zero/x0 as it is hard-wired zero. ld ra, 8*1(sp) // x1 // Skip slot 8*2(sp) for sp/x2 as it is set below. // Skip slot 8*3(sp) for platform-specific thread pointer gp/x3. // Skip slot 8*4(sp) for platform-specific global pointer tp/x4. // Skip slot 8*5(sp) for t0/x5 as it is clobbered below. // Skip slot 8*6(sp) for t1/x6 as it is clobbered below. ld t2, 8*7(sp) // x7 ld s0, 8*8(sp) // x8 ld s1, 8*9(sp) // x9 ld a0, 8*10(sp) // x10 ld a1, 8*11(sp) // x11 ld a2, 8*12(sp) // x12 ld a3, 8*13(sp) // x13 ld a4, 8*14(sp) // x14 ld a5, 8*15(sp) // x15 ld a6, 8*16(sp) // x16 ld a7, 8*17(sp) // x17 ld s2, 8*18(sp) // x18 ld s3, 8*19(sp) // x19 ld s4, 8*20(sp) // x20 ld s5, 8*21(sp) // x21 ld s6, 8*22(sp) // x22 ld s7, 8*23(sp) // x23 ld s8, 8*24(sp) // x24 ld s9, 8*25(sp) // x25 ld s10, 8*26(sp) // x26 ld s11, 8*27(sp) // x27 ld t3, 8*28(sp) // x28 ld t4, 8*29(sp) // x29 ld t5, 8*30(sp) // x30 ld t6, 8*31(sp) // x31 // Load sp to t0. ld t0, 8*2(sp) // Load PC to t1, it is in the last stack slot. ld t1, 8*32(sp) // Set sp. Do not access fprs_ and gprs_ from now, they are below sp. mv sp, t0 .cfi_def_cfa_offset 0 .cfi_register ra, t1 jr t1 END art_quick_do_long_jump .macro DEOPT_OR_RETURN temp, is_ref = 0 lwu \temp, THREAD_DEOPT_CHECK_REQUIRED_OFFSET(xSELF) bnez \temp, 2f ret 2: SETUP_SAVE_EVERYTHING_FRAME li a2, \is_ref // pass if result is a reference mv a1, a0 // pass the result mv a0, xSELF // pass Thread::Current call artDeoptimizeIfNeeded // (Thread*, uintptr_t, bool) CFI_REMEMBER_STATE bnez a0, 3f RESTORE_SAVE_EVERYTHING_FRAME ret 3: // Deoptimize CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING call art_quick_do_long_jump // (Context*) unimp // Unreached .endm .macro RETURN_OR_DEOPT_IF_RESULT_IS_NON_NULL_OR_DELIVER CFI_REMEMBER_STATE beqz a0, 1f DEOPT_OR_RETURN a1, /*is_ref=*/ 1 1: CFI_RESTORE_STATE_AND_DEF_CFA sp, 0 DELIVER_PENDING_EXCEPTION .endm .macro RETURN_OR_DEOPT_IF_INT_RESULT_IS_ZERO_OR_DELIVER CFI_REMEMBER_STATE bnez a0, 1f DEOPT_OR_RETURN a1 1: CFI_RESTORE_STATE_AND_DEF_CFA sp, 0 DELIVER_PENDING_EXCEPTION .endm .macro RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION is_ref = 0 lwu a1, THREAD_EXCEPTION_OFFSET(xSELF) // Get exception field. CFI_REMEMBER_STATE bnez a1, 1f DEOPT_OR_RETURN a1, \is_ref // Check if deopt is required. 1: CFI_RESTORE_STATE_AND_DEF_CFA sp, 0 DELIVER_PENDING_EXCEPTION // Deliver exception on current thread. .endm .macro RETURN_REF_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION /* is_ref= */ 1 .endm .macro DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_A0 temp, is_ref ld \temp, THREAD_DEOPT_CHECK_REQUIRED_OFFSET(xSELF) CFI_REMEMBER_STATE bnez \temp, 2f RESTORE_SAVE_EVERYTHING_FRAME /* load_a0= */ 0 ret 2: CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING sd a0, SAVE_EVERYTHING_FRAME_OFFSET_A0(sp) // update result in the frame li a2, \is_ref // pass if result is a reference mv a1, a0 // pass the result mv a0, xSELF // Thread::Current call artDeoptimizeIfNeeded CFI_REMEMBER_STATE bnez a0, 3f RESTORE_SAVE_EVERYTHING_FRAME ret 3: // Deoptimize CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING call art_quick_do_long_jump // (Context*) unimp // Unreached .endm // Entry from managed code that tries to lock the object in a fast path and // calls `artLockObjectFromCode()` for the difficult cases, may block for GC. // A0 holds the possibly null object to lock. ENTRY art_quick_lock_object LOCK_OBJECT_FAST_PATH a0, art_quick_lock_object_no_inline, /*can_be_null*/ 1 END art_quick_lock_object // Entry from managed code that calls `artLockObjectFromCode()`, may block for GC. // A0 holds the possibly null object to lock. .extern artLockObjectFromCode ENTRY art_quick_lock_object_no_inline // This is also the slow path for `art_quick_lock_object`. SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case we block mv a1, xSELF // pass Thread::Current call artLockObjectFromCode // (Object*, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME RETURN_OR_DEOPT_IF_INT_RESULT_IS_ZERO_OR_DELIVER END art_quick_lock_object_no_inline // Entry from managed code that tries to unlock the object in a fast path and calls // `artUnlockObjectFromCode()` for the difficult cases and delivers exception on failure. // A0 holds the possibly null object to unlock. ENTRY art_quick_unlock_object UNLOCK_OBJECT_FAST_PATH a0, art_quick_unlock_object_no_inline, /*can_be_null*/ 1 END art_quick_unlock_object // Entry from managed code that calls `artUnlockObjectFromCode()` // and delivers exception on failure. // A0 holds the possibly null object to unlock. .extern artUnlockObjectFromCode ENTRY art_quick_unlock_object_no_inline // This is also the slow path for `art_quick_unlock_object`. SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case exception allocation triggers GC mv a1, xSELF // pass Thread::Current call artUnlockObjectFromCode // (Object*, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME RETURN_OR_DEOPT_IF_INT_RESULT_IS_ZERO_OR_DELIVER END art_quick_unlock_object_no_inline // Called by managed code that is attempting to call a method on a proxy class. On entry a0 holds // the proxy method and a1 holds the receiver. The frame size of the invoked proxy method agrees // with kSaveRefsAndArgs frame. .extern artQuickProxyInvokeHandler ENTRY art_quick_proxy_invoke_handler SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_A0 // uint64_t artQuickProxyInvokeHandler(ArtMethod* proxy_method, // a0 // mirror::Object* receiver, // a1 // Thread* self, // a2 // ArtMethod** sp) // a3 mv a2, xSELF // pass Thread::Current mv a3, sp // pass sp call artQuickProxyInvokeHandler // (Method* proxy method, receiver, Thread*, sp) ld a2, THREAD_EXCEPTION_OFFSET(xSELF) bnez a2, .Lexception_in_proxy // success if no exception is pending CFI_REMEMBER_STATE RESTORE_SAVE_REFS_AND_ARGS_FRAME // Restore frame fmv.d.x fa0, a0 // Store result in fa0 in case it was float or double ret // return on success .Lexception_in_proxy: CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_REFS_AND_ARGS RESTORE_SAVE_REFS_AND_ARGS_FRAME DELIVER_PENDING_EXCEPTION END art_quick_proxy_invoke_handler // Compiled code has requested that we deoptimize into the interpreter. The deoptimization // will long jump to the upcall with a special exception of -1. .extern artDeoptimizeFromCompiledCode ENTRY art_quick_deoptimize_from_compiled_code SETUP_SAVE_EVERYTHING_FRAME mv a1, xSELF // Pass Thread::Current(). call artDeoptimizeFromCompiledCode // (DeoptimizationKind, Thread*) call art_quick_do_long_jump // (Context*) unimp // Unreached END art_quick_deoptimize_from_compiled_code .extern artStringBuilderAppend ENTRY art_quick_string_builder_append SETUP_SAVE_REFS_ONLY_FRAME // Save callee saves in case of GC. addi a1, sp, (FRAME_SIZE_SAVE_REFS_ONLY + __SIZEOF_POINTER__) // Pass args. mv a2, xSELF // Pass Thread::Current(). call artStringBuilderAppend // (uint32_t, const unit32_t*, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME RETURN_OR_DEOPT_IF_RESULT_IS_NON_NULL_OR_DELIVER END art_quick_string_builder_append // Entry from managed code that calls artInstanceOfFromCode and on failure calls // artThrowClassCastExceptionForObject. .extern artInstanceOfFromCode .extern artThrowClassCastExceptionForObject ENTRY art_quick_check_instance_of // Type check using the bit string passes null as the target class. In that case just throw. beqz a1, .Lthrow_class_cast_exception_for_bitstring_check // Store arguments and return address register. // Stack needs to be 16B aligned on calls. INCREASE_FRAME 32 sd a0, 0*8(sp) sd a1, 1*8(sp) SAVE_GPR ra, 3*8 // Call runtime code. call artInstanceOfFromCode // Restore RA. RESTORE_GPR ra, 3*8 // Check for exception. CFI_REMEMBER_STATE beqz a0, .Lthrow_class_cast_exception // Remove spill area and return (no need to restore A0 and A1). DECREASE_FRAME 32 ret .Lthrow_class_cast_exception: CFI_RESTORE_STATE_AND_DEF_CFA sp, 32 // Restore A0 and remove spill area. ld a0, 0*8(sp) ld a1, 1*8(sp) DECREASE_FRAME 32 .Lthrow_class_cast_exception_for_bitstring_check: SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // Save all registers as basis for long jump context. mv a2, xSELF // Pass Thread::Current(). call artThrowClassCastExceptionForObject // (Object*, Class*, Thread*) call art_quick_do_long_jump // (Context*) unimp // Unreached END art_quick_check_instance_of .macro N_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING n, c_name, cxx_name .extern \cxx_name ENTRY \c_name SETUP_SAVE_EVERYTHING_FRAME // save all registers as basis for long jump context. mv a\n, xSELF // pass Thread::Current. call \cxx_name // \cxx_name(args..., Thread*). call art_quick_do_long_jump // (Context*) unimp // Unreached END \c_name .endm .macro NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING c_name, cxx_name N_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING 0, \c_name, \cxx_name .endm .macro TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING c_name, cxx_name N_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING 2, \c_name, \cxx_name .endm .macro N_ARG_RUNTIME_EXCEPTION n, c_name, cxx_name .extern \cxx_name ENTRY \c_name SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context. mv a\n, xSELF // pass Thread::Current. call \cxx_name // \cxx_name(args..., Thread*). call art_quick_do_long_jump // (Context*) unimp // Unreached END \c_name .endm .macro NO_ARG_RUNTIME_EXCEPTION c_name, cxx_name N_ARG_RUNTIME_EXCEPTION 0, \c_name, \cxx_name .endm .macro ONE_ARG_RUNTIME_EXCEPTION c_name, cxx_name N_ARG_RUNTIME_EXCEPTION 1, \c_name, \cxx_name .endm // Called by managed code to create and deliver a NullPointerException. NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING \ art_quick_throw_null_pointer_exception, artThrowNullPointerExceptionFromCode // Call installed by a signal handler to create and deliver a NullPointerException. .extern artThrowNullPointerExceptionFromSignal ENTRY art_quick_throw_null_pointer_exception_from_signal // The fault handler pushes the gc map address, i.e. "return address", to stack // and passes the fault address in RA. So we need to set up the CFI info accordingly. .cfi_def_cfa_offset __SIZEOF_POINTER__ .cfi_rel_offset ra, 0 // Save all registers as basis for long jump context. INCREASE_FRAME (FRAME_SIZE_SAVE_EVERYTHING - __SIZEOF_POINTER__) SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_RA mv a0, ra // pass the fault address stored in RA by the fault handler. mv a1, xSELF // pass Thread::Current. call artThrowNullPointerExceptionFromSignal // (arg, Thread*). call art_quick_do_long_jump // (Context*) unimp // Unreached END art_quick_throw_null_pointer_exception_from_signal // Called by managed code to deliver an ArithmeticException. NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_div_zero, artThrowDivZeroFromCode // Called by managed code to create and deliver an ArrayIndexOutOfBoundsException. // Arg0 holds index, arg1 holds limit. TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_array_bounds, artThrowArrayBoundsFromCode // Called by managed code to create and deliver a StringIndexOutOfBoundsException // as if thrown from a call to String.charAt(). Arg0 holds index, arg1 holds limit. TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING \ art_quick_throw_string_bounds, artThrowStringBoundsFromCode // Called by managed code to create and deliver a StackOverflowError. NO_ARG_RUNTIME_EXCEPTION art_quick_throw_stack_overflow, artThrowStackOverflowFromCode // Called by managed code to deliver an exception. ONE_ARG_RUNTIME_EXCEPTION art_quick_deliver_exception, artDeliverExceptionFromCode // Called to attempt to execute an obsolete method. ONE_ARG_RUNTIME_EXCEPTION art_invoke_obsolete_method_stub, artInvokeObsoleteMethod ENTRY art_quick_resolution_trampoline SETUP_SAVE_REFS_AND_ARGS_FRAME // const void* artQuickResolutionTrampoline(ArtMethod* called, // a0 // mirror::Object* receiver, // a1 // Thread* self, // a2 // ArtMethod** sp) // a3 mv a2, xSELF mv a3, sp call artQuickResolutionTrampoline CFI_REMEMBER_STATE beqz a0, 1f mv t0, a0 // Remember returned code pointer in t0. ld a0, (sp) // artQuickResolutionTrampoline puts called method in *sp. RESTORE_SAVE_REFS_AND_ARGS_FRAME jr t0 1: CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_REFS_AND_ARGS RESTORE_SAVE_REFS_AND_ARGS_FRAME DELIVER_PENDING_EXCEPTION END art_quick_resolution_trampoline ENTRY art_quick_test_suspend SETUP_SAVE_EVERYTHING_FRAME \ RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET mv a0, xSELF call artTestSuspendFromCode CFI_REMEMBER_STATE bnez a0, .Ltest_suspend_deoptimize RESTORE_SAVE_EVERYTHING_FRAME ret .Ltest_suspend_deoptimize: // Deoptimize CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING call art_quick_do_long_jump // (Context*) unimp // Unreached END art_quick_test_suspend ENTRY art_quick_compile_optimized SETUP_SAVE_EVERYTHING_FRAME ld a0, FRAME_SIZE_SAVE_EVERYTHING(sp) // pass ArtMethod mv a1, xSELF // pass Thread::Current call artCompileOptimized // (ArtMethod*, Thread*) RESTORE_SAVE_EVERYTHING_FRAME // Note: If we implement implicit suspend checks or a marking register for GC, we don't need // to restore such registers here, as artCompileOptimized doesn't allow thread suspension. ret END art_quick_compile_optimized ENTRY art_quick_compile_baseline SETUP_SAVE_EVERYTHING_FRAME ld a0, FRAME_SIZE_SAVE_EVERYTHING(sp) // pass ArtMethod mv a1, xSELF // pass Thread::Current call artCompileBaseline // (ArtMethod*, Thread*) RESTORE_SAVE_EVERYTHING_FRAME // Note: If we implement implicit suspend checks or a marking register for GC, we don't need // to restore such registers here, as artCompileBaseline doesn't allow thread suspension. ret END art_quick_compile_baseline /* extern"C" void art_quick_osr_stub(void* stack, A0 * size_t stack_size_in_bytes, A1 * const uint8_t* native_pc, A2 * JValue* result, A3 * char* shorty, A4 * Thread* self) A5 */ ENTRY art_quick_osr_stub // Save all callee-save registers (we do not fill the spill area in the OSR frame, so we // need to preserve them here) and A3 (it will be needed after the OSR method returns). // Also add space for the `ArtMethod*` slot (null to indicate transition) and padding. SAVE_SIZE=(12 + 12 + /* RA */ 1 + /* A3 */ 1 + /* ArtMethod* */ 1 + /* padding */ 1) * 8 INCREASE_FRAME SAVE_SIZE sd zero, 0*8(sp) // Store null to the `ArtMethod*` slot to indicate transition. // Skip padding. SAVE_GPR a3, 2*8 // Save `result`. SAVE_FPR fs0, 3*8 SAVE_FPR fs1, 4*8 SAVE_FPR fs2, 5*8 SAVE_FPR fs3, 6*8 SAVE_FPR fs4, 7*8 SAVE_FPR fs5, 8*8 SAVE_FPR fs6, 9*8 SAVE_FPR fs7, 10*8 SAVE_FPR fs8, 11*8 SAVE_FPR fs9, 12*8 SAVE_FPR fs10, 13*8 SAVE_FPR fs11, 14*8 SAVE_GPR s2, 15*8 SAVE_GPR s3, 16*8 SAVE_GPR s4, 17*8 SAVE_GPR s5, 18*8 SAVE_GPR s6, 19*8 SAVE_GPR s7, 20*8 SAVE_GPR s8, 21*8 SAVE_GPR s9, 22*8 SAVE_GPR s10, 23*8 SAVE_GPR s11, 24*8 SAVE_GPR xSELF, 25*8 // Save xSELF/S1. SAVE_GPR fp, 26*8 // Save FP/S0. SAVE_GPR ra, 27*8 // Save return address. // Make the new FP point to the location where we stored the old FP. // Some stack-walking tools may rely on this simply-linked list of saved FPs. addi fp, sp, (26*8) // save frame pointer .cfi_def_cfa fp, SAVE_SIZE - (26*8) mv xSELF, a5 CFI_REMEMBER_STATE jal .Losr_entry // The called method removes the stack frame created in `.Losr_entry`. // The SP is already correctly restored, we do not need to restore it from FP. .cfi_def_cfa sp, SAVE_SIZE // Restore saved registers including the result address. RESTORE_GPR a3, 2*8 // Restore `result`. RESTORE_FPR fs0, 3*8 RESTORE_FPR fs1, 4*8 RESTORE_FPR fs2, 5*8 RESTORE_FPR fs3, 6*8 RESTORE_FPR fs4, 7*8 RESTORE_FPR fs5, 8*8 RESTORE_FPR fs6, 9*8 RESTORE_FPR fs7, 10*8 RESTORE_FPR fs8, 11*8 RESTORE_FPR fs9, 12*8 RESTORE_FPR fs10, 13*8 RESTORE_FPR fs11, 14*8 RESTORE_GPR s2, 15*8 RESTORE_GPR s3, 16*8 RESTORE_GPR s4, 17*8 RESTORE_GPR s5, 18*8 RESTORE_GPR s6, 19*8 RESTORE_GPR s7, 20*8 RESTORE_GPR s8, 21*8 RESTORE_GPR s9, 22*8 RESTORE_GPR s10, 23*8 RESTORE_GPR s11, 24*8 RESTORE_GPR xSELF, 25*8 // Restore xSELF/S1. RESTORE_GPR fp, 26*8 // Restore FP/S0. RESTORE_GPR ra, 27*8 // Restore return address. DECREASE_FRAME SAVE_SIZE // The compiler put the result in A0. Doesn't matter if it is 64 or 32 bits. sd a0, (a3) ret .Losr_entry: CFI_RESTORE_STATE_AND_DEF_CFA fp, SAVE_SIZE - (26*8) // Prepare the destination register for backward copy of arguments. addi t1, sp, -8 // Update stack pointer for the callee frame. sub sp, sp, a1 // Subtract the return address slot size from args size. addi a1, a1, -8 // Update return address slot expected by the callee. sd ra, (t1) // Prepare the source register for backward copy of arguments. add t0, a0, a1 // Copy arguments into stack frame. Use simple backward-copy routine for now. // There is always at least the `ArtMethod*` to to copy. // A0 - source address // A1 - args length // SP - destination address. // T0 - loop variable initialized to A0 + A1 for backward copy // T1 - loop variable initialized to SP + A1 for backward copy // T2 - temporary for holding the copied value .Losr_loop: addi t0, t0, -8 ld t2, (t0) addi t1, t1, -8 sd t2, (t1) bne t1, sp, .Losr_loop // Branch to the OSR entry point. jr a2 END art_quick_osr_stub /* * All generated callsites for interface invokes and invocation slow paths will load arguments * as usual - except instead of loading arg0/A0 with the target Method*, arg0/A0 will contain * the method_idx. This wrapper will call the appropriate C++ helper while preserving arguments * and allowing a moving GC to update references in callee-save registers. * NOTE: "this" is the first visible argument of the target, and so can be found in arg1/A1. * * The helper will attempt to locate the target and return a 128-bit result consisting of the * target `ArtMethod*` in A0 and its `entry_point_from_quick_compiled_code_` in A1. * * If unsuccessful, the helper will return null/null. There will be a pending exception * to deliver in the thread. * * On success this wrapper will restore arguments and *jump* to the target, leaving the RA * pointing back to the original caller. */ .macro INVOKE_TRAMPOLINE_BODY cxx_name .extern \cxx_name SETUP_SAVE_REFS_AND_ARGS_FRAME mv a2, xSELF // Pass Thread::Current(). mv a3, sp // Pass pointer to the saved frame context. call \cxx_name // (method_idx, this, Thread*, $sp) mv t0, a1 // Save method's code pointer in T0. RESTORE_SAVE_REFS_AND_ARGS_FRAME beq a0, zero, 1f jr t0 1: DELIVER_PENDING_EXCEPTION .endm .macro INVOKE_TRAMPOLINE c_name, cxx_name ENTRY \c_name INVOKE_TRAMPOLINE_BODY \cxx_name END \c_name .endm INVOKE_TRAMPOLINE art_quick_invoke_interface_trampoline_with_access_check, \ artInvokeInterfaceTrampolineWithAccessCheck INVOKE_TRAMPOLINE art_quick_invoke_static_trampoline_with_access_check, \ artInvokeStaticTrampolineWithAccessCheck INVOKE_TRAMPOLINE art_quick_invoke_direct_trampoline_with_access_check, \ artInvokeDirectTrampolineWithAccessCheck INVOKE_TRAMPOLINE art_quick_invoke_super_trampoline_with_access_check, \ artInvokeSuperTrampolineWithAccessCheck INVOKE_TRAMPOLINE art_quick_invoke_virtual_trampoline_with_access_check, \ artInvokeVirtualTrampolineWithAccessCheck /* * Polymorphic method invocation. * On entry: * A0 = unused * A1 = receiver */ .extern artInvokePolymorphic ENTRY art_quick_invoke_polymorphic SETUP_SAVE_REFS_AND_ARGS_FRAME mv a0, a1 // Pass the receiver. mv a1, xSELF // Pass Thread::Current(). mv a2, sp // Pass pointer to the saved frame context. call artInvokePolymorphic // artInvokePolymorphic(receiver, Thread*, context) RESTORE_SAVE_REFS_AND_ARGS_FRAME fmv.d.x fa0, a0 // Copy the result also to the FP return register. RETURN_OR_DELIVER_PENDING_EXCEPTION_REG t0 END art_quick_invoke_polymorphic /* * Slow path for MethodHandle.invokeExact intrinsic. * That intrinsic has a custom calling convention: the argument allocation doesn't start from * the receiver (MethodHandle) object, but from the argument following it. That's done to match * expectation of the underlying method when MethodHandle targets a method. That also affects * the way arguments are spilled onto the stack. */ .extern artInvokePolymorphicWithHiddenReceiver ENTRY art_quick_invoke_polymorphic_with_hidden_receiver // On entry: a0 := receiver SETUP_SAVE_REFS_AND_ARGS_FRAME // Save callee saves in case allocation triggers GC. mv a1, xSELF // Pass Thread::Current() mv a2, sp // Pass pointer to the saved frame context. call artInvokePolymorphicWithHiddenReceiver // invoke with (receiver, thread, save_area) RESTORE_SAVE_REFS_AND_ARGS_FRAME fmv.d.x fa0, a0 // Copy the result also to the FP return register. RETURN_OR_DELIVER_PENDING_EXCEPTION_REG t0 END art_quick_invoke_polymorphic_with_hidden_receiver /* * InvokeCustom invocation. * On entry: * A0 = call_site_idx */ .extern artInvokeCustom ENTRY art_quick_invoke_custom SETUP_SAVE_REFS_AND_ARGS_FRAME mv a1, xSELF // Pass Thread::Current(). mv a2, sp // Pass pointer to the saved frame context. call artInvokeCustom // artInvokeCustom(call_site_idx, Thread*, context) RESTORE_SAVE_REFS_AND_ARGS_FRAME fmv.d.x fa0, a0 // Copy the result also to the FP return register. RETURN_OR_DELIVER_PENDING_EXCEPTION_REG t0 END art_quick_invoke_custom /* * Called to resolve an imt conflict. * On entry: * A0 is the conflict ArtMethod. * T0 is a hidden argument that holds the target interface method's dex method index. */ ENTRY art_quick_imt_conflict_trampoline ld t1, ART_METHOD_JNI_OFFSET_64(a0) // Load ImtConflictTable ld a0, 0(t1) // Load first entry in ImtConflictTable. .Limt_table_iterate: // Branch if found. beq a0, t0, .Limt_table_found // If the entry is null, the interface method is not in the ImtConflictTable. beqz a0, .Lconflict_trampoline // Iterate over the entries of the ImtConflictTable. addi t1, t1, (2 * __SIZEOF_POINTER__) ld a0, 0(t1) j .Limt_table_iterate .Limt_table_found: // We successfully hit an entry in the table. Load the target method and jump to it. ld a0, __SIZEOF_POINTER__(t1) ld t1, ART_METHOD_QUICK_CODE_OFFSET_64(a0) jr t1 .Lconflict_trampoline: // Call the runtime stub to populate the ImtConflictTable and jump to the // resolved method. move a0, t0 // Load interface method INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline END art_quick_imt_conflict_trampoline .macro UPDATE_INLINE_CACHE_ENTRY class, entry, temp, loop_label, done_label, next_label \loop_label: lwu \temp, (\entry) beq \class, \temp, \done_label bnez \temp, \next_label lr.w \temp, (\entry) bnez \temp, \loop_label sc.w \temp, \class, (\entry) beqz \temp, \done_label j \loop_label .endm // A0 contains the class, T5 contains the inline cache. T6 can be used, T5 can be clobbered. ENTRY art_quick_update_inline_cache #if (INLINE_CACHE_SIZE != 5) #error "INLINE_CACHE_SIZE not as expected." #endif #if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) // Don't update the cache if we are marking. lwu t6, THREAD_IS_GC_MARKING_OFFSET(xSELF) bnez t6, .Ldone #endif addi t5, t5, INLINE_CACHE_CLASSES_OFFSET UPDATE_INLINE_CACHE_ENTRY a0, t5, t6, .Lentry1_loop, .Ldone, .Lentry2 .Lentry2: addi t5, t5, 4 UPDATE_INLINE_CACHE_ENTRY a0, t5, t6, .Lentry2_loop, .Ldone, .Lentry3 .Lentry3: addi t5, t5, 4 UPDATE_INLINE_CACHE_ENTRY a0, t5, t6, .Lentry3_loop, .Ldone, .Lentry4 .Lentry4: addi t5, t5, 4 UPDATE_INLINE_CACHE_ENTRY a0, t5, t6, .Lentry4_loop, .Ldone, .Lentry5 .Lentry5: // Unconditionally store, the inline cache is megamorphic. sw a0, 4(t5) .Ldone: ret END art_quick_update_inline_cache .macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL \ name, entrypoint, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET .extern \entrypoint ENTRY \name SETUP_SAVE_EVERYTHING_FRAME \runtime_method_offset // Save everything for stack crawl. mv a1, xSELF // Pass Thread::Current(). call \entrypoint // (uint32_t/Class* index/klass, Thread* self) beqz a0, 1f // If result is null, deliver the exception. DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_A0 /* temp= */ a1, /* is_ref= */ 1 1: DELIVER_PENDING_EXCEPTION_FRAME_READY END \name .endm .macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT name, entrypoint ONE_ARG_SAVE_EVERYTHING_DOWNCALL \ \name, \entrypoint, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET .endm ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT \ art_quick_initialize_static_storage, artInitializeStaticStorageFromCode ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_resolve_type, artResolveTypeFromCode ONE_ARG_SAVE_EVERYTHING_DOWNCALL \ art_quick_resolve_type_and_verify_access, artResolveTypeAndVerifyAccessFromCode ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_method_handle, artResolveMethodHandleFromCode ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_method_type, artResolveMethodTypeFromCode ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode // Helper macros for `art_quick_aput_obj`. #ifdef USE_READ_BARRIER #ifdef USE_BAKER_READ_BARRIER .macro BAKER_RB_CHECK_GRAY_BIT_AND_LOAD dest, obj, offset, gray_slow_path_label lw t6, MIRROR_OBJECT_LOCK_WORD_OFFSET(\obj) slliw t6, t6, 31 - LOCK_WORD_READ_BARRIER_STATE_SHIFT // Shift the state bit to sign bit. bltz t6, \gray_slow_path_label // False dependency to avoid needing load/load fence. xor t6, t6, t6 add \obj, \obj, t6 lwu \dest, \offset(\obj) // Heap reference = 32b; zero-extends to `dest`. UNPOISON_HEAP_REF \dest .endm .macro BAKER_RB_LOAD_AND_MARK dest, obj, offset, mark_function lwu \dest, \offset(\obj) // Heap reference = 32b; zero-extends to `dest`. UNPOISON_HEAP_REF \dest // Save RA in a register preserved by `art_quick_read_barrier_mark_regNN` // and unused by the `art_quick_aput_obj`. mv t2, ra call \mark_function mv ra, t2 // Restore RA. .endm #else // USE_BAKER_READ_BARRIER .extern artReadBarrierSlow .macro READ_BARRIER_SLOW dest, obj, offset // Store registers used in art_quick_aput_obj (a0-a4, RA), stack is 16B aligned. INCREASE_FRAME 48 SAVE_GPR a0, 0*8 SAVE_GPR a1, 1*8 SAVE_GPR a2, 2*8 SAVE_GPR a3, 3*8 SAVE_GPR a4, 4*8 SAVE_GPR ra, 5*8 // mv a0, \ref // Pass ref in A0 (no-op for now since parameter ref is unused). .ifnc \obj, a1 mv a1, \obj // Pass `obj`. .endif li a2, \offset // Pass offset. call artReadBarrierSlow // artReadBarrierSlow(ref, obj, offset) // No need to unpoison return value in A0, `artReadBarrierSlow()` would do the unpoisoning. .ifnc \dest, a0 mv \dest, a0 // save return value in dest .endif // Conditionally restore saved registers RESTORE_GPR_NE a0, 0*8, \dest RESTORE_GPR_NE a1, 1*8, \dest RESTORE_GPR_NE a2, 2*8, \dest RESTORE_GPR_NE a3, 3*8, \dest RESTORE_GPR_NE a4, 4*8, \dest RESTORE_GPR ra, 5*8 DECREASE_FRAME 48 .endm #endif // USE_BAKER_READ_BARRIER #endif // USE_READ_BARRIER ENTRY art_quick_aput_obj beqz a2, .Laput_obj_null #if defined(USE_READ_BARRIER) && !defined(USE_BAKER_READ_BARRIER) READ_BARRIER_SLOW a3, a0, MIRROR_OBJECT_CLASS_OFFSET READ_BARRIER_SLOW a3, a3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET READ_BARRIER_SLOW a4, a2, MIRROR_OBJECT_CLASS_OFFSET #else // !defined(USE_READ_BARRIER) || defined(USE_BAKER_READ_BARRIER) #ifdef USE_READ_BARRIER // TODO(riscv64): Define marking register to avoid this load. lw t6, THREAD_IS_GC_MARKING_OFFSET(xSELF) bnez t6, .Laput_obj_gc_marking #endif // USE_READ_BARRIER lwu a3, MIRROR_OBJECT_CLASS_OFFSET(a0) // Heap reference = 32b; zero-extends to a3. UNPOISON_HEAP_REF a3 lwu a3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET(a3) // Heap reference = 32b; zero-extends to a3. UNPOISON_HEAP_REF a3 lwu a4, MIRROR_OBJECT_CLASS_OFFSET(a2) // Heap reference = 32b; zero-extends to a4. UNPOISON_HEAP_REF a4 #endif // !defined(USE_READ_BARRIER) || defined(USE_BAKER_READ_BARRIER) // value's type == array's component type - trivial assignability bne a3, a4, .Laput_obj_check_assignability .Laput_obj_store: sh2add a3, a1, a0 POISON_HEAP_REF a2 sw a2, MIRROR_OBJECT_ARRAY_DATA_OFFSET(a3) // Heap reference = 32b. ld a3, THREAD_CARD_TABLE_OFFSET(xSELF) srli a0, a0, CARD_TABLE_CARD_SHIFT add a0, a0, a3 sb a3, (a0) ret .Laput_obj_null: sh2add a3, a1, a0 sw a2, MIRROR_OBJECT_ARRAY_DATA_OFFSET(a3) // Heap reference = 32b. ret .Laput_obj_check_assignability: // Store arguments and return register INCREASE_FRAME 32 SAVE_GPR a0, 0*8 SAVE_GPR a1, 1*8 SAVE_GPR a2, 2*8 SAVE_GPR ra, 3*8 // Call runtime code mv a0, a3 // Heap reference, 32b, "uncompress" = do nothing, already zero-extended. mv a1, a4 // Heap reference, 32b, "uncompress" = do nothing, already zero-extended. call artIsAssignableFromCode // Check for exception CFI_REMEMBER_STATE beqz a0, .Laput_obj_throw_array_store_exception // Restore RESTORE_GPR a0, 0*8 RESTORE_GPR a1, 1*8 RESTORE_GPR a2, 2*8 RESTORE_GPR ra, 3*8 DECREASE_FRAME 32 sh2add a3, a1, a0 POISON_HEAP_REF a2 sw a2, MIRROR_OBJECT_ARRAY_DATA_OFFSET(a3) // Heap reference = 32b. ld a3, THREAD_CARD_TABLE_OFFSET(xSELF) srli a0, a0, CARD_TABLE_CARD_SHIFT add a0, a0, a3 sb a3, (a0) ret .Laput_obj_throw_array_store_exception: CFI_RESTORE_STATE_AND_DEF_CFA sp, 32 RESTORE_GPR a0, 0*8 RESTORE_GPR a1, 1*8 RESTORE_GPR a2, 2*8 RESTORE_GPR ra, 3*8 DECREASE_FRAME 32 #if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) CFI_REMEMBER_STATE #endif // defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) SETUP_SAVE_ALL_CALLEE_SAVES_FRAME mv a1, a2 // Pass value. mv a2, xSELF // Pass Thread::Current(). call artThrowArrayStoreException // (Object*, Object*, Thread*). call art_quick_do_long_jump // (Context*) unimp // Unreached #if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) CFI_RESTORE_STATE_AND_DEF_CFA sp, 0 .Laput_obj_gc_marking: BAKER_RB_CHECK_GRAY_BIT_AND_LOAD \ a3, a0, MIRROR_OBJECT_CLASS_OFFSET, .Laput_obj_mark_array_class .Laput_obj_mark_array_class_continue: BAKER_RB_CHECK_GRAY_BIT_AND_LOAD \ a3, a3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET, .Laput_obj_mark_array_element .Laput_obj_mark_array_element_continue: BAKER_RB_CHECK_GRAY_BIT_AND_LOAD \ a4, a2, MIRROR_OBJECT_CLASS_OFFSET, .Laput_obj_mark_object_class .Laput_obj_mark_object_class_continue: // value's type == array's component type - trivial assignability bne a3, a4, .Laput_obj_check_assignability j .Laput_obj_store .Laput_obj_mark_array_class: BAKER_RB_LOAD_AND_MARK a3, a0, MIRROR_OBJECT_CLASS_OFFSET, art_quick_read_barrier_mark_reg13 j .Laput_obj_mark_array_class_continue .Laput_obj_mark_array_element: BAKER_RB_LOAD_AND_MARK \ a3, a3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET, art_quick_read_barrier_mark_reg13 j .Laput_obj_mark_array_element_continue .Laput_obj_mark_object_class: BAKER_RB_LOAD_AND_MARK a4, a2, MIRROR_OBJECT_CLASS_OFFSET, art_quick_read_barrier_mark_reg14 j .Laput_obj_mark_object_class_continue #endif // defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) END art_quick_aput_obj // Create a function `name` calling the art::ReadBarrier::Mark routine, getting its argument and // returning its result through \reg, saving and restoring all caller-save registers. // // The generated function follows a non-standard calling convention: // - register `reg` is used to pass the singleton argument, // - register `reg` is used to return the result, // - all other registers are callee-save (the values they hold are preserved). .macro READ_BARRIER_MARK_REG name, reg ENTRY \name beqz \reg, .Lrb_return_\name // early return if null // Save t5 and t6 onto stack to honor caller-save calling convention. INCREASE_FRAME 16 SAVE_GPR t5, (8*0) SAVE_GPR t6, (8*1) lw t5, MIRROR_OBJECT_LOCK_WORD_OFFSET(\reg) // t5 := lock word slliw t6, t5, 31-LOCK_WORD_MARK_BIT_SHIFT // mark bit into MSB bltz t6, .Lrb_tmp_restore_\name // Check if the top two bits are set. If so, it is a forwarding address. slliw t6, t5, 1 and t6, t6, t5 CFI_REMEMBER_STATE bgez t6, .Lrb_full_\name // Extract and zero-extend the forwarding address. slli \reg, t5, (LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT + 32) srli \reg, \reg, 32 .ifc \reg, t5 sd t5, (8*0)(sp) .endif .ifc \reg, t6 sd t6, (8*1)(sp) .endif .Lrb_tmp_restore_\name: RESTORE_GPR t5, (8*0) RESTORE_GPR t6, (8*1) DECREASE_FRAME 16 .Lrb_return_\name: ret .Lrb_full_\name: CFI_RESTORE_STATE_AND_DEF_CFA sp, 16 // Save remaining caller-save registers on stack. t5 and t6 already saved. // fa0-fa7, ft0-ft11, a0-a7, t0-t4, ra: 8 * (8 + 12 + 8 + 5 + 1) = 8 * 34 = 272 bytes INCREASE_FRAME 272 SAVE_FPR fa0, (8*0) SAVE_FPR fa1, (8*1) SAVE_FPR fa2, (8*2) SAVE_FPR fa3, (8*3) SAVE_FPR fa4, (8*4) SAVE_FPR fa5, (8*5) SAVE_FPR fa6, (8*6) SAVE_FPR fa7, (8*7) SAVE_FPR ft0, (8*8) SAVE_FPR ft1, (8*9) SAVE_FPR ft2, (8*10) SAVE_FPR ft3, (8*11) SAVE_FPR ft4, (8*12) SAVE_FPR ft5, (8*13) SAVE_FPR ft6, (8*14) SAVE_FPR ft7, (8*15) SAVE_FPR ft8, (8*16) SAVE_FPR ft9, (8*17) SAVE_FPR ft10, (8*18) SAVE_FPR ft11, (8*19) SAVE_GPR a0, (8*20) SAVE_GPR a1, (8*21) SAVE_GPR a2, (8*22) SAVE_GPR a3, (8*23) SAVE_GPR a4, (8*24) SAVE_GPR a5, (8*25) SAVE_GPR a6, (8*26) SAVE_GPR a7, (8*27) SAVE_GPR t0, (8*28) SAVE_GPR t1, (8*29) SAVE_GPR t2, (8*30) SAVE_GPR t3, (8*31) SAVE_GPR t4, (8*32) SAVE_GPR ra, (8*33) .ifc \reg, t5 ld a0, (8*34)(sp) .else .ifc \reg, t6 ld a0, (8*35)(sp) .else .ifnc \reg, a0 mv a0, \reg .endif .endif .endif call artReadBarrierMark .ifnc \reg, a0 mv \reg, a0 .endif // Restore all caller-save registers from stack, including t5 and t6. // fa0-fa7, ft0-ft11, ra, a0-a7, t0-t6: 8 * (8 + 12 + 1 + 8 + 7) = 8 * 36 = 288 bytes RESTORE_FPR fa0, (8*0) RESTORE_FPR fa1, (8*1) RESTORE_FPR fa2, (8*2) RESTORE_FPR fa3, (8*3) RESTORE_FPR fa4, (8*4) RESTORE_FPR fa5, (8*5) RESTORE_FPR fa6, (8*6) RESTORE_FPR fa7, (8*7) RESTORE_FPR ft0, (8*8) RESTORE_FPR ft1, (8*9) RESTORE_FPR ft2, (8*10) RESTORE_FPR ft3, (8*11) RESTORE_FPR ft4, (8*12) RESTORE_FPR ft5, (8*13) RESTORE_FPR ft6, (8*14) RESTORE_FPR ft7, (8*15) RESTORE_FPR ft8, (8*16) RESTORE_FPR ft9, (8*17) RESTORE_FPR ft10, (8*18) RESTORE_FPR ft11, (8*19) RESTORE_GPR_NE \reg, a0, (8*20) RESTORE_GPR_NE \reg, a1, (8*21) RESTORE_GPR_NE \reg, a2, (8*22) RESTORE_GPR_NE \reg, a3, (8*23) RESTORE_GPR_NE \reg, a4, (8*24) RESTORE_GPR_NE \reg, a5, (8*25) RESTORE_GPR_NE \reg, a6, (8*26) RESTORE_GPR_NE \reg, a7, (8*27) RESTORE_GPR_NE \reg, t0, (8*28) RESTORE_GPR_NE \reg, t1, (8*29) RESTORE_GPR_NE \reg, t2, (8*30) RESTORE_GPR_NE \reg, t3, (8*31) RESTORE_GPR_NE \reg, t4, (8*32) RESTORE_GPR_NE \reg, ra, (8*33) RESTORE_GPR_NE \reg, t5, (8*34) RESTORE_GPR_NE \reg, t6, (8*35) DECREASE_FRAME 288 ret END \name .endm // No read barrier for X0 (Zero), X1 (RA), X2 (SP), X3 (GP) and X4 (TP). READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, t0 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, t1 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg07, t2 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg08, s0 // No read barrier for X9 (S1/xSELF). READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg10, a0 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg11, a1 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg12, a2 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg13, a3 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg14, a4 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg15, a5 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg16, a6 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg17, a7 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg18, s2 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg19, s3 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg20, s4 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg21, s5 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg22, s6 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg23, s7 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg24, s8 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg25, s9 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg26, s10 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg27, s11 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg28, t3 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg29, t4 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg30, t5 READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg31, t6 .macro N_ARG_DOWNCALL n, name, entrypoint, return .extern \entrypoint ENTRY \name SETUP_SAVE_REFS_ONLY_FRAME // Save callee saves in case of GC. mv a\n, xSELF // Pass Thread::Current(). call \entrypoint // (, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME \return END \name .endm .macro ONE_ARG_DOWNCALL name, entrypoint, return N_ARG_DOWNCALL 1, \name, \entrypoint, \return .endm .macro TWO_ARG_DOWNCALL name, entrypoint, return N_ARG_DOWNCALL 2, \name, \entrypoint, \return .endm .macro THREE_ARG_DOWNCALL name, entrypoint, return N_ARG_DOWNCALL 3, \name, \entrypoint, \return .endm .macro FOUR_ARG_DOWNCALL name, entrypoint, return N_ARG_DOWNCALL 4, \name, \entrypoint, \return .endm // Entry from managed code that calls artHandleFillArrayDataFromCode and // delivers exception on failure. TWO_ARG_DOWNCALL art_quick_handle_fill_data, \ artHandleFillArrayDataFromCode, \ RETURN_OR_DEOPT_IF_INT_RESULT_IS_ZERO_OR_DELIVER // Generate the allocation entrypoints for each allocator. GENERATE_ALLOC_ENTRYPOINTS_FOR_NON_TLAB_ALLOCATORS // Comment out allocators that have riscv64 specific asm. // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_region_tlab, RegionTLAB) // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_region_tlab, RegionTLAB) GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_region_tlab, RegionTLAB) GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_OBJECT(_region_tlab, RegionTLAB) // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_region_tlab, RegionTLAB) // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_region_tlab, RegionTLAB) // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_region_tlab, RegionTLAB) // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_region_tlab, RegionTLAB) // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_region_tlab, RegionTLAB) GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_region_tlab, RegionTLAB) GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_region_tlab, RegionTLAB) GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_region_tlab, RegionTLAB) // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_tlab, TLAB) // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_tlab, TLAB) GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_tlab, TLAB) GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_OBJECT(_tlab, TLAB) // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_tlab, TLAB) // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_tlab, TLAB) // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_tlab, TLAB) // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_tlab, TLAB) // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_tlab, TLAB) GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_tlab, TLAB) GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_tlab, TLAB) GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_tlab, TLAB) // If isInitialized=1 then the compiler assumes the object's class has already been initialized. // If isInitialized=0 the compiler can only assume it's been at least resolved. .macro ART_QUICK_ALLOC_OBJECT_ROSALLOC c_name, cxx_name, isInitialized ENTRY \c_name // Fast path rosalloc allocation. // a0: type, xSELF(s1): Thread::Current // a1-a7: free. ld a3, THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET(xSELF) // Check if the thread local // allocation stack has room. // ldp won't work due to large offset. ld a4, THREAD_LOCAL_ALLOC_STACK_END_OFFSET(xSELF) bgeu a3, a4, .Lslow_path\c_name lwu a3, MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET(a0) // Load the object size (a3) li a5, ROSALLOC_MAX_THREAD_LOCAL_BRACKET_SIZE // Check if the size is for a thread // local allocation. // If the class is not yet visibly initialized, or it is finalizable, // the object size will be very large to force the branch below to be taken. // // See Class::SetStatus() in class.cc for more details. bgeu a3, a5, .Lslow_path\c_name // Compute the rosalloc bracket index // from the size. Since the size is // already aligned we can combine the // two shifts together. #if ROSALLOC_BRACKET_QUANTUM_SIZE_SHIFT != POINTER_SIZE_SHIFT #error "Unexpected ROSALLOC_BRACKET_QUANTUM_SIZE_SHIFT != POINTER_SIZE_SHIFT" #endif // No-op: srli a3, a3, (ROSALLOC_BRACKET_QUANTUM_SIZE_SHIFT - POINTER_SIZE_SHIFT) add a4, xSELF, a3 // Subtract pointer size since there // are no runs for 0 byte allocations // and the size is already aligned. ld a4, (THREAD_ROSALLOC_RUNS_OFFSET - __SIZEOF_POINTER__)(a4) // Load the free list head (a3). This // will be the return val. ld a3, (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)(a4) beqz a3, .Lslow_path\c_name // "Point of no slow path". Won't go to the slow path from here on. OK to clobber a0 and a1. ld a1, ROSALLOC_SLOT_NEXT_OFFSET(a3) // Load the next pointer of the head // and update the list head with the // next pointer. sd a1, (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)(a4) // Store the class pointer in the // header. This also overwrites the // next pointer. The offsets are // asserted to match. #if ROSALLOC_SLOT_NEXT_OFFSET != MIRROR_OBJECT_CLASS_OFFSET #error "Class pointer needs to overwrite next pointer." #endif POISON_HEAP_REF a0 sw a0, MIRROR_OBJECT_CLASS_OFFSET(a3) // Push the new object onto the thread // local allocation stack and // increment the thread local // allocation stack top. ld a1, THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET(xSELF) sw a3, (a1) addi a1, a1, COMPRESSED_REFERENCE_SIZE // Increment A1 to point to next slot. sd a1, THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET(xSELF) // Decrement the size of the free list. // After this "SD" the object is published to the thread local allocation stack, // and it will be observable from a runtime internal (eg. Heap::VisitObjects) point of view. // It is not yet visible to the running (user) compiled code until after the return. // // To avoid the memory barrier prior to the "SD", a trick is employed, by differentiating // the state of the allocation stack slot. It can be a pointer to one of: // 0) Null entry, because the stack was bumped but the new pointer wasn't written yet. // (The stack initial state is "null" pointers). // 1) A partially valid object, with an invalid class pointer to the next free rosalloc slot. // 2) A fully valid object, with a valid class pointer pointing to a real class. // Other states are not allowed. // // An object that is invalid only temporarily, and will eventually become valid. // The internal runtime code simply checks if the object is not null or is partial and then // ignores it. // // (Note: The actual check is done by seeing if a non-null object has a class pointer pointing // to ClassClass, and that the ClassClass's class pointer is self-cyclic. A rosalloc free slot // "next" pointer is not-cyclic.) // // See also b/28790624 for a listing of CLs dealing with this race. lwu a1, (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)(a4) addi a1, a1, -1 sw a1, (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)(a4) mv a0, a3 // Set the return value and return. // No barrier. The class is already observably initialized (otherwise the fast // path size check above would fail) and new-instance allocations are protected // from publishing by the compiler which inserts its own StoreStore barrier. ret .Lslow_path\c_name: SETUP_SAVE_REFS_ONLY_FRAME // Save callee saves in case of GC. mv a1, xSELF // Pass Thread::Current(). call \cxx_name RESTORE_SAVE_REFS_ONLY_FRAME RETURN_OR_DEOPT_IF_RESULT_IS_NON_NULL_OR_DELIVER END \c_name .endm ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_resolved_rosalloc, \ artAllocObjectFromCodeResolvedRosAlloc, /* isInitialized */ 0 ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_initialized_rosalloc, \ artAllocObjectFromCodeInitializedRosAlloc, /* isInitialized */ 1 // If isInitialized=1 then the compiler assumes the object's class has already been initialized. // If isInitialized=0 the compiler can only assume it's been at least resolved. .macro ALLOC_OBJECT_TLAB_FAST_PATH_RESOLVED slowPathLabel isInitialized ld a4, THREAD_LOCAL_POS_OFFSET(xSELF) ld a5, THREAD_LOCAL_END_OFFSET(xSELF) lwu a7, MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET(a0) // Load the object size (a7). add a6, a4, a7 // Add object size to tlab pos. // Check if it fits, overflow works // since the tlab pos and end are 32 // bit values. // If the class is not yet visibly initialized, or it is finalizable, // the object size will be very large to force the branch below to be taken. // // See Class::SetStatus() in class.cc for more details. bgtu a6, a5, \slowPathLabel sd a6, THREAD_LOCAL_POS_OFFSET(xSELF) // Store new thread_local_pos. POISON_HEAP_REF a0 sw a0, MIRROR_OBJECT_CLASS_OFFSET(a4) // Store the class pointer. mv a0, a4 // No barrier. The class is already observably initialized (otherwise the fast // path size check above would fail) and new-instance allocations are protected // from publishing by the compiler which inserts its own StoreStore barrier. ret .endm // The common code for art_quick_alloc_object_*region_tlab // Currently the implementation ignores isInitialized. TODO(b/172087402): clean this up. // Caller must execute a constructor fence after this. .macro GENERATE_ALLOC_OBJECT_RESOLVED_TLAB name, entrypoint, isInitialized ENTRY \name // Fast path region tlab allocation. // a0: type, xSELF(s1): Thread::Current // a1-a7: free. ALLOC_OBJECT_TLAB_FAST_PATH_RESOLVED .Lslow_path\name, \isInitialized .Lslow_path\name: SETUP_SAVE_REFS_ONLY_FRAME // Save callee saves in case of GC. mv a1, xSELF // Pass Thread::Current(). call \entrypoint // (mirror::Class*, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME RETURN_OR_DEOPT_IF_RESULT_IS_NON_NULL_OR_DELIVER END \name .endm GENERATE_ALLOC_OBJECT_RESOLVED_TLAB \ art_quick_alloc_object_resolved_region_tlab, \ artAllocObjectFromCodeResolvedRegionTLAB, /* isInitialized */ 0 GENERATE_ALLOC_OBJECT_RESOLVED_TLAB \ art_quick_alloc_object_initialized_region_tlab, \ artAllocObjectFromCodeInitializedRegionTLAB, /* isInitialized */ 1 GENERATE_ALLOC_OBJECT_RESOLVED_TLAB \ art_quick_alloc_object_resolved_tlab, \ artAllocObjectFromCodeResolvedTLAB, /* isInitialized */ 0 GENERATE_ALLOC_OBJECT_RESOLVED_TLAB \ art_quick_alloc_object_initialized_tlab, \ artAllocObjectFromCodeInitializedTLAB, /* isInitialized */ 1 .macro ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE \ slowPathLabel, class, count, temp0, temp1, temp2 andi \temp1, \temp1, OBJECT_ALIGNMENT_MASK_TOGGLED64 // Apply alignment mask // (addr + 7) & ~7. The mask must // be 64 bits to keep high bits in // case of overflow. // Negative sized arrays are handled here since xCount holds a zero extended 32 bit value. // Negative ints become large 64 bit unsigned ints which will always be larger than max signed // 32 bit int. Since the max shift for arrays is 3, it can not become a negative 64 bit int. li \temp2, MIN_LARGE_OBJECT_THRESHOLD // Possibly a large object, go slow bgeu \temp1, \temp2, \slowPathLabel // path. ld \temp0, THREAD_LOCAL_POS_OFFSET(xSELF) // Check tlab for space, note that // we use (end - begin) to handle // negative size arrays. It is // assumed that a negative size will // always be greater unsigned than // region size. ld \temp2, THREAD_LOCAL_END_OFFSET(xSELF) sub \temp2, \temp2, \temp0 // The array class is always initialized here. Unlike new-instance, // this does not act as a double test. bgtu \temp1, \temp2, \slowPathLabel // "Point of no slow path". Won't go to the slow path from here on. OK to clobber x0 and x1. // Move old thread_local_pos to x0 // for the return value. mv a0, \temp0 add \temp0, \temp0, \temp1 sd \temp0, THREAD_LOCAL_POS_OFFSET(xSELF) // Store new thread_local_pos. POISON_HEAP_REF \class sw \class, MIRROR_OBJECT_CLASS_OFFSET(a0) // Store the class pointer. sw \count, MIRROR_ARRAY_LENGTH_OFFSET(a0) // Store the array length. // new-array is special. The class is loaded and immediately goes to the Initialized state // before it is published. Therefore the only fence needed is for the publication of the object. // See ClassLinker::CreateArrayClass() for more details. // For publication of the new array, we don't need a 'fence w, w' here. // The compiler generates 'fence w, w' for all new-array insts. ret .endm // Caller must execute a constructor fence after this. .macro GENERATE_ALLOC_ARRAY_TLAB name, entrypoint, size_setup ENTRY \name // Fast path array allocation for region tlab allocation. // a0: mirror::Class* type // a1: int32_t component_count // a2-a7: free. mv a3, a0 \size_setup a3, a1, a4, a5, a6 ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE .Lslow_path\name, a3, a1, a4, a5, a6 .Lslow_path\name: // a0: mirror::Class* klass // a1: int32_t component_count // a2: Thread* self SETUP_SAVE_REFS_ONLY_FRAME // Save callee saves in case of GC. mv a2, xSELF // Pass Thread::Current(). call \entrypoint RESTORE_SAVE_REFS_ONLY_FRAME RETURN_OR_DEOPT_IF_RESULT_IS_NON_NULL_OR_DELIVER END \name .endm .macro COMPUTE_ARRAY_SIZE_UNKNOWN class, count, temp0, temp1, temp2 // Array classes are never finalizable or uninitialized, no need to check. lwu \temp0, MIRROR_CLASS_COMPONENT_TYPE_OFFSET(\class) // Load component type UNPOISON_HEAP_REF \temp0 lwu \temp0, MIRROR_CLASS_OBJECT_PRIMITIVE_TYPE_OFFSET(\temp0) srli \temp0, \temp0, PRIMITIVE_TYPE_SIZE_SHIFT_SHIFT // Component size shift is in high 16 // bits. zext.w \temp1, \count // From \count we use a 32 bit value, // it can not overflow. sll \temp1, \temp1, \temp0 // Calculate data size // Add array data offset and alignment. addi \temp1, \temp1, (MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) #if MIRROR_LONG_ARRAY_DATA_OFFSET != MIRROR_INT_ARRAY_DATA_OFFSET + 4 #error Long array data offset must be 4 greater than int array data offset. #endif addi \temp0, \temp0, 1 // Add 4 to the length only if the // component size shift is 3 // (for 64 bit alignment). andi \temp0, \temp0, 4 add \temp1, \temp1, \temp0 .endm .macro COMPUTE_ARRAY_SIZE_8 class, count, temp0, temp1, temp2 // Add array data offset and alignment adjustment to the `\count`. li \temp1, (MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) add.uw \temp1, \count, \temp1 .endm .macro COMPUTE_ARRAY_SIZE_16 class, count, temp0, temp1, temp2 // Add array data offset and alignment adjustment to the shifted `\count`. li \temp1, (MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) sh1add.uw \temp1, \count, \temp1 .endm .macro COMPUTE_ARRAY_SIZE_32 class, count, temp0, temp1, temp2 // Add array data offset and alignment adjustment to the shifted `\count`. li \temp1, (MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) sh2add.uw \temp1, \count, \temp1 .endm .macro COMPUTE_ARRAY_SIZE_64 class, count, temp0, temp1, temp2 // Add array data offset and alignment adjustment to the shifted `\count`. li \temp1, (MIRROR_WIDE_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) sh3add.uw \temp1, \count, \temp1 .endm // TODO(ngeoffray): art_quick_alloc_array_resolved_region_tlab is not used for arm64, remove // the entrypoint once all backends have been updated to use the size variants. GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_region_tlab, \ artAllocArrayFromCodeResolvedRegionTLAB, \ COMPUTE_ARRAY_SIZE_UNKNOWN GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_region_tlab, \ artAllocArrayFromCodeResolvedRegionTLAB, \ COMPUTE_ARRAY_SIZE_8 GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_region_tlab, \ artAllocArrayFromCodeResolvedRegionTLAB, \ COMPUTE_ARRAY_SIZE_16 GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_region_tlab, \ artAllocArrayFromCodeResolvedRegionTLAB, \ COMPUTE_ARRAY_SIZE_32 GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_region_tlab, \ artAllocArrayFromCodeResolvedRegionTLAB, \ COMPUTE_ARRAY_SIZE_64 GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_tlab, \ artAllocArrayFromCodeResolvedTLAB, \ COMPUTE_ARRAY_SIZE_UNKNOWN GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_tlab, \ artAllocArrayFromCodeResolvedTLAB, \ COMPUTE_ARRAY_SIZE_8 GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_tlab, \ artAllocArrayFromCodeResolvedTLAB, \ COMPUTE_ARRAY_SIZE_16 GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_tlab, \ artAllocArrayFromCodeResolvedTLAB, \ COMPUTE_ARRAY_SIZE_32 GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_tlab, \ artAllocArrayFromCodeResolvedTLAB, \ COMPUTE_ARRAY_SIZE_64 GENERATE_FIELD_ENTRYPOINTS // String's indexOf. // // TODO: Not very optimized. We should use the vector extension. // On entry: // a0: string object (known non-null) // a1: char to match (known <= 0xFFFF) // a2: Starting offset in string data ENTRY art_quick_indexof #if (STRING_COMPRESSION_FEATURE) lwu a4, MIRROR_STRING_COUNT_OFFSET(a0) #else lwu a3, MIRROR_STRING_COUNT_OFFSET(a0) #endif addi a0, a0, MIRROR_STRING_VALUE_OFFSET #if (STRING_COMPRESSION_FEATURE) /* Split the count into length (a3) and compression flag (a4) */ srliw a3, a4, 1 andi a4, a4, 1 #endif /* Clamp start to [0..count) */ sraiw a5, a2, 31 andn a2, a2, a5 bge a2, a3, .Lstring_indexof_nomatch #if (STRING_COMPRESSION_FEATURE) beqz a4, .Lstring_indexof_compressed #endif /* Build pointers to start and end of the data to compare */ sh1add a2, a2, a0 sh1add a3, a3, a0 /* * At this point we have: * a0: original start of string data * a1: char to compare * a2: start of the data to test * a3: end of the data to test */ .Lstring_indexof_loop: lhu a4, 0(a2) beq a4, a1, .Lstring_indexof_match addi a2, a2, 2 bne a2, a3, .Lstring_indexof_loop .Lstring_indexof_nomatch: li a0, -1 ret .Lstring_indexof_match: sub a0, a2, a0 srli a0, a0, 1 ret #if (STRING_COMPRESSION_FEATURE) // Comparing compressed string one character at a time with the input character. .Lstring_indexof_compressed: add a2, a2, a0 add a3, a3, a0 .Lstring_indexof_compressed_loop: lbu a4, (a2) beq a4, a1, .Lstring_indexof_compressed_match addi a2, a2, 1 bne a2, a3, .Lstring_indexof_compressed_loop li a0, -1 ret .Lstring_indexof_compressed_match: sub a0, a2, a0 ret #endif END art_quick_indexof