/* * Copyright (c) 2013-2016 Google Inc. All rights reserved * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files * (the "Software"), to deal in the Software without restriction, * including without limitation the rights to use, copy, modify, merge, * publish, distribute, sublicense, and/or sell copies of the Software, * and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define LOCAL_TRACE 0 struct sm_std_call_global_state { event_t event; /* * Queue of VMs with pending stdcalls to run. * A VM can be in one of the following states: * * * In the queue * Its stdcall is either running or queued to run later. * Once that happens, the stdcall thread will dequeue the VM and set its * stdcall_done flag. The VM is not safe to destroy while in the queue. * * * Out of the queue and stdcall_done==true * The queued stdcall has completed and the VM needs to use * GET_STDCALL_RET to retrieve its result. If the VM is destroyed, * the result is discarded under the assumption that the VM is not * running anymore. * * * Out of the queue and stdcall_done==false * The VM does not have any pending stdcall queued in Trusty. */ struct list_node vm_queue; /* * A list of all VMs with active_cpu != 1 */ struct list_node active_vms; }; /* Per-VM stdcall state */ struct sm_std_call_vm_state { struct list_node queue_node; struct list_node active_node; struct obj_ref self_ref; struct smc32_args args; long ret; bool done; int active_cpu; /* cpu that expects stdcall result */ int initial_cpu; /* Debug info: cpu that started stdcall */ int last_cpu; /* Debug info: most recent cpu expecting stdcall result */ int restart_count; }; extern unsigned long monitor_vector_table; extern ulong lk_boot_args[4]; static void* boot_args; static int boot_args_refcnt; static mutex_t boot_args_lock = MUTEX_INITIAL_VALUE(boot_args_lock); static atomic_uint_fast32_t sm_api_version; static atomic_uint_fast32_t sm_api_version_min; static atomic_uint_fast32_t sm_api_version_max = TRUSTY_API_VERSION_CURRENT; static spin_lock_t sm_api_version_lock; static atomic_bool platform_halted; #if LIB_SM_WITH_FFA_LOOP static bool sm_use_ffa = true; static atomic_bool sm_ffa_valid_call; #else static bool sm_use_ffa = false; #endif enum sm_vm_state { SM_VM_STATE_FRESH, SM_VM_STATE_AVAILABLE, SM_VM_STATE_DESTROY_NOTIFYING, SM_VM_STATE_DESTROY_NOTIFIED, SM_VM_STATE_READY_TO_FREE }; struct sm_vm { struct bst_node node; enum sm_vm_state state; ext_mem_obj_id_t client_id; struct list_node notifiers; struct obj refobj; struct obj_ref self_ref; struct sm_std_call_vm_state stdcall; }; /* * VM ID to create; can be one of two values: * * Non-negative 16-bit VM ID, or * * -1 when no VM needs to be created */ static int32_t sm_vm_to_create = -1; static struct bst_root sm_vm_tree = BST_ROOT_INITIAL_VALUE; static struct bst_root sm_vm_free_tree = BST_ROOT_INITIAL_VALUE; static spin_lock_t sm_vm_lock; static event_t sm_vm_event = EVENT_INITIAL_VALUE(sm_vm_event, 0, EVENT_FLAG_AUTOUNSIGNAL); static thread_t* sm_vm_notifier_thread; static atomic_uintptr_t sm_vm_active_notifier; static event_t sm_vm_notifier_done_event = EVENT_INITIAL_VALUE(sm_vm_notifier_done_event, 0, EVENT_FLAG_AUTOUNSIGNAL); /* * Placeholder compatibility VM for environments without hypervisors * and for the bootloader that may call Trusty before the hypervisor has * initialized. This pseudo-VM does not get creation or destruction messages * so we add and remove it from the tree manually. */ static struct sm_vm sm_vm_compat_vm = { .node = BST_NODE_INITIAL_VALUE, .state = SM_VM_STATE_FRESH, .client_id = 0, .notifiers = LIST_INITIAL_VALUE(sm_vm_compat_vm.notifiers), .self_ref = OBJ_REF_INITIAL_VALUE(sm_vm_compat_vm.self_ref), .stdcall = { .queue_node = LIST_INITIAL_CLEARED_VALUE, .active_node = LIST_INITIAL_CLEARED_VALUE, .self_ref = OBJ_REF_INITIAL_VALUE(sm_vm_compat_vm.stdcall.self_ref), .active_cpu = -1, .initial_cpu = -1, .last_cpu = -1, }, }; static event_t nsirqevent[SMP_MAX_CPUS]; static thread_t* nsirqthreads[SMP_MAX_CPUS]; static thread_t* nsidlethreads[SMP_MAX_CPUS]; static thread_t* stdcallthread; static bool irq_thread_ready[SMP_MAX_CPUS]; struct sm_std_call_global_state stdcallstate = { .event = EVENT_INITIAL_VALUE(stdcallstate.event, 0, 0), .vm_queue = LIST_INITIAL_VALUE(stdcallstate.vm_queue), .active_vms = LIST_INITIAL_VALUE(stdcallstate.active_vms), }; extern smc32_handler_t sm_stdcall_table[]; extern smc32_handler_t sm_nopcall_table[]; extern smc32_handler_t sm_fastcall_table[]; static long sm_get_stdcall_ret_locked(struct sm_vm* vm); static status_t sm_vm_get_locked(ext_mem_obj_id_t client_id, struct obj_ref* ref, struct sm_vm** out_vm); static void sm_vm_del_ref_locked(struct sm_vm* vm, struct obj_ref* ref); long smc_sm_api_version(struct smc32_args* args) { uint32_t api_version = args->params[0]; spin_lock(&sm_api_version_lock); LTRACEF("request api version %d\n", api_version); if (api_version > sm_api_version_max) { api_version = sm_api_version_max; } if (api_version < sm_api_version_min) { TRACEF("ERROR: Tried to select incompatible api version %d < %d, current version %d\n", api_version, sm_api_version_min, sm_api_version); api_version = sm_api_version; } else { /* Update and lock the version to prevent downgrade */ sm_api_version = api_version; sm_api_version_min = api_version; } spin_unlock(&sm_api_version_lock); LTRACEF("return api version %d\n", api_version); return api_version; } long smc_get_smp_max_cpus(struct smc32_args* args) { return SMP_MAX_CPUS; } uint32_t sm_get_api_version(void) { return sm_api_version; } bool sm_check_and_lock_api_version(uint32_t version_wanted) { spin_lock_saved_state_t state; DEBUG_ASSERT(version_wanted > 0); if (sm_api_version_min >= version_wanted) { return true; } if (sm_api_version_max < version_wanted) { return false; } spin_lock_save(&sm_api_version_lock, &state, SPIN_LOCK_FLAG_IRQ_FIQ); if (sm_api_version < version_wanted) { sm_api_version_max = MIN(sm_api_version_max, version_wanted - 1); TRACEF("max api version set: %d\n", sm_api_version_max); } else { sm_api_version_min = MAX(sm_api_version_min, version_wanted); TRACEF("min api version set: %d\n", sm_api_version_min); } DEBUG_ASSERT(sm_api_version_min <= sm_api_version_max); DEBUG_ASSERT(sm_api_version >= sm_api_version_min); DEBUG_ASSERT(sm_api_version <= sm_api_version_max); spin_unlock_restore(&sm_api_version_lock, state, SPIN_LOCK_FLAG_IRQ_FIQ); return sm_api_version_min >= version_wanted; } static int __NO_RETURN sm_stdcall_loop(void* arg) { long ret; spin_lock_saved_state_t state; struct sm_vm* vm; while (true) { LTRACEF("cpu %d, wait for stdcall\n", arch_curr_cpu_num()); event_wait(&stdcallstate.event); spin_lock_save(&sm_vm_lock, &state, SPIN_LOCK_FLAG_IRQ); while ((vm = list_peek_head_type(&stdcallstate.vm_queue, struct sm_vm, stdcall.queue_node))) { /* * vm->stdcall.self_ref is holding an active reference * to the VM, so there is no risk that the VM will get * deleted before the stdcall completes. We delete that * self-reference at the end of the loop below. * * The only fields of the VM structure we access without * holding the spin lock below are in vm->stdcall.args, * which should be safe to read because the arguments only * get written once in sm_queue_stdcall before queuing the * stdcall. Once we get here, the arguments should be fixed. */ spin_unlock_restore(&sm_vm_lock, state, SPIN_LOCK_FLAG_IRQ); /* Dispatch 'standard call' handler */ LTRACEF("cpu %d, got stdcall: 0x%x, 0x%x, 0x%x, 0x%x\n", arch_curr_cpu_num(), vm->stdcall.args.smc_nr, vm->stdcall.args.params[0], vm->stdcall.args.params[1], vm->stdcall.args.params[2]); ret = sm_stdcall_table[SMC_ENTITY(vm->stdcall.args.smc_nr)]( &vm->stdcall.args); LTRACEF("cpu %d, stdcall(0x%x, 0x%x, 0x%x, 0x%x) returned 0x%lx (%ld)\n", arch_curr_cpu_num(), vm->stdcall.args.smc_nr, vm->stdcall.args.params[0], vm->stdcall.args.params[1], vm->stdcall.args.params[2], ret, ret); spin_lock_save(&sm_vm_lock, &state, SPIN_LOCK_FLAG_IRQ); vm->stdcall.ret = ret; vm->stdcall.done = true; list_delete(&vm->stdcall.queue_node); sm_vm_del_ref_locked(vm, &vm->stdcall.self_ref); } event_unsignal(&stdcallstate.event); spin_unlock_restore(&sm_vm_lock, state, SPIN_LOCK_FLAG_IRQ); } } /* must be called with irqs disabled */ static long sm_queue_stdcall(struct smc32_args* args) { long ret; uint cpu = arch_curr_cpu_num(); struct sm_vm* vm = NULL; struct obj_ref vm_ref = OBJ_REF_INITIAL_VALUE(vm_ref); spin_lock(&sm_vm_lock); ret = sm_vm_get_locked(args->client_id, &vm_ref, &vm); if (ret) { dprintf(CRITICAL, "%s: could not get VM %" PRIx64 " (%ld)\n", __func__, args->client_id, ret); goto err; } if (list_in_list(&vm->stdcall.queue_node) || vm->stdcall.done) { if (args->smc_nr == SMC_SC_RESTART_LAST) { DEBUG_ASSERT(vm->stdcall.args.client_id == args->client_id); if (vm->stdcall.active_cpu == -1) { vm->stdcall.restart_count++; LTRACEF_LEVEL(3, "cpu %d, restart std call, restart_count %d\n", cpu, vm->stdcall.restart_count); goto restart_stdcall; } } dprintf(CRITICAL, "%s: cpu %d, std call busy\n", __func__, cpu); ret = SM_ERR_BUSY; goto err; } else { if (args->smc_nr == SMC_SC_RESTART_LAST) { dprintf(CRITICAL, "%s: cpu %d, unexpected restart, no std call active\n", __func__, arch_curr_cpu_num()); ret = SM_ERR_UNEXPECTED_RESTART; goto err; } } LTRACEF("cpu %d, queue std call 0x%x\n", cpu, args->smc_nr); list_add_tail(&stdcallstate.vm_queue, &vm->stdcall.queue_node); obj_ref_transfer(&vm->stdcall.self_ref, &vm_ref); vm->stdcall.initial_cpu = cpu; vm->stdcall.ret = SM_ERR_INTERNAL_FAILURE; vm->stdcall.args = *args; vm->stdcall.restart_count = 0; event_signal(&stdcallstate.event, false); restart_stdcall: if (!sm_use_ffa) { /* * On FF-A, we do not keep track of the active CPU since * get_stdcall_ret_locked is called by a separate direct message. */ vm->stdcall.active_cpu = cpu; list_add_tail(&stdcallstate.active_vms, &vm->stdcall.active_node); } ret = 0; err: if (obj_ref_active(&vm_ref)) { sm_vm_del_ref_locked(vm, &vm_ref); } spin_unlock(&sm_vm_lock); return ret; } #if LIB_SM_WITH_FFA_LOOP static long sm_ffa_handle_direct_req(long ret, struct smc_ret18* regs) { struct smc32_args args; uint16_t client_id = (regs->r1 >> 16) & 0xFFFFU; uint cpu = arch_curr_cpu_num(); struct sm_vm* vm; struct obj_ref vm_ref; switch (regs->r3) { case TRUSTY_FFA_MSG_RUN_FASTCALL: if (SMC_IS_SMC64(regs->r4)) { return SM_ERR_NOT_SUPPORTED; } if (!SMC_IS_FASTCALL(regs->r4)) { dprintf(CRITICAL, "Synchronous message is not a fastcall: %lx\n", regs->r4); return SM_ERR_INVALID_PARAMETERS; } args.smc_nr = regs->r4; args.params[0] = regs->r5; args.params[1] = regs->r6; args.params[2] = regs->r7; args.client_id = client_id; return sm_fastcall_table[SMC_ENTITY(args.smc_nr)](&args); case TRUSTY_FFA_MSG_QUEUE_STDCALL: if (SMC_IS_SMC64(regs->r4)) { return SM_ERR_NOT_SUPPORTED; } if (SMC_IS_FASTCALL(regs->r4)) { dprintf(CRITICAL, "Asynchronous message is a fastcall: %lx\n", regs->r4); return SM_ERR_INVALID_PARAMETERS; } args.smc_nr = regs->r4; args.params[0] = regs->r5; args.params[1] = regs->r6; args.params[2] = regs->r7; args.client_id = client_id; ret = sm_queue_stdcall(&args); if (!ret) { /* Ring the doorbell on the host so it queues a Trusty NOP */ sm_intc_raise_doorbell_irq(); } return ret; case TRUSTY_FFA_MSG_GET_STDCALL_RET: spin_lock(&sm_vm_lock); ret = sm_vm_get_locked(client_id, &vm_ref, &vm); if (ret) { dprintf(CRITICAL, "%s: could not get VM %" PRIx16 " (%ld)\n", __func__, client_id, ret); } else { ret = sm_get_stdcall_ret_locked(vm); sm_vm_del_ref_locked(vm, &vm_ref); } spin_unlock(&sm_vm_lock); return ret; case TRUSTY_FFA_MSG_RUN_NOPCALL: args.smc_nr = SMC_SC_NOP; args.params[0] = regs->r4; args.params[1] = regs->r5; args.params[2] = regs->r6; args.client_id = client_id; #if !ARM_MERGE_FIQ_IRQ #error "FF-A libsm requires ARM_MERGE_FIQ_IRQ" #endif ret = sm_nopcall_table[SMC_ENTITY(args.params[0])](&args); if (!ret) { /* Ring the doorbell on the host so it queues a Trusty NOP */ sm_intc_raise_doorbell_irq(); } return ret; case TRUSTY_FFA_MSG_IS_IDLE: return get_current_thread() == nsidlethreads[cpu]; default: dprintf(CRITICAL, "Unsupported FF-A message from client %" PRIu16 ": %lx\n", client_id, regs->r3); return SM_ERR_NOT_SUPPORTED; } } #endif static int sm_vm_compare_key(const struct bst_node* a, const void* b) { const struct sm_vm* vm = containerof(a, struct sm_vm, node); ext_mem_obj_id_t key = *(ext_mem_obj_id_t*)b; if (key > vm->client_id) { return 1; } else if (key < vm->client_id) { return -1; } else { return 0; } } static int sm_vm_compare(struct bst_node* a, struct bst_node* b) { const struct sm_vm* vm_b = containerof(b, struct sm_vm, node); return sm_vm_compare_key(a, &vm_b->client_id); } static void sm_vm_add_compat_vm_locked(ext_mem_client_id_t client_id) { DEBUG_ASSERT(spin_lock_held(&sm_vm_lock)); if (!bst_is_empty(&sm_vm_tree)) { /* * There is already a VM in the tree, so we don't need * to add the compatibility VM explicitly. */ return; } if (sm_vm_to_create != -1) { /* The tree is empty but we have a pending VM queued up for creation */ return; } DEBUG_ASSERT(sm_vm_compat_vm.state == SM_VM_STATE_FRESH || sm_vm_compat_vm.state == SM_VM_STATE_READY_TO_FREE); sm_vm_compat_vm.client_id = client_id; obj_init(&sm_vm_compat_vm.refobj, &sm_vm_compat_vm.self_ref); DEBUG_ASSERT(!obj_ref_active(&sm_vm_compat_vm.stdcall.self_ref)); DEBUG_ASSERT(sm_vm_compat_vm.stdcall.active_cpu == -1); sm_vm_compat_vm.stdcall.initial_cpu = -1; sm_vm_compat_vm.stdcall.last_cpu = -1; if (!bst_insert(&sm_vm_tree, &sm_vm_compat_vm.node, sm_vm_compare)) { panic("failed to insert compatibility VM\n"); } sm_vm_compat_vm.state = SM_VM_STATE_AVAILABLE; } status_t sm_vm_notifier_register(struct sm_vm_notifier* notif) { spin_lock_saved_state_t state; struct sm_vm* vm; status_t ret; if (!notif) { return ERR_INVALID_ARGS; } if (!notif->destroy) { return ERR_INVALID_ARGS; } spin_lock_irqsave(&sm_vm_lock, state); sm_vm_add_compat_vm_locked(notif->client_id); vm = bst_search_key_type(&sm_vm_tree, ¬if->client_id, sm_vm_compare_key, struct sm_vm, node); if (!vm) { ret = ERR_NOT_FOUND; } else if (vm->state == SM_VM_STATE_AVAILABLE) { list_add_tail(&vm->notifiers, ¬if->node); ret = NO_ERROR; } else { ret = ERR_BAD_STATE; } spin_unlock_irqrestore(&sm_vm_lock, state); return ret; } status_t sm_vm_notifier_unregister(struct sm_vm_notifier* notif) { spin_lock_saved_state_t state; status_t ret = NO_ERROR; struct sm_vm* vm; if (!notif) { return ERR_INVALID_ARGS; } spin_lock_irqsave(&sm_vm_lock, state); /* * Check the node with the lock held to avoid * it getting removed during the check */ if (!list_in_list(¬if->node)) { ret = ERR_NOT_FOUND; goto err_notif_not_in_list; } if ((uintptr_t)notif == atomic_load(&sm_vm_active_notifier)) { spin_unlock_irqrestore(&sm_vm_lock, state); /* The callback is currently running, wait for it to finish */ do { /* * If sm_vm_active_notifier is notif, that means that our * notifier is currently running; retry the event_wait * until the notifier actually completes in order to avoid * leftover wakeups. We use a global variable because the * notifier might have been destroyed by the handler by * the time it returns. */ event_wait(&sm_vm_notifier_done_event); } while ((uintptr_t)notif == atomic_load(&sm_vm_active_notifier)); /* Nothing else to do here, the notifier is already out of the list */ return NO_ERROR; } vm = bst_search_key_type(&sm_vm_tree, ¬if->client_id, sm_vm_compare_key, struct sm_vm, node); if (!vm) { ret = ERR_NOT_FOUND; goto err_no_vm; } list_delete(¬if->node); err_notif_not_in_list: err_no_vm: spin_unlock_irqrestore(&sm_vm_lock, state); return ret; } static status_t sm_vm_get_locked(ext_mem_obj_id_t client_id, struct obj_ref* ref, struct sm_vm** out_vm) { struct sm_vm* vm; DEBUG_ASSERT(spin_lock_held(&sm_vm_lock)); if (!ref) { return ERR_INVALID_ARGS; } if (obj_ref_active(ref)) { return ERR_INVALID_ARGS; } if (!out_vm) { return ERR_INVALID_ARGS; } sm_vm_add_compat_vm_locked(client_id); vm = bst_search_key_type(&sm_vm_tree, &client_id, sm_vm_compare_key, struct sm_vm, node); if (!vm) { return ERR_NOT_FOUND; } obj_add_ref(&vm->refobj, ref); *out_vm = vm; return NO_ERROR; } status_t sm_vm_get(ext_mem_obj_id_t client_id, struct obj_ref* ref, struct sm_vm** out_vm) { spin_lock_saved_state_t state; status_t ret; spin_lock_irqsave(&sm_vm_lock, state); ret = sm_vm_get_locked(client_id, ref, out_vm); spin_unlock_irqrestore(&sm_vm_lock, state); return ret; } void sm_vm_add_ref(struct sm_vm* vm, struct obj_ref* ref) { spin_lock_saved_state_t state; DEBUG_ASSERT(vm); DEBUG_ASSERT(ref); DEBUG_ASSERT(!obj_ref_active(ref)); spin_lock_irqsave(&sm_vm_lock, state); obj_add_ref(&vm->refobj, ref); spin_unlock_irqrestore(&sm_vm_lock, state); } static void sm_vm_del_ref_locked(struct sm_vm* vm, struct obj_ref* ref) { DEBUG_ASSERT(vm); DEBUG_ASSERT(ref); DEBUG_ASSERT(obj_ref_active(ref)); DEBUG_ASSERT(spin_lock_held(&sm_vm_lock)); obj_del_ref(&vm->refobj, ref, NULL); if (obj_has_only_ref(&vm->refobj, &vm->self_ref) && vm->state == SM_VM_STATE_DESTROY_NOTIFYING) { /* * This is the last reference to the VM and it is getting destroyed, * so wake up the notifier thread so it updates the state */ event_signal(&sm_vm_event, false); } } void sm_vm_del_ref(struct sm_vm* vm, struct obj_ref* ref) { spin_lock_saved_state_t state; spin_lock_irqsave(&sm_vm_lock, state); sm_vm_del_ref_locked(vm, ref); spin_unlock_irqrestore(&sm_vm_lock, state); } ext_mem_client_id_t sm_vm_get_id(struct sm_vm* vm) { DEBUG_ASSERT(vm); return vm->client_id; } #if LIB_SM_WITH_FFA_LOOP static long sm_ffa_handle_framework_msg(struct smc_ret18* regs) { uint32_t msg = regs->r2 & FFA_FRAMEWORK_MSG_MASK; ext_mem_obj_id_t client_id = regs->r5 & 0xffffU; struct sm_vm* vm; long ret; bool inserted; /* TODO: validate receiver */ switch (msg) { case FFA_FRAMEWORK_MSG_VM_CREATED_REQ: LTRACEF_LEVEL(1, "Got VM creation message for %" PRIu64 "\n", client_id); spin_lock(&sm_vm_lock); vm = bst_search_key_type(&sm_vm_tree, &client_id, sm_vm_compare_key, struct sm_vm, node); if (!vm) { if (sm_vm_to_create == -1) { sm_vm_to_create = client_id; event_signal(&sm_vm_event, false); sm_intc_raise_doorbell_irq(); } ret = FFA_ERROR_RETRY; } else if (vm->state == SM_VM_STATE_FRESH) { vm->state = SM_VM_STATE_AVAILABLE; ret = 0; } else { dprintf(CRITICAL, "Duplicate VM creation for %" PRIu64 "\n", client_id); ret = FFA_ERROR_INVALID_PARAMETERS; } spin_unlock(&sm_vm_lock); LTRACEF_LEVEL(1, "VM creation returning %ld\n", ret); regs->r2 = FFA_FRAMEWORK_MSG_VM_CREATED_RESP | FFA_FRAMEWORK_MSG_FLAG; return ret; case FFA_FRAMEWORK_MSG_VM_DESTROYED_REQ: LTRACEF_LEVEL(1, "Got VM destruction message for %" PRIu64 "\n", client_id); spin_lock(&sm_vm_lock); vm = bst_search_key_type(&sm_vm_tree, &client_id, sm_vm_compare_key, struct sm_vm, node); if (!vm) { dprintf(CRITICAL, "VM destruction for unknown %" PRIu64 "\n", client_id); ret = FFA_ERROR_INVALID_PARAMETERS; } else { DEBUG_ASSERT(vm->state != SM_VM_STATE_READY_TO_FREE); switch (vm->state) { case SM_VM_STATE_FRESH: /* * We got a creation request for this VM that we * returned RETRY on, but the hypervisor never retried * the request until we could report a success and now * it's sending us a destruction request for that VM. * * We could start destroying the VM instead, but this * is not correct hypervisor behavior so we are probably * better off returning an error. */ dprintf(CRITICAL, "Got early VM destroy for %" PRIu64 "\n", client_id); ret = FFA_ERROR_INVALID_PARAMETERS; break; case SM_VM_STATE_AVAILABLE: vm->state = SM_VM_STATE_DESTROY_NOTIFYING; /* * Signal the thread so it destroys the VM and ring * the doorbell on the host so it queues a Trusty NOP */ event_signal(&sm_vm_event, false); sm_intc_raise_doorbell_irq(); __FALLTHROUGH; case SM_VM_STATE_DESTROY_NOTIFYING: ret = FFA_ERROR_RETRY; break; case SM_VM_STATE_DESTROY_NOTIFIED: /* Mark the VM for freeing since we're done with it */ vm->state = SM_VM_STATE_READY_TO_FREE; bst_delete(&sm_vm_tree, &vm->node); inserted = bst_insert(&sm_vm_free_tree, &vm->node, sm_vm_compare); DEBUG_ASSERT(inserted); ret = 0; /* * Signal the event so the VM is freed later; we do not * need to ring the doorbell because this is not urgent, * so the freeing can happen whenever Trusty gets cycles next. */ event_signal(&sm_vm_event, false); break; default: panic("Invalid VM state: %d\n", vm->state); } } spin_unlock(&sm_vm_lock); LTRACEF_LEVEL(1, "VM destruction returning %ld\n", ret); regs->r2 = FFA_FRAMEWORK_MSG_VM_DESTROYED_RESP | FFA_FRAMEWORK_MSG_FLAG; return ret; default: dprintf(CRITICAL, "Unhandled FF-A framework message: %x\n", msg); return FFA_ERROR_NOT_SUPPORTED; } } #endif static int __NO_RETURN sm_vm_notifier_loop(void* arg) { spin_lock_saved_state_t state; struct sm_vm* vm; struct sm_vm_notifier* notif; status_t ret; while (true) { event_wait(&sm_vm_event); /* Create the new VM if a message came in */ while (true) { int32_t vm_id; struct sm_vm* vm; bool inserted; spin_lock_irqsave(&sm_vm_lock, state); if (sm_vm_to_create != -1 && sm_vm_compat_vm.state == SM_VM_STATE_AVAILABLE) { /* We got an actual VM, tear down the compatibility one */ sm_vm_compat_vm.state = SM_VM_STATE_DESTROY_NOTIFYING; /* * Signal the event so we continue the outer loop because * the remainder of the current iteration will handle the * new NOTIFYING state for the compatibility VM. The event * will be used at the start of the next iteration to get * back here and create the new VM. */ event_signal(&sm_vm_event, false); /* Defer creation of the new VM until compat_vm is gone */ vm_id = -1; } else { vm_id = sm_vm_to_create; } spin_unlock_irqrestore(&sm_vm_lock, state); LTRACEF_LEVEL(2, "Creating fresh VM %d\n", vm_id); if (vm_id == -1) { break; } vm = calloc(1, sizeof(struct sm_vm)); if (!vm) { dprintf(CRITICAL, "Out of memory for VMs\n"); continue; } vm->state = SM_VM_STATE_FRESH; vm->client_id = vm_id; list_initialize(&vm->notifiers); obj_init(&vm->refobj, &vm->self_ref); obj_ref_init(&vm->stdcall.self_ref); vm->stdcall.active_cpu = -1; vm->stdcall.initial_cpu = -1; vm->stdcall.last_cpu = -1; spin_lock_irqsave(&sm_vm_lock, state); sm_vm_to_create = -1; inserted = bst_insert(&sm_vm_tree, &vm->node, sm_vm_compare); spin_unlock_irqrestore(&sm_vm_lock, state); DEBUG_ASSERT(inserted); } /* Destroy all VMs on the free list */ while (true) { spin_lock_irqsave(&sm_vm_lock, state); vm = bst_next_type(&sm_vm_free_tree, NULL, struct sm_vm, node); if (vm) { bst_delete(&sm_vm_free_tree, &vm->node); } spin_unlock_irqrestore(&sm_vm_lock, state); if (!vm) { break; } LTRACEF_LEVEL(2, "Freeing VM %" PRIu64 "\n", vm->client_id); DEBUG_ASSERT(vm->state == SM_VM_STATE_READY_TO_FREE); obj_del_ref(&vm->refobj, &vm->self_ref, NULL); free(vm); } /* Call the next notifier */ while (true) { spin_lock_irqsave(&sm_vm_lock, state); notif = NULL; bst_for_every_entry(&sm_vm_tree, vm, struct sm_vm, node) { if (vm->state == SM_VM_STATE_DESTROY_NOTIFYING) { if (!list_is_empty(&vm->notifiers)) { notif = list_remove_head_type( &vm->notifiers, struct sm_vm_notifier, node); atomic_store(&sm_vm_active_notifier, (uintptr_t)notif); break; } if (!obj_has_only_ref(&vm->refobj, &vm->self_ref)) { /* There are active references to this VM */ continue; } /* * No more notifiers or references, we can mark the VM * as "destroy-notified" and move on to the next one. * * This is thread-safe because only the current thread * runs the notifiers, and no new nodes can be added * while in the SM_VM_STATE_DESTROY_NOTIFYING. */ vm->state = SM_VM_STATE_DESTROY_NOTIFIED; if (vm == &sm_vm_compat_vm) { /* * We are done with the compatibility VM, * remove it from the tree permanently. */ vm->state = SM_VM_STATE_READY_TO_FREE; bst_delete(&sm_vm_tree, &vm->node); DEBUG_ASSERT(sm_vm_event.signaled); } } } spin_unlock_irqrestore(&sm_vm_lock, state); if (!notif) { break; } LTRACEF_LEVEL(2, "Calling VM destroy handler for %" PRIu64 "\n", notif->client_id); DEBUG_ASSERT(notif->destroy); ret = notif->destroy(notif); if (ret) { TRACEF("VM destroy handler returned error (%d)\n", ret); } atomic_store(&sm_vm_active_notifier, 0); event_signal(&sm_vm_notifier_done_event, true); } } } #if LIB_SM_WITH_FFA_LOOP __WEAK void sm_memlog_notify_clients(bool raise_doorbell) { /* Do nothing if lib/memlog is not included */ } static void sm_ffa_loop(long ret, struct smc32_args* args) { struct smc_ret18 regs = {0}; uint64_t extended_args[ARM_FFA_MSG_EXTENDED_ARGS_COUNT]; STATIC_ASSERT(sizeof extended_args == sizeof regs.req2_params); enum arm_ffa_init_state ffa_init_state = arm_ffa_init_state(); if (atomic_load(&platform_halted)) { regs = arm_ffa_call_error(FFA_ERROR_ABORTED); } else if (ffa_init_state == ARM_FFA_INIT_UNINIT) { panic("FF-A not initialized before main loop\n"); } else if (ffa_init_state == ARM_FFA_INIT_FAILED) { TRACEF("FF-A failed to initialize, " "falling back to legacy SPD SMCs\n"); sm_use_ffa = false; return; } else { /* * Linux will check the shadow priority next and * give us more cycles if it's anything other than IDLE */ LTRACEF_LEVEL(5, "Calling FFA_MSG_WAIT (%ld)\n", ret); /* * Update the memlog notification state without raising the doorbell * because trusty_log_call_notify will check the log in the host anyway */ sm_memlog_notify_clients(false); regs = arm_ffa_call_msg_wait(); } while (true) { LTRACEF_LEVEL(5, "Incoming FF-A SMC (%lx)\n", regs.r0); switch ((uint32_t)regs.r0) { case SMC_FC_FFA_MSG_SEND_DIRECT_REQ: case SMC_FC64_FFA_MSG_SEND_DIRECT_REQ: if (atomic_load(&platform_halted)) { /* Return to NS since we have nothing to do */ regs = arm_ffa_call_error(FFA_ERROR_ABORTED); break; } atomic_store(&sm_ffa_valid_call, true); if (regs.r2 & FFA_FRAMEWORK_MSG_FLAG) { ret = sm_ffa_handle_framework_msg(®s); } else { ret = sm_ffa_handle_direct_req(ret, ®s); } LTRACEF_LEVEL(5, "Calling FFA_MSG_SEND_DIRECT_RESP (%ld)\n", ret); sm_memlog_notify_clients(true); regs = arm_ffa_msg_send_direct_resp(®s, (ulong)ret, 0, 0, 0, 0); break; case SMC_FC64_FFA_MSG_SEND_DIRECT_REQ2: if (atomic_load(&platform_halted)) { regs = arm_ffa_call_error(FFA_ERROR_ABORTED); break; } ret = arm_ffa_handle_direct_req2(®s); /* * Whereas sm_ffa_handle_direct_req returns secure monitor error * codes, arm_ffa_handle_direct_req2 can fail with Trusty error * codes not understood by the caller, e.g., if no handler is found. */ if (ret) { dprintf(CRITICAL, "Failed to handle FFA_MSG_SEND_DIRECT_REQ2: %lx\n", ret); regs = arm_ffa_call_error(FFA_ERROR_ABORTED); break; } LTRACEF_LEVEL(5, "Calling FFA_MSG_SEND_DIRECT_RESP2 (%ld)\n", ret); sm_memlog_notify_clients(true); /* * copy req2_params into a fresh buffer `args` since the former can * be overwritten at any time by the callee. */ memcpy(extended_args, regs.req2_params, sizeof extended_args); regs = arm_ffa_msg_send_direct_resp2(®s, extended_args); break; case SMC_FC_FFA_RUN: if (atomic_load(&platform_halted)) { /* Return to NS since we have nothing to do */ regs = arm_ffa_call_error(FFA_ERROR_ABORTED); break; } atomic_store(&sm_ffa_valid_call, true); args->smc_nr = SMC_SC_NOP; args->params[0] = args->params[1] = args->params[2] = 0; return; case SMC_FC_FFA_INTERRUPT: atomic_store(&sm_ffa_valid_call, true); sm_intc_fiq_enter(); /* * sm_intc_fiq_enter rings the doorbell, * so we do not need to do it again here. */ regs = arm_ffa_call_msg_wait(); break; case SMC_FC_FFA_ERROR: if (atomic_load(&platform_halted)) { /* * Loop forever if we halted and * got back here from FFA_ERROR_ABORTED, * there is not much else we can do */ break; } if ((int32_t)regs.r2 == FFA_ERROR_NOT_SUPPORTED && !atomic_load(&sm_ffa_valid_call)) { TRACEF("Using legacy SPD SMCs\n"); sm_use_ffa = false; return; } panic("Received FFA_ERROR from SPMC: (%lx, %lx)\n", regs.r1, regs.r2); case SMC_UNKNOWN: if (atomic_load(&sm_ffa_valid_call)) { /* We already got a valid FF-A call earlier */ panic("Received SMC_UNKNOWN from SPMC\n"); } TRACEF("Using legacy SPD SMCs\n"); sm_use_ffa = false; return; default: dprintf(CRITICAL, "Unhandled FF-A SMC: %lx\n", regs.r0); regs = arm_ffa_call_error(FFA_ERROR_NOT_SUPPORTED); } } } #endif static void sm_sched_nonsecure_fiq_loop(long ret, struct smc32_args* args) { #if LIB_SM_WITH_FFA_LOOP if (sm_use_ffa) { sm_ffa_loop(ret, args); /* Check again in case we switched to the legacy SPD SMCs */ if (sm_use_ffa) { return; } } #endif while (true) { if (atomic_load(&platform_halted)) { ret = SM_ERR_PANIC; } sm_sched_nonsecure(ret, args); if (atomic_load(&platform_halted) && args->smc_nr != SMC_FC_FIQ_ENTER) { continue; } if (SMC_IS_SMC64(args->smc_nr)) { ret = SM_ERR_NOT_SUPPORTED; continue; } if (!SMC_IS_FASTCALL(args->smc_nr)) { break; } ret = sm_fastcall_table[SMC_ENTITY(args->smc_nr)](args); } } /* must be called with irqs disabled */ static enum handler_return sm_return_and_wait_for_next_stdcall(long ret, int cpu) { struct smc32_args args = SMC32_ARGS_INITIAL_VALUE(args); do { #if ARCH_HAS_FIQ arch_disable_fiqs(); #endif sm_sched_nonsecure_fiq_loop(ret, &args); #if ARCH_HAS_FIQ arch_enable_fiqs(); #endif /* Allow concurrent SMC_SC_NOP calls on multiple cpus */ if (args.smc_nr == SMC_SC_NOP) { LTRACEF_LEVEL(3, "cpu %d, got nop\n", cpu); ret = sm_nopcall_table[SMC_ENTITY(args.params[0])](&args); } else { DEBUG_ASSERT(!sm_use_ffa); ret = sm_queue_stdcall(&args); } } while (ret); return sm_intc_enable_interrupts(); } static struct sm_vm* sm_vm_find_by_stdcall_active_cpu_locked(int cpu) { struct sm_vm* vm; DEBUG_ASSERT(spin_lock_held(&sm_vm_lock)); list_for_every_entry(&stdcallstate.active_vms, vm, struct sm_vm, stdcall.active_node) { if (vm->stdcall.active_cpu == cpu) { return vm; } } return NULL; } static void sm_irq_return_ns(void) { long ret; int cpu; struct sm_vm* vm; cpu = arch_curr_cpu_num(); spin_lock(&sm_vm_lock); /* TODO: remove? */ vm = sm_vm_find_by_stdcall_active_cpu_locked(cpu); LTRACEF_LEVEL(2, "got irq on cpu %d, stdcallcpu %d\n", cpu, vm ? vm->stdcall.active_cpu : -1); if (vm) { vm->stdcall.last_cpu = vm->stdcall.active_cpu; vm->stdcall.active_cpu = -1; if (list_in_list(&vm->stdcall.active_node)) { list_delete(&vm->stdcall.active_node); } ret = SM_ERR_INTERRUPTED; } else { ret = SM_ERR_NOP_INTERRUPTED; } LTRACEF_LEVEL(2, "got irq on cpu %d, return %ld\n", cpu, ret); spin_unlock(&sm_vm_lock); sm_return_and_wait_for_next_stdcall(ret, cpu); } static int __NO_RETURN sm_irq_loop(void* arg) { int cpu; /* cpu that requested this thread, the current cpu could be different */ int eventcpu = (uintptr_t)arg; /* * Run this thread with interrupts masked, so we don't reenter the * interrupt handler. The interrupt handler for non-secure interrupts * returns to this thread with the interrupt still pending. */ arch_disable_ints(); irq_thread_ready[eventcpu] = true; cpu = arch_curr_cpu_num(); LTRACEF("wait for irqs for cpu %d, on cpu %d\n", eventcpu, cpu); while (true) { event_wait(&nsirqevent[eventcpu]); sm_irq_return_ns(); } } /* must be called with irqs disabled and sm_vm_lock held */ static long sm_get_stdcall_ret_locked(struct sm_vm* vm) { long ret; uint cpu = arch_curr_cpu_num(); DEBUG_ASSERT(spin_lock_held(&sm_vm_lock)); if (!sm_use_ffa && vm->stdcall.active_cpu != (int)cpu) { dprintf(CRITICAL, "%s: stdcallcpu, a%d != curr-cpu %d, l%d, i%d\n", __func__, vm->stdcall.active_cpu, cpu, vm->stdcall.last_cpu, vm->stdcall.initial_cpu); ret = SM_ERR_INTERNAL_FAILURE; goto err; } if (!list_in_list(&vm->stdcall.queue_node) && !vm->stdcall.done) { dprintf(CRITICAL, "%s: stdcall bad state: neither queued nor done\n", __func__); ret = SM_ERR_NOT_ALLOWED; goto err; } vm->stdcall.last_cpu = (int)cpu; vm->stdcall.active_cpu = -1; if (list_in_list(&vm->stdcall.active_node)) { list_delete(&vm->stdcall.active_node); } if (vm->stdcall.done) { vm->stdcall.done = false; ret = vm->stdcall.ret; LTRACEF("cpu %d, return stdcall result, %ld, initial cpu %d\n", cpu, vm->stdcall.ret, vm->stdcall.initial_cpu); } else { if (sm_check_and_lock_api_version(TRUSTY_API_VERSION_SMP)) ret = SM_ERR_CPU_IDLE; /* ns using smp api */ else if (vm->stdcall.restart_count) ret = SM_ERR_BUSY; else ret = SM_ERR_INTERRUPTED; LTRACEF("cpu %d, initial cpu %d, restart_count %d, std call not finished, return %ld\n", cpu, vm->stdcall.initial_cpu, vm->stdcall.restart_count, ret); } err: return ret; } static uint enter_smcall_critical_section(void) { /* * Disable interrupts so vm->stdcall.active_cpu does not * change to or from this cpu after checking it in the critical * section. */ arch_disable_ints(); /* Switch to sm-stdcall if sm_queue_stdcall woke it up */ thread_yield(); spin_lock(&sm_vm_lock); return arch_curr_cpu_num(); } static void exit_smcall_critical_section(long ret, uint cpu) { enum handler_return resched; spin_unlock(&sm_vm_lock); resched = sm_return_and_wait_for_next_stdcall(ret, cpu); if (resched == INT_RESCHEDULE) thread_preempt(); /* Re-enable interrupts (needed for SMC_SC_NOP) */ arch_enable_ints(); } static int sm_wait_for_smcall(void* arg) { int cpu; long ret = 0; struct sm_vm* vm; LTRACEF("wait for stdcalls, on cpu %d\n", arch_curr_cpu_num()); while (true) { cpu = enter_smcall_critical_section(); vm = sm_vm_find_by_stdcall_active_cpu_locked(cpu); if (vm) ret = sm_get_stdcall_ret_locked(vm); else ret = SM_ERR_NOP_DONE; exit_smcall_critical_section(ret, cpu); } } #if WITH_LIB_SM_MONITOR /* per-cpu secure monitor initialization */ static void sm_mon_percpu_init(uint level) { /* let normal world enable SMP, lock TLB, access CP10/11 */ __asm__ volatile( "mrc p15, 0, r1, c1, c1, 2 \n" "orr r1, r1, #0xC00 \n" "orr r1, r1, #0x60000 \n" "mcr p15, 0, r1, c1, c1, 2 @ NSACR \n" : : : "r1"); __asm__ volatile("mcr p15, 0, %0, c12, c0, 1 \n" : : "r"(&monitor_vector_table)); } LK_INIT_HOOK_FLAGS(libsm_mon_perrcpu, sm_mon_percpu_init, LK_INIT_LEVEL_PLATFORM - 3, LK_INIT_FLAG_ALL_CPUS); #endif static void sm_init(uint level) { status_t err; char name[32]; mutex_acquire(&boot_args_lock); /* Map the boot arguments if supplied by the bootloader */ if (lk_boot_args[1] && lk_boot_args[2]) { ulong offset = lk_boot_args[1] & (PAGE_SIZE - 1); paddr_t paddr = round_down(lk_boot_args[1], PAGE_SIZE); size_t size = round_up(lk_boot_args[2] + offset, PAGE_SIZE); void* vptr; err = vmm_alloc_physical(vmm_get_kernel_aspace(), "sm", size, &vptr, PAGE_SIZE_SHIFT, paddr, 0, ARCH_MMU_FLAG_NS | ARCH_MMU_FLAG_PERM_NO_EXECUTE | ARCH_MMU_FLAG_CACHED); if (!err) { boot_args = (uint8_t*)vptr + offset; boot_args_refcnt++; } else { boot_args = NULL; TRACEF("Error mapping boot parameter block: %d\n", err); } } mutex_release(&boot_args_lock); for (int cpu = 0; cpu < SMP_MAX_CPUS; cpu++) { event_init(&nsirqevent[cpu], false, EVENT_FLAG_AUTOUNSIGNAL); snprintf(name, sizeof(name), "irq-ns-switch-%d", cpu); nsirqthreads[cpu] = thread_create(name, sm_irq_loop, (void*)(uintptr_t)cpu, HIGHEST_PRIORITY, DEFAULT_STACK_SIZE); if (!nsirqthreads[cpu]) { panic("failed to create irq NS switcher thread for cpu %d!\n", cpu); } thread_set_pinned_cpu(nsirqthreads[cpu], cpu); thread_set_real_time(nsirqthreads[cpu]); snprintf(name, sizeof(name), "idle-ns-switch-%d", cpu); nsidlethreads[cpu] = thread_create(name, sm_wait_for_smcall, NULL, LOWEST_PRIORITY + 1, DEFAULT_STACK_SIZE); if (!nsidlethreads[cpu]) { panic("failed to create idle NS switcher thread for cpu %d!\n", cpu); } thread_set_pinned_cpu(nsidlethreads[cpu], cpu); thread_set_real_time(nsidlethreads[cpu]); } stdcallthread = thread_create("sm-stdcall", sm_stdcall_loop, NULL, LOWEST_PRIORITY + 2, DEFAULT_STACK_SIZE); if (!stdcallthread) { panic("failed to create sm-stdcall thread!\n"); } thread_set_real_time(stdcallthread); thread_resume(stdcallthread); sm_vm_notifier_thread = thread_create("sm-vm-notifier", sm_vm_notifier_loop, NULL, HIGH_PRIORITY, DEFAULT_STACK_SIZE); if (!sm_vm_notifier_thread) { panic("failed to create sm-vm-notifier thread!\n"); } thread_resume(sm_vm_notifier_thread); } LK_INIT_HOOK(libsm, sm_init, LK_INIT_LEVEL_PLATFORM - 1); enum handler_return sm_handle_irq(void) { int cpu = arch_curr_cpu_num(); if (irq_thread_ready[cpu]) { event_signal(&nsirqevent[cpu], false); } else { TRACEF("warning: got ns irq before irq thread is ready\n"); sm_irq_return_ns(); } return INT_RESCHEDULE; } void sm_handle_fiq(void) { uint32_t expected_return; struct smc32_args args = SMC32_ARGS_INITIAL_VALUE(args); DEBUG_ASSERT(!sm_use_ffa); if (sm_check_and_lock_api_version(TRUSTY_API_VERSION_RESTART_FIQ)) { sm_sched_nonsecure_fiq_loop(SM_ERR_FIQ_INTERRUPTED, &args); expected_return = SMC_SC_RESTART_FIQ; } else { sm_sched_nonsecure_fiq_loop(SM_ERR_INTERRUPTED, &args); expected_return = SMC_SC_RESTART_LAST; } if (args.smc_nr != expected_return) { TRACEF("got bad restart smc %x, expected %x\n", args.smc_nr, expected_return); while (args.smc_nr != expected_return) sm_sched_nonsecure_fiq_loop(SM_ERR_INTERLEAVED_SMC, &args); } } void platform_halt(platform_halt_action suggested_action, platform_halt_reason reason) { bool already_halted; struct smc32_args args = SMC32_ARGS_INITIAL_VALUE(args); arch_disable_ints(); already_halted = atomic_exchange(&platform_halted, true); if (!already_halted) { for (int cpu = 0; cpu < SMP_MAX_CPUS; cpu++) { if (nsirqthreads[cpu]) { event_signal(&nsirqevent[cpu], false); } } dprintf(ALWAYS, "%s\n", lk_version); dprintf(ALWAYS, "HALT: (reason = %d)\n", reason); } #if ARCH_HAS_FIQ arch_disable_fiqs(); #endif while (true) sm_sched_nonsecure_fiq_loop(SM_ERR_PANIC, &args); } status_t sm_get_boot_args(void** boot_argsp, size_t* args_sizep) { status_t err = NO_ERROR; if (!boot_argsp || !args_sizep) return ERR_INVALID_ARGS; mutex_acquire(&boot_args_lock); if (!boot_args) { err = ERR_NOT_CONFIGURED; goto unlock; } boot_args_refcnt++; *boot_argsp = boot_args; *args_sizep = lk_boot_args[2]; unlock: mutex_release(&boot_args_lock); return err; } static void resume_nsthreads(void) { int i; for (i = 0; i < SMP_MAX_CPUS; i++) { DEBUG_ASSERT(nsirqthreads[i]); DEBUG_ASSERT(nsidlethreads[i]); thread_resume(nsirqthreads[i]); thread_resume(nsidlethreads[i]); } } void sm_put_boot_args(void) { mutex_acquire(&boot_args_lock); if (!boot_args) { TRACEF("WARNING: caller does not own " "a reference to boot parameters\n"); goto unlock; } boot_args_refcnt--; if (boot_args_refcnt == 0) { vmm_free_region(vmm_get_kernel_aspace(), (vaddr_t)boot_args); boot_args = NULL; resume_nsthreads(); } unlock: mutex_release(&boot_args_lock); } static void sm_release_boot_args(uint level) { if (boot_args) { sm_put_boot_args(); } else { /* we need to resume the ns-switcher here if * the boot loader didn't pass bootargs */ resume_nsthreads(); } if (boot_args) TRACEF("WARNING: outstanding reference to boot args" "at the end of initialzation!\n"); } LK_INIT_HOOK(libsm_bootargs, sm_release_boot_args, LK_INIT_LEVEL_LAST);