// SPDX-License-Identifier: MIT /* * Copyright © 2021 Intel Corporation */ #include #include #include #include #include #include #include #include #include #include "igt.h" #include "igt_map.h" #include "intel_allocator.h" #include "intel_allocator_msgchannel.h" #include "intel_pat.h" #include "xe/xe_query.h" #include "xe/xe_util.h" #ifdef ALLOCDBG #define alloc_info igt_info #define alloc_debug igt_debug static const char *reqtype_str[] = { [REQ_STOP] = "stop", [REQ_OPEN] = "open", [REQ_CLOSE] = "close", [REQ_ADDRESS_RANGE] = "address range", [REQ_ALLOC] = "alloc", [REQ_FREE] = "free", [REQ_IS_ALLOCATED] = "is allocated", [REQ_RESERVE] = "reserve", [REQ_UNRESERVE] = "unreserve", [REQ_RESERVE_IF_NOT_ALLOCATED] = "reserve-ina", [REQ_IS_RESERVED] = "is reserved", }; static inline const char *reqstr(enum reqtype request_type) { igt_assert(request_type >= REQ_STOP && request_type <= REQ_IS_RESERVED); return reqtype_str[request_type]; } #else #define alloc_info(...) {} #define alloc_debug(...) {} #endif #ifdef ALLOCBINDDBG #define bind_info igt_info #define bind_debug igt_debug #else #define bind_info(...) {} #define bind_debug(...) {} #endif /* * We limit allocator space to avoid hang when batch would be * pinned in the last page. */ #define RESERVED 4096 struct allocator { int fd; uint32_t ctx; uint32_t vm; _Atomic(int32_t) refcount; struct intel_allocator *ial; }; struct handle_entry { uint64_t handle; struct allocator *al; }; /* For tracking alloc()/free() for Xe */ struct ahnd_info { int fd; uint64_t ahnd; uint32_t vm; enum intel_driver driver; struct igt_map *bind_map; pthread_mutex_t bind_map_mutex; }; enum allocator_bind_op { BOUND, TO_BIND, TO_UNBIND, }; struct allocator_object { uint32_t handle; uint64_t offset; uint64_t size; uint8_t pat_index; enum allocator_bind_op bind_op; }; struct intel_allocator * intel_allocator_reloc_create(int fd, uint64_t start, uint64_t end); struct intel_allocator * intel_allocator_random_create(int fd, uint64_t start, uint64_t end); struct intel_allocator * intel_allocator_simple_create(int fd, uint64_t start, uint64_t end, enum allocator_strategy strategy); /* * Instead of trying to find first empty handle just get new one. Assuming * our counter is incremented 2^32 times per second (4GHz clock and handle * assignment takes single clock) 64-bit counter would wrap around after * ~68 years. * * allocator * handles intel allocator * +-----+ +--------+ +-------------+ * | 1 +---------->+ fd: 3 +--------->+ data: ... | * +-----+ +---->+ ctx: 1 | | refcount: 2 | * | 2 +-----+ | ref: 2 | +-------------+ * +-----+ +--------+ * | 3 +--+ +--------+ intel allocator * +-----+ | | fd: 3 | +-------------+ * | ... | +------->| ctx: 2 +--------->+ data: ... | * +-----+ | ref: 1 | | refcount: 1 | * | n +--------+ +--------+ +-------------+ * +-----+ | * | ... +-----+ | allocator * +-----+ | | intel allocator * | ... +--+ | | +--------+ +-------------+ * + + | | +->+ fd: 3 +-----+--->+ data: ... | * | +---->+ vm: 1 | | | refcount: 3 | * | | ref: 2 | | +-------------+ * | +--------+ | * | +--------+ | * | | fd: 3 | | * +------->+ vm: 2 +-----+ * | ref: 1 | * +--------+ */ static _Atomic(uint64_t) next_handle; static struct igt_map *handles; static struct igt_map *ctx_map; static struct igt_map *vm_map; static pthread_mutex_t map_mutex = PTHREAD_MUTEX_INITIALIZER; #define GET_MAP(vm) ((vm) ? vm_map : ctx_map) static bool multiprocess; static pthread_t allocator_thread; static bool allocator_thread_running; static bool warn_if_not_empty; /* For allocator purposes we need to track pid/tid */ static pid_t allocator_pid = -1; extern pid_t child_pid; extern __thread pid_t child_tid; /* * Track alloc()/free() requires storing in local process which has * an access to real drm fd it can work on. */ extern struct igt_map *ahnd_map; extern pthread_mutex_t ahnd_map_mutex; /* * - for parent process we have child_pid == -1 * - for child which calls intel_allocator_init() allocator_pid == child_pid */ static inline bool is_same_process(void) { return child_pid == -1 || allocator_pid == child_pid; } static struct msg_channel *channel; static int send_alloc_stop(struct msg_channel *msgchan) { struct alloc_req req = {0}; req.request_type = REQ_STOP; return msgchan->send_req(msgchan, &req); } static int send_req(struct msg_channel *msgchan, pid_t tid, struct alloc_req *request) { request->tid = tid; return msgchan->send_req(msgchan, request); } static int recv_req(struct msg_channel *msgchan, struct alloc_req *request) { return msgchan->recv_req(msgchan, request); } static int send_resp(struct msg_channel *msgchan, pid_t tid, struct alloc_resp *response) { response->tid = tid; return msgchan->send_resp(msgchan, response); } static int recv_resp(struct msg_channel *msgchan, pid_t tid, struct alloc_resp *response) { response->tid = tid; return msgchan->recv_resp(msgchan, response); } static inline void map_entry_free_func(struct igt_map_entry *entry) { free(entry->data); } static bool can_report_gtt_size(int fd) { struct drm_i915_gem_context_param p = { .param = I915_CONTEXT_PARAM_GTT_SIZE }; return (__gem_context_get_param(fd, &p) == 0); } static uint64_t __handle_create(struct allocator *al) { struct handle_entry *h = malloc(sizeof(*h)); igt_assert(h); h->handle = atomic_fetch_add(&next_handle, 1); h->al = al; igt_map_insert(handles, h, h); return h->handle; } static void __handle_destroy(uint64_t handle) { struct handle_entry he = { .handle = handle }; igt_map_remove(handles, &he, map_entry_free_func); } static struct allocator *__allocator_find(int fd, uint32_t ctx, uint32_t vm) { struct allocator al = { .fd = fd, .ctx = ctx, .vm = vm }; struct igt_map *map = GET_MAP(vm); return igt_map_search(map, &al); } static struct allocator *__allocator_find_by_handle(uint64_t handle) { struct handle_entry *h, he = { .handle = handle }; h = igt_map_search(handles, &he); if (!h) return NULL; return h->al; } static struct allocator *__allocator_create(int fd, uint32_t ctx, uint32_t vm, struct intel_allocator *ial) { struct igt_map *map = GET_MAP(vm); struct allocator *al = malloc(sizeof(*al)); igt_assert(al); igt_assert(fd == ial->fd); al->fd = fd; al->ctx = ctx; al->vm = vm; atomic_init(&al->refcount, 0); al->ial = ial; igt_map_insert(map, al, al); return al; } static void __allocator_destroy(struct allocator *al) { struct igt_map *map = GET_MAP(al->vm); igt_map_remove(map, al, map_entry_free_func); } static int __allocator_get(struct allocator *al) { struct intel_allocator *ial = al->ial; int refcount; atomic_fetch_add(&al->refcount, 1); refcount = atomic_fetch_add(&ial->refcount, 1); igt_assert(refcount >= 0); return refcount; } static bool __allocator_put(struct allocator *al) { struct intel_allocator *ial = al->ial; bool released = false; int refcount, al_refcount; al_refcount = atomic_fetch_sub(&al->refcount, 1); refcount = atomic_fetch_sub(&ial->refcount, 1); igt_assert(refcount >= 1); if (refcount == 1) { if (!ial->is_empty(ial) && warn_if_not_empty) igt_warn("Allocator not clear before destroy!\n"); /* Check allocator has also refcount == 1 */ igt_assert_eq(al_refcount, 1); released = true; } return released; } static struct intel_allocator *intel_allocator_create(int fd, uint64_t start, uint64_t end, uint8_t allocator_type, uint8_t allocator_strategy, uint64_t default_alignment) { struct intel_allocator *ial = NULL; switch (allocator_type) { /* * Few words of explanation is required here. * * INTEL_ALLOCATOR_NONE allows keeping information in the code (intel-bb * is an example) we're not using IGT allocator itself and likely * we rely on relocations. * So trying to create NONE allocator doesn't makes sense and below * assertion catches such invalid usage. */ case INTEL_ALLOCATOR_NONE: igt_assert_f(allocator_type != INTEL_ALLOCATOR_NONE, "We cannot use NONE allocator\n"); break; case INTEL_ALLOCATOR_RELOC: ial = intel_allocator_reloc_create(fd, start, end); break; case INTEL_ALLOCATOR_SIMPLE: ial = intel_allocator_simple_create(fd, start, end, allocator_strategy); break; default: igt_assert_f(ial, "Allocator type %d not implemented\n", allocator_type); break; } igt_assert(ial); ial->type = allocator_type; ial->strategy = allocator_strategy; ial->default_alignment = default_alignment; pthread_mutex_init(&ial->mutex, NULL); return ial; } static void intel_allocator_destroy(struct intel_allocator *ial) { alloc_info("Destroying allocator (empty: %d)\n", ial->is_empty(ial)); ial->destroy(ial); } static struct allocator *allocator_open(int fd, uint32_t ctx, uint32_t vm, uint64_t start, uint64_t end, uint8_t allocator_type, uint8_t allocator_strategy, uint64_t default_alignment, uint64_t *ahndp) { struct intel_allocator *ial; struct allocator *al; const char *idstr = vm ? "vm" : "ctx"; igt_assert(ahndp); al = __allocator_find(fd, ctx, vm); if (!al) { alloc_info("Allocator fd: %d, ctx: %u, vm: %u, <0x%llx : 0x%llx>, " "default alignment: 0x%llx " "not found, creating one\n", fd, ctx, vm, (long long) start, (long long) end, (long long) default_alignment); ial = intel_allocator_create(fd, start, end, allocator_type, allocator_strategy, default_alignment); al = __allocator_create(fd, ctx, vm, ial); } ial = al->ial; igt_assert_f(ial->type == allocator_type, "Allocator type must be same for fd/%s\n", idstr); igt_assert_f(ial->strategy == allocator_strategy, "Allocator strategy must be same or fd/%s\n", idstr); igt_assert_f(ial->default_alignment == default_alignment, "Allocator default alignment must be same or fd/%s\n", idstr); __allocator_get(al); *ahndp = __handle_create(al); return al; } static bool allocator_close(uint64_t ahnd) { struct allocator *al; bool released, is_empty = false; al = __allocator_find_by_handle(ahnd); if (!al) { igt_warn("Cannot find handle: %llx\n", (long long) ahnd); return false; } released = __allocator_put(al); if (released) { is_empty = al->ial->is_empty(al->ial); intel_allocator_destroy(al->ial); } if (!atomic_load(&al->refcount)) __allocator_destroy(al); __handle_destroy(ahnd); return is_empty; } static int send_req_recv_resp(struct msg_channel *msgchan, struct alloc_req *request, struct alloc_resp *response) { int ret; ret = send_req(msgchan, child_tid, request); if (ret < 0) { igt_warn("Error sending request [type: %d]: err = %d [%s]\n", request->request_type, errno, strerror(errno)); return ret; } ret = recv_resp(msgchan, child_tid, response); if (ret < 0) igt_warn("Error receiving response [type: %d]: err = %d [%s]\n", request->request_type, errno, strerror(errno)); /* * This is main assumption - we receive message which size must be > 0. * If this is fulfilled we return 0 as a success. */ if (ret > 0) ret = 0; return ret; } static int handle_request(struct alloc_req *req, struct alloc_resp *resp) { int ret; long refcnt; memset(resp, 0, sizeof(*resp)); if (is_same_process()) { struct intel_allocator *ial; struct allocator *al; uint64_t start, end, size, ahnd; uint32_t ctx, vm; bool allocated, reserved, unreserved; /* Used when debug is on, so avoid compilation warnings */ (void) ctx; (void) vm; (void) refcnt; /* * Mutex only work on allocator instance, not stop/open/close */ if (req->request_type > REQ_CLOSE) { /* * We have to lock map mutex because concurrent open * can lead to resizing the map. */ pthread_mutex_lock(&map_mutex); al = __allocator_find_by_handle(req->allocator_handle); pthread_mutex_unlock(&map_mutex); igt_assert(al); ial = al->ial; igt_assert(ial); pthread_mutex_lock(&ial->mutex); } switch (req->request_type) { case REQ_STOP: alloc_info("\n"); break; case REQ_OPEN: pthread_mutex_lock(&map_mutex); al = allocator_open(req->open.fd, req->open.ctx, req->open.vm, req->open.start, req->open.end, req->open.allocator_type, req->open.allocator_strategy, req->open.default_alignment, &ahnd); refcnt = atomic_load(&al->refcount); ret = atomic_load(&al->ial->refcount); pthread_mutex_unlock(&map_mutex); resp->response_type = RESP_OPEN; resp->open.allocator_handle = ahnd; alloc_info(" [tid: %ld] fd: %d, ahnd: %" PRIx64 ", ctx: %u, vm: %u" ", alloc_type: %u, defalign: %llx" ", al->refcnt: %ld->%ld" ", refcnt: %d->%d\n", (long) req->tid, req->open.fd, ahnd, req->open.ctx, req->open.vm, req->open.allocator_type, (long long) req->open.default_alignment, refcnt - 1, refcnt, ret - 1, ret); break; case REQ_CLOSE: pthread_mutex_lock(&map_mutex); al = __allocator_find_by_handle(req->allocator_handle); resp->response_type = RESP_CLOSE; if (!al) { alloc_info(" [tid: %ld] ahnd: %" PRIx64 " -> no handle\n", (long) req->tid, req->allocator_handle); pthread_mutex_unlock(&map_mutex); break; } resp->response_type = RESP_CLOSE; ctx = al->ctx; vm = al->vm; refcnt = atomic_load(&al->refcount); ret = atomic_load(&al->ial->refcount); resp->close.is_empty = allocator_close(req->allocator_handle); pthread_mutex_unlock(&map_mutex); alloc_info(" [tid: %ld] ahnd: %" PRIx64 ", ctx: %u, vm: %u" ", is_empty: %d, al->refcount: %ld->%ld" ", refcnt: %d->%d\n", (long) req->tid, req->allocator_handle, ctx, vm, resp->close.is_empty, refcnt, refcnt - 1, ret, ret - 1); break; case REQ_ADDRESS_RANGE: resp->response_type = RESP_ADDRESS_RANGE; ial->get_address_range(ial, &start, &end); resp->address_range.start = start; resp->address_range.end = end; alloc_info("
[tid: %ld] ahnd: %" PRIx64 ", ctx: %u, vm: %u" ", start: 0x%" PRIx64 ", end: 0x%" PRId64 "\n", (long) req->tid, req->allocator_handle, al->ctx, al->vm, start, end); break; case REQ_ALLOC: req->alloc.alignment = max(ial->default_alignment, req->alloc.alignment); resp->response_type = RESP_ALLOC; resp->alloc.offset = ial->alloc(ial, req->alloc.handle, req->alloc.size, req->alloc.alignment, req->alloc.pat_index, req->alloc.strategy); alloc_info(" [tid: %ld] ahnd: %" PRIx64 ", ctx: %u, vm: %u, handle: %u" ", size: 0x%" PRIx64 ", offset: 0x%" PRIx64 ", alignment: 0x%" PRIx64 ", pat_index: %u, strategy: %u\n", (long) req->tid, req->allocator_handle, al->ctx, al->vm, req->alloc.handle, req->alloc.size, resp->alloc.offset, req->alloc.alignment, req->alloc.pat_index, req->alloc.strategy); break; case REQ_FREE: resp->response_type = RESP_FREE; resp->free.freed = ial->free(ial, req->free.handle); alloc_info(" [tid: %ld] ahnd: %" PRIx64 ", ctx: %u, vm: %u" ", handle: %u, freed: %d\n", (long) req->tid, req->allocator_handle, al->ctx, al->vm, req->free.handle, resp->free.freed); break; case REQ_IS_ALLOCATED: resp->response_type = RESP_IS_ALLOCATED; allocated = ial->is_allocated(ial, req->is_allocated.handle, req->is_allocated.size, req->is_allocated.offset); resp->is_allocated.allocated = allocated; alloc_info(" [tid: %ld] ahnd: %" PRIx64 ", ctx: %u, vm: %u" ", offset: 0x%" PRIx64 ", allocated: %d\n", (long) req->tid, req->allocator_handle, al->ctx, al->vm, req->is_allocated.offset, allocated); break; case REQ_RESERVE: resp->response_type = RESP_RESERVE; reserved = ial->reserve(ial, req->reserve.handle, req->reserve.start, req->reserve.end); resp->reserve.reserved = reserved; alloc_info(" [tid: %ld] ahnd: %" PRIx64 ", ctx: %u, vm: %u, handle: %u" ", start: 0x%" PRIx64 ", end: 0x%" PRIx64 ", reserved: %d\n", (long) req->tid, req->allocator_handle, al->ctx, al->vm, req->reserve.handle, req->reserve.start, req->reserve.end, reserved); break; case REQ_UNRESERVE: resp->response_type = RESP_UNRESERVE; unreserved = ial->unreserve(ial, req->unreserve.handle, req->unreserve.start, req->unreserve.end); resp->unreserve.unreserved = unreserved; alloc_info(" [tid: %ld] ahnd: %" PRIx64 ", ctx: %u, vm: %u, handle: %u" ", start: 0x%" PRIx64 ", end: 0x%" PRIx64 ", unreserved: %d\n", (long) req->tid, req->allocator_handle, al->ctx, al->vm, req->unreserve.handle, req->unreserve.start, req->unreserve.end, unreserved); break; case REQ_IS_RESERVED: resp->response_type = RESP_IS_RESERVED; reserved = ial->is_reserved(ial, req->is_reserved.start, req->is_reserved.end); resp->is_reserved.reserved = reserved; alloc_info(" [tid: %ld] ahnd: %" PRIx64 ", ctx: %u, vm: %u" ", start: 0x%" PRIx64 ", end: 0x%" PRIx64 ", reserved: %d\n", (long) req->tid, req->allocator_handle, al->ctx, al->vm, req->is_reserved.start, req->is_reserved.end, reserved); break; case REQ_RESERVE_IF_NOT_ALLOCATED: resp->response_type = RESP_RESERVE_IF_NOT_ALLOCATED; size = req->reserve.end - req->reserve.start; allocated = ial->is_allocated(ial, req->reserve.handle, size, req->reserve.start); if (allocated) { resp->reserve_if_not_allocated.allocated = allocated; alloc_info(" [tid: %ld] " "ahnd: %" PRIx64 ", ctx: %u, vm: %u" ", handle: %u, size: 0x%lx" ", start: 0x%" PRIx64 ", end: 0x%" PRIx64 ", allocated: %d, reserved: %d\n", (long) req->tid, req->allocator_handle, al->ctx, al->vm, req->reserve.handle, (long) size, req->reserve.start, req->reserve.end, allocated, false); break; } reserved = ial->reserve(ial, req->reserve.handle, req->reserve.start, req->reserve.end); resp->reserve_if_not_allocated.reserved = reserved; alloc_info(" [tid: %ld] " "ahnd: %" PRIx64 ", ctx: %u, vm: %u" ", handle: %u, start: 0x%" PRIx64 ", end: 0x%" PRIx64 ", allocated: %d, reserved: %d\n", (long) req->tid, req->allocator_handle, al->ctx, al->vm, req->reserve.handle, req->reserve.start, req->reserve.end, false, reserved); break; } if (req->request_type > REQ_CLOSE) pthread_mutex_unlock(&ial->mutex); return 0; } igt_assert_f(channel->ready, "Allocator must be called in multiprocess mode, " "use intel_allocator_multiprocess_(start|stop)()\n"); ret = send_req_recv_resp(channel, req, resp); if (ret < 0) exit(0); return ret; } static void *allocator_thread_loop(void *data) { struct alloc_req req; struct alloc_resp resp; int ret; (void) data; alloc_info("Allocator pid: %ld, tid: %ld\n", (long) allocator_pid, (long) gettid()); alloc_info("Entering allocator loop\n"); WRITE_ONCE(allocator_thread_running, true); while (1) { ret = recv_req(channel, &req); if (ret == -1) { igt_warn("Error receiving request in thread, ret = %d [%s]\n", ret, strerror(errno)); igt_waitchildren_timeout(1, "Stopping children, error receiving request\n"); return (void *) -1; } /* Fake message to stop the thread */ if (req.request_type == REQ_STOP) { alloc_info("\n"); break; } ret = handle_request(&req, &resp); if (ret) { igt_warn("Error handling request in thread, ret = %d [%s]\n", ret, strerror(errno)); break; } ret = send_resp(channel, req.tid, &resp); if (ret) { igt_warn("Error sending response in thread, ret = %d [%s]\n", ret, strerror(errno)); igt_waitchildren_timeout(1, "Stopping children, error sending response\n"); return (void *) -1; } } WRITE_ONCE(allocator_thread_running, false); return NULL; } /** * __intel_allocator_multiprocess_prepare: * * Prepares allocator infrastructure to work in multiprocess mode. * * Some description is required why prepare/start steps are separated. * When we write the code and we don't use address sanitizer simple * intel_allocator_multiprocess_start() call is enough. With address * sanitizer and using forking we can encounter situation where one * forked child called allocator alloc() (so parent has some poisoned * memory in shadow map), then second fork occurs. Second child will * get poisoned shadow map from parent (there allocator thread reside). * Checking shadow map in this child will report memory leak. * * How to separate initialization steps take a look into api_intel_allocator.c * fork_simple_stress() function. */ void __intel_allocator_multiprocess_prepare(void) { intel_allocator_init(); multiprocess = true; channel->init(channel); } #define START_TIMEOUT_MS 100 void __intel_allocator_multiprocess_start(void) { int time_left = START_TIMEOUT_MS; pthread_create(&allocator_thread, NULL, allocator_thread_loop, NULL); /* Wait unless allocator thread get started */ while (time_left-- > 0 && !READ_ONCE(allocator_thread_running)) usleep(1000); } /** * intel_allocator_multiprocess_start: * * Function turns on intel_allocator multiprocess mode what means * all allocations from children processes are performed in a separate thread * within main igt process. Children are aware of the situation and use * some interprocess communication channel to send/receive messages * (open, close, alloc, free, ...) to/from allocator thread. * * Must be used when you want to use an allocator in non single-process code. * All allocations in threads spawned in main igt process are handled by * mutexing, not by sending/receiving messages to/from allocator thread. * * Note. This destroys all previously created allocators and theirs content. */ void intel_allocator_multiprocess_start(void) { alloc_info("allocator multiprocess start\n"); igt_assert_f(child_pid == -1, "Allocator thread can be spawned only in main IGT process\n"); __intel_allocator_multiprocess_prepare(); __intel_allocator_multiprocess_start(); } /** * intel_allocator_multiprocess_stop: * * Function turns off intel_allocator multiprocess mode what means * stopping allocator thread and deinitializing its data. */ #define STOP_TIMEOUT_MS 100 void intel_allocator_multiprocess_stop(void) { int time_left = STOP_TIMEOUT_MS; alloc_info("allocator multiprocess stop\n"); if (multiprocess) { send_alloc_stop(channel); /* Give allocator thread time to complete */ while (time_left-- > 0 && READ_ONCE(allocator_thread_running)) usleep(1000); /* coarse calculation */ /* Deinit, this should stop all blocked syscalls, if any */ channel->deinit(channel); pthread_join(allocator_thread, NULL); /* But we're not sure does child will stuck */ igt_waitchildren_timeout(5, "Stopping children"); multiprocess = false; } } static void track_ahnd(int fd, uint64_t ahnd, uint32_t vm) { struct ahnd_info *ainfo; pthread_mutex_lock(&ahnd_map_mutex); ainfo = igt_map_search(ahnd_map, &ahnd); if (!ainfo) { ainfo = malloc(sizeof(*ainfo)); ainfo->fd = fd; ainfo->ahnd = ahnd; ainfo->vm = vm; ainfo->driver = get_intel_driver(fd); ainfo->bind_map = igt_map_create(igt_map_hash_32, igt_map_equal_32); pthread_mutex_init(&ainfo->bind_map_mutex, NULL); bind_debug("[TRACK AHND] pid: %d, tid: %d, create \n", getpid(), gettid(), ainfo->fd, (long long)ainfo->ahnd, ainfo->vm, ainfo->driver, ahnd_map, ainfo->bind_map); igt_map_insert(ahnd_map, &ainfo->ahnd, ainfo); } pthread_mutex_unlock(&ahnd_map_mutex); } static void untrack_ahnd(uint64_t ahnd) { struct ahnd_info *ainfo; pthread_mutex_lock(&ahnd_map_mutex); ainfo = igt_map_search(ahnd_map, &ahnd); if (ainfo) { bind_debug("[UNTRACK AHND]: pid: %d, tid: %d, removing ahnd: %llx\n", getpid(), gettid(), (long long)ahnd); igt_map_remove(ahnd_map, &ahnd, map_entry_free_func); } pthread_mutex_unlock(&ahnd_map_mutex); } static uint64_t __intel_allocator_open_full(int fd, uint32_t ctx, uint32_t vm, uint64_t start, uint64_t end, uint8_t allocator_type, enum allocator_strategy strategy, uint64_t default_alignment) { struct alloc_req req = { .request_type = REQ_OPEN, .open.fd = fd, .open.ctx = ctx, .open.vm = vm, .open.start = start, .open.end = end, .open.allocator_type = allocator_type, .open.allocator_strategy = strategy, .open.default_alignment = default_alignment }; struct alloc_resp resp; uint64_t gtt_size; if (is_i915_device(fd)) { if (!start) req.open.start = gem_detect_safe_start_offset(fd); if (!end) { igt_assert_f(can_report_gtt_size(fd), "Invalid fd\n"); gtt_size = gem_aperture_size(fd); if (!gem_uses_full_ppgtt(fd)) gtt_size /= 2; else gtt_size -= RESERVED; req.open.end = gtt_size; } if (!default_alignment) req.open.default_alignment = gem_detect_safe_alignment(fd); req.open.start = ALIGN(req.open.start, req.open.default_alignment); } else { struct xe_device *xe_dev = xe_device_get(fd); igt_assert(xe_dev); if (!default_alignment) req.open.default_alignment = xe_get_default_alignment(fd); if (!end) req.open.end = 1ull << xe_dev->va_bits; } /* Get child_tid only once at open() */ if (child_tid == -1) child_tid = gettid(); igt_assert(handle_request(&req, &resp) == 0); igt_assert(resp.open.allocator_handle); igt_assert(resp.response_type == RESP_OPEN); /* * Igts mostly uses ctx as id when opening the allocator (i915 legacy). * If ctx is passed let's use it as an vm id, otherwise use vm. */ track_ahnd(fd, resp.open.allocator_handle, ctx ?: vm); return resp.open.allocator_handle; } /** * intel_allocator_open_full: * @fd: i915 descriptor * @ctx: context * @start: address of the beginning * @end: address of the end * @allocator_type: one of INTEL_ALLOCATOR_* define * @strategy: passed to the allocator to define the strategy (like order * of allocation, see notes below) * @default_alignment: default objects alignment - power-of-two requested * alignment, if 0 then safe alignment will be chosen * * Function opens an allocator instance within <@start, @end) vm for given * @fd and @ctx and returns its handle. If the allocator for such pair * doesn't exist it is created with refcount = 1. * Parallel opens returns same handle bumping its refcount. * * Returns: unique handle to the currently opened allocator. * * Notes: * * If start = end = 0, the allocator is opened for the whole available gtt. * * Strategy is generally used internally by the underlying allocator: * * For SIMPLE allocator: * - ALLOC_STRATEGY_HIGH_TO_LOW means topmost addresses are allocated first, * - ALLOC_STRATEGY_LOW_TO_HIGH opposite, allocation starts from lowest * addresses. * * For RANDOM allocator: * - no strategy is currently implemented. */ uint64_t intel_allocator_open_full(int fd, uint32_t ctx, uint64_t start, uint64_t end, uint8_t allocator_type, enum allocator_strategy strategy, uint64_t default_alignment) { return __intel_allocator_open_full(fd, ctx, 0, start, end, allocator_type, strategy, default_alignment); } uint64_t intel_allocator_open_vm_full(int fd, uint32_t vm, uint64_t start, uint64_t end, uint8_t allocator_type, enum allocator_strategy strategy, uint64_t default_alignment) { igt_assert(vm != 0); return __intel_allocator_open_full(fd, 0, vm, start, end, allocator_type, strategy, default_alignment); } /** * intel_allocator_open: * @fd: i915 or xe descriptor * @ctx: context * @allocator_type: one of INTEL_ALLOCATOR_* define * * Function opens an allocator instance for given @fd and @ctx and returns * its handle. If the allocator for such pair doesn't exist it is created * with refcount = 1. Parallel opens returns same handle bumping its refcount. * * Returns: unique handle to the currently opened allocator. * * Notes: we pass ALLOC_STRATEGY_HIGH_TO_LOW as default, playing with higher * addresses makes easier to find addressing issues (like passing non-canonical * offsets, which won't be catched unless 47-bit is set). */ uint64_t intel_allocator_open(int fd, uint32_t ctx, uint8_t allocator_type) { return intel_allocator_open_full(fd, ctx, 0, 0, allocator_type, ALLOC_STRATEGY_HIGH_TO_LOW, 0); } uint64_t intel_allocator_open_vm(int fd, uint32_t vm, uint8_t allocator_type) { return intel_allocator_open_vm_full(fd, vm, 0, 0, allocator_type, ALLOC_STRATEGY_HIGH_TO_LOW, 0); } /** * intel_allocator_close: * @allocator_handle: handle to the allocator that will be closed * * Function decreases an allocator refcount for the given @handle. * When refcount reaches zero allocator is closed (destroyed) and all * allocated / reserved areas are freed. * * Returns: true if closed allocator was empty, false otherwise. */ bool intel_allocator_close(uint64_t allocator_handle) { struct alloc_req req = { .request_type = REQ_CLOSE, .allocator_handle = allocator_handle }; struct alloc_resp resp; igt_assert(handle_request(&req, &resp) == 0); igt_assert(resp.response_type == RESP_CLOSE); untrack_ahnd(allocator_handle); return resp.close.is_empty; } /** * intel_allocator_get_address_range: * @allocator_handle: handle to an allocator * @startp: pointer to the variable where function writes starting offset * @endp: pointer to the variable where function writes ending offset * * Function fills @startp, @endp with respectively, starting and ending offset * of the allocator working virtual address space range. * * Note. Allocators working ranges can differ depending on the device or * the allocator type so before reserving a specific offset a good practise * is to ensure that address is between accepted range. */ void intel_allocator_get_address_range(uint64_t allocator_handle, uint64_t *startp, uint64_t *endp) { struct alloc_req req = { .request_type = REQ_ADDRESS_RANGE, .allocator_handle = allocator_handle }; struct alloc_resp resp; igt_assert(handle_request(&req, &resp) == 0); igt_assert(resp.response_type == RESP_ADDRESS_RANGE); if (startp) *startp = resp.address_range.start; if (endp) *endp = resp.address_range.end; } static bool is_same(struct allocator_object *obj, uint32_t handle, uint64_t offset, uint64_t size, uint8_t pat_index, enum allocator_bind_op bind_op) { return obj->handle == handle && obj->offset == offset && obj->size == size && obj->pat_index == pat_index && (obj->bind_op == bind_op || obj->bind_op == BOUND); } static void track_object(uint64_t allocator_handle, uint32_t handle, uint64_t offset, uint64_t size, uint8_t pat_index, enum allocator_bind_op bind_op) { struct ahnd_info *ainfo; struct allocator_object *obj; bind_debug("[TRACK OBJECT]: [%s] pid: %d, tid: %d, ahnd: %llx, handle: %u, offset: %llx, size: %llx, pat_index: %u\n", bind_op == TO_BIND ? "BIND" : "UNBIND", getpid(), gettid(), (long long)allocator_handle, handle, (long long)offset, (long long)size, pat_index); if (offset == ALLOC_INVALID_ADDRESS) { bind_debug("[TRACK OBJECT] => invalid address %llx, skipping tracking\n", (long long)offset); return; } pthread_mutex_lock(&ahnd_map_mutex); ainfo = igt_map_search(ahnd_map, &allocator_handle); pthread_mutex_unlock(&ahnd_map_mutex); igt_assert_f(ainfo, "[TRACK OBJECT] => MISSING ahnd %llx <=\n", (long long)allocator_handle); if (ainfo->driver == INTEL_DRIVER_I915) return; /* no-op for i915, at least for now */ pthread_mutex_lock(&ainfo->bind_map_mutex); obj = igt_map_search(ainfo->bind_map, &handle); if (obj) { /* * User may call alloc() couple of times, check object is the * same. For free() there's simple case, just remove from * bind_map. */ if (bind_op == TO_BIND) { igt_assert_eq(is_same(obj, handle, offset, size, pat_index, bind_op), true); } else if (bind_op == TO_UNBIND) { if (obj->bind_op == TO_BIND) igt_map_remove(ainfo->bind_map, &obj->handle, map_entry_free_func); else if (obj->bind_op == BOUND) obj->bind_op = bind_op; } } else { /* Ignore to unbind bo which wasn't previously inserted */ if (bind_op == TO_UNBIND) goto out; obj = calloc(1, sizeof(*obj)); obj->handle = handle; obj->offset = offset; obj->size = size; obj->pat_index = pat_index; obj->bind_op = bind_op; igt_map_insert(ainfo->bind_map, &obj->handle, obj); } out: pthread_mutex_unlock(&ainfo->bind_map_mutex); } /** * __intel_allocator_alloc: * @allocator_handle: handle to an allocator * @handle: handle to an object * @size: size of an object * @alignment: determines object alignment * @pat_index: chosen pat_index for the binding * @strategy: chosen allocator strategy * * Function finds and returns the most suitable offset with given @alignment * for an object with @size identified by the @handle. * * Returns: currently assigned address for a given object. If an object was * already allocated returns same address. If allocator can't find suitable * range returns ALLOC_INVALID_ADDRESS. */ uint64_t __intel_allocator_alloc(uint64_t allocator_handle, uint32_t handle, uint64_t size, uint64_t alignment, uint8_t pat_index, enum allocator_strategy strategy) { struct alloc_req req = { .request_type = REQ_ALLOC, .allocator_handle = allocator_handle, .alloc.handle = handle, .alloc.size = size, .alloc.strategy = strategy, .alloc.alignment = alignment, .alloc.pat_index = pat_index, }; struct alloc_resp resp; igt_assert((alignment & (alignment-1)) == 0); igt_assert(handle_request(&req, &resp) == 0); igt_assert(resp.response_type == RESP_ALLOC); track_object(allocator_handle, handle, resp.alloc.offset, size, pat_index, TO_BIND); return resp.alloc.offset; } /** * intel_allocator_alloc: * @allocator_handle: handle to an allocator * @handle: handle to an object * @size: size of an object * @alignment: determines object alignment * * Same as __intel_allocator_alloc() but asserts if allocator can't return * valid address. Uses default allocation strategy chosen during opening * the allocator. */ uint64_t intel_allocator_alloc(uint64_t allocator_handle, uint32_t handle, uint64_t size, uint64_t alignment) { uint64_t offset; offset = __intel_allocator_alloc(allocator_handle, handle, size, alignment, DEFAULT_PAT_INDEX, ALLOC_STRATEGY_NONE); igt_assert(offset != ALLOC_INVALID_ADDRESS); return offset; } /** * intel_allocator_alloc_with_strategy: * @allocator_handle: handle to an allocator * @handle: handle to an object * @size: size of an object * @alignment: determines object alignment * @strategy: strategy of allocation * * Same as __intel_allocator_alloc() but asserts if allocator can't return * valid address. Use @strategy instead of default chosen during opening * the allocator. */ uint64_t intel_allocator_alloc_with_strategy(uint64_t allocator_handle, uint32_t handle, uint64_t size, uint64_t alignment, enum allocator_strategy strategy) { uint64_t offset; offset = __intel_allocator_alloc(allocator_handle, handle, size, alignment, DEFAULT_PAT_INDEX, strategy); igt_assert(offset != ALLOC_INVALID_ADDRESS); return offset; } /** * intel_allocator_free: * @allocator_handle: handle to an allocator * @handle: handle to an object to be freed * * Function free object identified by the @handle in allocator what makes it * offset again allocable. * * Note. Reserved objects can only be freed by an #intel_allocator_unreserve * function. * * Returns: true if the object was successfully freed, otherwise false. */ bool intel_allocator_free(uint64_t allocator_handle, uint32_t handle) { struct alloc_req req = { .request_type = REQ_FREE, .allocator_handle = allocator_handle, .free.handle = handle }; struct alloc_resp resp; igt_assert(handle_request(&req, &resp) == 0); igt_assert(resp.response_type == RESP_FREE); track_object(allocator_handle, handle, 0, 0, 0, TO_UNBIND); return resp.free.freed; } /** * intel_allocator_is_allocated: * @allocator_handle: handle to an allocator * @handle: handle to an object * @size: size of an object * @offset: address of an object * * Function checks whether the object identified by the @handle and @size * is allocated at the @offset. * * Returns: true if the object is currently allocated at the @offset, * otherwise false. */ bool intel_allocator_is_allocated(uint64_t allocator_handle, uint32_t handle, uint64_t size, uint64_t offset) { struct alloc_req req = { .request_type = REQ_IS_ALLOCATED, .allocator_handle = allocator_handle, .is_allocated.handle = handle, .is_allocated.size = size, .is_allocated.offset = offset }; struct alloc_resp resp; igt_assert(handle_request(&req, &resp) == 0); igt_assert(resp.response_type == RESP_IS_ALLOCATED); return resp.is_allocated.allocated; } /** * intel_allocator_reserve: * @allocator_handle: handle to an allocator * @handle: handle to an object * @size: size of an object * @offset: address of an object * * Function reserves space that starts at the @offset and has @size. * Optionally we can pass @handle to mark that space is for a specific * object, otherwise pass -1. * * Note. Reserved space is identified by offset and size, not a handle. * So an object can have multiple reserved spaces with its handle. * * Returns: true if space is successfully reserved, otherwise false. */ bool intel_allocator_reserve(uint64_t allocator_handle, uint32_t handle, uint64_t size, uint64_t offset) { struct alloc_req req = { .request_type = REQ_RESERVE, .allocator_handle = allocator_handle, .reserve.handle = handle, .reserve.start = offset, .reserve.end = offset + size }; struct alloc_resp resp; igt_assert(handle_request(&req, &resp) == 0); igt_assert(resp.response_type == RESP_RESERVE); return resp.reserve.reserved; } /** * intel_allocator_unreserve: * @allocator_handle: handle to an allocator * @handle: handle to an object * @size: size of an object * @offset: address of an object * * Function unreserves space that starts at the @offset, @size and @handle. * * Note. @handle, @size and @offset have to match those used in reservation. * i.e. check with the same offset but even smaller size will fail. * * Returns: true if the space is successfully unreserved, otherwise false. */ bool intel_allocator_unreserve(uint64_t allocator_handle, uint32_t handle, uint64_t size, uint64_t offset) { struct alloc_req req = { .request_type = REQ_UNRESERVE, .allocator_handle = allocator_handle, .unreserve.handle = handle, .unreserve.start = offset, .unreserve.end = offset + size }; struct alloc_resp resp; igt_assert(handle_request(&req, &resp) == 0); igt_assert(resp.response_type == RESP_UNRESERVE); return resp.unreserve.unreserved; } /** * intel_allocator_is_reserved: * @allocator_handle: handle to an allocator * @size: size of an object * @offset: address of an object * * Function checks whether space starting at the @offset and @size is * currently under reservation. * * Note. @size and @offset have to match those used in reservation, * i.e. check with the same offset but even smaller size will fail. * * Returns: true if space is reserved, othwerise false. */ bool intel_allocator_is_reserved(uint64_t allocator_handle, uint64_t size, uint64_t offset) { struct alloc_req req = { .request_type = REQ_IS_RESERVED, .allocator_handle = allocator_handle, .is_reserved.start = offset, .is_reserved.end = offset + size }; struct alloc_resp resp; igt_assert(handle_request(&req, &resp) == 0); igt_assert(resp.response_type == RESP_IS_RESERVED); return resp.is_reserved.reserved; } /** * intel_allocator_reserve_if_not_allocated: * @allocator_handle: handle to an allocator * @handle: handle to an object * @size: size of an object * @offset: address of an object * @is_allocatedp: if not NULL function writes there object allocation status * (true/false) * * Function checks whether the object identified by the @handle and @size * is allocated at the @offset and writes the result to @is_allocatedp. * If it's not it reserves it at the given @offset. * * Returns: true if the space for an object was reserved, otherwise false. */ bool intel_allocator_reserve_if_not_allocated(uint64_t allocator_handle, uint32_t handle, uint64_t size, uint64_t offset, bool *is_allocatedp) { struct alloc_req req = { .request_type = REQ_RESERVE_IF_NOT_ALLOCATED, .allocator_handle = allocator_handle, .reserve.handle = handle, .reserve.start = offset, .reserve.end = offset + size }; struct alloc_resp resp; igt_assert(handle_request(&req, &resp) == 0); igt_assert(resp.response_type == RESP_RESERVE_IF_NOT_ALLOCATED); if (is_allocatedp) *is_allocatedp = resp.reserve_if_not_allocated.allocated; return resp.reserve_if_not_allocated.reserved; } /** * intel_allocator_print: * @allocator_handle: handle to an allocator * * Function prints statistics and content of the allocator. * Mainly for debugging purposes. * * Note. Printing possible only in the main process. **/ void intel_allocator_print(uint64_t allocator_handle) { igt_assert(allocator_handle); if (!multiprocess || is_same_process()) { struct allocator *al; al = __allocator_find_by_handle(allocator_handle); pthread_mutex_lock(&map_mutex); al->ial->print(al->ial, true); pthread_mutex_unlock(&map_mutex); } else { igt_warn("Print stats is in main process only\n"); } } static void __xe_op_bind(struct ahnd_info *ainfo, uint32_t sync_in, uint32_t sync_out) { struct allocator_object *obj; struct igt_map_entry *pos; struct igt_list_head obj_list; struct xe_object *entry, *tmp; IGT_INIT_LIST_HEAD(&obj_list); pthread_mutex_lock(&ainfo->bind_map_mutex); igt_map_foreach(ainfo->bind_map, pos) { obj = pos->data; if (obj->bind_op == BOUND) continue; bind_info("= [vm: %u] %s => %u %lx %lx %u\n", ainfo->vm, obj->bind_op == TO_BIND ? "TO BIND" : "TO UNBIND", obj->handle, obj->offset, obj->size, obj->pat_index); entry = malloc(sizeof(*entry)); entry->handle = obj->handle; entry->offset = obj->offset; entry->size = obj->size; entry->pat_index = obj->pat_index; entry->bind_op = obj->bind_op == TO_BIND ? XE_OBJECT_BIND : XE_OBJECT_UNBIND; igt_list_add(&entry->link, &obj_list); /* * We clean bind_map even before calling bind/unbind * as all binding operations asserts in case of error. */ if (obj->bind_op == TO_BIND) obj->bind_op = BOUND; else igt_map_remove(ainfo->bind_map, &obj->handle, map_entry_free_func); } xe_bind_unbind_async(ainfo->fd, ainfo->vm, 0, &obj_list, sync_in, sync_out); pthread_mutex_unlock(&ainfo->bind_map_mutex); igt_list_for_each_entry_safe(entry, tmp, &obj_list, link) { igt_list_del(&entry->link); free(entry); } } uint64_t get_offset_pat_index(uint64_t ahnd, uint32_t handle, uint64_t size, uint64_t alignment, uint8_t pat_index) { uint64_t offset; offset = __intel_allocator_alloc(ahnd, handle, size, alignment, pat_index, ALLOC_STRATEGY_NONE); igt_assert(offset != ALLOC_INVALID_ADDRESS); return offset; } /** * intel_allocator_bind: * @allocator_handle: handle to an allocator * @sync_in: syncobj (fence-in) * @sync_out: syncobj (fence-out) * * Function binds and unbinds all objects added to the allocator which weren't * previously binded/unbinded. * **/ void intel_allocator_bind(uint64_t allocator_handle, uint32_t sync_in, uint32_t sync_out) { struct ahnd_info *ainfo; pthread_mutex_lock(&ahnd_map_mutex); ainfo = igt_map_search(ahnd_map, &allocator_handle); pthread_mutex_unlock(&ahnd_map_mutex); igt_assert(ainfo); /* * We collect bind/unbind operations on alloc()/free() to do group * operation getting @sync_in as syncobj handle (fence-in). If user * passes 0 as @sync_out we bind/unbind synchronously. */ __xe_op_bind(ainfo, sync_in, sync_out); } static int equal_handles(const void *key1, const void *key2) { const struct handle_entry *h1 = key1, *h2 = key2; return h1->handle == h2->handle; } static int equal_ctx(const void *key1, const void *key2) { const struct allocator *a1 = key1, *a2 = key2; return a1->fd == a2->fd && a1->ctx == a2->ctx; } static int equal_vm(const void *key1, const void *key2) { const struct allocator *a1 = key1, *a2 = key2; return a1->fd == a2->fd && a1->vm == a2->vm; } /* 2^31 + 2^29 - 2^25 + 2^22 - 2^19 - 2^16 + 1 */ #define GOLDEN_RATIO_PRIME_32 0x9e370001UL static inline uint32_t hash_handles(const void *val) { uint32_t hash = ((struct handle_entry *) val)->handle; hash = hash * GOLDEN_RATIO_PRIME_32; return hash; } static inline uint32_t hash_instance(const void *val) { uint64_t hash = ((struct allocator *) val)->fd; hash = hash * GOLDEN_RATIO_PRIME_32; return hash; } static void __free_maps(struct igt_map *map, bool close_allocators) { struct igt_map_entry *pos; const struct handle_entry *h; if (!map) return; if (close_allocators) igt_map_foreach(map, pos) { h = pos->key; allocator_close(h->handle); } igt_map_destroy(map, map_entry_free_func); } static void __free_ahnd_map(void) { struct igt_map_entry *pos; struct ahnd_info *ainfo; if (!ahnd_map) return; igt_map_foreach(ahnd_map, pos) { ainfo = pos->data; igt_map_destroy(ainfo->bind_map, map_entry_free_func); } igt_map_destroy(ahnd_map, map_entry_free_func); } /** * intel_allocator_init: * * Function initializes the allocators infrastructure. The second call will * override current infra and destroy existing there allocators. It is called * in igt_constructor. **/ void intel_allocator_init(void) { alloc_info("Prepare an allocator infrastructure\n"); allocator_pid = getpid(); alloc_info("Allocator pid: %ld\n", (long) allocator_pid); __free_maps(handles, true); __free_maps(ctx_map, false); __free_maps(vm_map, false); __free_ahnd_map(); atomic_init(&next_handle, 1); handles = igt_map_create(hash_handles, equal_handles); ctx_map = igt_map_create(hash_instance, equal_ctx); vm_map = igt_map_create(hash_instance, equal_vm); pthread_mutex_init(&ahnd_map_mutex, NULL); ahnd_map = igt_map_create(igt_map_hash_64, igt_map_equal_64); igt_assert(handles && ctx_map && vm_map && ahnd_map); channel = intel_allocator_get_msgchannel(CHANNEL_SYSVIPC_MSGQUEUE); } igt_constructor { intel_allocator_init(); }