| /* |
| * Copyright 2020 Advanced Micro Devices, Inc. |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice shall be included in |
| * all copies or substantial portions of the Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
| * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
| * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
| * OTHER DEALINGS IN THE SOFTWARE. |
| * |
| * Authors: Christian König |
| */ |
| |
| #include <linux/iosys-map.h> |
| #include <linux/io-mapping.h> |
| #include <linux/scatterlist.h> |
| |
| #include <drm/ttm/ttm_resource.h> |
| #include <drm/ttm/ttm_bo_driver.h> |
| |
| /** |
| * ttm_lru_bulk_move_init - initialize a bulk move structure |
| * @bulk: the structure to init |
| * |
| * For now just memset the structure to zero. |
| */ |
| void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk) |
| { |
| memset(bulk, 0, sizeof(*bulk)); |
| } |
| EXPORT_SYMBOL(ttm_lru_bulk_move_init); |
| |
| /** |
| * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail. |
| * |
| * @bulk: bulk move structure |
| * |
| * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that |
| * resource order never changes. Should be called with &ttm_device.lru_lock held. |
| */ |
| void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk) |
| { |
| unsigned i, j; |
| |
| for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) { |
| for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) { |
| struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j]; |
| struct ttm_resource_manager *man; |
| |
| if (!pos->first) |
| continue; |
| |
| lockdep_assert_held(&pos->first->bo->bdev->lru_lock); |
| dma_resv_assert_held(pos->first->bo->base.resv); |
| dma_resv_assert_held(pos->last->bo->base.resv); |
| |
| man = ttm_manager_type(pos->first->bo->bdev, i); |
| list_bulk_move_tail(&man->lru[j], &pos->first->lru, |
| &pos->last->lru); |
| } |
| } |
| } |
| EXPORT_SYMBOL(ttm_lru_bulk_move_tail); |
| |
| /* Return the bulk move pos object for this resource */ |
| static struct ttm_lru_bulk_move_pos * |
| ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res) |
| { |
| return &bulk->pos[res->mem_type][res->bo->priority]; |
| } |
| |
| /* Move the resource to the tail of the bulk move range */ |
| static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos, |
| struct ttm_resource *res) |
| { |
| if (pos->last != res) { |
| list_move(&res->lru, &pos->last->lru); |
| pos->last = res; |
| } |
| } |
| |
| /* Add the resource to a bulk_move cursor */ |
| static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk, |
| struct ttm_resource *res) |
| { |
| struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res); |
| |
| if (!pos->first) { |
| pos->first = res; |
| pos->last = res; |
| } else { |
| ttm_lru_bulk_move_pos_tail(pos, res); |
| } |
| } |
| |
| /* Remove the resource from a bulk_move range */ |
| static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk, |
| struct ttm_resource *res) |
| { |
| struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res); |
| |
| if (unlikely(pos->first == res && pos->last == res)) { |
| pos->first = NULL; |
| pos->last = NULL; |
| } else if (pos->first == res) { |
| pos->first = list_next_entry(res, lru); |
| } else if (pos->last == res) { |
| pos->last = list_prev_entry(res, lru); |
| } else { |
| list_move(&res->lru, &pos->last->lru); |
| } |
| } |
| |
| /* Add the resource to a bulk move if the BO is configured for it */ |
| void ttm_resource_add_bulk_move(struct ttm_resource *res, |
| struct ttm_buffer_object *bo) |
| { |
| if (bo->bulk_move && !bo->pin_count) |
| ttm_lru_bulk_move_add(bo->bulk_move, res); |
| } |
| |
| /* Remove the resource from a bulk move if the BO is configured for it */ |
| void ttm_resource_del_bulk_move(struct ttm_resource *res, |
| struct ttm_buffer_object *bo) |
| { |
| if (bo->bulk_move && !bo->pin_count) |
| ttm_lru_bulk_move_del(bo->bulk_move, res); |
| } |
| |
| /* Move a resource to the LRU or bulk tail */ |
| void ttm_resource_move_to_lru_tail(struct ttm_resource *res) |
| { |
| struct ttm_buffer_object *bo = res->bo; |
| struct ttm_device *bdev = bo->bdev; |
| |
| lockdep_assert_held(&bo->bdev->lru_lock); |
| |
| if (bo->pin_count) { |
| list_move_tail(&res->lru, &bdev->pinned); |
| |
| } else if (bo->bulk_move) { |
| struct ttm_lru_bulk_move_pos *pos = |
| ttm_lru_bulk_move_pos(bo->bulk_move, res); |
| |
| ttm_lru_bulk_move_pos_tail(pos, res); |
| } else { |
| struct ttm_resource_manager *man; |
| |
| man = ttm_manager_type(bdev, res->mem_type); |
| list_move_tail(&res->lru, &man->lru[bo->priority]); |
| } |
| } |
| |
| /** |
| * ttm_resource_init - resource object constructure |
| * @bo: buffer object this resources is allocated for |
| * @place: placement of the resource |
| * @res: the resource object to inistilize |
| * |
| * Initialize a new resource object. Counterpart of ttm_resource_fini(). |
| */ |
| void ttm_resource_init(struct ttm_buffer_object *bo, |
| const struct ttm_place *place, |
| struct ttm_resource *res) |
| { |
| struct ttm_resource_manager *man; |
| |
| res->start = 0; |
| res->num_pages = PFN_UP(bo->base.size); |
| res->mem_type = place->mem_type; |
| res->placement = place->flags; |
| res->bus.addr = NULL; |
| res->bus.offset = 0; |
| res->bus.is_iomem = false; |
| res->bus.caching = ttm_cached; |
| res->bo = bo; |
| |
| man = ttm_manager_type(bo->bdev, place->mem_type); |
| spin_lock(&bo->bdev->lru_lock); |
| if (bo->pin_count) |
| list_add_tail(&res->lru, &bo->bdev->pinned); |
| else |
| list_add_tail(&res->lru, &man->lru[bo->priority]); |
| man->usage += res->num_pages << PAGE_SHIFT; |
| spin_unlock(&bo->bdev->lru_lock); |
| } |
| EXPORT_SYMBOL(ttm_resource_init); |
| |
| /** |
| * ttm_resource_fini - resource destructor |
| * @man: the resource manager this resource belongs to |
| * @res: the resource to clean up |
| * |
| * Should be used by resource manager backends to clean up the TTM resource |
| * objects before freeing the underlying structure. Makes sure the resource is |
| * removed from the LRU before destruction. |
| * Counterpart of ttm_resource_init(). |
| */ |
| void ttm_resource_fini(struct ttm_resource_manager *man, |
| struct ttm_resource *res) |
| { |
| struct ttm_device *bdev = man->bdev; |
| |
| spin_lock(&bdev->lru_lock); |
| list_del_init(&res->lru); |
| man->usage -= res->num_pages << PAGE_SHIFT; |
| spin_unlock(&bdev->lru_lock); |
| } |
| EXPORT_SYMBOL(ttm_resource_fini); |
| |
| int ttm_resource_alloc(struct ttm_buffer_object *bo, |
| const struct ttm_place *place, |
| struct ttm_resource **res_ptr) |
| { |
| struct ttm_resource_manager *man = |
| ttm_manager_type(bo->bdev, place->mem_type); |
| int ret; |
| |
| ret = man->func->alloc(man, bo, place, res_ptr); |
| if (ret) |
| return ret; |
| |
| spin_lock(&bo->bdev->lru_lock); |
| ttm_resource_add_bulk_move(*res_ptr, bo); |
| spin_unlock(&bo->bdev->lru_lock); |
| return 0; |
| } |
| |
| void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res) |
| { |
| struct ttm_resource_manager *man; |
| |
| if (!*res) |
| return; |
| |
| spin_lock(&bo->bdev->lru_lock); |
| ttm_resource_del_bulk_move(*res, bo); |
| spin_unlock(&bo->bdev->lru_lock); |
| man = ttm_manager_type(bo->bdev, (*res)->mem_type); |
| man->func->free(man, *res); |
| *res = NULL; |
| } |
| EXPORT_SYMBOL(ttm_resource_free); |
| |
| /** |
| * ttm_resource_intersects - test for intersection |
| * |
| * @bdev: TTM device structure |
| * @res: The resource to test |
| * @place: The placement to test |
| * @size: How many bytes the new allocation needs. |
| * |
| * Test if @res intersects with @place and @size. Used for testing if evictions |
| * are valueable or not. |
| * |
| * Returns true if the res placement intersects with @place and @size. |
| */ |
| bool ttm_resource_intersects(struct ttm_device *bdev, |
| struct ttm_resource *res, |
| const struct ttm_place *place, |
| size_t size) |
| { |
| struct ttm_resource_manager *man; |
| |
| if (!res) |
| return false; |
| |
| man = ttm_manager_type(bdev, res->mem_type); |
| if (!place || !man->func->intersects) |
| return true; |
| |
| return man->func->intersects(man, res, place, size); |
| } |
| |
| /** |
| * ttm_resource_compatible - test for compatibility |
| * |
| * @bdev: TTM device structure |
| * @res: The resource to test |
| * @place: The placement to test |
| * @size: How many bytes the new allocation needs. |
| * |
| * Test if @res compatible with @place and @size. |
| * |
| * Returns true if the res placement compatible with @place and @size. |
| */ |
| bool ttm_resource_compatible(struct ttm_device *bdev, |
| struct ttm_resource *res, |
| const struct ttm_place *place, |
| size_t size) |
| { |
| struct ttm_resource_manager *man; |
| |
| if (!res || !place) |
| return false; |
| |
| man = ttm_manager_type(bdev, res->mem_type); |
| if (!man->func->compatible) |
| return true; |
| |
| return man->func->compatible(man, res, place, size); |
| } |
| |
| static bool ttm_resource_places_compat(struct ttm_resource *res, |
| const struct ttm_place *places, |
| unsigned num_placement) |
| { |
| struct ttm_buffer_object *bo = res->bo; |
| struct ttm_device *bdev = bo->bdev; |
| unsigned i; |
| |
| if (res->placement & TTM_PL_FLAG_TEMPORARY) |
| return false; |
| |
| for (i = 0; i < num_placement; i++) { |
| const struct ttm_place *heap = &places[i]; |
| |
| if (!ttm_resource_compatible(bdev, res, heap, bo->base.size)) |
| continue; |
| |
| if ((res->mem_type == heap->mem_type) && |
| (!(heap->flags & TTM_PL_FLAG_CONTIGUOUS) || |
| (res->placement & TTM_PL_FLAG_CONTIGUOUS))) |
| return true; |
| } |
| return false; |
| } |
| |
| /** |
| * ttm_resource_compat - check if resource is compatible with placement |
| * |
| * @res: the resource to check |
| * @placement: the placement to check against |
| * |
| * Returns true if the placement is compatible. |
| */ |
| bool ttm_resource_compat(struct ttm_resource *res, |
| struct ttm_placement *placement) |
| { |
| if (ttm_resource_places_compat(res, placement->placement, |
| placement->num_placement)) |
| return true; |
| |
| if ((placement->busy_placement != placement->placement || |
| placement->num_busy_placement > placement->num_placement) && |
| ttm_resource_places_compat(res, placement->busy_placement, |
| placement->num_busy_placement)) |
| return true; |
| |
| return false; |
| } |
| EXPORT_SYMBOL(ttm_resource_compat); |
| |
| void ttm_resource_set_bo(struct ttm_resource *res, |
| struct ttm_buffer_object *bo) |
| { |
| spin_lock(&bo->bdev->lru_lock); |
| res->bo = bo; |
| spin_unlock(&bo->bdev->lru_lock); |
| } |
| |
| /** |
| * ttm_resource_manager_init |
| * |
| * @man: memory manager object to init |
| * @bdev: ttm device this manager belongs to |
| * @size: size of managed resources in arbitrary units |
| * |
| * Initialise core parts of a manager object. |
| */ |
| void ttm_resource_manager_init(struct ttm_resource_manager *man, |
| struct ttm_device *bdev, |
| uint64_t size) |
| { |
| unsigned i; |
| |
| spin_lock_init(&man->move_lock); |
| man->bdev = bdev; |
| man->size = size; |
| man->usage = 0; |
| |
| for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) |
| INIT_LIST_HEAD(&man->lru[i]); |
| man->move = NULL; |
| } |
| EXPORT_SYMBOL(ttm_resource_manager_init); |
| |
| /* |
| * ttm_resource_manager_evict_all |
| * |
| * @bdev - device to use |
| * @man - manager to use |
| * |
| * Evict all the objects out of a memory manager until it is empty. |
| * Part of memory manager cleanup sequence. |
| */ |
| int ttm_resource_manager_evict_all(struct ttm_device *bdev, |
| struct ttm_resource_manager *man) |
| { |
| struct ttm_operation_ctx ctx = { |
| .interruptible = false, |
| .no_wait_gpu = false, |
| .force_alloc = true |
| }; |
| struct dma_fence *fence; |
| int ret; |
| unsigned i; |
| |
| /* |
| * Can't use standard list traversal since we're unlocking. |
| */ |
| |
| spin_lock(&bdev->lru_lock); |
| for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { |
| while (!list_empty(&man->lru[i])) { |
| spin_unlock(&bdev->lru_lock); |
| ret = ttm_mem_evict_first(bdev, man, NULL, &ctx, |
| NULL); |
| if (ret) |
| return ret; |
| spin_lock(&bdev->lru_lock); |
| } |
| } |
| spin_unlock(&bdev->lru_lock); |
| |
| spin_lock(&man->move_lock); |
| fence = dma_fence_get(man->move); |
| spin_unlock(&man->move_lock); |
| |
| if (fence) { |
| ret = dma_fence_wait(fence, false); |
| dma_fence_put(fence); |
| if (ret) |
| return ret; |
| } |
| |
| return 0; |
| } |
| EXPORT_SYMBOL(ttm_resource_manager_evict_all); |
| |
| /** |
| * ttm_resource_manager_usage |
| * |
| * @man: A memory manager object. |
| * |
| * Return how many resources are currently used. |
| */ |
| uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man) |
| { |
| uint64_t usage; |
| |
| spin_lock(&man->bdev->lru_lock); |
| usage = man->usage; |
| spin_unlock(&man->bdev->lru_lock); |
| return usage; |
| } |
| EXPORT_SYMBOL(ttm_resource_manager_usage); |
| |
| /** |
| * ttm_resource_manager_debug |
| * |
| * @man: manager type to dump. |
| * @p: printer to use for debug. |
| */ |
| void ttm_resource_manager_debug(struct ttm_resource_manager *man, |
| struct drm_printer *p) |
| { |
| drm_printf(p, " use_type: %d\n", man->use_type); |
| drm_printf(p, " use_tt: %d\n", man->use_tt); |
| drm_printf(p, " size: %llu\n", man->size); |
| drm_printf(p, " usage: %llu\n", ttm_resource_manager_usage(man)); |
| if (man->func->debug) |
| man->func->debug(man, p); |
| } |
| EXPORT_SYMBOL(ttm_resource_manager_debug); |
| |
| /** |
| * ttm_resource_manager_first |
| * |
| * @man: resource manager to iterate over |
| * @cursor: cursor to record the position |
| * |
| * Returns the first resource from the resource manager. |
| */ |
| struct ttm_resource * |
| ttm_resource_manager_first(struct ttm_resource_manager *man, |
| struct ttm_resource_cursor *cursor) |
| { |
| struct ttm_resource *res; |
| |
| lockdep_assert_held(&man->bdev->lru_lock); |
| |
| for (cursor->priority = 0; cursor->priority < TTM_MAX_BO_PRIORITY; |
| ++cursor->priority) |
| list_for_each_entry(res, &man->lru[cursor->priority], lru) |
| return res; |
| |
| return NULL; |
| } |
| |
| /** |
| * ttm_resource_manager_next |
| * |
| * @man: resource manager to iterate over |
| * @cursor: cursor to record the position |
| * @res: the current resource pointer |
| * |
| * Returns the next resource from the resource manager. |
| */ |
| struct ttm_resource * |
| ttm_resource_manager_next(struct ttm_resource_manager *man, |
| struct ttm_resource_cursor *cursor, |
| struct ttm_resource *res) |
| { |
| lockdep_assert_held(&man->bdev->lru_lock); |
| |
| list_for_each_entry_continue(res, &man->lru[cursor->priority], lru) |
| return res; |
| |
| for (++cursor->priority; cursor->priority < TTM_MAX_BO_PRIORITY; |
| ++cursor->priority) |
| list_for_each_entry(res, &man->lru[cursor->priority], lru) |
| return res; |
| |
| return NULL; |
| } |
| |
| static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter, |
| struct iosys_map *dmap, |
| pgoff_t i) |
| { |
| struct ttm_kmap_iter_iomap *iter_io = |
| container_of(iter, typeof(*iter_io), base); |
| void __iomem *addr; |
| |
| retry: |
| while (i >= iter_io->cache.end) { |
| iter_io->cache.sg = iter_io->cache.sg ? |
| sg_next(iter_io->cache.sg) : iter_io->st->sgl; |
| iter_io->cache.i = iter_io->cache.end; |
| iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >> |
| PAGE_SHIFT; |
| iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) - |
| iter_io->start; |
| } |
| |
| if (i < iter_io->cache.i) { |
| iter_io->cache.end = 0; |
| iter_io->cache.sg = NULL; |
| goto retry; |
| } |
| |
| addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs + |
| (((resource_size_t)i - iter_io->cache.i) |
| << PAGE_SHIFT)); |
| iosys_map_set_vaddr_iomem(dmap, addr); |
| } |
| |
| static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter, |
| struct iosys_map *map) |
| { |
| io_mapping_unmap_local(map->vaddr_iomem); |
| } |
| |
| static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = { |
| .map_local = ttm_kmap_iter_iomap_map_local, |
| .unmap_local = ttm_kmap_iter_iomap_unmap_local, |
| .maps_tt = false, |
| }; |
| |
| /** |
| * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap |
| * @iter_io: The struct ttm_kmap_iter_iomap to initialize. |
| * @iomap: The struct io_mapping representing the underlying linear io_memory. |
| * @st: sg_table into @iomap, representing the memory of the struct |
| * ttm_resource. |
| * @start: Offset that needs to be subtracted from @st to make |
| * sg_dma_address(st->sgl) - @start == 0 for @iomap start. |
| * |
| * Return: Pointer to the embedded struct ttm_kmap_iter. |
| */ |
| struct ttm_kmap_iter * |
| ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io, |
| struct io_mapping *iomap, |
| struct sg_table *st, |
| resource_size_t start) |
| { |
| iter_io->base.ops = &ttm_kmap_iter_io_ops; |
| iter_io->iomap = iomap; |
| iter_io->st = st; |
| iter_io->start = start; |
| memset(&iter_io->cache, 0, sizeof(iter_io->cache)); |
| |
| return &iter_io->base; |
| } |
| EXPORT_SYMBOL(ttm_kmap_iter_iomap_init); |
| |
| /** |
| * DOC: Linear io iterator |
| * |
| * This code should die in the not too near future. Best would be if we could |
| * make io-mapping use memremap for all io memory, and have memremap |
| * implement a kmap_local functionality. We could then strip a huge amount of |
| * code. These linear io iterators are implemented to mimic old functionality, |
| * and they don't use kmap_local semantics at all internally. Rather ioremap or |
| * friends, and at least on 32-bit they add global TLB flushes and points |
| * of failure. |
| */ |
| |
| static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter, |
| struct iosys_map *dmap, |
| pgoff_t i) |
| { |
| struct ttm_kmap_iter_linear_io *iter_io = |
| container_of(iter, typeof(*iter_io), base); |
| |
| *dmap = iter_io->dmap; |
| iosys_map_incr(dmap, i * PAGE_SIZE); |
| } |
| |
| static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = { |
| .map_local = ttm_kmap_iter_linear_io_map_local, |
| .maps_tt = false, |
| }; |
| |
| /** |
| * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory |
| * @iter_io: The iterator to initialize |
| * @bdev: The TTM device |
| * @mem: The ttm resource representing the iomap. |
| * |
| * This function is for internal TTM use only. It sets up a memcpy kmap iterator |
| * pointing at a linear chunk of io memory. |
| * |
| * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on |
| * failure. |
| */ |
| struct ttm_kmap_iter * |
| ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, |
| struct ttm_device *bdev, |
| struct ttm_resource *mem) |
| { |
| int ret; |
| |
| ret = ttm_mem_io_reserve(bdev, mem); |
| if (ret) |
| goto out_err; |
| if (!mem->bus.is_iomem) { |
| ret = -EINVAL; |
| goto out_io_free; |
| } |
| |
| if (mem->bus.addr) { |
| iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr); |
| iter_io->needs_unmap = false; |
| } else { |
| size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT; |
| |
| iter_io->needs_unmap = true; |
| memset(&iter_io->dmap, 0, sizeof(iter_io->dmap)); |
| if (mem->bus.caching == ttm_write_combined) |
| iosys_map_set_vaddr_iomem(&iter_io->dmap, |
| ioremap_wc(mem->bus.offset, |
| bus_size)); |
| else if (mem->bus.caching == ttm_cached) |
| iosys_map_set_vaddr(&iter_io->dmap, |
| memremap(mem->bus.offset, bus_size, |
| MEMREMAP_WB | |
| MEMREMAP_WT | |
| MEMREMAP_WC)); |
| |
| /* If uncached requested or if mapping cached or wc failed */ |
| if (iosys_map_is_null(&iter_io->dmap)) |
| iosys_map_set_vaddr_iomem(&iter_io->dmap, |
| ioremap(mem->bus.offset, |
| bus_size)); |
| |
| if (iosys_map_is_null(&iter_io->dmap)) { |
| ret = -ENOMEM; |
| goto out_io_free; |
| } |
| } |
| |
| iter_io->base.ops = &ttm_kmap_iter_linear_io_ops; |
| return &iter_io->base; |
| |
| out_io_free: |
| ttm_mem_io_free(bdev, mem); |
| out_err: |
| return ERR_PTR(ret); |
| } |
| |
| /** |
| * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory |
| * @iter_io: The iterator to initialize |
| * @bdev: The TTM device |
| * @mem: The ttm resource representing the iomap. |
| * |
| * This function is for internal TTM use only. It cleans up a memcpy kmap |
| * iterator initialized by ttm_kmap_iter_linear_io_init. |
| */ |
| void |
| ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io, |
| struct ttm_device *bdev, |
| struct ttm_resource *mem) |
| { |
| if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) { |
| if (iter_io->dmap.is_iomem) |
| iounmap(iter_io->dmap.vaddr_iomem); |
| else |
| memunmap(iter_io->dmap.vaddr); |
| } |
| |
| ttm_mem_io_free(bdev, mem); |
| } |
| |
| #if defined(CONFIG_DEBUG_FS) |
| |
| static int ttm_resource_manager_show(struct seq_file *m, void *unused) |
| { |
| struct ttm_resource_manager *man = |
| (struct ttm_resource_manager *)m->private; |
| struct drm_printer p = drm_seq_file_printer(m); |
| ttm_resource_manager_debug(man, &p); |
| return 0; |
| } |
| DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager); |
| |
| #endif |
| |
| /** |
| * ttm_resource_manager_create_debugfs - Create debugfs entry for specified |
| * resource manager. |
| * @man: The TTM resource manager for which the debugfs stats file be creates |
| * @parent: debugfs directory in which the file will reside |
| * @name: The filename to create. |
| * |
| * This function setups up a debugfs file that can be used to look |
| * at debug statistics of the specified ttm_resource_manager. |
| */ |
| void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man, |
| struct dentry * parent, |
| const char *name) |
| { |
| #if defined(CONFIG_DEBUG_FS) |
| debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops); |
| #endif |
| } |
| EXPORT_SYMBOL(ttm_resource_manager_create_debugfs); |