| /* |
| * Copyright (c) 2016 Hisilicon Limited. |
| * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. |
| * |
| * This software is available to you under a choice of one of two |
| * licenses. You may choose to be licensed under the terms of the GNU |
| * General Public License (GPL) Version 2, available from the file |
| * COPYING in the main directory of this source tree, or the |
| * OpenIB.org BSD license below: |
| * |
| * Redistribution and use in source and binary forms, with or |
| * without modification, are permitted provided that the following |
| * conditions are met: |
| * |
| * - Redistributions of source code must retain the above |
| * copyright notice, this list of conditions and the following |
| * disclaimer. |
| * |
| * - Redistributions in binary form must reproduce the above |
| * copyright notice, this list of conditions and the following |
| * disclaimer in the documentation and/or other materials |
| * provided with the distribution. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| * SOFTWARE. |
| */ |
| |
| #include <linux/platform_device.h> |
| #include <linux/vmalloc.h> |
| #include "hns_roce_device.h" |
| #include <rdma/ib_umem.h> |
| |
| int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj) |
| { |
| int ret = 0; |
| |
| spin_lock(&bitmap->lock); |
| *obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); |
| if (*obj >= bitmap->max) { |
| bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) |
| & bitmap->mask; |
| *obj = find_first_zero_bit(bitmap->table, bitmap->max); |
| } |
| |
| if (*obj < bitmap->max) { |
| set_bit(*obj, bitmap->table); |
| bitmap->last = (*obj + 1); |
| if (bitmap->last == bitmap->max) |
| bitmap->last = 0; |
| *obj |= bitmap->top; |
| } else { |
| ret = -EINVAL; |
| } |
| |
| spin_unlock(&bitmap->lock); |
| |
| return ret; |
| } |
| |
| void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj, |
| int rr) |
| { |
| hns_roce_bitmap_free_range(bitmap, obj, 1, rr); |
| } |
| |
| int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt, |
| int align, unsigned long *obj) |
| { |
| int ret = 0; |
| int i; |
| |
| if (likely(cnt == 1 && align == 1)) |
| return hns_roce_bitmap_alloc(bitmap, obj); |
| |
| spin_lock(&bitmap->lock); |
| |
| *obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, |
| bitmap->last, cnt, align - 1); |
| if (*obj >= bitmap->max) { |
| bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) |
| & bitmap->mask; |
| *obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, 0, |
| cnt, align - 1); |
| } |
| |
| if (*obj < bitmap->max) { |
| for (i = 0; i < cnt; i++) |
| set_bit(*obj + i, bitmap->table); |
| |
| if (*obj == bitmap->last) { |
| bitmap->last = (*obj + cnt); |
| if (bitmap->last >= bitmap->max) |
| bitmap->last = 0; |
| } |
| *obj |= bitmap->top; |
| } else { |
| ret = -EINVAL; |
| } |
| |
| spin_unlock(&bitmap->lock); |
| |
| return ret; |
| } |
| |
| void hns_roce_bitmap_free_range(struct hns_roce_bitmap *bitmap, |
| unsigned long obj, int cnt, |
| int rr) |
| { |
| int i; |
| |
| obj &= bitmap->max + bitmap->reserved_top - 1; |
| |
| spin_lock(&bitmap->lock); |
| for (i = 0; i < cnt; i++) |
| clear_bit(obj + i, bitmap->table); |
| |
| if (!rr) |
| bitmap->last = min(bitmap->last, obj); |
| bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) |
| & bitmap->mask; |
| spin_unlock(&bitmap->lock); |
| } |
| |
| int hns_roce_bitmap_init(struct hns_roce_bitmap *bitmap, u32 num, u32 mask, |
| u32 reserved_bot, u32 reserved_top) |
| { |
| u32 i; |
| |
| if (num != roundup_pow_of_two(num)) |
| return -EINVAL; |
| |
| bitmap->last = 0; |
| bitmap->top = 0; |
| bitmap->max = num - reserved_top; |
| bitmap->mask = mask; |
| bitmap->reserved_top = reserved_top; |
| spin_lock_init(&bitmap->lock); |
| bitmap->table = kcalloc(BITS_TO_LONGS(bitmap->max), sizeof(long), |
| GFP_KERNEL); |
| if (!bitmap->table) |
| return -ENOMEM; |
| |
| for (i = 0; i < reserved_bot; ++i) |
| set_bit(i, bitmap->table); |
| |
| return 0; |
| } |
| |
| void hns_roce_bitmap_cleanup(struct hns_roce_bitmap *bitmap) |
| { |
| kfree(bitmap->table); |
| } |
| |
| void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf) |
| { |
| struct hns_roce_buf_list *trunks; |
| u32 i; |
| |
| if (!buf) |
| return; |
| |
| trunks = buf->trunk_list; |
| if (trunks) { |
| buf->trunk_list = NULL; |
| for (i = 0; i < buf->ntrunks; i++) |
| dma_free_coherent(hr_dev->dev, 1 << buf->trunk_shift, |
| trunks[i].buf, trunks[i].map); |
| |
| kfree(trunks); |
| } |
| |
| kfree(buf); |
| } |
| |
| /* |
| * Allocate the dma buffer for storing ROCEE table entries |
| * |
| * @size: required size |
| * @page_shift: the unit size in a continuous dma address range |
| * @flags: HNS_ROCE_BUF_ flags to control the allocation flow. |
| */ |
| struct hns_roce_buf *hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, |
| u32 page_shift, u32 flags) |
| { |
| u32 trunk_size, page_size, alloced_size; |
| struct hns_roce_buf_list *trunks; |
| struct hns_roce_buf *buf; |
| gfp_t gfp_flags; |
| u32 ntrunk, i; |
| |
| /* The minimum shift of the page accessed by hw is HNS_HW_PAGE_SHIFT */ |
| if (WARN_ON(page_shift < HNS_HW_PAGE_SHIFT)) |
| return ERR_PTR(-EINVAL); |
| |
| gfp_flags = (flags & HNS_ROCE_BUF_NOSLEEP) ? GFP_ATOMIC : GFP_KERNEL; |
| buf = kzalloc(sizeof(*buf), gfp_flags); |
| if (!buf) |
| return ERR_PTR(-ENOMEM); |
| |
| buf->page_shift = page_shift; |
| page_size = 1 << buf->page_shift; |
| |
| /* Calc the trunk size and num by required size and page_shift */ |
| if (flags & HNS_ROCE_BUF_DIRECT) { |
| buf->trunk_shift = ilog2(ALIGN(size, PAGE_SIZE)); |
| ntrunk = 1; |
| } else { |
| buf->trunk_shift = ilog2(ALIGN(page_size, PAGE_SIZE)); |
| ntrunk = DIV_ROUND_UP(size, 1 << buf->trunk_shift); |
| } |
| |
| trunks = kcalloc(ntrunk, sizeof(*trunks), gfp_flags); |
| if (!trunks) { |
| kfree(buf); |
| return ERR_PTR(-ENOMEM); |
| } |
| |
| trunk_size = 1 << buf->trunk_shift; |
| alloced_size = 0; |
| for (i = 0; i < ntrunk; i++) { |
| trunks[i].buf = dma_alloc_coherent(hr_dev->dev, trunk_size, |
| &trunks[i].map, gfp_flags); |
| if (!trunks[i].buf) |
| break; |
| |
| alloced_size += trunk_size; |
| } |
| |
| buf->ntrunks = i; |
| |
| /* In nofail mode, it's only failed when the alloced size is 0 */ |
| if ((flags & HNS_ROCE_BUF_NOFAIL) ? i == 0 : i != ntrunk) { |
| for (i = 0; i < buf->ntrunks; i++) |
| dma_free_coherent(hr_dev->dev, trunk_size, |
| trunks[i].buf, trunks[i].map); |
| |
| kfree(trunks); |
| kfree(buf); |
| return ERR_PTR(-ENOMEM); |
| } |
| |
| buf->npages = DIV_ROUND_UP(alloced_size, page_size); |
| buf->trunk_list = trunks; |
| |
| return buf; |
| } |
| |
| int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, |
| int buf_cnt, int start, struct hns_roce_buf *buf) |
| { |
| int i, end; |
| int total; |
| |
| end = start + buf_cnt; |
| if (end > buf->npages) { |
| dev_err(hr_dev->dev, |
| "failed to check kmem bufs, end %d + %d total %u!\n", |
| start, buf_cnt, buf->npages); |
| return -EINVAL; |
| } |
| |
| total = 0; |
| for (i = start; i < end; i++) |
| bufs[total++] = hns_roce_buf_page(buf, i); |
| |
| return total; |
| } |
| |
| int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, |
| int buf_cnt, int start, struct ib_umem *umem, |
| unsigned int page_shift) |
| { |
| struct ib_block_iter biter; |
| int total = 0; |
| int idx = 0; |
| u64 addr; |
| |
| if (page_shift < HNS_HW_PAGE_SHIFT) { |
| dev_err(hr_dev->dev, "failed to check umem page shift %u!\n", |
| page_shift); |
| return -EINVAL; |
| } |
| |
| /* convert system page cnt to hw page cnt */ |
| rdma_umem_for_each_dma_block(umem, &biter, 1 << page_shift) { |
| addr = rdma_block_iter_dma_address(&biter); |
| if (idx >= start) { |
| bufs[total++] = addr; |
| if (total >= buf_cnt) |
| goto done; |
| } |
| idx++; |
| } |
| |
| done: |
| return total; |
| } |
| |
| void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev) |
| { |
| if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) |
| hns_roce_cleanup_srq_table(hr_dev); |
| hns_roce_cleanup_qp_table(hr_dev); |
| hns_roce_cleanup_cq_table(hr_dev); |
| hns_roce_cleanup_mr_table(hr_dev); |
| hns_roce_cleanup_pd_table(hr_dev); |
| hns_roce_cleanup_uar_table(hr_dev); |
| } |