| // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
| /* Copyright (c) 2019 Mellanox Technologies. */ |
| |
| #include "dr_types.h" |
| |
| #define DR_ICM_MODIFY_HDR_ALIGN_BASE 64 |
| #define DR_ICM_POOL_STE_HOT_MEM_PERCENT 25 |
| #define DR_ICM_POOL_MODIFY_HDR_PTRN_HOT_MEM_PERCENT 50 |
| #define DR_ICM_POOL_MODIFY_ACTION_HOT_MEM_PERCENT 90 |
| |
| struct mlx5dr_icm_hot_chunk { |
| struct mlx5dr_icm_buddy_mem *buddy_mem; |
| unsigned int seg; |
| enum mlx5dr_icm_chunk_size size; |
| }; |
| |
| struct mlx5dr_icm_pool { |
| enum mlx5dr_icm_type icm_type; |
| enum mlx5dr_icm_chunk_size max_log_chunk_sz; |
| struct mlx5dr_domain *dmn; |
| struct kmem_cache *chunks_kmem_cache; |
| |
| /* memory management */ |
| struct mutex mutex; /* protect the ICM pool and ICM buddy */ |
| struct list_head buddy_mem_list; |
| |
| /* Hardware may be accessing this memory but at some future, |
| * undetermined time, it might cease to do so. |
| * sync_ste command sets them free. |
| */ |
| struct mlx5dr_icm_hot_chunk *hot_chunks_arr; |
| u32 hot_chunks_num; |
| u64 hot_memory_size; |
| /* hot memory size threshold for triggering sync */ |
| u64 th; |
| }; |
| |
| struct mlx5dr_icm_dm { |
| u32 obj_id; |
| enum mlx5_sw_icm_type type; |
| phys_addr_t addr; |
| size_t length; |
| }; |
| |
| struct mlx5dr_icm_mr { |
| u32 mkey; |
| struct mlx5dr_icm_dm dm; |
| struct mlx5dr_domain *dmn; |
| size_t length; |
| u64 icm_start_addr; |
| }; |
| |
| static int dr_icm_create_dm_mkey(struct mlx5_core_dev *mdev, |
| u32 pd, u64 length, u64 start_addr, int mode, |
| u32 *mkey) |
| { |
| u32 inlen = MLX5_ST_SZ_BYTES(create_mkey_in); |
| u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {}; |
| void *mkc; |
| |
| mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); |
| |
| MLX5_SET(mkc, mkc, access_mode_1_0, mode); |
| MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7); |
| MLX5_SET(mkc, mkc, lw, 1); |
| MLX5_SET(mkc, mkc, lr, 1); |
| if (mode == MLX5_MKC_ACCESS_MODE_SW_ICM) { |
| MLX5_SET(mkc, mkc, rw, 1); |
| MLX5_SET(mkc, mkc, rr, 1); |
| } |
| |
| MLX5_SET64(mkc, mkc, len, length); |
| MLX5_SET(mkc, mkc, pd, pd); |
| MLX5_SET(mkc, mkc, qpn, 0xffffff); |
| MLX5_SET64(mkc, mkc, start_addr, start_addr); |
| |
| return mlx5_core_create_mkey(mdev, mkey, in, inlen); |
| } |
| |
| u64 mlx5dr_icm_pool_get_chunk_mr_addr(struct mlx5dr_icm_chunk *chunk) |
| { |
| u32 offset = mlx5dr_icm_pool_dm_type_to_entry_size(chunk->buddy_mem->pool->icm_type); |
| |
| return (u64)offset * chunk->seg; |
| } |
| |
| u32 mlx5dr_icm_pool_get_chunk_rkey(struct mlx5dr_icm_chunk *chunk) |
| { |
| return chunk->buddy_mem->icm_mr->mkey; |
| } |
| |
| u64 mlx5dr_icm_pool_get_chunk_icm_addr(struct mlx5dr_icm_chunk *chunk) |
| { |
| u32 size = mlx5dr_icm_pool_dm_type_to_entry_size(chunk->buddy_mem->pool->icm_type); |
| |
| return (u64)chunk->buddy_mem->icm_mr->icm_start_addr + size * chunk->seg; |
| } |
| |
| u32 mlx5dr_icm_pool_get_chunk_byte_size(struct mlx5dr_icm_chunk *chunk) |
| { |
| return mlx5dr_icm_pool_chunk_size_to_byte(chunk->size, |
| chunk->buddy_mem->pool->icm_type); |
| } |
| |
| u32 mlx5dr_icm_pool_get_chunk_num_of_entries(struct mlx5dr_icm_chunk *chunk) |
| { |
| return mlx5dr_icm_pool_chunk_size_to_entries(chunk->size); |
| } |
| |
| static struct mlx5dr_icm_mr * |
| dr_icm_pool_mr_create(struct mlx5dr_icm_pool *pool) |
| { |
| struct mlx5_core_dev *mdev = pool->dmn->mdev; |
| enum mlx5_sw_icm_type dm_type = 0; |
| struct mlx5dr_icm_mr *icm_mr; |
| size_t log_align_base = 0; |
| int err; |
| |
| icm_mr = kvzalloc(sizeof(*icm_mr), GFP_KERNEL); |
| if (!icm_mr) |
| return NULL; |
| |
| icm_mr->dmn = pool->dmn; |
| |
| icm_mr->dm.length = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz, |
| pool->icm_type); |
| |
| switch (pool->icm_type) { |
| case DR_ICM_TYPE_STE: |
| dm_type = MLX5_SW_ICM_TYPE_STEERING; |
| log_align_base = ilog2(icm_mr->dm.length); |
| break; |
| case DR_ICM_TYPE_MODIFY_ACTION: |
| dm_type = MLX5_SW_ICM_TYPE_HEADER_MODIFY; |
| /* Align base is 64B */ |
| log_align_base = ilog2(DR_ICM_MODIFY_HDR_ALIGN_BASE); |
| break; |
| case DR_ICM_TYPE_MODIFY_HDR_PTRN: |
| dm_type = MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN; |
| /* Align base is 64B */ |
| log_align_base = ilog2(DR_ICM_MODIFY_HDR_ALIGN_BASE); |
| break; |
| default: |
| WARN_ON(pool->icm_type); |
| } |
| |
| icm_mr->dm.type = dm_type; |
| |
| err = mlx5_dm_sw_icm_alloc(mdev, icm_mr->dm.type, icm_mr->dm.length, |
| log_align_base, 0, &icm_mr->dm.addr, |
| &icm_mr->dm.obj_id); |
| if (err) { |
| mlx5dr_err(pool->dmn, "Failed to allocate SW ICM memory, err (%d)\n", err); |
| goto free_icm_mr; |
| } |
| |
| /* Register device memory */ |
| err = dr_icm_create_dm_mkey(mdev, pool->dmn->pdn, |
| icm_mr->dm.length, |
| icm_mr->dm.addr, |
| MLX5_MKC_ACCESS_MODE_SW_ICM, |
| &icm_mr->mkey); |
| if (err) { |
| mlx5dr_err(pool->dmn, "Failed to create SW ICM MKEY, err (%d)\n", err); |
| goto free_dm; |
| } |
| |
| icm_mr->icm_start_addr = icm_mr->dm.addr; |
| |
| if (icm_mr->icm_start_addr & (BIT(log_align_base) - 1)) { |
| mlx5dr_err(pool->dmn, "Failed to get Aligned ICM mem (asked: %zu)\n", |
| log_align_base); |
| goto free_mkey; |
| } |
| |
| return icm_mr; |
| |
| free_mkey: |
| mlx5_core_destroy_mkey(mdev, icm_mr->mkey); |
| free_dm: |
| mlx5_dm_sw_icm_dealloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 0, |
| icm_mr->dm.addr, icm_mr->dm.obj_id); |
| free_icm_mr: |
| kvfree(icm_mr); |
| return NULL; |
| } |
| |
| static void dr_icm_pool_mr_destroy(struct mlx5dr_icm_mr *icm_mr) |
| { |
| struct mlx5_core_dev *mdev = icm_mr->dmn->mdev; |
| struct mlx5dr_icm_dm *dm = &icm_mr->dm; |
| |
| mlx5_core_destroy_mkey(mdev, icm_mr->mkey); |
| mlx5_dm_sw_icm_dealloc(mdev, dm->type, dm->length, 0, |
| dm->addr, dm->obj_id); |
| kvfree(icm_mr); |
| } |
| |
| static int dr_icm_buddy_get_ste_size(struct mlx5dr_icm_buddy_mem *buddy) |
| { |
| /* We support only one type of STE size, both for ConnectX-5 and later |
| * devices. Once the support for match STE which has a larger tag is |
| * added (32B instead of 16B), the STE size for devices later than |
| * ConnectX-5 needs to account for that. |
| */ |
| return DR_STE_SIZE_REDUCED; |
| } |
| |
| static void dr_icm_chunk_ste_init(struct mlx5dr_icm_chunk *chunk, int offset) |
| { |
| int num_of_entries = mlx5dr_icm_pool_get_chunk_num_of_entries(chunk); |
| struct mlx5dr_icm_buddy_mem *buddy = chunk->buddy_mem; |
| int ste_size = dr_icm_buddy_get_ste_size(buddy); |
| int index = offset / DR_STE_SIZE; |
| |
| chunk->ste_arr = &buddy->ste_arr[index]; |
| chunk->miss_list = &buddy->miss_list[index]; |
| chunk->hw_ste_arr = buddy->hw_ste_arr + index * ste_size; |
| |
| memset(chunk->hw_ste_arr, 0, num_of_entries * ste_size); |
| memset(chunk->ste_arr, 0, |
| num_of_entries * sizeof(chunk->ste_arr[0])); |
| } |
| |
| static int dr_icm_buddy_init_ste_cache(struct mlx5dr_icm_buddy_mem *buddy) |
| { |
| int num_of_entries = |
| mlx5dr_icm_pool_chunk_size_to_entries(buddy->pool->max_log_chunk_sz); |
| |
| buddy->ste_arr = kvcalloc(num_of_entries, |
| sizeof(struct mlx5dr_ste), GFP_KERNEL); |
| if (!buddy->ste_arr) |
| return -ENOMEM; |
| |
| /* Preallocate full STE size on non-ConnectX-5 devices since |
| * we need to support both full and reduced with the same cache. |
| */ |
| buddy->hw_ste_arr = kvcalloc(num_of_entries, |
| dr_icm_buddy_get_ste_size(buddy), GFP_KERNEL); |
| if (!buddy->hw_ste_arr) |
| goto free_ste_arr; |
| |
| buddy->miss_list = kvmalloc(num_of_entries * sizeof(struct list_head), GFP_KERNEL); |
| if (!buddy->miss_list) |
| goto free_hw_ste_arr; |
| |
| return 0; |
| |
| free_hw_ste_arr: |
| kvfree(buddy->hw_ste_arr); |
| free_ste_arr: |
| kvfree(buddy->ste_arr); |
| return -ENOMEM; |
| } |
| |
| static void dr_icm_buddy_cleanup_ste_cache(struct mlx5dr_icm_buddy_mem *buddy) |
| { |
| kvfree(buddy->ste_arr); |
| kvfree(buddy->hw_ste_arr); |
| kvfree(buddy->miss_list); |
| } |
| |
| static int dr_icm_buddy_create(struct mlx5dr_icm_pool *pool) |
| { |
| struct mlx5dr_icm_buddy_mem *buddy; |
| struct mlx5dr_icm_mr *icm_mr; |
| |
| icm_mr = dr_icm_pool_mr_create(pool); |
| if (!icm_mr) |
| return -ENOMEM; |
| |
| buddy = kvzalloc(sizeof(*buddy), GFP_KERNEL); |
| if (!buddy) |
| goto free_mr; |
| |
| if (mlx5dr_buddy_init(buddy, pool->max_log_chunk_sz)) |
| goto err_free_buddy; |
| |
| buddy->icm_mr = icm_mr; |
| buddy->pool = pool; |
| |
| if (pool->icm_type == DR_ICM_TYPE_STE) { |
| /* Reduce allocations by preallocating and reusing the STE structures */ |
| if (dr_icm_buddy_init_ste_cache(buddy)) |
| goto err_cleanup_buddy; |
| } |
| |
| /* add it to the -start- of the list in order to search in it first */ |
| list_add(&buddy->list_node, &pool->buddy_mem_list); |
| |
| pool->dmn->num_buddies[pool->icm_type]++; |
| |
| return 0; |
| |
| err_cleanup_buddy: |
| mlx5dr_buddy_cleanup(buddy); |
| err_free_buddy: |
| kvfree(buddy); |
| free_mr: |
| dr_icm_pool_mr_destroy(icm_mr); |
| return -ENOMEM; |
| } |
| |
| static void dr_icm_buddy_destroy(struct mlx5dr_icm_buddy_mem *buddy) |
| { |
| enum mlx5dr_icm_type icm_type = buddy->pool->icm_type; |
| |
| dr_icm_pool_mr_destroy(buddy->icm_mr); |
| |
| mlx5dr_buddy_cleanup(buddy); |
| |
| if (icm_type == DR_ICM_TYPE_STE) |
| dr_icm_buddy_cleanup_ste_cache(buddy); |
| |
| buddy->pool->dmn->num_buddies[icm_type]--; |
| |
| kvfree(buddy); |
| } |
| |
| static void |
| dr_icm_chunk_init(struct mlx5dr_icm_chunk *chunk, |
| struct mlx5dr_icm_pool *pool, |
| enum mlx5dr_icm_chunk_size chunk_size, |
| struct mlx5dr_icm_buddy_mem *buddy_mem_pool, |
| unsigned int seg) |
| { |
| int offset; |
| |
| chunk->seg = seg; |
| chunk->size = chunk_size; |
| chunk->buddy_mem = buddy_mem_pool; |
| |
| if (pool->icm_type == DR_ICM_TYPE_STE) { |
| offset = mlx5dr_icm_pool_dm_type_to_entry_size(pool->icm_type) * seg; |
| dr_icm_chunk_ste_init(chunk, offset); |
| } |
| |
| buddy_mem_pool->used_memory += mlx5dr_icm_pool_get_chunk_byte_size(chunk); |
| } |
| |
| static bool dr_icm_pool_is_sync_required(struct mlx5dr_icm_pool *pool) |
| { |
| return pool->hot_memory_size > pool->th; |
| } |
| |
| static void dr_icm_pool_clear_hot_chunks_arr(struct mlx5dr_icm_pool *pool) |
| { |
| struct mlx5dr_icm_hot_chunk *hot_chunk; |
| u32 i, num_entries; |
| |
| for (i = 0; i < pool->hot_chunks_num; i++) { |
| hot_chunk = &pool->hot_chunks_arr[i]; |
| num_entries = mlx5dr_icm_pool_chunk_size_to_entries(hot_chunk->size); |
| mlx5dr_buddy_free_mem(hot_chunk->buddy_mem, |
| hot_chunk->seg, ilog2(num_entries)); |
| hot_chunk->buddy_mem->used_memory -= |
| mlx5dr_icm_pool_chunk_size_to_byte(hot_chunk->size, |
| pool->icm_type); |
| } |
| |
| pool->hot_chunks_num = 0; |
| pool->hot_memory_size = 0; |
| } |
| |
| static int dr_icm_pool_sync_all_buddy_pools(struct mlx5dr_icm_pool *pool) |
| { |
| struct mlx5dr_icm_buddy_mem *buddy, *tmp_buddy; |
| int err; |
| |
| err = mlx5dr_cmd_sync_steering(pool->dmn->mdev); |
| if (err) { |
| mlx5dr_err(pool->dmn, "Failed to sync to HW (err: %d)\n", err); |
| return err; |
| } |
| |
| dr_icm_pool_clear_hot_chunks_arr(pool); |
| |
| list_for_each_entry_safe(buddy, tmp_buddy, &pool->buddy_mem_list, list_node) { |
| if (!buddy->used_memory && pool->icm_type == DR_ICM_TYPE_STE) |
| dr_icm_buddy_destroy(buddy); |
| } |
| |
| return 0; |
| } |
| |
| static int dr_icm_handle_buddies_get_mem(struct mlx5dr_icm_pool *pool, |
| enum mlx5dr_icm_chunk_size chunk_size, |
| struct mlx5dr_icm_buddy_mem **buddy, |
| unsigned int *seg) |
| { |
| struct mlx5dr_icm_buddy_mem *buddy_mem_pool; |
| bool new_mem = false; |
| int err; |
| |
| alloc_buddy_mem: |
| /* find the next free place from the buddy list */ |
| list_for_each_entry(buddy_mem_pool, &pool->buddy_mem_list, list_node) { |
| err = mlx5dr_buddy_alloc_mem(buddy_mem_pool, |
| chunk_size, seg); |
| if (!err) |
| goto found; |
| |
| if (WARN_ON(new_mem)) { |
| /* We have new memory pool, first in the list */ |
| mlx5dr_err(pool->dmn, |
| "No memory for order: %d\n", |
| chunk_size); |
| goto out; |
| } |
| } |
| |
| /* no more available allocators in that pool, create new */ |
| err = dr_icm_buddy_create(pool); |
| if (err) { |
| mlx5dr_err(pool->dmn, |
| "Failed creating buddy for order %d\n", |
| chunk_size); |
| goto out; |
| } |
| |
| /* mark we have new memory, first in list */ |
| new_mem = true; |
| goto alloc_buddy_mem; |
| |
| found: |
| *buddy = buddy_mem_pool; |
| out: |
| return err; |
| } |
| |
| /* Allocate an ICM chunk, each chunk holds a piece of ICM memory and |
| * also memory used for HW STE management for optimizations. |
| */ |
| struct mlx5dr_icm_chunk * |
| mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool, |
| enum mlx5dr_icm_chunk_size chunk_size) |
| { |
| struct mlx5dr_icm_chunk *chunk = NULL; |
| struct mlx5dr_icm_buddy_mem *buddy; |
| unsigned int seg; |
| int ret; |
| |
| if (chunk_size > pool->max_log_chunk_sz) |
| return NULL; |
| |
| mutex_lock(&pool->mutex); |
| /* find mem, get back the relevant buddy pool and seg in that mem */ |
| ret = dr_icm_handle_buddies_get_mem(pool, chunk_size, &buddy, &seg); |
| if (ret) |
| goto out; |
| |
| chunk = kmem_cache_alloc(pool->chunks_kmem_cache, GFP_KERNEL); |
| if (!chunk) |
| goto out_err; |
| |
| dr_icm_chunk_init(chunk, pool, chunk_size, buddy, seg); |
| |
| goto out; |
| |
| out_err: |
| mlx5dr_buddy_free_mem(buddy, seg, chunk_size); |
| out: |
| mutex_unlock(&pool->mutex); |
| return chunk; |
| } |
| |
| void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk) |
| { |
| struct mlx5dr_icm_buddy_mem *buddy = chunk->buddy_mem; |
| struct mlx5dr_icm_pool *pool = buddy->pool; |
| struct mlx5dr_icm_hot_chunk *hot_chunk; |
| struct kmem_cache *chunks_cache; |
| |
| chunks_cache = pool->chunks_kmem_cache; |
| |
| /* move the chunk to the waiting chunks array, AKA "hot" memory */ |
| mutex_lock(&pool->mutex); |
| |
| pool->hot_memory_size += mlx5dr_icm_pool_get_chunk_byte_size(chunk); |
| |
| hot_chunk = &pool->hot_chunks_arr[pool->hot_chunks_num++]; |
| hot_chunk->buddy_mem = chunk->buddy_mem; |
| hot_chunk->seg = chunk->seg; |
| hot_chunk->size = chunk->size; |
| |
| kmem_cache_free(chunks_cache, chunk); |
| |
| /* Check if we have chunks that are waiting for sync-ste */ |
| if (dr_icm_pool_is_sync_required(pool)) |
| dr_icm_pool_sync_all_buddy_pools(pool); |
| |
| mutex_unlock(&pool->mutex); |
| } |
| |
| struct mlx5dr_ste_htbl *mlx5dr_icm_pool_alloc_htbl(struct mlx5dr_icm_pool *pool) |
| { |
| return kmem_cache_alloc(pool->dmn->htbls_kmem_cache, GFP_KERNEL); |
| } |
| |
| void mlx5dr_icm_pool_free_htbl(struct mlx5dr_icm_pool *pool, struct mlx5dr_ste_htbl *htbl) |
| { |
| kmem_cache_free(pool->dmn->htbls_kmem_cache, htbl); |
| } |
| |
| struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn, |
| enum mlx5dr_icm_type icm_type) |
| { |
| u32 num_of_chunks, entry_size; |
| struct mlx5dr_icm_pool *pool; |
| u32 max_hot_size = 0; |
| |
| pool = kvzalloc(sizeof(*pool), GFP_KERNEL); |
| if (!pool) |
| return NULL; |
| |
| pool->dmn = dmn; |
| pool->icm_type = icm_type; |
| pool->chunks_kmem_cache = dmn->chunks_kmem_cache; |
| |
| INIT_LIST_HEAD(&pool->buddy_mem_list); |
| mutex_init(&pool->mutex); |
| |
| switch (icm_type) { |
| case DR_ICM_TYPE_STE: |
| pool->max_log_chunk_sz = dmn->info.max_log_sw_icm_sz; |
| max_hot_size = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz, |
| pool->icm_type) * |
| DR_ICM_POOL_STE_HOT_MEM_PERCENT / 100; |
| break; |
| case DR_ICM_TYPE_MODIFY_ACTION: |
| pool->max_log_chunk_sz = dmn->info.max_log_action_icm_sz; |
| max_hot_size = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz, |
| pool->icm_type) * |
| DR_ICM_POOL_MODIFY_ACTION_HOT_MEM_PERCENT / 100; |
| break; |
| case DR_ICM_TYPE_MODIFY_HDR_PTRN: |
| pool->max_log_chunk_sz = dmn->info.max_log_modify_hdr_pattern_icm_sz; |
| max_hot_size = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz, |
| pool->icm_type) * |
| DR_ICM_POOL_MODIFY_HDR_PTRN_HOT_MEM_PERCENT / 100; |
| break; |
| default: |
| WARN_ON(icm_type); |
| } |
| |
| entry_size = mlx5dr_icm_pool_dm_type_to_entry_size(pool->icm_type); |
| |
| num_of_chunks = DIV_ROUND_UP(max_hot_size, entry_size) + 1; |
| pool->th = max_hot_size; |
| |
| pool->hot_chunks_arr = kvcalloc(num_of_chunks, |
| sizeof(struct mlx5dr_icm_hot_chunk), |
| GFP_KERNEL); |
| if (!pool->hot_chunks_arr) |
| goto free_pool; |
| |
| return pool; |
| |
| free_pool: |
| kvfree(pool); |
| return NULL; |
| } |
| |
| void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool) |
| { |
| struct mlx5dr_icm_buddy_mem *buddy, *tmp_buddy; |
| |
| dr_icm_pool_clear_hot_chunks_arr(pool); |
| |
| list_for_each_entry_safe(buddy, tmp_buddy, &pool->buddy_mem_list, list_node) |
| dr_icm_buddy_destroy(buddy); |
| |
| kvfree(pool->hot_chunks_arr); |
| mutex_destroy(&pool->mutex); |
| kvfree(pool); |
| } |