blob: 48729bf171e04478b83f705ddcbe7e5ed33c8436 [file] [log] [blame]
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
/* Copyright (C) 2017-2018 Netronome Systems, Inc. */
#include <linux/hash.h>
#include <linux/hashtable.h>
#include <linux/jhash.h>
#include <linux/vmalloc.h>
#include <net/pkt_cls.h>
#include "cmsg.h"
#include "main.h"
#include "../nfp_app.h"
struct nfp_mask_id_table {
struct hlist_node link;
u32 hash_key;
u32 ref_cnt;
u8 mask_id;
};
struct nfp_fl_flow_table_cmp_arg {
struct net_device *netdev;
unsigned long cookie;
__be32 host_ctx;
};
static int nfp_release_stats_entry(struct nfp_app *app, u32 stats_context_id)
{
struct nfp_flower_priv *priv = app->priv;
struct circ_buf *ring;
ring = &priv->stats_ids.free_list;
/* Check if buffer is full. */
if (!CIRC_SPACE(ring->head, ring->tail,
priv->stats_ring_size * NFP_FL_STATS_ELEM_RS -
NFP_FL_STATS_ELEM_RS + 1))
return -ENOBUFS;
memcpy(&ring->buf[ring->head], &stats_context_id, NFP_FL_STATS_ELEM_RS);
ring->head = (ring->head + NFP_FL_STATS_ELEM_RS) %
(priv->stats_ring_size * NFP_FL_STATS_ELEM_RS);
return 0;
}
static int nfp_get_stats_entry(struct nfp_app *app, u32 *stats_context_id)
{
struct nfp_flower_priv *priv = app->priv;
u32 freed_stats_id, temp_stats_id;
struct circ_buf *ring;
ring = &priv->stats_ids.free_list;
freed_stats_id = priv->stats_ring_size;
/* Check for unallocated entries first. */
if (priv->stats_ids.init_unalloc > 0) {
*stats_context_id = priv->stats_ids.init_unalloc - 1;
priv->stats_ids.init_unalloc--;
return 0;
}
/* Check if buffer is empty. */
if (ring->head == ring->tail) {
*stats_context_id = freed_stats_id;
return -ENOENT;
}
memcpy(&temp_stats_id, &ring->buf[ring->tail], NFP_FL_STATS_ELEM_RS);
*stats_context_id = temp_stats_id;
memcpy(&ring->buf[ring->tail], &freed_stats_id, NFP_FL_STATS_ELEM_RS);
ring->tail = (ring->tail + NFP_FL_STATS_ELEM_RS) %
(priv->stats_ring_size * NFP_FL_STATS_ELEM_RS);
return 0;
}
/* Must be called with either RTNL or rcu_read_lock */
struct nfp_fl_payload *
nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie,
struct net_device *netdev, __be32 host_ctx)
{
struct nfp_fl_flow_table_cmp_arg flower_cmp_arg;
struct nfp_flower_priv *priv = app->priv;
flower_cmp_arg.netdev = netdev;
flower_cmp_arg.cookie = tc_flower_cookie;
flower_cmp_arg.host_ctx = host_ctx;
return rhashtable_lookup_fast(&priv->flow_table, &flower_cmp_arg,
nfp_flower_table_params);
}
void nfp_flower_rx_flow_stats(struct nfp_app *app, struct sk_buff *skb)
{
unsigned int msg_len = nfp_flower_cmsg_get_data_len(skb);
struct nfp_flower_priv *priv = app->priv;
struct nfp_fl_stats_frame *stats;
unsigned char *msg;
u32 ctx_id;
int i;
msg = nfp_flower_cmsg_get_data(skb);
spin_lock(&priv->stats_lock);
for (i = 0; i < msg_len / sizeof(*stats); i++) {
stats = (struct nfp_fl_stats_frame *)msg + i;
ctx_id = be32_to_cpu(stats->stats_con_id);
priv->stats[ctx_id].pkts += be32_to_cpu(stats->pkt_count);
priv->stats[ctx_id].bytes += be64_to_cpu(stats->byte_count);
priv->stats[ctx_id].used = jiffies;
}
spin_unlock(&priv->stats_lock);
}
static int nfp_release_mask_id(struct nfp_app *app, u8 mask_id)
{
struct nfp_flower_priv *priv = app->priv;
struct circ_buf *ring;
ring = &priv->mask_ids.mask_id_free_list;
/* Checking if buffer is full. */
if (CIRC_SPACE(ring->head, ring->tail, NFP_FLOWER_MASK_ENTRY_RS) == 0)
return -ENOBUFS;
memcpy(&ring->buf[ring->head], &mask_id, NFP_FLOWER_MASK_ELEMENT_RS);
ring->head = (ring->head + NFP_FLOWER_MASK_ELEMENT_RS) %
(NFP_FLOWER_MASK_ENTRY_RS * NFP_FLOWER_MASK_ELEMENT_RS);
priv->mask_ids.last_used[mask_id] = ktime_get();
return 0;
}
static int nfp_mask_alloc(struct nfp_app *app, u8 *mask_id)
{
struct nfp_flower_priv *priv = app->priv;
ktime_t reuse_timeout;
struct circ_buf *ring;
u8 temp_id, freed_id;
ring = &priv->mask_ids.mask_id_free_list;
freed_id = NFP_FLOWER_MASK_ENTRY_RS - 1;
/* Checking for unallocated entries first. */
if (priv->mask_ids.init_unallocated > 0) {
*mask_id = priv->mask_ids.init_unallocated;
priv->mask_ids.init_unallocated--;
return 0;
}
/* Checking if buffer is empty. */
if (ring->head == ring->tail)
goto err_not_found;
memcpy(&temp_id, &ring->buf[ring->tail], NFP_FLOWER_MASK_ELEMENT_RS);
*mask_id = temp_id;
reuse_timeout = ktime_add_ns(priv->mask_ids.last_used[*mask_id],
NFP_FL_MASK_REUSE_TIME_NS);
if (ktime_before(ktime_get(), reuse_timeout))
goto err_not_found;
memcpy(&ring->buf[ring->tail], &freed_id, NFP_FLOWER_MASK_ELEMENT_RS);
ring->tail = (ring->tail + NFP_FLOWER_MASK_ELEMENT_RS) %
(NFP_FLOWER_MASK_ENTRY_RS * NFP_FLOWER_MASK_ELEMENT_RS);
return 0;
err_not_found:
*mask_id = freed_id;
return -ENOENT;
}
static int
nfp_add_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len)
{
struct nfp_flower_priv *priv = app->priv;
struct nfp_mask_id_table *mask_entry;
unsigned long hash_key;
u8 mask_id;
if (nfp_mask_alloc(app, &mask_id))
return -ENOENT;
mask_entry = kmalloc(sizeof(*mask_entry), GFP_KERNEL);
if (!mask_entry) {
nfp_release_mask_id(app, mask_id);
return -ENOMEM;
}
INIT_HLIST_NODE(&mask_entry->link);
mask_entry->mask_id = mask_id;
hash_key = jhash(mask_data, mask_len, priv->mask_id_seed);
mask_entry->hash_key = hash_key;
mask_entry->ref_cnt = 1;
hash_add(priv->mask_table, &mask_entry->link, hash_key);
return mask_id;
}
static struct nfp_mask_id_table *
nfp_search_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len)
{
struct nfp_flower_priv *priv = app->priv;
struct nfp_mask_id_table *mask_entry;
unsigned long hash_key;
hash_key = jhash(mask_data, mask_len, priv->mask_id_seed);
hash_for_each_possible(priv->mask_table, mask_entry, link, hash_key)
if (mask_entry->hash_key == hash_key)
return mask_entry;
return NULL;
}
static int
nfp_find_in_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len)
{
struct nfp_mask_id_table *mask_entry;
mask_entry = nfp_search_mask_table(app, mask_data, mask_len);
if (!mask_entry)
return -ENOENT;
mask_entry->ref_cnt++;
/* Casting u8 to int for later use. */
return mask_entry->mask_id;
}
static bool
nfp_check_mask_add(struct nfp_app *app, char *mask_data, u32 mask_len,
u8 *meta_flags, u8 *mask_id)
{
int id;
id = nfp_find_in_mask_table(app, mask_data, mask_len);
if (id < 0) {
id = nfp_add_mask_table(app, mask_data, mask_len);
if (id < 0)
return false;
*meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK;
}
*mask_id = id;
return true;
}
static bool
nfp_check_mask_remove(struct nfp_app *app, char *mask_data, u32 mask_len,
u8 *meta_flags, u8 *mask_id)
{
struct nfp_mask_id_table *mask_entry;
mask_entry = nfp_search_mask_table(app, mask_data, mask_len);
if (!mask_entry)
return false;
if (meta_flags)
*meta_flags &= ~NFP_FL_META_FLAG_MANAGE_MASK;
*mask_id = mask_entry->mask_id;
mask_entry->ref_cnt--;
if (!mask_entry->ref_cnt) {
hash_del(&mask_entry->link);
nfp_release_mask_id(app, *mask_id);
kfree(mask_entry);
if (meta_flags)
*meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK;
}
return true;
}
int nfp_compile_flow_metadata(struct nfp_app *app,
struct tc_cls_flower_offload *flow,
struct nfp_fl_payload *nfp_flow,
struct net_device *netdev)
{
struct nfp_flower_priv *priv = app->priv;
struct nfp_fl_payload *check_entry;
u8 new_mask_id;
u32 stats_cxt;
if (nfp_get_stats_entry(app, &stats_cxt))
return -ENOENT;
nfp_flow->meta.host_ctx_id = cpu_to_be32(stats_cxt);
nfp_flow->meta.host_cookie = cpu_to_be64(flow->cookie);
new_mask_id = 0;
if (!nfp_check_mask_add(app, nfp_flow->mask_data,
nfp_flow->meta.mask_len,
&nfp_flow->meta.flags, &new_mask_id)) {
if (nfp_release_stats_entry(app, stats_cxt))
return -EINVAL;
return -ENOENT;
}
nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version);
priv->flower_version++;
/* Update flow payload with mask ids. */
nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id;
priv->stats[stats_cxt].pkts = 0;
priv->stats[stats_cxt].bytes = 0;
priv->stats[stats_cxt].used = jiffies;
check_entry = nfp_flower_search_fl_table(app, flow->cookie, netdev,
NFP_FL_STATS_CTX_DONT_CARE);
if (check_entry) {
if (nfp_release_stats_entry(app, stats_cxt))
return -EINVAL;
if (!nfp_check_mask_remove(app, nfp_flow->mask_data,
nfp_flow->meta.mask_len,
NULL, &new_mask_id))
return -EINVAL;
return -EEXIST;
}
return 0;
}
int nfp_modify_flow_metadata(struct nfp_app *app,
struct nfp_fl_payload *nfp_flow)
{
struct nfp_flower_priv *priv = app->priv;
u8 new_mask_id = 0;
u32 temp_ctx_id;
nfp_check_mask_remove(app, nfp_flow->mask_data,
nfp_flow->meta.mask_len, &nfp_flow->meta.flags,
&new_mask_id);
nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version);
priv->flower_version++;
/* Update flow payload with mask ids. */
nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id;
/* Release the stats ctx id. */
temp_ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id);
return nfp_release_stats_entry(app, temp_ctx_id);
}
static int nfp_fl_obj_cmpfn(struct rhashtable_compare_arg *arg,
const void *obj)
{
const struct nfp_fl_flow_table_cmp_arg *cmp_arg = arg->key;
const struct nfp_fl_payload *flow_entry = obj;
if ((!cmp_arg->netdev || flow_entry->ingress_dev == cmp_arg->netdev) &&
(cmp_arg->host_ctx == NFP_FL_STATS_CTX_DONT_CARE ||
flow_entry->meta.host_ctx_id == cmp_arg->host_ctx))
return flow_entry->tc_flower_cookie != cmp_arg->cookie;
return 1;
}
static u32 nfp_fl_obj_hashfn(const void *data, u32 len, u32 seed)
{
const struct nfp_fl_payload *flower_entry = data;
return jhash2((u32 *)&flower_entry->tc_flower_cookie,
sizeof(flower_entry->tc_flower_cookie) / sizeof(u32),
seed);
}
static u32 nfp_fl_key_hashfn(const void *data, u32 len, u32 seed)
{
const struct nfp_fl_flow_table_cmp_arg *cmp_arg = data;
return jhash2((u32 *)&cmp_arg->cookie,
sizeof(cmp_arg->cookie) / sizeof(u32), seed);
}
const struct rhashtable_params nfp_flower_table_params = {
.head_offset = offsetof(struct nfp_fl_payload, fl_node),
.hashfn = nfp_fl_key_hashfn,
.obj_cmpfn = nfp_fl_obj_cmpfn,
.obj_hashfn = nfp_fl_obj_hashfn,
.automatic_shrinking = true,
};
int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count)
{
struct nfp_flower_priv *priv = app->priv;
int err;
hash_init(priv->mask_table);
err = rhashtable_init(&priv->flow_table, &nfp_flower_table_params);
if (err)
return err;
get_random_bytes(&priv->mask_id_seed, sizeof(priv->mask_id_seed));
/* Init ring buffer and unallocated mask_ids. */
priv->mask_ids.mask_id_free_list.buf =
kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS,
NFP_FLOWER_MASK_ELEMENT_RS, GFP_KERNEL);
if (!priv->mask_ids.mask_id_free_list.buf)
goto err_free_flow_table;
priv->mask_ids.init_unallocated = NFP_FLOWER_MASK_ENTRY_RS - 1;
/* Init timestamps for mask id*/
priv->mask_ids.last_used =
kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS,
sizeof(*priv->mask_ids.last_used), GFP_KERNEL);
if (!priv->mask_ids.last_used)
goto err_free_mask_id;
/* Init ring buffer and unallocated stats_ids. */
priv->stats_ids.free_list.buf =
vmalloc(array_size(NFP_FL_STATS_ELEM_RS,
priv->stats_ring_size));
if (!priv->stats_ids.free_list.buf)
goto err_free_last_used;
priv->stats_ids.init_unalloc = host_ctx_count;
priv->stats = kvmalloc_array(priv->stats_ring_size,
sizeof(struct nfp_fl_stats), GFP_KERNEL);
if (!priv->stats)
goto err_free_ring_buf;
spin_lock_init(&priv->stats_lock);
return 0;
err_free_ring_buf:
vfree(priv->stats_ids.free_list.buf);
err_free_last_used:
kfree(priv->mask_ids.last_used);
err_free_mask_id:
kfree(priv->mask_ids.mask_id_free_list.buf);
err_free_flow_table:
rhashtable_destroy(&priv->flow_table);
return -ENOMEM;
}
void nfp_flower_metadata_cleanup(struct nfp_app *app)
{
struct nfp_flower_priv *priv = app->priv;
if (!priv)
return;
rhashtable_free_and_destroy(&priv->flow_table,
nfp_check_rhashtable_empty, NULL);
kvfree(priv->stats);
kfree(priv->mask_ids.mask_id_free_list.buf);
kfree(priv->mask_ids.last_used);
vfree(priv->stats_ids.free_list.buf);
}