| // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
| /* Copyright (C) 2015-2018 Netronome Systems, Inc. */ |
| |
| /* |
| * nfp_cppcore.c |
| * Provides low-level access to the NFP's internal CPP bus |
| * Authors: Jakub Kicinski <jakub.kicinski@netronome.com> |
| * Jason McMullan <jason.mcmullan@netronome.com> |
| * Rolf Neugebauer <rolf.neugebauer@netronome.com> |
| */ |
| |
| #include <asm/unaligned.h> |
| #include <linux/delay.h> |
| #include <linux/device.h> |
| #include <linux/ioport.h> |
| #include <linux/kernel.h> |
| #include <linux/module.h> |
| #include <linux/mutex.h> |
| #include <linux/sched.h> |
| #include <linux/slab.h> |
| #include <linux/wait.h> |
| |
| #include "nfp_arm.h" |
| #include "nfp_cpp.h" |
| #include "nfp6000/nfp6000.h" |
| |
| #define NFP_ARM_GCSR_SOFTMODEL2 0x0000014c |
| #define NFP_ARM_GCSR_SOFTMODEL3 0x00000150 |
| |
| struct nfp_cpp_resource { |
| struct list_head list; |
| const char *name; |
| u32 cpp_id; |
| u64 start; |
| u64 end; |
| }; |
| |
| /** |
| * struct nfp_cpp - main nfpcore device structure |
| * Following fields are read-only after probe() exits or netdevs are spawned. |
| * @dev: embedded device structure |
| * @op: low-level implementation ops |
| * @priv: private data of the low-level implementation |
| * @model: chip model |
| * @interface: chip interface id we are using to reach it |
| * @serial: chip serial number |
| * @imb_cat_table: CPP Mapping Table |
| * @mu_locality_lsb: MU access type bit offset |
| * |
| * Following fields use explicit locking: |
| * @resource_list: NFP CPP resource list |
| * @resource_lock: protects @resource_list |
| * |
| * @area_cache_list: cached areas for cpp/xpb read/write speed up |
| * @area_cache_mutex: protects @area_cache_list |
| * |
| * @waitq: area wait queue |
| */ |
| struct nfp_cpp { |
| struct device dev; |
| |
| void *priv; |
| |
| u32 model; |
| u16 interface; |
| u8 serial[NFP_SERIAL_LEN]; |
| |
| const struct nfp_cpp_operations *op; |
| struct list_head resource_list; |
| rwlock_t resource_lock; |
| wait_queue_head_t waitq; |
| |
| u32 imb_cat_table[16]; |
| unsigned int mu_locality_lsb; |
| |
| struct mutex area_cache_mutex; |
| struct list_head area_cache_list; |
| }; |
| |
| /* Element of the area_cache_list */ |
| struct nfp_cpp_area_cache { |
| struct list_head entry; |
| u32 id; |
| u64 addr; |
| u32 size; |
| struct nfp_cpp_area *area; |
| }; |
| |
| struct nfp_cpp_area { |
| struct nfp_cpp *cpp; |
| struct kref kref; |
| atomic_t refcount; |
| struct mutex mutex; /* Lock for the area's refcount */ |
| unsigned long long offset; |
| unsigned long size; |
| struct nfp_cpp_resource resource; |
| void __iomem *iomem; |
| /* Here follows the 'priv' part of nfp_cpp_area. */ |
| }; |
| |
| struct nfp_cpp_explicit { |
| struct nfp_cpp *cpp; |
| struct nfp_cpp_explicit_command cmd; |
| /* Here follows the 'priv' part of nfp_cpp_area. */ |
| }; |
| |
| static void __resource_add(struct list_head *head, struct nfp_cpp_resource *res) |
| { |
| struct nfp_cpp_resource *tmp; |
| struct list_head *pos; |
| |
| list_for_each(pos, head) { |
| tmp = container_of(pos, struct nfp_cpp_resource, list); |
| |
| if (tmp->cpp_id > res->cpp_id) |
| break; |
| |
| if (tmp->cpp_id == res->cpp_id && tmp->start > res->start) |
| break; |
| } |
| |
| list_add_tail(&res->list, pos); |
| } |
| |
| static void __resource_del(struct nfp_cpp_resource *res) |
| { |
| list_del_init(&res->list); |
| } |
| |
| static void __release_cpp_area(struct kref *kref) |
| { |
| struct nfp_cpp_area *area = |
| container_of(kref, struct nfp_cpp_area, kref); |
| struct nfp_cpp *cpp = nfp_cpp_area_cpp(area); |
| |
| if (area->cpp->op->area_cleanup) |
| area->cpp->op->area_cleanup(area); |
| |
| write_lock(&cpp->resource_lock); |
| __resource_del(&area->resource); |
| write_unlock(&cpp->resource_lock); |
| kfree(area); |
| } |
| |
| static void nfp_cpp_area_put(struct nfp_cpp_area *area) |
| { |
| kref_put(&area->kref, __release_cpp_area); |
| } |
| |
| static struct nfp_cpp_area *nfp_cpp_area_get(struct nfp_cpp_area *area) |
| { |
| kref_get(&area->kref); |
| |
| return area; |
| } |
| |
| /** |
| * nfp_cpp_free() - free the CPP handle |
| * @cpp: CPP handle |
| */ |
| void nfp_cpp_free(struct nfp_cpp *cpp) |
| { |
| struct nfp_cpp_area_cache *cache, *ctmp; |
| struct nfp_cpp_resource *res, *rtmp; |
| |
| /* Remove all caches */ |
| list_for_each_entry_safe(cache, ctmp, &cpp->area_cache_list, entry) { |
| list_del(&cache->entry); |
| if (cache->id) |
| nfp_cpp_area_release(cache->area); |
| nfp_cpp_area_free(cache->area); |
| kfree(cache); |
| } |
| |
| /* There should be no dangling areas at this point */ |
| WARN_ON(!list_empty(&cpp->resource_list)); |
| |
| /* .. but if they weren't, try to clean up. */ |
| list_for_each_entry_safe(res, rtmp, &cpp->resource_list, list) { |
| struct nfp_cpp_area *area = container_of(res, |
| struct nfp_cpp_area, |
| resource); |
| |
| dev_err(cpp->dev.parent, "Dangling area: %d:%d:%d:0x%0llx-0x%0llx%s%s\n", |
| NFP_CPP_ID_TARGET_of(res->cpp_id), |
| NFP_CPP_ID_ACTION_of(res->cpp_id), |
| NFP_CPP_ID_TOKEN_of(res->cpp_id), |
| res->start, res->end, |
| res->name ? " " : "", |
| res->name ? res->name : ""); |
| |
| if (area->cpp->op->area_release) |
| area->cpp->op->area_release(area); |
| |
| __release_cpp_area(&area->kref); |
| } |
| |
| if (cpp->op->free) |
| cpp->op->free(cpp); |
| |
| device_unregister(&cpp->dev); |
| |
| kfree(cpp); |
| } |
| |
| /** |
| * nfp_cpp_model() - Retrieve the Model ID of the NFP |
| * @cpp: NFP CPP handle |
| * |
| * Return: NFP CPP Model ID |
| */ |
| u32 nfp_cpp_model(struct nfp_cpp *cpp) |
| { |
| return cpp->model; |
| } |
| |
| /** |
| * nfp_cpp_interface() - Retrieve the Interface ID of the NFP |
| * @cpp: NFP CPP handle |
| * |
| * Return: NFP CPP Interface ID |
| */ |
| u16 nfp_cpp_interface(struct nfp_cpp *cpp) |
| { |
| return cpp->interface; |
| } |
| |
| /** |
| * nfp_cpp_serial() - Retrieve the Serial ID of the NFP |
| * @cpp: NFP CPP handle |
| * @serial: Pointer to NFP serial number |
| * |
| * Return: Length of NFP serial number |
| */ |
| int nfp_cpp_serial(struct nfp_cpp *cpp, const u8 **serial) |
| { |
| *serial = &cpp->serial[0]; |
| return sizeof(cpp->serial); |
| } |
| |
| #define NFP_IMB_TGTADDRESSMODECFG_MODE_of(_x) (((_x) >> 13) & 0x7) |
| #define NFP_IMB_TGTADDRESSMODECFG_ADDRMODE BIT(12) |
| #define NFP_IMB_TGTADDRESSMODECFG_ADDRMODE_32_BIT 0 |
| #define NFP_IMB_TGTADDRESSMODECFG_ADDRMODE_40_BIT BIT(12) |
| |
| static int nfp_cpp_set_mu_locality_lsb(struct nfp_cpp *cpp) |
| { |
| unsigned int mode, addr40; |
| u32 imbcppat; |
| int res; |
| |
| imbcppat = cpp->imb_cat_table[NFP_CPP_TARGET_MU]; |
| mode = NFP_IMB_TGTADDRESSMODECFG_MODE_of(imbcppat); |
| addr40 = !!(imbcppat & NFP_IMB_TGTADDRESSMODECFG_ADDRMODE); |
| |
| res = nfp_cppat_mu_locality_lsb(mode, addr40); |
| if (res < 0) |
| return res; |
| cpp->mu_locality_lsb = res; |
| |
| return 0; |
| } |
| |
| unsigned int nfp_cpp_mu_locality_lsb(struct nfp_cpp *cpp) |
| { |
| return cpp->mu_locality_lsb; |
| } |
| |
| /** |
| * nfp_cpp_area_alloc_with_name() - allocate a new CPP area |
| * @cpp: CPP device handle |
| * @dest: NFP CPP ID |
| * @name: Name of region |
| * @address: Address of region |
| * @size: Size of region |
| * |
| * Allocate and initialize a CPP area structure. The area must later |
| * be locked down with an 'acquire' before it can be safely accessed. |
| * |
| * NOTE: @address and @size must be 32-bit aligned values. |
| * |
| * Return: NFP CPP area handle, or NULL |
| */ |
| struct nfp_cpp_area * |
| nfp_cpp_area_alloc_with_name(struct nfp_cpp *cpp, u32 dest, const char *name, |
| unsigned long long address, unsigned long size) |
| { |
| struct nfp_cpp_area *area; |
| u64 tmp64 = address; |
| int err, name_len; |
| |
| /* Remap from cpp_island to cpp_target */ |
| err = nfp_target_cpp(dest, tmp64, &dest, &tmp64, cpp->imb_cat_table); |
| if (err < 0) |
| return NULL; |
| |
| address = tmp64; |
| |
| if (!name) |
| name = "(reserved)"; |
| |
| name_len = strlen(name) + 1; |
| area = kzalloc(sizeof(*area) + cpp->op->area_priv_size + name_len, |
| GFP_KERNEL); |
| if (!area) |
| return NULL; |
| |
| area->cpp = cpp; |
| area->resource.name = (void *)area + sizeof(*area) + |
| cpp->op->area_priv_size; |
| memcpy((char *)area->resource.name, name, name_len); |
| |
| area->resource.cpp_id = dest; |
| area->resource.start = address; |
| area->resource.end = area->resource.start + size - 1; |
| INIT_LIST_HEAD(&area->resource.list); |
| |
| atomic_set(&area->refcount, 0); |
| kref_init(&area->kref); |
| mutex_init(&area->mutex); |
| |
| if (cpp->op->area_init) { |
| int err; |
| |
| err = cpp->op->area_init(area, dest, address, size); |
| if (err < 0) { |
| kfree(area); |
| return NULL; |
| } |
| } |
| |
| write_lock(&cpp->resource_lock); |
| __resource_add(&cpp->resource_list, &area->resource); |
| write_unlock(&cpp->resource_lock); |
| |
| area->offset = address; |
| area->size = size; |
| |
| return area; |
| } |
| |
| /** |
| * nfp_cpp_area_alloc() - allocate a new CPP area |
| * @cpp: CPP handle |
| * @dest: CPP id |
| * @address: Start address on CPP target |
| * @size: Size of area in bytes |
| * |
| * Allocate and initialize a CPP area structure. The area must later |
| * be locked down with an 'acquire' before it can be safely accessed. |
| * |
| * NOTE: @address and @size must be 32-bit aligned values. |
| * |
| * Return: NFP CPP Area handle, or NULL |
| */ |
| struct nfp_cpp_area * |
| nfp_cpp_area_alloc(struct nfp_cpp *cpp, u32 dest, |
| unsigned long long address, unsigned long size) |
| { |
| return nfp_cpp_area_alloc_with_name(cpp, dest, NULL, address, size); |
| } |
| |
| /** |
| * nfp_cpp_area_alloc_acquire() - allocate a new CPP area and lock it down |
| * @cpp: CPP handle |
| * @name: Name of region |
| * @dest: CPP id |
| * @address: Start address on CPP target |
| * @size: Size of area |
| * |
| * Allocate and initialize a CPP area structure, and lock it down so |
| * that it can be accessed directly. |
| * |
| * NOTE: @address and @size must be 32-bit aligned values. |
| * The area must also be 'released' when the structure is freed. |
| * |
| * Return: NFP CPP Area handle, or NULL |
| */ |
| struct nfp_cpp_area * |
| nfp_cpp_area_alloc_acquire(struct nfp_cpp *cpp, const char *name, u32 dest, |
| unsigned long long address, unsigned long size) |
| { |
| struct nfp_cpp_area *area; |
| |
| area = nfp_cpp_area_alloc_with_name(cpp, dest, name, address, size); |
| if (!area) |
| return NULL; |
| |
| if (nfp_cpp_area_acquire(area)) { |
| nfp_cpp_area_free(area); |
| return NULL; |
| } |
| |
| return area; |
| } |
| |
| /** |
| * nfp_cpp_area_free() - free up the CPP area |
| * @area: CPP area handle |
| * |
| * Frees up memory resources held by the CPP area. |
| */ |
| void nfp_cpp_area_free(struct nfp_cpp_area *area) |
| { |
| if (atomic_read(&area->refcount)) |
| nfp_warn(area->cpp, "Warning: freeing busy area\n"); |
| nfp_cpp_area_put(area); |
| } |
| |
| static bool nfp_cpp_area_acquire_try(struct nfp_cpp_area *area, int *status) |
| { |
| *status = area->cpp->op->area_acquire(area); |
| |
| return *status != -EAGAIN; |
| } |
| |
| static int __nfp_cpp_area_acquire(struct nfp_cpp_area *area) |
| { |
| int err, status; |
| |
| if (atomic_inc_return(&area->refcount) > 1) |
| return 0; |
| |
| if (!area->cpp->op->area_acquire) |
| return 0; |
| |
| err = wait_event_interruptible(area->cpp->waitq, |
| nfp_cpp_area_acquire_try(area, &status)); |
| if (!err) |
| err = status; |
| if (err) { |
| nfp_warn(area->cpp, "Warning: area wait failed: %d\n", err); |
| atomic_dec(&area->refcount); |
| return err; |
| } |
| |
| nfp_cpp_area_get(area); |
| |
| return 0; |
| } |
| |
| /** |
| * nfp_cpp_area_acquire() - lock down a CPP area for access |
| * @area: CPP area handle |
| * |
| * Locks down the CPP area for a potential long term activity. Area |
| * must always be locked down before being accessed. |
| * |
| * Return: 0, or -ERRNO |
| */ |
| int nfp_cpp_area_acquire(struct nfp_cpp_area *area) |
| { |
| int ret; |
| |
| mutex_lock(&area->mutex); |
| ret = __nfp_cpp_area_acquire(area); |
| mutex_unlock(&area->mutex); |
| |
| return ret; |
| } |
| |
| /** |
| * nfp_cpp_area_acquire_nonblocking() - lock down a CPP area for access |
| * @area: CPP area handle |
| * |
| * Locks down the CPP area for a potential long term activity. Area |
| * must always be locked down before being accessed. |
| * |
| * NOTE: Returns -EAGAIN is no area is available |
| * |
| * Return: 0, or -ERRNO |
| */ |
| int nfp_cpp_area_acquire_nonblocking(struct nfp_cpp_area *area) |
| { |
| mutex_lock(&area->mutex); |
| if (atomic_inc_return(&area->refcount) == 1) { |
| if (area->cpp->op->area_acquire) { |
| int err; |
| |
| err = area->cpp->op->area_acquire(area); |
| if (err < 0) { |
| atomic_dec(&area->refcount); |
| mutex_unlock(&area->mutex); |
| return err; |
| } |
| } |
| } |
| mutex_unlock(&area->mutex); |
| |
| nfp_cpp_area_get(area); |
| return 0; |
| } |
| |
| /** |
| * nfp_cpp_area_release() - release a locked down CPP area |
| * @area: CPP area handle |
| * |
| * Releases a previously locked down CPP area. |
| */ |
| void nfp_cpp_area_release(struct nfp_cpp_area *area) |
| { |
| mutex_lock(&area->mutex); |
| /* Only call the release on refcount == 0 */ |
| if (atomic_dec_and_test(&area->refcount)) { |
| if (area->cpp->op->area_release) { |
| area->cpp->op->area_release(area); |
| /* Let anyone waiting for a BAR try to get one.. */ |
| wake_up_interruptible_all(&area->cpp->waitq); |
| } |
| } |
| mutex_unlock(&area->mutex); |
| |
| nfp_cpp_area_put(area); |
| } |
| |
| /** |
| * nfp_cpp_area_release_free() - release CPP area and free it |
| * @area: CPP area handle |
| * |
| * Releases CPP area and frees up memory resources held by the it. |
| */ |
| void nfp_cpp_area_release_free(struct nfp_cpp_area *area) |
| { |
| nfp_cpp_area_release(area); |
| nfp_cpp_area_free(area); |
| } |
| |
| /** |
| * nfp_cpp_area_read() - read data from CPP area |
| * @area: CPP area handle |
| * @offset: offset into CPP area |
| * @kernel_vaddr: kernel address to put data into |
| * @length: number of bytes to read |
| * |
| * Read data from indicated CPP region. |
| * |
| * NOTE: @offset and @length must be 32-bit aligned values. |
| * Area must have been locked down with an 'acquire'. |
| * |
| * Return: length of io, or -ERRNO |
| */ |
| int nfp_cpp_area_read(struct nfp_cpp_area *area, |
| unsigned long offset, void *kernel_vaddr, |
| size_t length) |
| { |
| return area->cpp->op->area_read(area, kernel_vaddr, offset, length); |
| } |
| |
| /** |
| * nfp_cpp_area_write() - write data to CPP area |
| * @area: CPP area handle |
| * @offset: offset into CPP area |
| * @kernel_vaddr: kernel address to read data from |
| * @length: number of bytes to write |
| * |
| * Write data to indicated CPP region. |
| * |
| * NOTE: @offset and @length must be 32-bit aligned values. |
| * Area must have been locked down with an 'acquire'. |
| * |
| * Return: length of io, or -ERRNO |
| */ |
| int nfp_cpp_area_write(struct nfp_cpp_area *area, |
| unsigned long offset, const void *kernel_vaddr, |
| size_t length) |
| { |
| return area->cpp->op->area_write(area, kernel_vaddr, offset, length); |
| } |
| |
| /** |
| * nfp_cpp_area_size() - return size of a CPP area |
| * @cpp_area: CPP area handle |
| * |
| * Return: Size of the area |
| */ |
| size_t nfp_cpp_area_size(struct nfp_cpp_area *cpp_area) |
| { |
| return cpp_area->size; |
| } |
| |
| /** |
| * nfp_cpp_area_name() - return name of a CPP area |
| * @cpp_area: CPP area handle |
| * |
| * Return: Name of the area, or NULL |
| */ |
| const char *nfp_cpp_area_name(struct nfp_cpp_area *cpp_area) |
| { |
| return cpp_area->resource.name; |
| } |
| |
| /** |
| * nfp_cpp_area_priv() - return private struct for CPP area |
| * @cpp_area: CPP area handle |
| * |
| * Return: Private data for the CPP area |
| */ |
| void *nfp_cpp_area_priv(struct nfp_cpp_area *cpp_area) |
| { |
| return &cpp_area[1]; |
| } |
| |
| /** |
| * nfp_cpp_area_cpp() - return CPP handle for CPP area |
| * @cpp_area: CPP area handle |
| * |
| * Return: NFP CPP handle |
| */ |
| struct nfp_cpp *nfp_cpp_area_cpp(struct nfp_cpp_area *cpp_area) |
| { |
| return cpp_area->cpp; |
| } |
| |
| /** |
| * nfp_cpp_area_resource() - get resource |
| * @area: CPP area handle |
| * |
| * NOTE: Area must have been locked down with an 'acquire'. |
| * |
| * Return: struct resource pointer, or NULL |
| */ |
| struct resource *nfp_cpp_area_resource(struct nfp_cpp_area *area) |
| { |
| struct resource *res = NULL; |
| |
| if (area->cpp->op->area_resource) |
| res = area->cpp->op->area_resource(area); |
| |
| return res; |
| } |
| |
| /** |
| * nfp_cpp_area_phys() - get physical address of CPP area |
| * @area: CPP area handle |
| * |
| * NOTE: Area must have been locked down with an 'acquire'. |
| * |
| * Return: phy_addr_t of the area, or NULL |
| */ |
| phys_addr_t nfp_cpp_area_phys(struct nfp_cpp_area *area) |
| { |
| phys_addr_t addr = ~0; |
| |
| if (area->cpp->op->area_phys) |
| addr = area->cpp->op->area_phys(area); |
| |
| return addr; |
| } |
| |
| /** |
| * nfp_cpp_area_iomem() - get IOMEM region for CPP area |
| * @area: CPP area handle |
| * |
| * Returns an iomem pointer for use with readl()/writel() style |
| * operations. |
| * |
| * NOTE: Area must have been locked down with an 'acquire'. |
| * |
| * Return: __iomem pointer to the area, or NULL |
| */ |
| void __iomem *nfp_cpp_area_iomem(struct nfp_cpp_area *area) |
| { |
| void __iomem *iomem = NULL; |
| |
| if (area->cpp->op->area_iomem) |
| iomem = area->cpp->op->area_iomem(area); |
| |
| return iomem; |
| } |
| |
| /** |
| * nfp_cpp_area_readl() - Read a u32 word from an area |
| * @area: CPP Area handle |
| * @offset: Offset into area |
| * @value: Pointer to read buffer |
| * |
| * Return: 0 on success, or -ERRNO |
| */ |
| int nfp_cpp_area_readl(struct nfp_cpp_area *area, |
| unsigned long offset, u32 *value) |
| { |
| u8 tmp[4]; |
| int n; |
| |
| n = nfp_cpp_area_read(area, offset, &tmp, sizeof(tmp)); |
| if (n != sizeof(tmp)) |
| return n < 0 ? n : -EIO; |
| |
| *value = get_unaligned_le32(tmp); |
| return 0; |
| } |
| |
| /** |
| * nfp_cpp_area_writel() - Write a u32 word to an area |
| * @area: CPP Area handle |
| * @offset: Offset into area |
| * @value: Value to write |
| * |
| * Return: 0 on success, or -ERRNO |
| */ |
| int nfp_cpp_area_writel(struct nfp_cpp_area *area, |
| unsigned long offset, u32 value) |
| { |
| u8 tmp[4]; |
| int n; |
| |
| put_unaligned_le32(value, tmp); |
| n = nfp_cpp_area_write(area, offset, &tmp, sizeof(tmp)); |
| |
| return n == sizeof(tmp) ? 0 : n < 0 ? n : -EIO; |
| } |
| |
| /** |
| * nfp_cpp_area_readq() - Read a u64 word from an area |
| * @area: CPP Area handle |
| * @offset: Offset into area |
| * @value: Pointer to read buffer |
| * |
| * Return: 0 on success, or -ERRNO |
| */ |
| int nfp_cpp_area_readq(struct nfp_cpp_area *area, |
| unsigned long offset, u64 *value) |
| { |
| u8 tmp[8]; |
| int n; |
| |
| n = nfp_cpp_area_read(area, offset, &tmp, sizeof(tmp)); |
| if (n != sizeof(tmp)) |
| return n < 0 ? n : -EIO; |
| |
| *value = get_unaligned_le64(tmp); |
| return 0; |
| } |
| |
| /** |
| * nfp_cpp_area_writeq() - Write a u64 word to an area |
| * @area: CPP Area handle |
| * @offset: Offset into area |
| * @value: Value to write |
| * |
| * Return: 0 on success, or -ERRNO |
| */ |
| int nfp_cpp_area_writeq(struct nfp_cpp_area *area, |
| unsigned long offset, u64 value) |
| { |
| u8 tmp[8]; |
| int n; |
| |
| put_unaligned_le64(value, tmp); |
| n = nfp_cpp_area_write(area, offset, &tmp, sizeof(tmp)); |
| |
| return n == sizeof(tmp) ? 0 : n < 0 ? n : -EIO; |
| } |
| |
| /** |
| * nfp_cpp_area_fill() - fill a CPP area with a value |
| * @area: CPP area |
| * @offset: offset into CPP area |
| * @value: value to fill with |
| * @length: length of area to fill |
| * |
| * Fill indicated area with given value. |
| * |
| * Return: length of io, or -ERRNO |
| */ |
| int nfp_cpp_area_fill(struct nfp_cpp_area *area, |
| unsigned long offset, u32 value, size_t length) |
| { |
| u8 tmp[4]; |
| size_t i; |
| int k; |
| |
| put_unaligned_le32(value, tmp); |
| |
| if (offset % sizeof(tmp) || length % sizeof(tmp)) |
| return -EINVAL; |
| |
| for (i = 0; i < length; i += sizeof(tmp)) { |
| k = nfp_cpp_area_write(area, offset + i, &tmp, sizeof(tmp)); |
| if (k < 0) |
| return k; |
| } |
| |
| return i; |
| } |
| |
| /** |
| * nfp_cpp_area_cache_add() - Permanently reserve and area for the hot cache |
| * @cpp: NFP CPP handle |
| * @size: Size of the area - MUST BE A POWER OF 2. |
| */ |
| int nfp_cpp_area_cache_add(struct nfp_cpp *cpp, size_t size) |
| { |
| struct nfp_cpp_area_cache *cache; |
| struct nfp_cpp_area *area; |
| |
| /* Allocate an area - we use the MU target's base as a placeholder, |
| * as all supported chips have a MU. |
| */ |
| area = nfp_cpp_area_alloc(cpp, NFP_CPP_ID(7, NFP_CPP_ACTION_RW, 0), |
| 0, size); |
| if (!area) |
| return -ENOMEM; |
| |
| cache = kzalloc(sizeof(*cache), GFP_KERNEL); |
| if (!cache) |
| return -ENOMEM; |
| |
| cache->id = 0; |
| cache->addr = 0; |
| cache->size = size; |
| cache->area = area; |
| mutex_lock(&cpp->area_cache_mutex); |
| list_add_tail(&cache->entry, &cpp->area_cache_list); |
| mutex_unlock(&cpp->area_cache_mutex); |
| |
| return 0; |
| } |
| |
| static struct nfp_cpp_area_cache * |
| area_cache_get(struct nfp_cpp *cpp, u32 id, |
| u64 addr, unsigned long *offset, size_t length) |
| { |
| struct nfp_cpp_area_cache *cache; |
| int err; |
| |
| /* Early exit when length == 0, which prevents |
| * the need for special case code below when |
| * checking against available cache size. |
| */ |
| if (length == 0 || id == 0) |
| return NULL; |
| |
| /* Remap from cpp_island to cpp_target */ |
| err = nfp_target_cpp(id, addr, &id, &addr, cpp->imb_cat_table); |
| if (err < 0) |
| return NULL; |
| |
| mutex_lock(&cpp->area_cache_mutex); |
| |
| if (list_empty(&cpp->area_cache_list)) { |
| mutex_unlock(&cpp->area_cache_mutex); |
| return NULL; |
| } |
| |
| addr += *offset; |
| |
| /* See if we have a match */ |
| list_for_each_entry(cache, &cpp->area_cache_list, entry) { |
| if (id == cache->id && |
| addr >= cache->addr && |
| addr + length <= cache->addr + cache->size) |
| goto exit; |
| } |
| |
| /* No matches - inspect the tail of the LRU */ |
| cache = list_entry(cpp->area_cache_list.prev, |
| struct nfp_cpp_area_cache, entry); |
| |
| /* Can we fit in the cache entry? */ |
| if (round_down(addr + length - 1, cache->size) != |
| round_down(addr, cache->size)) { |
| mutex_unlock(&cpp->area_cache_mutex); |
| return NULL; |
| } |
| |
| /* If id != 0, we will need to release it */ |
| if (cache->id) { |
| nfp_cpp_area_release(cache->area); |
| cache->id = 0; |
| cache->addr = 0; |
| } |
| |
| /* Adjust the start address to be cache size aligned */ |
| cache->id = id; |
| cache->addr = addr & ~(u64)(cache->size - 1); |
| |
| /* Re-init to the new ID and address */ |
| if (cpp->op->area_init) { |
| err = cpp->op->area_init(cache->area, |
| id, cache->addr, cache->size); |
| if (err < 0) { |
| mutex_unlock(&cpp->area_cache_mutex); |
| return NULL; |
| } |
| } |
| |
| /* Attempt to acquire */ |
| err = nfp_cpp_area_acquire(cache->area); |
| if (err < 0) { |
| mutex_unlock(&cpp->area_cache_mutex); |
| return NULL; |
| } |
| |
| exit: |
| /* Adjust offset */ |
| *offset = addr - cache->addr; |
| return cache; |
| } |
| |
| static void |
| area_cache_put(struct nfp_cpp *cpp, struct nfp_cpp_area_cache *cache) |
| { |
| if (!cache) |
| return; |
| |
| /* Move to front of LRU */ |
| list_move(&cache->entry, &cpp->area_cache_list); |
| |
| mutex_unlock(&cpp->area_cache_mutex); |
| } |
| |
| static int __nfp_cpp_read(struct nfp_cpp *cpp, u32 destination, |
| unsigned long long address, void *kernel_vaddr, |
| size_t length) |
| { |
| struct nfp_cpp_area_cache *cache; |
| struct nfp_cpp_area *area; |
| unsigned long offset = 0; |
| int err; |
| |
| cache = area_cache_get(cpp, destination, address, &offset, length); |
| if (cache) { |
| area = cache->area; |
| } else { |
| area = nfp_cpp_area_alloc(cpp, destination, address, length); |
| if (!area) |
| return -ENOMEM; |
| |
| err = nfp_cpp_area_acquire(area); |
| if (err) { |
| nfp_cpp_area_free(area); |
| return err; |
| } |
| } |
| |
| err = nfp_cpp_area_read(area, offset, kernel_vaddr, length); |
| |
| if (cache) |
| area_cache_put(cpp, cache); |
| else |
| nfp_cpp_area_release_free(area); |
| |
| return err; |
| } |
| |
| /** |
| * nfp_cpp_read() - read from CPP target |
| * @cpp: CPP handle |
| * @destination: CPP id |
| * @address: offset into CPP target |
| * @kernel_vaddr: kernel buffer for result |
| * @length: number of bytes to read |
| * |
| * Return: length of io, or -ERRNO |
| */ |
| int nfp_cpp_read(struct nfp_cpp *cpp, u32 destination, |
| unsigned long long address, void *kernel_vaddr, |
| size_t length) |
| { |
| size_t n, offset; |
| int ret; |
| |
| for (offset = 0; offset < length; offset += n) { |
| unsigned long long r_addr = address + offset; |
| |
| /* make first read smaller to align to safe window */ |
| n = min_t(size_t, length - offset, |
| ALIGN(r_addr + 1, NFP_CPP_SAFE_AREA_SIZE) - r_addr); |
| |
| ret = __nfp_cpp_read(cpp, destination, address + offset, |
| kernel_vaddr + offset, n); |
| if (ret < 0) |
| return ret; |
| if (ret != n) |
| return offset + n; |
| } |
| |
| return length; |
| } |
| |
| static int __nfp_cpp_write(struct nfp_cpp *cpp, u32 destination, |
| unsigned long long address, |
| const void *kernel_vaddr, size_t length) |
| { |
| struct nfp_cpp_area_cache *cache; |
| struct nfp_cpp_area *area; |
| unsigned long offset = 0; |
| int err; |
| |
| cache = area_cache_get(cpp, destination, address, &offset, length); |
| if (cache) { |
| area = cache->area; |
| } else { |
| area = nfp_cpp_area_alloc(cpp, destination, address, length); |
| if (!area) |
| return -ENOMEM; |
| |
| err = nfp_cpp_area_acquire(area); |
| if (err) { |
| nfp_cpp_area_free(area); |
| return err; |
| } |
| } |
| |
| err = nfp_cpp_area_write(area, offset, kernel_vaddr, length); |
| |
| if (cache) |
| area_cache_put(cpp, cache); |
| else |
| nfp_cpp_area_release_free(area); |
| |
| return err; |
| } |
| |
| /** |
| * nfp_cpp_write() - write to CPP target |
| * @cpp: CPP handle |
| * @destination: CPP id |
| * @address: offset into CPP target |
| * @kernel_vaddr: kernel buffer to read from |
| * @length: number of bytes to write |
| * |
| * Return: length of io, or -ERRNO |
| */ |
| int nfp_cpp_write(struct nfp_cpp *cpp, u32 destination, |
| unsigned long long address, |
| const void *kernel_vaddr, size_t length) |
| { |
| size_t n, offset; |
| int ret; |
| |
| for (offset = 0; offset < length; offset += n) { |
| unsigned long long w_addr = address + offset; |
| |
| /* make first write smaller to align to safe window */ |
| n = min_t(size_t, length - offset, |
| ALIGN(w_addr + 1, NFP_CPP_SAFE_AREA_SIZE) - w_addr); |
| |
| ret = __nfp_cpp_write(cpp, destination, address + offset, |
| kernel_vaddr + offset, n); |
| if (ret < 0) |
| return ret; |
| if (ret != n) |
| return offset + n; |
| } |
| |
| return length; |
| } |
| |
| /* Return the correct CPP address, and fixup xpb_addr as needed. */ |
| static u32 nfp_xpb_to_cpp(struct nfp_cpp *cpp, u32 *xpb_addr) |
| { |
| int island; |
| u32 xpb; |
| |
| xpb = NFP_CPP_ID(14, NFP_CPP_ACTION_RW, 0); |
| /* Ensure that non-local XPB accesses go |
| * out through the global XPBM bus. |
| */ |
| island = (*xpb_addr >> 24) & 0x3f; |
| if (!island) |
| return xpb; |
| |
| if (island != 1) { |
| *xpb_addr |= 1 << 30; |
| return xpb; |
| } |
| |
| /* Accesses to the ARM Island overlay uses Island 0 / Global Bit */ |
| *xpb_addr &= ~0x7f000000; |
| if (*xpb_addr < 0x60000) { |
| *xpb_addr |= 1 << 30; |
| } else { |
| /* And only non-ARM interfaces use the island id = 1 */ |
| if (NFP_CPP_INTERFACE_TYPE_of(nfp_cpp_interface(cpp)) |
| != NFP_CPP_INTERFACE_TYPE_ARM) |
| *xpb_addr |= 1 << 24; |
| } |
| |
| return xpb; |
| } |
| |
| /** |
| * nfp_xpb_readl() - Read a u32 word from a XPB location |
| * @cpp: CPP device handle |
| * @xpb_addr: Address for operation |
| * @value: Pointer to read buffer |
| * |
| * Return: 0 on success, or -ERRNO |
| */ |
| int nfp_xpb_readl(struct nfp_cpp *cpp, u32 xpb_addr, u32 *value) |
| { |
| u32 cpp_dest = nfp_xpb_to_cpp(cpp, &xpb_addr); |
| |
| return nfp_cpp_readl(cpp, cpp_dest, xpb_addr, value); |
| } |
| |
| /** |
| * nfp_xpb_writel() - Write a u32 word to a XPB location |
| * @cpp: CPP device handle |
| * @xpb_addr: Address for operation |
| * @value: Value to write |
| * |
| * Return: 0 on success, or -ERRNO |
| */ |
| int nfp_xpb_writel(struct nfp_cpp *cpp, u32 xpb_addr, u32 value) |
| { |
| u32 cpp_dest = nfp_xpb_to_cpp(cpp, &xpb_addr); |
| |
| return nfp_cpp_writel(cpp, cpp_dest, xpb_addr, value); |
| } |
| |
| /** |
| * nfp_xpb_writelm() - Modify bits of a 32-bit value from the XPB bus |
| * @cpp: NFP CPP device handle |
| * @xpb_tgt: XPB target and address |
| * @mask: mask of bits to alter |
| * @value: value to modify |
| * |
| * KERNEL: This operation is safe to call in interrupt or softirq context. |
| * |
| * Return: 0 on success, or -ERRNO |
| */ |
| int nfp_xpb_writelm(struct nfp_cpp *cpp, u32 xpb_tgt, |
| u32 mask, u32 value) |
| { |
| int err; |
| u32 tmp; |
| |
| err = nfp_xpb_readl(cpp, xpb_tgt, &tmp); |
| if (err < 0) |
| return err; |
| |
| tmp &= ~mask; |
| tmp |= mask & value; |
| return nfp_xpb_writel(cpp, xpb_tgt, tmp); |
| } |
| |
| /* Lockdep markers */ |
| static struct lock_class_key nfp_cpp_resource_lock_key; |
| |
| static void nfp_cpp_dev_release(struct device *dev) |
| { |
| /* Nothing to do here - it just makes the kernel happy */ |
| } |
| |
| /** |
| * nfp_cpp_from_operations() - Create a NFP CPP handle |
| * from an operations structure |
| * @ops: NFP CPP operations structure |
| * @parent: Parent device |
| * @priv: Private data of low-level implementation |
| * |
| * NOTE: On failure, cpp_ops->free will be called! |
| * |
| * Return: NFP CPP handle on success, ERR_PTR on failure |
| */ |
| struct nfp_cpp * |
| nfp_cpp_from_operations(const struct nfp_cpp_operations *ops, |
| struct device *parent, void *priv) |
| { |
| const u32 arm = NFP_CPP_ID(NFP_CPP_TARGET_ARM, NFP_CPP_ACTION_RW, 0); |
| struct nfp_cpp *cpp; |
| int ifc, err; |
| u32 mask[2]; |
| u32 xpbaddr; |
| size_t tgt; |
| |
| cpp = kzalloc(sizeof(*cpp), GFP_KERNEL); |
| if (!cpp) { |
| err = -ENOMEM; |
| goto err_malloc; |
| } |
| |
| cpp->op = ops; |
| cpp->priv = priv; |
| |
| ifc = ops->get_interface(parent); |
| if (ifc < 0) { |
| err = ifc; |
| goto err_free_cpp; |
| } |
| cpp->interface = ifc; |
| if (ops->read_serial) { |
| err = ops->read_serial(parent, cpp->serial); |
| if (err) |
| goto err_free_cpp; |
| } |
| |
| rwlock_init(&cpp->resource_lock); |
| init_waitqueue_head(&cpp->waitq); |
| lockdep_set_class(&cpp->resource_lock, &nfp_cpp_resource_lock_key); |
| INIT_LIST_HEAD(&cpp->resource_list); |
| INIT_LIST_HEAD(&cpp->area_cache_list); |
| mutex_init(&cpp->area_cache_mutex); |
| cpp->dev.init_name = "cpp"; |
| cpp->dev.parent = parent; |
| cpp->dev.release = nfp_cpp_dev_release; |
| err = device_register(&cpp->dev); |
| if (err < 0) { |
| put_device(&cpp->dev); |
| goto err_free_cpp; |
| } |
| |
| dev_set_drvdata(&cpp->dev, cpp); |
| |
| /* NOTE: cpp_lock is NOT locked for op->init, |
| * since it may call NFP CPP API operations |
| */ |
| if (cpp->op->init) { |
| err = cpp->op->init(cpp); |
| if (err < 0) { |
| dev_err(parent, |
| "NFP interface initialization failed\n"); |
| goto err_out; |
| } |
| } |
| |
| err = nfp_cpp_model_autodetect(cpp, &cpp->model); |
| if (err < 0) { |
| dev_err(parent, "NFP model detection failed\n"); |
| goto err_out; |
| } |
| |
| for (tgt = 0; tgt < ARRAY_SIZE(cpp->imb_cat_table); tgt++) { |
| /* Hardcoded XPB IMB Base, island 0 */ |
| xpbaddr = 0x000a0000 + (tgt * 4); |
| err = nfp_xpb_readl(cpp, xpbaddr, |
| &cpp->imb_cat_table[tgt]); |
| if (err < 0) { |
| dev_err(parent, |
| "Can't read CPP mapping from device\n"); |
| goto err_out; |
| } |
| } |
| |
| nfp_cpp_readl(cpp, arm, NFP_ARM_GCSR + NFP_ARM_GCSR_SOFTMODEL2, |
| &mask[0]); |
| nfp_cpp_readl(cpp, arm, NFP_ARM_GCSR + NFP_ARM_GCSR_SOFTMODEL3, |
| &mask[1]); |
| |
| err = nfp_cpp_set_mu_locality_lsb(cpp); |
| if (err < 0) { |
| dev_err(parent, "Can't calculate MU locality bit offset\n"); |
| goto err_out; |
| } |
| |
| dev_info(cpp->dev.parent, "Model: 0x%08x, SN: %pM, Ifc: 0x%04x\n", |
| nfp_cpp_model(cpp), cpp->serial, nfp_cpp_interface(cpp)); |
| |
| return cpp; |
| |
| err_out: |
| device_unregister(&cpp->dev); |
| err_free_cpp: |
| kfree(cpp); |
| err_malloc: |
| return ERR_PTR(err); |
| } |
| |
| /** |
| * nfp_cpp_priv() - Get the operations private data of a CPP handle |
| * @cpp: CPP handle |
| * |
| * Return: Private data for the NFP CPP handle |
| */ |
| void *nfp_cpp_priv(struct nfp_cpp *cpp) |
| { |
| return cpp->priv; |
| } |
| |
| /** |
| * nfp_cpp_device() - Get the Linux device handle of a CPP handle |
| * @cpp: CPP handle |
| * |
| * Return: Device for the NFP CPP bus |
| */ |
| struct device *nfp_cpp_device(struct nfp_cpp *cpp) |
| { |
| return &cpp->dev; |
| } |
| |
| #define NFP_EXPL_OP(func, expl, args...) \ |
| ({ \ |
| struct nfp_cpp *cpp = nfp_cpp_explicit_cpp(expl); \ |
| int err = -ENODEV; \ |
| \ |
| if (cpp->op->func) \ |
| err = cpp->op->func(expl, ##args); \ |
| err; \ |
| }) |
| |
| #define NFP_EXPL_OP_NR(func, expl, args...) \ |
| ({ \ |
| struct nfp_cpp *cpp = nfp_cpp_explicit_cpp(expl); \ |
| \ |
| if (cpp->op->func) \ |
| cpp->op->func(expl, ##args); \ |
| \ |
| }) |
| |
| /** |
| * nfp_cpp_explicit_acquire() - Acquire explicit access handle |
| * @cpp: NFP CPP handle |
| * |
| * The 'data_ref' and 'signal_ref' values are useful when |
| * constructing the NFP_EXPL_CSR1 and NFP_EXPL_POST values. |
| * |
| * Return: NFP CPP explicit handle |
| */ |
| struct nfp_cpp_explicit *nfp_cpp_explicit_acquire(struct nfp_cpp *cpp) |
| { |
| struct nfp_cpp_explicit *expl; |
| int err; |
| |
| expl = kzalloc(sizeof(*expl) + cpp->op->explicit_priv_size, GFP_KERNEL); |
| if (!expl) |
| return NULL; |
| |
| expl->cpp = cpp; |
| err = NFP_EXPL_OP(explicit_acquire, expl); |
| if (err < 0) { |
| kfree(expl); |
| return NULL; |
| } |
| |
| return expl; |
| } |
| |
| /** |
| * nfp_cpp_explicit_set_target() - Set target fields for explicit |
| * @expl: Explicit handle |
| * @cpp_id: CPP ID field |
| * @len: CPP Length field |
| * @mask: CPP Mask field |
| * |
| * Return: 0, or -ERRNO |
| */ |
| int nfp_cpp_explicit_set_target(struct nfp_cpp_explicit *expl, |
| u32 cpp_id, u8 len, u8 mask) |
| { |
| expl->cmd.cpp_id = cpp_id; |
| expl->cmd.len = len; |
| expl->cmd.byte_mask = mask; |
| |
| return 0; |
| } |
| |
| /** |
| * nfp_cpp_explicit_set_data() - Set data fields for explicit |
| * @expl: Explicit handle |
| * @data_master: CPP Data Master field |
| * @data_ref: CPP Data Ref field |
| * |
| * Return: 0, or -ERRNO |
| */ |
| int nfp_cpp_explicit_set_data(struct nfp_cpp_explicit *expl, |
| u8 data_master, u16 data_ref) |
| { |
| expl->cmd.data_master = data_master; |
| expl->cmd.data_ref = data_ref; |
| |
| return 0; |
| } |
| |
| /** |
| * nfp_cpp_explicit_set_signal() - Set signal fields for explicit |
| * @expl: Explicit handle |
| * @signal_master: CPP Signal Master field |
| * @signal_ref: CPP Signal Ref field |
| * |
| * Return: 0, or -ERRNO |
| */ |
| int nfp_cpp_explicit_set_signal(struct nfp_cpp_explicit *expl, |
| u8 signal_master, u8 signal_ref) |
| { |
| expl->cmd.signal_master = signal_master; |
| expl->cmd.signal_ref = signal_ref; |
| |
| return 0; |
| } |
| |
| /** |
| * nfp_cpp_explicit_set_posted() - Set completion fields for explicit |
| * @expl: Explicit handle |
| * @posted: True for signaled completion, false otherwise |
| * @siga: CPP Signal A field |
| * @siga_mode: CPP Signal A Mode field |
| * @sigb: CPP Signal B field |
| * @sigb_mode: CPP Signal B Mode field |
| * |
| * Return: 0, or -ERRNO |
| */ |
| int nfp_cpp_explicit_set_posted(struct nfp_cpp_explicit *expl, int posted, |
| u8 siga, |
| enum nfp_cpp_explicit_signal_mode siga_mode, |
| u8 sigb, |
| enum nfp_cpp_explicit_signal_mode sigb_mode) |
| { |
| expl->cmd.posted = posted; |
| expl->cmd.siga = siga; |
| expl->cmd.sigb = sigb; |
| expl->cmd.siga_mode = siga_mode; |
| expl->cmd.sigb_mode = sigb_mode; |
| |
| return 0; |
| } |
| |
| /** |
| * nfp_cpp_explicit_put() - Set up the write (pull) data for a explicit access |
| * @expl: NFP CPP Explicit handle |
| * @buff: Data to have the target pull in the transaction |
| * @len: Length of data, in bytes |
| * |
| * The 'len' parameter must be less than or equal to 128 bytes. |
| * |
| * If this function is called before the configuration |
| * registers are set, it will return -EINVAL. |
| * |
| * Return: 0, or -ERRNO |
| */ |
| int nfp_cpp_explicit_put(struct nfp_cpp_explicit *expl, |
| const void *buff, size_t len) |
| { |
| return NFP_EXPL_OP(explicit_put, expl, buff, len); |
| } |
| |
| /** |
| * nfp_cpp_explicit_do() - Execute a transaction, and wait for it to complete |
| * @expl: NFP CPP Explicit handle |
| * @address: Address to send in the explicit transaction |
| * |
| * If this function is called before the configuration |
| * registers are set, it will return -1, with an errno of EINVAL. |
| * |
| * Return: 0, or -ERRNO |
| */ |
| int nfp_cpp_explicit_do(struct nfp_cpp_explicit *expl, u64 address) |
| { |
| return NFP_EXPL_OP(explicit_do, expl, &expl->cmd, address); |
| } |
| |
| /** |
| * nfp_cpp_explicit_get() - Get the 'push' (read) data from a explicit access |
| * @expl: NFP CPP Explicit handle |
| * @buff: Data that the target pushed in the transaction |
| * @len: Length of data, in bytes |
| * |
| * The 'len' parameter must be less than or equal to 128 bytes. |
| * |
| * If this function is called before all three configuration |
| * registers are set, it will return -1, with an errno of EINVAL. |
| * |
| * If this function is called before nfp_cpp_explicit_do() |
| * has completed, it will return -1, with an errno of EBUSY. |
| * |
| * Return: 0, or -ERRNO |
| */ |
| int nfp_cpp_explicit_get(struct nfp_cpp_explicit *expl, void *buff, size_t len) |
| { |
| return NFP_EXPL_OP(explicit_get, expl, buff, len); |
| } |
| |
| /** |
| * nfp_cpp_explicit_release() - Release explicit access handle |
| * @expl: NFP CPP Explicit handle |
| * |
| */ |
| void nfp_cpp_explicit_release(struct nfp_cpp_explicit *expl) |
| { |
| NFP_EXPL_OP_NR(explicit_release, expl); |
| kfree(expl); |
| } |
| |
| /** |
| * nfp_cpp_explicit_cpp() - return CPP handle for CPP explicit |
| * @cpp_explicit: CPP explicit handle |
| * |
| * Return: NFP CPP handle of the explicit |
| */ |
| struct nfp_cpp *nfp_cpp_explicit_cpp(struct nfp_cpp_explicit *cpp_explicit) |
| { |
| return cpp_explicit->cpp; |
| } |
| |
| /** |
| * nfp_cpp_explicit_priv() - return private struct for CPP explicit |
| * @cpp_explicit: CPP explicit handle |
| * |
| * Return: private data of the explicit, or NULL |
| */ |
| void *nfp_cpp_explicit_priv(struct nfp_cpp_explicit *cpp_explicit) |
| { |
| return &cpp_explicit[1]; |
| } |