| // SPDX-License-Identifier: GPL-2.0 |
| /* |
| * Resource Director Technology (RDT) |
| * |
| * Pseudo-locking support built on top of Cache Allocation Technology (CAT) |
| * |
| * Copyright (C) 2018 Intel Corporation |
| * |
| * Author: Reinette Chatre <reinette.chatre@intel.com> |
| */ |
| |
| #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
| |
| #include <linux/cacheinfo.h> |
| #include <linux/cpu.h> |
| #include <linux/cpumask.h> |
| #include <linux/debugfs.h> |
| #include <linux/kthread.h> |
| #include <linux/mman.h> |
| #include <linux/perf_event.h> |
| #include <linux/pm_qos.h> |
| #include <linux/slab.h> |
| #include <linux/uaccess.h> |
| |
| #include <asm/cacheflush.h> |
| #include <asm/intel-family.h> |
| #include <asm/intel_rdt_sched.h> |
| #include <asm/perf_event.h> |
| |
| #include "../../events/perf_event.h" /* For X86_CONFIG() */ |
| #include "intel_rdt.h" |
| |
| #define CREATE_TRACE_POINTS |
| #include "intel_rdt_pseudo_lock_event.h" |
| |
| /* |
| * MSR_MISC_FEATURE_CONTROL register enables the modification of hardware |
| * prefetcher state. Details about this register can be found in the MSR |
| * tables for specific platforms found in Intel's SDM. |
| */ |
| #define MSR_MISC_FEATURE_CONTROL 0x000001a4 |
| |
| /* |
| * The bits needed to disable hardware prefetching varies based on the |
| * platform. During initialization we will discover which bits to use. |
| */ |
| static u64 prefetch_disable_bits; |
| |
| /* |
| * Major number assigned to and shared by all devices exposing |
| * pseudo-locked regions. |
| */ |
| static unsigned int pseudo_lock_major; |
| static unsigned long pseudo_lock_minor_avail = GENMASK(MINORBITS, 0); |
| static struct class *pseudo_lock_class; |
| |
| /** |
| * get_prefetch_disable_bits - prefetch disable bits of supported platforms |
| * |
| * Capture the list of platforms that have been validated to support |
| * pseudo-locking. This includes testing to ensure pseudo-locked regions |
| * with low cache miss rates can be created under variety of load conditions |
| * as well as that these pseudo-locked regions can maintain their low cache |
| * miss rates under variety of load conditions for significant lengths of time. |
| * |
| * After a platform has been validated to support pseudo-locking its |
| * hardware prefetch disable bits are included here as they are documented |
| * in the SDM. |
| * |
| * When adding a platform here also add support for its cache events to |
| * measure_cycles_perf_fn() |
| * |
| * Return: |
| * If platform is supported, the bits to disable hardware prefetchers, 0 |
| * if platform is not supported. |
| */ |
| static u64 get_prefetch_disable_bits(void) |
| { |
| if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL || |
| boot_cpu_data.x86 != 6) |
| return 0; |
| |
| switch (boot_cpu_data.x86_model) { |
| case INTEL_FAM6_BROADWELL_X: |
| /* |
| * SDM defines bits of MSR_MISC_FEATURE_CONTROL register |
| * as: |
| * 0 L2 Hardware Prefetcher Disable (R/W) |
| * 1 L2 Adjacent Cache Line Prefetcher Disable (R/W) |
| * 2 DCU Hardware Prefetcher Disable (R/W) |
| * 3 DCU IP Prefetcher Disable (R/W) |
| * 63:4 Reserved |
| */ |
| return 0xF; |
| case INTEL_FAM6_ATOM_GOLDMONT: |
| case INTEL_FAM6_ATOM_GOLDMONT_PLUS: |
| /* |
| * SDM defines bits of MSR_MISC_FEATURE_CONTROL register |
| * as: |
| * 0 L2 Hardware Prefetcher Disable (R/W) |
| * 1 Reserved |
| * 2 DCU Hardware Prefetcher Disable (R/W) |
| * 63:3 Reserved |
| */ |
| return 0x5; |
| } |
| |
| return 0; |
| } |
| |
| /** |
| * pseudo_lock_minor_get - Obtain available minor number |
| * @minor: Pointer to where new minor number will be stored |
| * |
| * A bitmask is used to track available minor numbers. Here the next free |
| * minor number is marked as unavailable and returned. |
| * |
| * Return: 0 on success, <0 on failure. |
| */ |
| static int pseudo_lock_minor_get(unsigned int *minor) |
| { |
| unsigned long first_bit; |
| |
| first_bit = find_first_bit(&pseudo_lock_minor_avail, MINORBITS); |
| |
| if (first_bit == MINORBITS) |
| return -ENOSPC; |
| |
| __clear_bit(first_bit, &pseudo_lock_minor_avail); |
| *minor = first_bit; |
| |
| return 0; |
| } |
| |
| /** |
| * pseudo_lock_minor_release - Return minor number to available |
| * @minor: The minor number made available |
| */ |
| static void pseudo_lock_minor_release(unsigned int minor) |
| { |
| __set_bit(minor, &pseudo_lock_minor_avail); |
| } |
| |
| /** |
| * region_find_by_minor - Locate a pseudo-lock region by inode minor number |
| * @minor: The minor number of the device representing pseudo-locked region |
| * |
| * When the character device is accessed we need to determine which |
| * pseudo-locked region it belongs to. This is done by matching the minor |
| * number of the device to the pseudo-locked region it belongs. |
| * |
| * Minor numbers are assigned at the time a pseudo-locked region is associated |
| * with a cache instance. |
| * |
| * Return: On success return pointer to resource group owning the pseudo-locked |
| * region, NULL on failure. |
| */ |
| static struct rdtgroup *region_find_by_minor(unsigned int minor) |
| { |
| struct rdtgroup *rdtgrp, *rdtgrp_match = NULL; |
| |
| list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) { |
| if (rdtgrp->plr && rdtgrp->plr->minor == minor) { |
| rdtgrp_match = rdtgrp; |
| break; |
| } |
| } |
| return rdtgrp_match; |
| } |
| |
| /** |
| * pseudo_lock_pm_req - A power management QoS request list entry |
| * @list: Entry within the @pm_reqs list for a pseudo-locked region |
| * @req: PM QoS request |
| */ |
| struct pseudo_lock_pm_req { |
| struct list_head list; |
| struct dev_pm_qos_request req; |
| }; |
| |
| static void pseudo_lock_cstates_relax(struct pseudo_lock_region *plr) |
| { |
| struct pseudo_lock_pm_req *pm_req, *next; |
| |
| list_for_each_entry_safe(pm_req, next, &plr->pm_reqs, list) { |
| dev_pm_qos_remove_request(&pm_req->req); |
| list_del(&pm_req->list); |
| kfree(pm_req); |
| } |
| } |
| |
| /** |
| * pseudo_lock_cstates_constrain - Restrict cores from entering C6 |
| * |
| * To prevent the cache from being affected by power management entering |
| * C6 has to be avoided. This is accomplished by requesting a latency |
| * requirement lower than lowest C6 exit latency of all supported |
| * platforms as found in the cpuidle state tables in the intel_idle driver. |
| * At this time it is possible to do so with a single latency requirement |
| * for all supported platforms. |
| * |
| * Since Goldmont is supported, which is affected by X86_BUG_MONITOR, |
| * the ACPI latencies need to be considered while keeping in mind that C2 |
| * may be set to map to deeper sleep states. In this case the latency |
| * requirement needs to prevent entering C2 also. |
| */ |
| static int pseudo_lock_cstates_constrain(struct pseudo_lock_region *plr) |
| { |
| struct pseudo_lock_pm_req *pm_req; |
| int cpu; |
| int ret; |
| |
| for_each_cpu(cpu, &plr->d->cpu_mask) { |
| pm_req = kzalloc(sizeof(*pm_req), GFP_KERNEL); |
| if (!pm_req) { |
| rdt_last_cmd_puts("fail allocating mem for PM QoS\n"); |
| ret = -ENOMEM; |
| goto out_err; |
| } |
| ret = dev_pm_qos_add_request(get_cpu_device(cpu), |
| &pm_req->req, |
| DEV_PM_QOS_RESUME_LATENCY, |
| 30); |
| if (ret < 0) { |
| rdt_last_cmd_printf("fail to add latency req cpu%d\n", |
| cpu); |
| kfree(pm_req); |
| ret = -1; |
| goto out_err; |
| } |
| list_add(&pm_req->list, &plr->pm_reqs); |
| } |
| |
| return 0; |
| |
| out_err: |
| pseudo_lock_cstates_relax(plr); |
| return ret; |
| } |
| |
| /** |
| * pseudo_lock_region_clear - Reset pseudo-lock region data |
| * @plr: pseudo-lock region |
| * |
| * All content of the pseudo-locked region is reset - any memory allocated |
| * freed. |
| * |
| * Return: void |
| */ |
| static void pseudo_lock_region_clear(struct pseudo_lock_region *plr) |
| { |
| plr->size = 0; |
| plr->line_size = 0; |
| kfree(plr->kmem); |
| plr->kmem = NULL; |
| plr->r = NULL; |
| if (plr->d) |
| plr->d->plr = NULL; |
| plr->d = NULL; |
| plr->cbm = 0; |
| plr->debugfs_dir = NULL; |
| } |
| |
| /** |
| * pseudo_lock_region_init - Initialize pseudo-lock region information |
| * @plr: pseudo-lock region |
| * |
| * Called after user provided a schemata to be pseudo-locked. From the |
| * schemata the &struct pseudo_lock_region is on entry already initialized |
| * with the resource, domain, and capacity bitmask. Here the information |
| * required for pseudo-locking is deduced from this data and &struct |
| * pseudo_lock_region initialized further. This information includes: |
| * - size in bytes of the region to be pseudo-locked |
| * - cache line size to know the stride with which data needs to be accessed |
| * to be pseudo-locked |
| * - a cpu associated with the cache instance on which the pseudo-locking |
| * flow can be executed |
| * |
| * Return: 0 on success, <0 on failure. Descriptive error will be written |
| * to last_cmd_status buffer. |
| */ |
| static int pseudo_lock_region_init(struct pseudo_lock_region *plr) |
| { |
| struct cpu_cacheinfo *ci; |
| int ret; |
| int i; |
| |
| /* Pick the first cpu we find that is associated with the cache. */ |
| plr->cpu = cpumask_first(&plr->d->cpu_mask); |
| |
| if (!cpu_online(plr->cpu)) { |
| rdt_last_cmd_printf("cpu %u associated with cache not online\n", |
| plr->cpu); |
| ret = -ENODEV; |
| goto out_region; |
| } |
| |
| ci = get_cpu_cacheinfo(plr->cpu); |
| |
| plr->size = rdtgroup_cbm_to_size(plr->r, plr->d, plr->cbm); |
| |
| for (i = 0; i < ci->num_leaves; i++) { |
| if (ci->info_list[i].level == plr->r->cache_level) { |
| plr->line_size = ci->info_list[i].coherency_line_size; |
| return 0; |
| } |
| } |
| |
| ret = -1; |
| rdt_last_cmd_puts("unable to determine cache line size\n"); |
| out_region: |
| pseudo_lock_region_clear(plr); |
| return ret; |
| } |
| |
| /** |
| * pseudo_lock_init - Initialize a pseudo-lock region |
| * @rdtgrp: resource group to which new pseudo-locked region will belong |
| * |
| * A pseudo-locked region is associated with a resource group. When this |
| * association is created the pseudo-locked region is initialized. The |
| * details of the pseudo-locked region are not known at this time so only |
| * allocation is done and association established. |
| * |
| * Return: 0 on success, <0 on failure |
| */ |
| static int pseudo_lock_init(struct rdtgroup *rdtgrp) |
| { |
| struct pseudo_lock_region *plr; |
| |
| plr = kzalloc(sizeof(*plr), GFP_KERNEL); |
| if (!plr) |
| return -ENOMEM; |
| |
| init_waitqueue_head(&plr->lock_thread_wq); |
| INIT_LIST_HEAD(&plr->pm_reqs); |
| rdtgrp->plr = plr; |
| return 0; |
| } |
| |
| /** |
| * pseudo_lock_region_alloc - Allocate kernel memory that will be pseudo-locked |
| * @plr: pseudo-lock region |
| * |
| * Initialize the details required to set up the pseudo-locked region and |
| * allocate the contiguous memory that will be pseudo-locked to the cache. |
| * |
| * Return: 0 on success, <0 on failure. Descriptive error will be written |
| * to last_cmd_status buffer. |
| */ |
| static int pseudo_lock_region_alloc(struct pseudo_lock_region *plr) |
| { |
| int ret; |
| |
| ret = pseudo_lock_region_init(plr); |
| if (ret < 0) |
| return ret; |
| |
| /* |
| * We do not yet support contiguous regions larger than |
| * KMALLOC_MAX_SIZE. |
| */ |
| if (plr->size > KMALLOC_MAX_SIZE) { |
| rdt_last_cmd_puts("requested region exceeds maximum size\n"); |
| ret = -E2BIG; |
| goto out_region; |
| } |
| |
| plr->kmem = kzalloc(plr->size, GFP_KERNEL); |
| if (!plr->kmem) { |
| rdt_last_cmd_puts("unable to allocate memory\n"); |
| ret = -ENOMEM; |
| goto out_region; |
| } |
| |
| ret = 0; |
| goto out; |
| out_region: |
| pseudo_lock_region_clear(plr); |
| out: |
| return ret; |
| } |
| |
| /** |
| * pseudo_lock_free - Free a pseudo-locked region |
| * @rdtgrp: resource group to which pseudo-locked region belonged |
| * |
| * The pseudo-locked region's resources have already been released, or not |
| * yet created at this point. Now it can be freed and disassociated from the |
| * resource group. |
| * |
| * Return: void |
| */ |
| static void pseudo_lock_free(struct rdtgroup *rdtgrp) |
| { |
| pseudo_lock_region_clear(rdtgrp->plr); |
| kfree(rdtgrp->plr); |
| rdtgrp->plr = NULL; |
| } |
| |
| /** |
| * pseudo_lock_fn - Load kernel memory into cache |
| * @_rdtgrp: resource group to which pseudo-lock region belongs |
| * |
| * This is the core pseudo-locking flow. |
| * |
| * First we ensure that the kernel memory cannot be found in the cache. |
| * Then, while taking care that there will be as little interference as |
| * possible, the memory to be loaded is accessed while core is running |
| * with class of service set to the bitmask of the pseudo-locked region. |
| * After this is complete no future CAT allocations will be allowed to |
| * overlap with this bitmask. |
| * |
| * Local register variables are utilized to ensure that the memory region |
| * to be locked is the only memory access made during the critical locking |
| * loop. |
| * |
| * Return: 0. Waiter on waitqueue will be woken on completion. |
| */ |
| static int pseudo_lock_fn(void *_rdtgrp) |
| { |
| struct rdtgroup *rdtgrp = _rdtgrp; |
| struct pseudo_lock_region *plr = rdtgrp->plr; |
| u32 rmid_p, closid_p; |
| unsigned long i; |
| #ifdef CONFIG_KASAN |
| /* |
| * The registers used for local register variables are also used |
| * when KASAN is active. When KASAN is active we use a regular |
| * variable to ensure we always use a valid pointer, but the cost |
| * is that this variable will enter the cache through evicting the |
| * memory we are trying to lock into the cache. Thus expect lower |
| * pseudo-locking success rate when KASAN is active. |
| */ |
| unsigned int line_size; |
| unsigned int size; |
| void *mem_r; |
| #else |
| register unsigned int line_size asm("esi"); |
| register unsigned int size asm("edi"); |
| #ifdef CONFIG_X86_64 |
| register void *mem_r asm("rbx"); |
| #else |
| register void *mem_r asm("ebx"); |
| #endif /* CONFIG_X86_64 */ |
| #endif /* CONFIG_KASAN */ |
| |
| /* |
| * Make sure none of the allocated memory is cached. If it is we |
| * will get a cache hit in below loop from outside of pseudo-locked |
| * region. |
| * wbinvd (as opposed to clflush/clflushopt) is required to |
| * increase likelihood that allocated cache portion will be filled |
| * with associated memory. |
| */ |
| native_wbinvd(); |
| |
| /* |
| * Always called with interrupts enabled. By disabling interrupts |
| * ensure that we will not be preempted during this critical section. |
| */ |
| local_irq_disable(); |
| |
| /* |
| * Call wrmsr and rdmsr as directly as possible to avoid tracing |
| * clobbering local register variables or affecting cache accesses. |
| * |
| * Disable the hardware prefetcher so that when the end of the memory |
| * being pseudo-locked is reached the hardware will not read beyond |
| * the buffer and evict pseudo-locked memory read earlier from the |
| * cache. |
| */ |
| __wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0); |
| closid_p = this_cpu_read(pqr_state.cur_closid); |
| rmid_p = this_cpu_read(pqr_state.cur_rmid); |
| mem_r = plr->kmem; |
| size = plr->size; |
| line_size = plr->line_size; |
| /* |
| * Critical section begin: start by writing the closid associated |
| * with the capacity bitmask of the cache region being |
| * pseudo-locked followed by reading of kernel memory to load it |
| * into the cache. |
| */ |
| __wrmsr(IA32_PQR_ASSOC, rmid_p, rdtgrp->closid); |
| /* |
| * Cache was flushed earlier. Now access kernel memory to read it |
| * into cache region associated with just activated plr->closid. |
| * Loop over data twice: |
| * - In first loop the cache region is shared with the page walker |
| * as it populates the paging structure caches (including TLB). |
| * - In the second loop the paging structure caches are used and |
| * cache region is populated with the memory being referenced. |
| */ |
| for (i = 0; i < size; i += PAGE_SIZE) { |
| /* |
| * Add a barrier to prevent speculative execution of this |
| * loop reading beyond the end of the buffer. |
| */ |
| rmb(); |
| asm volatile("mov (%0,%1,1), %%eax\n\t" |
| : |
| : "r" (mem_r), "r" (i) |
| : "%eax", "memory"); |
| } |
| for (i = 0; i < size; i += line_size) { |
| /* |
| * Add a barrier to prevent speculative execution of this |
| * loop reading beyond the end of the buffer. |
| */ |
| rmb(); |
| asm volatile("mov (%0,%1,1), %%eax\n\t" |
| : |
| : "r" (mem_r), "r" (i) |
| : "%eax", "memory"); |
| } |
| /* |
| * Critical section end: restore closid with capacity bitmask that |
| * does not overlap with pseudo-locked region. |
| */ |
| __wrmsr(IA32_PQR_ASSOC, rmid_p, closid_p); |
| |
| /* Re-enable the hardware prefetcher(s) */ |
| wrmsr(MSR_MISC_FEATURE_CONTROL, 0x0, 0x0); |
| local_irq_enable(); |
| |
| plr->thread_done = 1; |
| wake_up_interruptible(&plr->lock_thread_wq); |
| return 0; |
| } |
| |
| /** |
| * rdtgroup_monitor_in_progress - Test if monitoring in progress |
| * @r: resource group being queried |
| * |
| * Return: 1 if monitor groups have been created for this resource |
| * group, 0 otherwise. |
| */ |
| static int rdtgroup_monitor_in_progress(struct rdtgroup *rdtgrp) |
| { |
| return !list_empty(&rdtgrp->mon.crdtgrp_list); |
| } |
| |
| /** |
| * rdtgroup_locksetup_user_restrict - Restrict user access to group |
| * @rdtgrp: resource group needing access restricted |
| * |
| * A resource group used for cache pseudo-locking cannot have cpus or tasks |
| * assigned to it. This is communicated to the user by restricting access |
| * to all the files that can be used to make such changes. |
| * |
| * Permissions restored with rdtgroup_locksetup_user_restore() |
| * |
| * Return: 0 on success, <0 on failure. If a failure occurs during the |
| * restriction of access an attempt will be made to restore permissions but |
| * the state of the mode of these files will be uncertain when a failure |
| * occurs. |
| */ |
| static int rdtgroup_locksetup_user_restrict(struct rdtgroup *rdtgrp) |
| { |
| int ret; |
| |
| ret = rdtgroup_kn_mode_restrict(rdtgrp, "tasks"); |
| if (ret) |
| return ret; |
| |
| ret = rdtgroup_kn_mode_restrict(rdtgrp, "cpus"); |
| if (ret) |
| goto err_tasks; |
| |
| ret = rdtgroup_kn_mode_restrict(rdtgrp, "cpus_list"); |
| if (ret) |
| goto err_cpus; |
| |
| if (rdt_mon_capable) { |
| ret = rdtgroup_kn_mode_restrict(rdtgrp, "mon_groups"); |
| if (ret) |
| goto err_cpus_list; |
| } |
| |
| ret = 0; |
| goto out; |
| |
| err_cpus_list: |
| rdtgroup_kn_mode_restore(rdtgrp, "cpus_list", 0777); |
| err_cpus: |
| rdtgroup_kn_mode_restore(rdtgrp, "cpus", 0777); |
| err_tasks: |
| rdtgroup_kn_mode_restore(rdtgrp, "tasks", 0777); |
| out: |
| return ret; |
| } |
| |
| /** |
| * rdtgroup_locksetup_user_restore - Restore user access to group |
| * @rdtgrp: resource group needing access restored |
| * |
| * Restore all file access previously removed using |
| * rdtgroup_locksetup_user_restrict() |
| * |
| * Return: 0 on success, <0 on failure. If a failure occurs during the |
| * restoration of access an attempt will be made to restrict permissions |
| * again but the state of the mode of these files will be uncertain when |
| * a failure occurs. |
| */ |
| static int rdtgroup_locksetup_user_restore(struct rdtgroup *rdtgrp) |
| { |
| int ret; |
| |
| ret = rdtgroup_kn_mode_restore(rdtgrp, "tasks", 0777); |
| if (ret) |
| return ret; |
| |
| ret = rdtgroup_kn_mode_restore(rdtgrp, "cpus", 0777); |
| if (ret) |
| goto err_tasks; |
| |
| ret = rdtgroup_kn_mode_restore(rdtgrp, "cpus_list", 0777); |
| if (ret) |
| goto err_cpus; |
| |
| if (rdt_mon_capable) { |
| ret = rdtgroup_kn_mode_restore(rdtgrp, "mon_groups", 0777); |
| if (ret) |
| goto err_cpus_list; |
| } |
| |
| ret = 0; |
| goto out; |
| |
| err_cpus_list: |
| rdtgroup_kn_mode_restrict(rdtgrp, "cpus_list"); |
| err_cpus: |
| rdtgroup_kn_mode_restrict(rdtgrp, "cpus"); |
| err_tasks: |
| rdtgroup_kn_mode_restrict(rdtgrp, "tasks"); |
| out: |
| return ret; |
| } |
| |
| /** |
| * rdtgroup_locksetup_enter - Resource group enters locksetup mode |
| * @rdtgrp: resource group requested to enter locksetup mode |
| * |
| * A resource group enters locksetup mode to reflect that it would be used |
| * to represent a pseudo-locked region and is in the process of being set |
| * up to do so. A resource group used for a pseudo-locked region would |
| * lose the closid associated with it so we cannot allow it to have any |
| * tasks or cpus assigned nor permit tasks or cpus to be assigned in the |
| * future. Monitoring of a pseudo-locked region is not allowed either. |
| * |
| * The above and more restrictions on a pseudo-locked region are checked |
| * for and enforced before the resource group enters the locksetup mode. |
| * |
| * Returns: 0 if the resource group successfully entered locksetup mode, <0 |
| * on failure. On failure the last_cmd_status buffer is updated with text to |
| * communicate details of failure to the user. |
| */ |
| int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp) |
| { |
| int ret; |
| |
| /* |
| * The default resource group can neither be removed nor lose the |
| * default closid associated with it. |
| */ |
| if (rdtgrp == &rdtgroup_default) { |
| rdt_last_cmd_puts("cannot pseudo-lock default group\n"); |
| return -EINVAL; |
| } |
| |
| /* |
| * Cache Pseudo-locking not supported when CDP is enabled. |
| * |
| * Some things to consider if you would like to enable this |
| * support (using L3 CDP as example): |
| * - When CDP is enabled two separate resources are exposed, |
| * L3DATA and L3CODE, but they are actually on the same cache. |
| * The implication for pseudo-locking is that if a |
| * pseudo-locked region is created on a domain of one |
| * resource (eg. L3CODE), then a pseudo-locked region cannot |
| * be created on that same domain of the other resource |
| * (eg. L3DATA). This is because the creation of a |
| * pseudo-locked region involves a call to wbinvd that will |
| * affect all cache allocations on particular domain. |
| * - Considering the previous, it may be possible to only |
| * expose one of the CDP resources to pseudo-locking and |
| * hide the other. For example, we could consider to only |
| * expose L3DATA and since the L3 cache is unified it is |
| * still possible to place instructions there are execute it. |
| * - If only one region is exposed to pseudo-locking we should |
| * still keep in mind that availability of a portion of cache |
| * for pseudo-locking should take into account both resources. |
| * Similarly, if a pseudo-locked region is created in one |
| * resource, the portion of cache used by it should be made |
| * unavailable to all future allocations from both resources. |
| */ |
| if (rdt_resources_all[RDT_RESOURCE_L3DATA].alloc_enabled || |
| rdt_resources_all[RDT_RESOURCE_L2DATA].alloc_enabled) { |
| rdt_last_cmd_puts("CDP enabled\n"); |
| return -EINVAL; |
| } |
| |
| /* |
| * Not knowing the bits to disable prefetching implies that this |
| * platform does not support Cache Pseudo-Locking. |
| */ |
| prefetch_disable_bits = get_prefetch_disable_bits(); |
| if (prefetch_disable_bits == 0) { |
| rdt_last_cmd_puts("pseudo-locking not supported\n"); |
| return -EINVAL; |
| } |
| |
| if (rdtgroup_monitor_in_progress(rdtgrp)) { |
| rdt_last_cmd_puts("monitoring in progress\n"); |
| return -EINVAL; |
| } |
| |
| if (rdtgroup_tasks_assigned(rdtgrp)) { |
| rdt_last_cmd_puts("tasks assigned to resource group\n"); |
| return -EINVAL; |
| } |
| |
| if (!cpumask_empty(&rdtgrp->cpu_mask)) { |
| rdt_last_cmd_puts("CPUs assigned to resource group\n"); |
| return -EINVAL; |
| } |
| |
| if (rdtgroup_locksetup_user_restrict(rdtgrp)) { |
| rdt_last_cmd_puts("unable to modify resctrl permissions\n"); |
| return -EIO; |
| } |
| |
| ret = pseudo_lock_init(rdtgrp); |
| if (ret) { |
| rdt_last_cmd_puts("unable to init pseudo-lock region\n"); |
| goto out_release; |
| } |
| |
| /* |
| * If this system is capable of monitoring a rmid would have been |
| * allocated when the control group was created. This is not needed |
| * anymore when this group would be used for pseudo-locking. This |
| * is safe to call on platforms not capable of monitoring. |
| */ |
| free_rmid(rdtgrp->mon.rmid); |
| |
| ret = 0; |
| goto out; |
| |
| out_release: |
| rdtgroup_locksetup_user_restore(rdtgrp); |
| out: |
| return ret; |
| } |
| |
| /** |
| * rdtgroup_locksetup_exit - resource group exist locksetup mode |
| * @rdtgrp: resource group |
| * |
| * When a resource group exits locksetup mode the earlier restrictions are |
| * lifted. |
| * |
| * Return: 0 on success, <0 on failure |
| */ |
| int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp) |
| { |
| int ret; |
| |
| if (rdt_mon_capable) { |
| ret = alloc_rmid(); |
| if (ret < 0) { |
| rdt_last_cmd_puts("out of RMIDs\n"); |
| return ret; |
| } |
| rdtgrp->mon.rmid = ret; |
| } |
| |
| ret = rdtgroup_locksetup_user_restore(rdtgrp); |
| if (ret) { |
| free_rmid(rdtgrp->mon.rmid); |
| return ret; |
| } |
| |
| pseudo_lock_free(rdtgrp); |
| return 0; |
| } |
| |
| /** |
| * rdtgroup_cbm_overlaps_pseudo_locked - Test if CBM or portion is pseudo-locked |
| * @d: RDT domain |
| * @cbm: CBM to test |
| * |
| * @d represents a cache instance and @cbm a capacity bitmask that is |
| * considered for it. Determine if @cbm overlaps with any existing |
| * pseudo-locked region on @d. |
| * |
| * @cbm is unsigned long, even if only 32 bits are used, to make the |
| * bitmap functions work correctly. |
| * |
| * Return: true if @cbm overlaps with pseudo-locked region on @d, false |
| * otherwise. |
| */ |
| bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_domain *d, unsigned long cbm) |
| { |
| unsigned int cbm_len; |
| unsigned long cbm_b; |
| |
| if (d->plr) { |
| cbm_len = d->plr->r->cache.cbm_len; |
| cbm_b = d->plr->cbm; |
| if (bitmap_intersects(&cbm, &cbm_b, cbm_len)) |
| return true; |
| } |
| return false; |
| } |
| |
| /** |
| * rdtgroup_pseudo_locked_in_hierarchy - Pseudo-locked region in cache hierarchy |
| * @d: RDT domain under test |
| * |
| * The setup of a pseudo-locked region affects all cache instances within |
| * the hierarchy of the region. It is thus essential to know if any |
| * pseudo-locked regions exist within a cache hierarchy to prevent any |
| * attempts to create new pseudo-locked regions in the same hierarchy. |
| * |
| * Return: true if a pseudo-locked region exists in the hierarchy of @d or |
| * if it is not possible to test due to memory allocation issue, |
| * false otherwise. |
| */ |
| bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d) |
| { |
| cpumask_var_t cpu_with_psl; |
| struct rdt_resource *r; |
| struct rdt_domain *d_i; |
| bool ret = false; |
| |
| if (!zalloc_cpumask_var(&cpu_with_psl, GFP_KERNEL)) |
| return true; |
| |
| /* |
| * First determine which cpus have pseudo-locked regions |
| * associated with them. |
| */ |
| for_each_alloc_enabled_rdt_resource(r) { |
| list_for_each_entry(d_i, &r->domains, list) { |
| if (d_i->plr) |
| cpumask_or(cpu_with_psl, cpu_with_psl, |
| &d_i->cpu_mask); |
| } |
| } |
| |
| /* |
| * Next test if new pseudo-locked region would intersect with |
| * existing region. |
| */ |
| if (cpumask_intersects(&d->cpu_mask, cpu_with_psl)) |
| ret = true; |
| |
| free_cpumask_var(cpu_with_psl); |
| return ret; |
| } |
| |
| /** |
| * measure_cycles_lat_fn - Measure cycle latency to read pseudo-locked memory |
| * @_plr: pseudo-lock region to measure |
| * |
| * There is no deterministic way to test if a memory region is cached. One |
| * way is to measure how long it takes to read the memory, the speed of |
| * access is a good way to learn how close to the cpu the data was. Even |
| * more, if the prefetcher is disabled and the memory is read at a stride |
| * of half the cache line, then a cache miss will be easy to spot since the |
| * read of the first half would be significantly slower than the read of |
| * the second half. |
| * |
| * Return: 0. Waiter on waitqueue will be woken on completion. |
| */ |
| static int measure_cycles_lat_fn(void *_plr) |
| { |
| struct pseudo_lock_region *plr = _plr; |
| unsigned long i; |
| u64 start, end; |
| void *mem_r; |
| |
| local_irq_disable(); |
| /* |
| * Disable hardware prefetchers. |
| */ |
| wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0); |
| mem_r = READ_ONCE(plr->kmem); |
| /* |
| * Dummy execute of the time measurement to load the needed |
| * instructions into the L1 instruction cache. |
| */ |
| start = rdtsc_ordered(); |
| for (i = 0; i < plr->size; i += 32) { |
| start = rdtsc_ordered(); |
| asm volatile("mov (%0,%1,1), %%eax\n\t" |
| : |
| : "r" (mem_r), "r" (i) |
| : "%eax", "memory"); |
| end = rdtsc_ordered(); |
| trace_pseudo_lock_mem_latency((u32)(end - start)); |
| } |
| wrmsr(MSR_MISC_FEATURE_CONTROL, 0x0, 0x0); |
| local_irq_enable(); |
| plr->thread_done = 1; |
| wake_up_interruptible(&plr->lock_thread_wq); |
| return 0; |
| } |
| |
| /* |
| * Create a perf_event_attr for the hit and miss perf events that will |
| * be used during the performance measurement. A perf_event maintains |
| * a pointer to its perf_event_attr so a unique attribute structure is |
| * created for each perf_event. |
| * |
| * The actual configuration of the event is set right before use in order |
| * to use the X86_CONFIG macro. |
| */ |
| static struct perf_event_attr perf_miss_attr = { |
| .type = PERF_TYPE_RAW, |
| .size = sizeof(struct perf_event_attr), |
| .pinned = 1, |
| .disabled = 0, |
| .exclude_user = 1, |
| }; |
| |
| static struct perf_event_attr perf_hit_attr = { |
| .type = PERF_TYPE_RAW, |
| .size = sizeof(struct perf_event_attr), |
| .pinned = 1, |
| .disabled = 0, |
| .exclude_user = 1, |
| }; |
| |
| struct residency_counts { |
| u64 miss_before, hits_before; |
| u64 miss_after, hits_after; |
| }; |
| |
| static int measure_residency_fn(struct perf_event_attr *miss_attr, |
| struct perf_event_attr *hit_attr, |
| struct pseudo_lock_region *plr, |
| struct residency_counts *counts) |
| { |
| u64 hits_before = 0, hits_after = 0, miss_before = 0, miss_after = 0; |
| struct perf_event *miss_event, *hit_event; |
| int hit_pmcnum, miss_pmcnum; |
| unsigned int line_size; |
| unsigned int size; |
| unsigned long i; |
| void *mem_r; |
| u64 tmp; |
| |
| miss_event = perf_event_create_kernel_counter(miss_attr, plr->cpu, |
| NULL, NULL, NULL); |
| if (IS_ERR(miss_event)) |
| goto out; |
| |
| hit_event = perf_event_create_kernel_counter(hit_attr, plr->cpu, |
| NULL, NULL, NULL); |
| if (IS_ERR(hit_event)) |
| goto out_miss; |
| |
| local_irq_disable(); |
| /* |
| * Check any possible error state of events used by performing |
| * one local read. |
| */ |
| if (perf_event_read_local(miss_event, &tmp, NULL, NULL)) { |
| local_irq_enable(); |
| goto out_hit; |
| } |
| if (perf_event_read_local(hit_event, &tmp, NULL, NULL)) { |
| local_irq_enable(); |
| goto out_hit; |
| } |
| |
| /* |
| * Disable hardware prefetchers. |
| */ |
| wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0); |
| |
| /* Initialize rest of local variables */ |
| /* |
| * Performance event has been validated right before this with |
| * interrupts disabled - it is thus safe to read the counter index. |
| */ |
| miss_pmcnum = x86_perf_rdpmc_index(miss_event); |
| hit_pmcnum = x86_perf_rdpmc_index(hit_event); |
| line_size = READ_ONCE(plr->line_size); |
| mem_r = READ_ONCE(plr->kmem); |
| size = READ_ONCE(plr->size); |
| |
| /* |
| * Read counter variables twice - first to load the instructions |
| * used in L1 cache, second to capture accurate value that does not |
| * include cache misses incurred because of instruction loads. |
| */ |
| rdpmcl(hit_pmcnum, hits_before); |
| rdpmcl(miss_pmcnum, miss_before); |
| /* |
| * From SDM: Performing back-to-back fast reads are not guaranteed |
| * to be monotonic. |
| * Use LFENCE to ensure all previous instructions are retired |
| * before proceeding. |
| */ |
| rmb(); |
| rdpmcl(hit_pmcnum, hits_before); |
| rdpmcl(miss_pmcnum, miss_before); |
| /* |
| * Use LFENCE to ensure all previous instructions are retired |
| * before proceeding. |
| */ |
| rmb(); |
| for (i = 0; i < size; i += line_size) { |
| /* |
| * Add a barrier to prevent speculative execution of this |
| * loop reading beyond the end of the buffer. |
| */ |
| rmb(); |
| asm volatile("mov (%0,%1,1), %%eax\n\t" |
| : |
| : "r" (mem_r), "r" (i) |
| : "%eax", "memory"); |
| } |
| /* |
| * Use LFENCE to ensure all previous instructions are retired |
| * before proceeding. |
| */ |
| rmb(); |
| rdpmcl(hit_pmcnum, hits_after); |
| rdpmcl(miss_pmcnum, miss_after); |
| /* |
| * Use LFENCE to ensure all previous instructions are retired |
| * before proceeding. |
| */ |
| rmb(); |
| /* Re-enable hardware prefetchers */ |
| wrmsr(MSR_MISC_FEATURE_CONTROL, 0x0, 0x0); |
| local_irq_enable(); |
| out_hit: |
| perf_event_release_kernel(hit_event); |
| out_miss: |
| perf_event_release_kernel(miss_event); |
| out: |
| /* |
| * All counts will be zero on failure. |
| */ |
| counts->miss_before = miss_before; |
| counts->hits_before = hits_before; |
| counts->miss_after = miss_after; |
| counts->hits_after = hits_after; |
| return 0; |
| } |
| |
| static int measure_l2_residency(void *_plr) |
| { |
| struct pseudo_lock_region *plr = _plr; |
| struct residency_counts counts = {0}; |
| |
| /* |
| * Non-architectural event for the Goldmont Microarchitecture |
| * from Intel x86 Architecture Software Developer Manual (SDM): |
| * MEM_LOAD_UOPS_RETIRED D1H (event number) |
| * Umask values: |
| * L2_HIT 02H |
| * L2_MISS 10H |
| */ |
| switch (boot_cpu_data.x86_model) { |
| case INTEL_FAM6_ATOM_GOLDMONT: |
| case INTEL_FAM6_ATOM_GOLDMONT_PLUS: |
| perf_miss_attr.config = X86_CONFIG(.event = 0xd1, |
| .umask = 0x10); |
| perf_hit_attr.config = X86_CONFIG(.event = 0xd1, |
| .umask = 0x2); |
| break; |
| default: |
| goto out; |
| } |
| |
| measure_residency_fn(&perf_miss_attr, &perf_hit_attr, plr, &counts); |
| /* |
| * If a failure prevented the measurements from succeeding |
| * tracepoints will still be written and all counts will be zero. |
| */ |
| trace_pseudo_lock_l2(counts.hits_after - counts.hits_before, |
| counts.miss_after - counts.miss_before); |
| out: |
| plr->thread_done = 1; |
| wake_up_interruptible(&plr->lock_thread_wq); |
| return 0; |
| } |
| |
| static int measure_l3_residency(void *_plr) |
| { |
| struct pseudo_lock_region *plr = _plr; |
| struct residency_counts counts = {0}; |
| |
| /* |
| * On Broadwell Microarchitecture the MEM_LOAD_UOPS_RETIRED event |
| * has two "no fix" errata associated with it: BDM35 and BDM100. On |
| * this platform the following events are used instead: |
| * LONGEST_LAT_CACHE 2EH (Documented in SDM) |
| * REFERENCE 4FH |
| * MISS 41H |
| */ |
| |
| switch (boot_cpu_data.x86_model) { |
| case INTEL_FAM6_BROADWELL_X: |
| /* On BDW the hit event counts references, not hits */ |
| perf_hit_attr.config = X86_CONFIG(.event = 0x2e, |
| .umask = 0x4f); |
| perf_miss_attr.config = X86_CONFIG(.event = 0x2e, |
| .umask = 0x41); |
| break; |
| default: |
| goto out; |
| } |
| |
| measure_residency_fn(&perf_miss_attr, &perf_hit_attr, plr, &counts); |
| /* |
| * If a failure prevented the measurements from succeeding |
| * tracepoints will still be written and all counts will be zero. |
| */ |
| |
| counts.miss_after -= counts.miss_before; |
| if (boot_cpu_data.x86_model == INTEL_FAM6_BROADWELL_X) { |
| /* |
| * On BDW references and misses are counted, need to adjust. |
| * Sometimes the "hits" counter is a bit more than the |
| * references, for example, x references but x + 1 hits. |
| * To not report invalid hit values in this case we treat |
| * that as misses equal to references. |
| */ |
| /* First compute the number of cache references measured */ |
| counts.hits_after -= counts.hits_before; |
| /* Next convert references to cache hits */ |
| counts.hits_after -= min(counts.miss_after, counts.hits_after); |
| } else { |
| counts.hits_after -= counts.hits_before; |
| } |
| |
| trace_pseudo_lock_l3(counts.hits_after, counts.miss_after); |
| out: |
| plr->thread_done = 1; |
| wake_up_interruptible(&plr->lock_thread_wq); |
| return 0; |
| } |
| |
| /** |
| * pseudo_lock_measure_cycles - Trigger latency measure to pseudo-locked region |
| * |
| * The measurement of latency to access a pseudo-locked region should be |
| * done from a cpu that is associated with that pseudo-locked region. |
| * Determine which cpu is associated with this region and start a thread on |
| * that cpu to perform the measurement, wait for that thread to complete. |
| * |
| * Return: 0 on success, <0 on failure |
| */ |
| static int pseudo_lock_measure_cycles(struct rdtgroup *rdtgrp, int sel) |
| { |
| struct pseudo_lock_region *plr = rdtgrp->plr; |
| struct task_struct *thread; |
| unsigned int cpu; |
| int ret = -1; |
| |
| cpus_read_lock(); |
| mutex_lock(&rdtgroup_mutex); |
| |
| if (rdtgrp->flags & RDT_DELETED) { |
| ret = -ENODEV; |
| goto out; |
| } |
| |
| if (!plr->d) { |
| ret = -ENODEV; |
| goto out; |
| } |
| |
| plr->thread_done = 0; |
| cpu = cpumask_first(&plr->d->cpu_mask); |
| if (!cpu_online(cpu)) { |
| ret = -ENODEV; |
| goto out; |
| } |
| |
| plr->cpu = cpu; |
| |
| if (sel == 1) |
| thread = kthread_create_on_node(measure_cycles_lat_fn, plr, |
| cpu_to_node(cpu), |
| "pseudo_lock_measure/%u", |
| cpu); |
| else if (sel == 2) |
| thread = kthread_create_on_node(measure_l2_residency, plr, |
| cpu_to_node(cpu), |
| "pseudo_lock_measure/%u", |
| cpu); |
| else if (sel == 3) |
| thread = kthread_create_on_node(measure_l3_residency, plr, |
| cpu_to_node(cpu), |
| "pseudo_lock_measure/%u", |
| cpu); |
| else |
| goto out; |
| |
| if (IS_ERR(thread)) { |
| ret = PTR_ERR(thread); |
| goto out; |
| } |
| kthread_bind(thread, cpu); |
| wake_up_process(thread); |
| |
| ret = wait_event_interruptible(plr->lock_thread_wq, |
| plr->thread_done == 1); |
| if (ret < 0) |
| goto out; |
| |
| ret = 0; |
| |
| out: |
| mutex_unlock(&rdtgroup_mutex); |
| cpus_read_unlock(); |
| return ret; |
| } |
| |
| static ssize_t pseudo_lock_measure_trigger(struct file *file, |
| const char __user *user_buf, |
| size_t count, loff_t *ppos) |
| { |
| struct rdtgroup *rdtgrp = file->private_data; |
| size_t buf_size; |
| char buf[32]; |
| int ret; |
| int sel; |
| |
| buf_size = min(count, (sizeof(buf) - 1)); |
| if (copy_from_user(buf, user_buf, buf_size)) |
| return -EFAULT; |
| |
| buf[buf_size] = '\0'; |
| ret = kstrtoint(buf, 10, &sel); |
| if (ret == 0) { |
| if (sel != 1 && sel != 2 && sel != 3) |
| return -EINVAL; |
| ret = debugfs_file_get(file->f_path.dentry); |
| if (ret) |
| return ret; |
| ret = pseudo_lock_measure_cycles(rdtgrp, sel); |
| if (ret == 0) |
| ret = count; |
| debugfs_file_put(file->f_path.dentry); |
| } |
| |
| return ret; |
| } |
| |
| static const struct file_operations pseudo_measure_fops = { |
| .write = pseudo_lock_measure_trigger, |
| .open = simple_open, |
| .llseek = default_llseek, |
| }; |
| |
| /** |
| * rdtgroup_pseudo_lock_create - Create a pseudo-locked region |
| * @rdtgrp: resource group to which pseudo-lock region belongs |
| * |
| * Called when a resource group in the pseudo-locksetup mode receives a |
| * valid schemata that should be pseudo-locked. Since the resource group is |
| * in pseudo-locksetup mode the &struct pseudo_lock_region has already been |
| * allocated and initialized with the essential information. If a failure |
| * occurs the resource group remains in the pseudo-locksetup mode with the |
| * &struct pseudo_lock_region associated with it, but cleared from all |
| * information and ready for the user to re-attempt pseudo-locking by |
| * writing the schemata again. |
| * |
| * Return: 0 if the pseudo-locked region was successfully pseudo-locked, <0 |
| * on failure. Descriptive error will be written to last_cmd_status buffer. |
| */ |
| int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp) |
| { |
| struct pseudo_lock_region *plr = rdtgrp->plr; |
| struct task_struct *thread; |
| unsigned int new_minor; |
| struct device *dev; |
| int ret; |
| |
| ret = pseudo_lock_region_alloc(plr); |
| if (ret < 0) |
| return ret; |
| |
| ret = pseudo_lock_cstates_constrain(plr); |
| if (ret < 0) { |
| ret = -EINVAL; |
| goto out_region; |
| } |
| |
| plr->thread_done = 0; |
| |
| thread = kthread_create_on_node(pseudo_lock_fn, rdtgrp, |
| cpu_to_node(plr->cpu), |
| "pseudo_lock/%u", plr->cpu); |
| if (IS_ERR(thread)) { |
| ret = PTR_ERR(thread); |
| rdt_last_cmd_printf("locking thread returned error %d\n", ret); |
| goto out_cstates; |
| } |
| |
| kthread_bind(thread, plr->cpu); |
| wake_up_process(thread); |
| |
| ret = wait_event_interruptible(plr->lock_thread_wq, |
| plr->thread_done == 1); |
| if (ret < 0) { |
| /* |
| * If the thread does not get on the CPU for whatever |
| * reason and the process which sets up the region is |
| * interrupted then this will leave the thread in runnable |
| * state and once it gets on the CPU it will derefence |
| * the cleared, but not freed, plr struct resulting in an |
| * empty pseudo-locking loop. |
| */ |
| rdt_last_cmd_puts("locking thread interrupted\n"); |
| goto out_cstates; |
| } |
| |
| ret = pseudo_lock_minor_get(&new_minor); |
| if (ret < 0) { |
| rdt_last_cmd_puts("unable to obtain a new minor number\n"); |
| goto out_cstates; |
| } |
| |
| /* |
| * Unlock access but do not release the reference. The |
| * pseudo-locked region will still be here on return. |
| * |
| * The mutex has to be released temporarily to avoid a potential |
| * deadlock with the mm->mmap_sem semaphore which is obtained in |
| * the device_create() and debugfs_create_dir() callpath below |
| * as well as before the mmap() callback is called. |
| */ |
| mutex_unlock(&rdtgroup_mutex); |
| |
| if (!IS_ERR_OR_NULL(debugfs_resctrl)) { |
| plr->debugfs_dir = debugfs_create_dir(rdtgrp->kn->name, |
| debugfs_resctrl); |
| if (!IS_ERR_OR_NULL(plr->debugfs_dir)) |
| debugfs_create_file("pseudo_lock_measure", 0200, |
| plr->debugfs_dir, rdtgrp, |
| &pseudo_measure_fops); |
| } |
| |
| dev = device_create(pseudo_lock_class, NULL, |
| MKDEV(pseudo_lock_major, new_minor), |
| rdtgrp, "%s", rdtgrp->kn->name); |
| |
| mutex_lock(&rdtgroup_mutex); |
| |
| if (IS_ERR(dev)) { |
| ret = PTR_ERR(dev); |
| rdt_last_cmd_printf("failed to create character device: %d\n", |
| ret); |
| goto out_debugfs; |
| } |
| |
| /* We released the mutex - check if group was removed while we did so */ |
| if (rdtgrp->flags & RDT_DELETED) { |
| ret = -ENODEV; |
| goto out_device; |
| } |
| |
| plr->minor = new_minor; |
| |
| rdtgrp->mode = RDT_MODE_PSEUDO_LOCKED; |
| closid_free(rdtgrp->closid); |
| rdtgroup_kn_mode_restore(rdtgrp, "cpus", 0444); |
| rdtgroup_kn_mode_restore(rdtgrp, "cpus_list", 0444); |
| |
| ret = 0; |
| goto out; |
| |
| out_device: |
| device_destroy(pseudo_lock_class, MKDEV(pseudo_lock_major, new_minor)); |
| out_debugfs: |
| debugfs_remove_recursive(plr->debugfs_dir); |
| pseudo_lock_minor_release(new_minor); |
| out_cstates: |
| pseudo_lock_cstates_relax(plr); |
| out_region: |
| pseudo_lock_region_clear(plr); |
| out: |
| return ret; |
| } |
| |
| /** |
| * rdtgroup_pseudo_lock_remove - Remove a pseudo-locked region |
| * @rdtgrp: resource group to which the pseudo-locked region belongs |
| * |
| * The removal of a pseudo-locked region can be initiated when the resource |
| * group is removed from user space via a "rmdir" from userspace or the |
| * unmount of the resctrl filesystem. On removal the resource group does |
| * not go back to pseudo-locksetup mode before it is removed, instead it is |
| * removed directly. There is thus assymmetry with the creation where the |
| * &struct pseudo_lock_region is removed here while it was not created in |
| * rdtgroup_pseudo_lock_create(). |
| * |
| * Return: void |
| */ |
| void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp) |
| { |
| struct pseudo_lock_region *plr = rdtgrp->plr; |
| |
| if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { |
| /* |
| * Default group cannot be a pseudo-locked region so we can |
| * free closid here. |
| */ |
| closid_free(rdtgrp->closid); |
| goto free; |
| } |
| |
| pseudo_lock_cstates_relax(plr); |
| debugfs_remove_recursive(rdtgrp->plr->debugfs_dir); |
| device_destroy(pseudo_lock_class, MKDEV(pseudo_lock_major, plr->minor)); |
| pseudo_lock_minor_release(plr->minor); |
| |
| free: |
| pseudo_lock_free(rdtgrp); |
| } |
| |
| static int pseudo_lock_dev_open(struct inode *inode, struct file *filp) |
| { |
| struct rdtgroup *rdtgrp; |
| |
| mutex_lock(&rdtgroup_mutex); |
| |
| rdtgrp = region_find_by_minor(iminor(inode)); |
| if (!rdtgrp) { |
| mutex_unlock(&rdtgroup_mutex); |
| return -ENODEV; |
| } |
| |
| filp->private_data = rdtgrp; |
| atomic_inc(&rdtgrp->waitcount); |
| /* Perform a non-seekable open - llseek is not supported */ |
| filp->f_mode &= ~(FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE); |
| |
| mutex_unlock(&rdtgroup_mutex); |
| |
| return 0; |
| } |
| |
| static int pseudo_lock_dev_release(struct inode *inode, struct file *filp) |
| { |
| struct rdtgroup *rdtgrp; |
| |
| mutex_lock(&rdtgroup_mutex); |
| rdtgrp = filp->private_data; |
| WARN_ON(!rdtgrp); |
| if (!rdtgrp) { |
| mutex_unlock(&rdtgroup_mutex); |
| return -ENODEV; |
| } |
| filp->private_data = NULL; |
| atomic_dec(&rdtgrp->waitcount); |
| mutex_unlock(&rdtgroup_mutex); |
| return 0; |
| } |
| |
| static int pseudo_lock_dev_mremap(struct vm_area_struct *area) |
| { |
| /* Not supported */ |
| return -EINVAL; |
| } |
| |
| static const struct vm_operations_struct pseudo_mmap_ops = { |
| .mremap = pseudo_lock_dev_mremap, |
| }; |
| |
| static int pseudo_lock_dev_mmap(struct file *filp, struct vm_area_struct *vma) |
| { |
| unsigned long vsize = vma->vm_end - vma->vm_start; |
| unsigned long off = vma->vm_pgoff << PAGE_SHIFT; |
| struct pseudo_lock_region *plr; |
| struct rdtgroup *rdtgrp; |
| unsigned long physical; |
| unsigned long psize; |
| |
| mutex_lock(&rdtgroup_mutex); |
| |
| rdtgrp = filp->private_data; |
| WARN_ON(!rdtgrp); |
| if (!rdtgrp) { |
| mutex_unlock(&rdtgroup_mutex); |
| return -ENODEV; |
| } |
| |
| plr = rdtgrp->plr; |
| |
| if (!plr->d) { |
| mutex_unlock(&rdtgroup_mutex); |
| return -ENODEV; |
| } |
| |
| /* |
| * Task is required to run with affinity to the cpus associated |
| * with the pseudo-locked region. If this is not the case the task |
| * may be scheduled elsewhere and invalidate entries in the |
| * pseudo-locked region. |
| */ |
| if (!cpumask_subset(¤t->cpus_allowed, &plr->d->cpu_mask)) { |
| mutex_unlock(&rdtgroup_mutex); |
| return -EINVAL; |
| } |
| |
| physical = __pa(plr->kmem) >> PAGE_SHIFT; |
| psize = plr->size - off; |
| |
| if (off > plr->size) { |
| mutex_unlock(&rdtgroup_mutex); |
| return -ENOSPC; |
| } |
| |
| /* |
| * Ensure changes are carried directly to the memory being mapped, |
| * do not allow copy-on-write mapping. |
| */ |
| if (!(vma->vm_flags & VM_SHARED)) { |
| mutex_unlock(&rdtgroup_mutex); |
| return -EINVAL; |
| } |
| |
| if (vsize > psize) { |
| mutex_unlock(&rdtgroup_mutex); |
| return -ENOSPC; |
| } |
| |
| memset(plr->kmem + off, 0, vsize); |
| |
| if (remap_pfn_range(vma, vma->vm_start, physical + vma->vm_pgoff, |
| vsize, vma->vm_page_prot)) { |
| mutex_unlock(&rdtgroup_mutex); |
| return -EAGAIN; |
| } |
| vma->vm_ops = &pseudo_mmap_ops; |
| mutex_unlock(&rdtgroup_mutex); |
| return 0; |
| } |
| |
| static const struct file_operations pseudo_lock_dev_fops = { |
| .owner = THIS_MODULE, |
| .llseek = no_llseek, |
| .read = NULL, |
| .write = NULL, |
| .open = pseudo_lock_dev_open, |
| .release = pseudo_lock_dev_release, |
| .mmap = pseudo_lock_dev_mmap, |
| }; |
| |
| static char *pseudo_lock_devnode(struct device *dev, umode_t *mode) |
| { |
| struct rdtgroup *rdtgrp; |
| |
| rdtgrp = dev_get_drvdata(dev); |
| if (mode) |
| *mode = 0600; |
| return kasprintf(GFP_KERNEL, "pseudo_lock/%s", rdtgrp->kn->name); |
| } |
| |
| int rdt_pseudo_lock_init(void) |
| { |
| int ret; |
| |
| ret = register_chrdev(0, "pseudo_lock", &pseudo_lock_dev_fops); |
| if (ret < 0) |
| return ret; |
| |
| pseudo_lock_major = ret; |
| |
| pseudo_lock_class = class_create(THIS_MODULE, "pseudo_lock"); |
| if (IS_ERR(pseudo_lock_class)) { |
| ret = PTR_ERR(pseudo_lock_class); |
| unregister_chrdev(pseudo_lock_major, "pseudo_lock"); |
| return ret; |
| } |
| |
| pseudo_lock_class->devnode = pseudo_lock_devnode; |
| return 0; |
| } |
| |
| void rdt_pseudo_lock_release(void) |
| { |
| class_destroy(pseudo_lock_class); |
| pseudo_lock_class = NULL; |
| unregister_chrdev(pseudo_lock_major, "pseudo_lock"); |
| pseudo_lock_major = 0; |
| } |