blob: 1798f0f14d58a3ec555ac9bcbf5fb4132ba167d7 [file] [log] [blame]
// SPDX-License-Identifier: GPL-2.0-only
/*
* Support for Partition Mobility/Migration
*
* Copyright (C) 2010 Nathan Fontenot
* Copyright (C) 2010 IBM Corporation
*/
#define pr_fmt(fmt) "mobility: " fmt
#include <linux/cpu.h>
#include <linux/kernel.h>
#include <linux/kobject.h>
#include <linux/nmi.h>
#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/stat.h>
#include <linux/stop_machine.h>
#include <linux/completion.h>
#include <linux/device.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/stringify.h>
#include <asm/machdep.h>
#include <asm/nmi.h>
#include <asm/rtas.h>
#include "pseries.h"
#include "vas.h" /* vas_migration_handler() */
#include "../../kernel/cacheinfo.h"
static struct kobject *mobility_kobj;
struct update_props_workarea {
__be32 phandle;
__be32 state;
__be64 reserved;
__be32 nprops;
} __packed;
#define NODE_ACTION_MASK 0xff000000
#define NODE_COUNT_MASK 0x00ffffff
#define DELETE_DT_NODE 0x01000000
#define UPDATE_DT_NODE 0x02000000
#define ADD_DT_NODE 0x03000000
#define MIGRATION_SCOPE (1)
#define PRRN_SCOPE -2
#ifdef CONFIG_PPC_WATCHDOG
static unsigned int nmi_wd_lpm_factor = 200;
#ifdef CONFIG_SYSCTL
static struct ctl_table nmi_wd_lpm_factor_ctl_table[] = {
{
.procname = "nmi_wd_lpm_factor",
.data = &nmi_wd_lpm_factor,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_douintvec_minmax,
},
};
static int __init register_nmi_wd_lpm_factor_sysctl(void)
{
register_sysctl("kernel", nmi_wd_lpm_factor_ctl_table);
return 0;
}
device_initcall(register_nmi_wd_lpm_factor_sysctl);
#endif /* CONFIG_SYSCTL */
#endif /* CONFIG_PPC_WATCHDOG */
static int mobility_rtas_call(int token, char *buf, s32 scope)
{
int rc;
spin_lock(&rtas_data_buf_lock);
memcpy(rtas_data_buf, buf, RTAS_DATA_BUF_SIZE);
rc = rtas_call(token, 2, 1, NULL, rtas_data_buf, scope);
memcpy(buf, rtas_data_buf, RTAS_DATA_BUF_SIZE);
spin_unlock(&rtas_data_buf_lock);
return rc;
}
static int delete_dt_node(struct device_node *dn)
{
struct device_node *pdn;
bool is_platfac;
pdn = of_get_parent(dn);
is_platfac = of_node_is_type(dn, "ibm,platform-facilities") ||
of_node_is_type(pdn, "ibm,platform-facilities");
of_node_put(pdn);
/*
* The drivers that bind to nodes in the platform-facilities
* hierarchy don't support node removal, and the removal directive
* from firmware is always followed by an add of an equivalent
* node. The capability (e.g. RNG, encryption, compression)
* represented by the node is never interrupted by the migration.
* So ignore changes to this part of the tree.
*/
if (is_platfac) {
pr_notice("ignoring remove operation for %pOFfp\n", dn);
return 0;
}
pr_debug("removing node %pOFfp\n", dn);
dlpar_detach_node(dn);
return 0;
}
static int update_dt_property(struct device_node *dn, struct property **prop,
const char *name, u32 vd, char *value)
{
struct property *new_prop = *prop;
int more = 0;
/* A negative 'vd' value indicates that only part of the new property
* value is contained in the buffer and we need to call
* ibm,update-properties again to get the rest of the value.
*
* A negative value is also the two's compliment of the actual value.
*/
if (vd & 0x80000000) {
vd = ~vd + 1;
more = 1;
}
if (new_prop) {
/* partial property fixup */
char *new_data = kzalloc(new_prop->length + vd, GFP_KERNEL);
if (!new_data)
return -ENOMEM;
memcpy(new_data, new_prop->value, new_prop->length);
memcpy(new_data + new_prop->length, value, vd);
kfree(new_prop->value);
new_prop->value = new_data;
new_prop->length += vd;
} else {
new_prop = kzalloc(sizeof(*new_prop), GFP_KERNEL);
if (!new_prop)
return -ENOMEM;
new_prop->name = kstrdup(name, GFP_KERNEL);
if (!new_prop->name) {
kfree(new_prop);
return -ENOMEM;
}
new_prop->length = vd;
new_prop->value = kzalloc(new_prop->length, GFP_KERNEL);
if (!new_prop->value) {
kfree(new_prop->name);
kfree(new_prop);
return -ENOMEM;
}
memcpy(new_prop->value, value, vd);
*prop = new_prop;
}
if (!more) {
pr_debug("updating node %pOF property %s\n", dn, name);
of_update_property(dn, new_prop);
*prop = NULL;
}
return 0;
}
static int update_dt_node(struct device_node *dn, s32 scope)
{
struct update_props_workarea *upwa;
struct property *prop = NULL;
int i, rc, rtas_rc;
char *prop_data;
char *rtas_buf;
int update_properties_token;
u32 nprops;
u32 vd;
update_properties_token = rtas_function_token(RTAS_FN_IBM_UPDATE_PROPERTIES);
if (update_properties_token == RTAS_UNKNOWN_SERVICE)
return -EINVAL;
rtas_buf = kzalloc(RTAS_DATA_BUF_SIZE, GFP_KERNEL);
if (!rtas_buf)
return -ENOMEM;
upwa = (struct update_props_workarea *)&rtas_buf[0];
upwa->phandle = cpu_to_be32(dn->phandle);
do {
rtas_rc = mobility_rtas_call(update_properties_token, rtas_buf,
scope);
if (rtas_rc < 0)
break;
prop_data = rtas_buf + sizeof(*upwa);
nprops = be32_to_cpu(upwa->nprops);
/* On the first call to ibm,update-properties for a node the
* first property value descriptor contains an empty
* property name, the property value length encoded as u32,
* and the property value is the node path being updated.
*/
if (*prop_data == 0) {
prop_data++;
vd = be32_to_cpu(*(__be32 *)prop_data);
prop_data += vd + sizeof(vd);
nprops--;
}
for (i = 0; i < nprops; i++) {
char *prop_name;
prop_name = prop_data;
prop_data += strlen(prop_name) + 1;
vd = be32_to_cpu(*(__be32 *)prop_data);
prop_data += sizeof(vd);
switch (vd) {
case 0x00000000:
/* name only property, nothing to do */
break;
case 0x80000000:
of_remove_property(dn, of_find_property(dn,
prop_name, NULL));
prop = NULL;
break;
default:
rc = update_dt_property(dn, &prop, prop_name,
vd, prop_data);
if (rc) {
pr_err("updating %s property failed: %d\n",
prop_name, rc);
}
prop_data += vd;
break;
}
cond_resched();
}
cond_resched();
} while (rtas_rc == 1);
kfree(rtas_buf);
return 0;
}
static int add_dt_node(struct device_node *parent_dn, __be32 drc_index)
{
struct device_node *dn;
int rc;
dn = dlpar_configure_connector(drc_index, parent_dn);
if (!dn)
return -ENOENT;
/*
* Since delete_dt_node() ignores this node type, this is the
* necessary counterpart. We also know that a platform-facilities
* node returned from dlpar_configure_connector() has children
* attached, and dlpar_attach_node() only adds the parent, leaking
* the children. So ignore these on the add side for now.
*/
if (of_node_is_type(dn, "ibm,platform-facilities")) {
pr_notice("ignoring add operation for %pOF\n", dn);
dlpar_free_cc_nodes(dn);
return 0;
}
rc = dlpar_attach_node(dn, parent_dn);
if (rc)
dlpar_free_cc_nodes(dn);
pr_debug("added node %pOFfp\n", dn);
return rc;
}
static int pseries_devicetree_update(s32 scope)
{
char *rtas_buf;
__be32 *data;
int update_nodes_token;
int rc;
update_nodes_token = rtas_function_token(RTAS_FN_IBM_UPDATE_NODES);
if (update_nodes_token == RTAS_UNKNOWN_SERVICE)
return 0;
rtas_buf = kzalloc(RTAS_DATA_BUF_SIZE, GFP_KERNEL);
if (!rtas_buf)
return -ENOMEM;
do {
rc = mobility_rtas_call(update_nodes_token, rtas_buf, scope);
if (rc && rc != 1)
break;
data = (__be32 *)rtas_buf + 4;
while (be32_to_cpu(*data) & NODE_ACTION_MASK) {
int i;
u32 action = be32_to_cpu(*data) & NODE_ACTION_MASK;
u32 node_count = be32_to_cpu(*data) & NODE_COUNT_MASK;
data++;
for (i = 0; i < node_count; i++) {
struct device_node *np;
__be32 phandle = *data++;
__be32 drc_index;
np = of_find_node_by_phandle(be32_to_cpu(phandle));
if (!np) {
pr_warn("Failed lookup: phandle 0x%x for action 0x%x\n",
be32_to_cpu(phandle), action);
continue;
}
switch (action) {
case DELETE_DT_NODE:
delete_dt_node(np);
break;
case UPDATE_DT_NODE:
update_dt_node(np, scope);
break;
case ADD_DT_NODE:
drc_index = *data++;
add_dt_node(np, drc_index);
break;
}
of_node_put(np);
cond_resched();
}
}
cond_resched();
} while (rc == 1);
kfree(rtas_buf);
return rc;
}
void post_mobility_fixup(void)
{
int rc;
rtas_activate_firmware();
/*
* We don't want CPUs to go online/offline while the device
* tree is being updated.
*/
cpus_read_lock();
/*
* It's common for the destination firmware to replace cache
* nodes. Release all of the cacheinfo hierarchy's references
* before updating the device tree.
*/
cacheinfo_teardown();
rc = pseries_devicetree_update(MIGRATION_SCOPE);
if (rc)
pr_err("device tree update failed: %d\n", rc);
cacheinfo_rebuild();
cpus_read_unlock();
/* Possibly switch to a new L1 flush type */
pseries_setup_security_mitigations();
/* Reinitialise system information for hv-24x7 */
read_24x7_sys_info();
return;
}
static int poll_vasi_state(u64 handle, unsigned long *res)
{
unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
long hvrc;
int ret;
hvrc = plpar_hcall(H_VASI_STATE, retbuf, handle);
switch (hvrc) {
case H_SUCCESS:
ret = 0;
*res = retbuf[0];
break;
case H_PARAMETER:
ret = -EINVAL;
break;
case H_FUNCTION:
ret = -EOPNOTSUPP;
break;
case H_HARDWARE:
default:
pr_err("unexpected H_VASI_STATE result %ld\n", hvrc);
ret = -EIO;
break;
}
return ret;
}
static int wait_for_vasi_session_suspending(u64 handle)
{
unsigned long state;
int ret;
/*
* Wait for transition from H_VASI_ENABLED to
* H_VASI_SUSPENDING. Treat anything else as an error.
*/
while (true) {
ret = poll_vasi_state(handle, &state);
if (ret != 0 || state == H_VASI_SUSPENDING) {
break;
} else if (state == H_VASI_ENABLED) {
ssleep(1);
} else {
pr_err("unexpected H_VASI_STATE result %lu\n", state);
ret = -EIO;
break;
}
}
/*
* Proceed even if H_VASI_STATE is unavailable. If H_JOIN or
* ibm,suspend-me are also unimplemented, we'll recover then.
*/
if (ret == -EOPNOTSUPP)
ret = 0;
return ret;
}
static void wait_for_vasi_session_completed(u64 handle)
{
unsigned long state = 0;
int ret;
pr_info("waiting for memory transfer to complete...\n");
/*
* Wait for transition from H_VASI_RESUMED to H_VASI_COMPLETED.
*/
while (true) {
ret = poll_vasi_state(handle, &state);
/*
* If the memory transfer is already complete and the migration
* has been cleaned up by the hypervisor, H_PARAMETER is return,
* which is translate in EINVAL by poll_vasi_state().
*/
if (ret == -EINVAL || (!ret && state == H_VASI_COMPLETED)) {
pr_info("memory transfer completed.\n");
break;
}
if (ret) {
pr_err("H_VASI_STATE return error (%d)\n", ret);
break;
}
if (state != H_VASI_RESUMED) {
pr_err("unexpected H_VASI_STATE result %lu\n", state);
break;
}
msleep(500);
}
}
static void prod_single(unsigned int target_cpu)
{
long hvrc;
int hwid;
hwid = get_hard_smp_processor_id(target_cpu);
hvrc = plpar_hcall_norets(H_PROD, hwid);
if (hvrc == H_SUCCESS)
return;
pr_err_ratelimited("H_PROD of CPU %u (hwid %d) error: %ld\n",
target_cpu, hwid, hvrc);
}
static void prod_others(void)
{
unsigned int cpu;
for_each_online_cpu(cpu) {
if (cpu != smp_processor_id())
prod_single(cpu);
}
}
static u16 clamp_slb_size(void)
{
#ifdef CONFIG_PPC_64S_HASH_MMU
u16 prev = mmu_slb_size;
slb_set_size(SLB_MIN_SIZE);
return prev;
#else
return 0;
#endif
}
static int do_suspend(void)
{
u16 saved_slb_size;
int status;
int ret;
pr_info("calling ibm,suspend-me on CPU %i\n", smp_processor_id());
/*
* The destination processor model may have fewer SLB entries
* than the source. We reduce mmu_slb_size to a safe minimum
* before suspending in order to minimize the possibility of
* programming non-existent entries on the destination. If
* suspend fails, we restore it before returning. On success
* the OF reconfig path will update it from the new device
* tree after resuming on the destination.
*/
saved_slb_size = clamp_slb_size();
ret = rtas_ibm_suspend_me(&status);
if (ret != 0) {
pr_err("ibm,suspend-me error: %d\n", status);
slb_set_size(saved_slb_size);
}
return ret;
}
/**
* struct pseries_suspend_info - State shared between CPUs for join/suspend.
* @counter: Threads are to increment this upon resuming from suspend
* or if an error is received from H_JOIN. The thread which performs
* the first increment (i.e. sets it to 1) is responsible for
* waking the other threads.
* @done: False if join/suspend is in progress. True if the operation is
* complete (successful or not).
*/
struct pseries_suspend_info {
atomic_t counter;
bool done;
};
static int do_join(void *arg)
{
struct pseries_suspend_info *info = arg;
atomic_t *counter = &info->counter;
long hvrc;
int ret;
retry:
/* Must ensure MSR.EE off for H_JOIN. */
hard_irq_disable();
hvrc = plpar_hcall_norets(H_JOIN);
switch (hvrc) {
case H_CONTINUE:
/*
* All other CPUs are offline or in H_JOIN. This CPU
* attempts the suspend.
*/
ret = do_suspend();
break;
case H_SUCCESS:
/*
* The suspend is complete and this cpu has received a
* prod, or we've received a stray prod from unrelated
* code (e.g. paravirt spinlocks) and we need to join
* again.
*
* This barrier orders the return from H_JOIN above vs
* the load of info->done. It pairs with the barrier
* in the wakeup/prod path below.
*/
smp_mb();
if (READ_ONCE(info->done) == false) {
pr_info_ratelimited("premature return from H_JOIN on CPU %i, retrying",
smp_processor_id());
goto retry;
}
ret = 0;
break;
case H_BAD_MODE:
case H_HARDWARE:
default:
ret = -EIO;
pr_err_ratelimited("H_JOIN error %ld on CPU %i\n",
hvrc, smp_processor_id());
break;
}
if (atomic_inc_return(counter) == 1) {
pr_info("CPU %u waking all threads\n", smp_processor_id());
WRITE_ONCE(info->done, true);
/*
* This barrier orders the store to info->done vs subsequent
* H_PRODs to wake the other CPUs. It pairs with the barrier
* in the H_SUCCESS case above.
*/
smp_mb();
prod_others();
}
/*
* Execution may have been suspended for several seconds, so reset
* the watchdogs. touch_nmi_watchdog() also touches the soft lockup
* watchdog.
*/
rcu_cpu_stall_reset();
touch_nmi_watchdog();
return ret;
}
/*
* Abort reason code byte 0. We use only the 'Migrating partition' value.
*/
enum vasi_aborting_entity {
ORCHESTRATOR = 1,
VSP_SOURCE = 2,
PARTITION_FIRMWARE = 3,
PLATFORM_FIRMWARE = 4,
VSP_TARGET = 5,
MIGRATING_PARTITION = 6,
};
static void pseries_cancel_migration(u64 handle, int err)
{
u32 reason_code;
u32 detail;
u8 entity;
long hvrc;
entity = MIGRATING_PARTITION;
detail = abs(err) & 0xffffff;
reason_code = (entity << 24) | detail;
hvrc = plpar_hcall_norets(H_VASI_SIGNAL, handle,
H_VASI_SIGNAL_CANCEL, reason_code);
if (hvrc)
pr_err("H_VASI_SIGNAL error: %ld\n", hvrc);
}
static int pseries_suspend(u64 handle)
{
const unsigned int max_attempts = 5;
unsigned int retry_interval_ms = 1;
unsigned int attempt = 1;
int ret;
while (true) {
struct pseries_suspend_info info;
unsigned long vasi_state;
int vasi_err;
info = (struct pseries_suspend_info) {
.counter = ATOMIC_INIT(0),
.done = false,
};
ret = stop_machine(do_join, &info, cpu_online_mask);
if (ret == 0)
break;
/*
* Encountered an error. If the VASI stream is still
* in Suspending state, it's likely a transient
* condition related to some device in the partition
* and we can retry in the hope that the cause has
* cleared after some delay.
*
* A better design would allow drivers etc to prepare
* for the suspend and avoid conditions which prevent
* the suspend from succeeding. For now, we have this
* mitigation.
*/
pr_notice("Partition suspend attempt %u of %u error: %d\n",
attempt, max_attempts, ret);
if (attempt == max_attempts)
break;
vasi_err = poll_vasi_state(handle, &vasi_state);
if (vasi_err == 0) {
if (vasi_state != H_VASI_SUSPENDING) {
pr_notice("VASI state %lu after failed suspend\n",
vasi_state);
break;
}
} else if (vasi_err != -EOPNOTSUPP) {
pr_err("VASI state poll error: %d", vasi_err);
break;
}
pr_notice("Will retry partition suspend after %u ms\n",
retry_interval_ms);
msleep(retry_interval_ms);
retry_interval_ms *= 10;
attempt++;
}
return ret;
}
static int pseries_migrate_partition(u64 handle)
{
int ret;
unsigned int factor = 0;
#ifdef CONFIG_PPC_WATCHDOG
factor = nmi_wd_lpm_factor;
#endif
/*
* When the migration is initiated, the hypervisor changes VAS
* mappings to prepare before OS gets the notification and
* closes all VAS windows. NX generates continuous faults during
* this time and the user space can not differentiate these
* faults from the migration event. So reduce this time window
* by closing VAS windows at the beginning of this function.
*/
vas_migration_handler(VAS_SUSPEND);
ret = wait_for_vasi_session_suspending(handle);
if (ret)
goto out;
if (factor)
watchdog_hardlockup_set_timeout_pct(factor);
ret = pseries_suspend(handle);
if (ret == 0) {
post_mobility_fixup();
/*
* Wait until the memory transfer is complete, so that the user
* space process returns from the syscall after the transfer is
* complete. This allows the user hooks to be executed at the
* right time.
*/
wait_for_vasi_session_completed(handle);
} else
pseries_cancel_migration(handle, ret);
if (factor)
watchdog_hardlockup_set_timeout_pct(0);
out:
vas_migration_handler(VAS_RESUME);
return ret;
}
int rtas_syscall_dispatch_ibm_suspend_me(u64 handle)
{
return pseries_migrate_partition(handle);
}
static ssize_t migration_store(const struct class *class,
const struct class_attribute *attr, const char *buf,
size_t count)
{
u64 streamid;
int rc;
rc = kstrtou64(buf, 0, &streamid);
if (rc)
return rc;
rc = pseries_migrate_partition(streamid);
if (rc)
return rc;
return count;
}
/*
* Used by drmgr to determine the kernel behavior of the migration interface.
*
* Version 1: Performs all PAPR requirements for migration including
* firmware activation and device tree update.
*/
#define MIGRATION_API_VERSION 1
static CLASS_ATTR_WO(migration);
static CLASS_ATTR_STRING(api_version, 0444, __stringify(MIGRATION_API_VERSION));
static int __init mobility_sysfs_init(void)
{
int rc;
mobility_kobj = kobject_create_and_add("mobility", kernel_kobj);
if (!mobility_kobj)
return -ENOMEM;
rc = sysfs_create_file(mobility_kobj, &class_attr_migration.attr);
if (rc)
pr_err("unable to create migration sysfs file (%d)\n", rc);
rc = sysfs_create_file(mobility_kobj, &class_attr_api_version.attr.attr);
if (rc)
pr_err("unable to create api_version sysfs file (%d)\n", rc);
return 0;
}
machine_device_initcall(pseries, mobility_sysfs_init);