drivers/dpll/dpll_core.c

// SPDX-License-Identifier: GPL-2.0
/*
 *  dpll_core.c - DPLL subsystem kernel-space interface implementation.
 *
 *  Copyright (c) 2023 Meta Platforms, Inc. and affiliates
 *  Copyright (c) 2023 Intel Corporation.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/device.h>
#include <linux/err.h>
#include <linux/idr.h>
#include <linux/module.h>
#include <linux/property.h>
#include <linux/slab.h>
#include <linux/string.h>

#include "dpll_core.h"
#include "dpll_netlink.h"

/* Mutex lock to protect DPLL subsystem devices and pins */
DEFINE_MUTEX(dpll_lock);

DEFINE_XARRAY_FLAGS(dpll_device_xa, XA_FLAGS_ALLOC);
DEFINE_XARRAY_FLAGS(dpll_pin_xa, XA_FLAGS_ALLOC);

static RAW_NOTIFIER_HEAD(dpll_notifier_chain);
static DEFINE_IDA(dpll_pin_idx_ida);

static u32 dpll_device_xa_id;
static u32 dpll_pin_xa_id;

#define ASSERT_DPLL_REGISTERED(d)	\
	WARN_ON_ONCE(!xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
#define ASSERT_DPLL_NOT_REGISTERED(d)	\
	WARN_ON_ONCE(xa_get_mark(&dpll_device_xa, (d)->id, DPLL_REGISTERED))
#define ASSERT_DPLL_PIN_REGISTERED(p) \
	WARN_ON_ONCE(!xa_get_mark(&dpll_pin_xa, (p)->id, DPLL_REGISTERED))

struct dpll_device_registration {
	struct list_head list;
	const struct dpll_device_ops *ops;
	void *priv;
	dpll_tracker tracker;
};

struct dpll_pin_registration {
	struct list_head list;
	const struct dpll_pin_ops *ops;
	void *priv;
	void *cookie;
	dpll_tracker tracker;
};

static int call_dpll_notifiers(unsigned long action, void *info)
{
	lockdep_assert_held(&dpll_lock);
	return raw_notifier_call_chain(&dpll_notifier_chain, action, info);
}

void dpll_device_notify(struct dpll_device *dpll, unsigned long action)
{
	struct dpll_device_notifier_info info = {
		.dpll = dpll,
		.id = dpll->id,
		.idx = dpll->device_idx,
		.clock_id = dpll->clock_id,
		.type = dpll->type,
	};

	call_dpll_notifiers(action, &info);
}

void dpll_pin_notify(struct dpll_pin *pin, u64 src_clock_id,
		     unsigned long action)
{
	struct dpll_pin_notifier_info info = {
		.pin = pin,
		.id = pin->id,
		.idx = pin->pin_idx,
		.clock_id = pin->clock_id,
		.fwnode = pin->fwnode,
		.prop = &pin->prop,
		.src_clock_id = src_clock_id,
	};

	call_dpll_notifiers(action, &info);
}

static void dpll_device_tracker_alloc(struct dpll_device *dpll,
				      dpll_tracker *tracker)
{
#ifdef CONFIG_DPLL_REFCNT_TRACKER
	ref_tracker_alloc(&dpll->refcnt_tracker, tracker, GFP_KERNEL);
#endif
}

static void dpll_device_tracker_free(struct dpll_device *dpll,
				     dpll_tracker *tracker)
{
#ifdef CONFIG_DPLL_REFCNT_TRACKER
	ref_tracker_free(&dpll->refcnt_tracker, tracker);
#endif
}

static void __dpll_device_hold(struct dpll_device *dpll, dpll_tracker *tracker)
{
	dpll_device_tracker_alloc(dpll, tracker);
	refcount_inc(&dpll->refcount);
}

static void __dpll_device_put(struct dpll_device *dpll, dpll_tracker *tracker)
{
	dpll_device_tracker_free(dpll, tracker);
	if (refcount_dec_and_test(&dpll->refcount)) {
		ASSERT_DPLL_NOT_REGISTERED(dpll);
		WARN_ON_ONCE(!xa_empty(&dpll->pin_refs));
		xa_destroy(&dpll->pin_refs);
		xa_erase(&dpll_device_xa, dpll->id);
		WARN_ON(!list_empty(&dpll->registration_list));
		ref_tracker_dir_exit(&dpll->refcnt_tracker);
		kfree(dpll);
	}
}

static void dpll_pin_tracker_alloc(struct dpll_pin *pin, dpll_tracker *tracker)
{
#ifdef CONFIG_DPLL_REFCNT_TRACKER
	ref_tracker_alloc(&pin->refcnt_tracker, tracker, GFP_KERNEL);
#endif
}

static void dpll_pin_tracker_free(struct dpll_pin *pin, dpll_tracker *tracker)
{
#ifdef CONFIG_DPLL_REFCNT_TRACKER
	ref_tracker_free(&pin->refcnt_tracker, tracker);
#endif
}

static void __dpll_pin_hold(struct dpll_pin *pin, dpll_tracker *tracker)
{
	dpll_pin_tracker_alloc(pin, tracker);
	refcount_inc(&pin->refcount);
}

static void dpll_pin_idx_free(u32 pin_idx);
static void dpll_pin_prop_free(struct dpll_pin_properties *prop);

static void __dpll_pin_put(struct dpll_pin *pin, dpll_tracker *tracker)
{
	dpll_pin_tracker_free(pin, tracker);
	if (refcount_dec_and_test(&pin->refcount)) {
		xa_erase(&dpll_pin_xa, pin->id);
		xa_destroy(&pin->dpll_refs);
		xa_destroy(&pin->parent_refs);
		xa_destroy(&pin->ref_sync_pins);
		dpll_pin_prop_free(&pin->prop);
		fwnode_handle_put(pin->fwnode);
		dpll_pin_idx_free(pin->pin_idx);
		ref_tracker_dir_exit(&pin->refcnt_tracker);
		kfree_rcu(pin, rcu);
	}
}

struct dpll_device *dpll_device_get_by_id(int id)
{
	if (xa_get_mark(&dpll_device_xa, id, DPLL_REGISTERED))
		return xa_load(&dpll_device_xa, id);

	return NULL;
}

static struct dpll_pin_registration *
dpll_pin_registration_find(struct dpll_pin_ref *ref,
			   const struct dpll_pin_ops *ops, void *priv,
			   void *cookie)
{
	struct dpll_pin_registration *reg;

	list_for_each_entry(reg, &ref->registration_list, list) {
		if (reg->ops == ops && reg->priv == priv &&
		    reg->cookie == cookie)
			return reg;
	}
	return NULL;
}

static int
dpll_xa_ref_pin_add(struct xarray *xa_pins, struct dpll_pin *pin,
		    const struct dpll_pin_ops *ops, void *priv,
		    void *cookie)
{
	struct dpll_pin_registration *reg;
	struct dpll_pin_ref *ref;
	bool ref_exists = false;
	unsigned long i;
	int ret;

	xa_for_each(xa_pins, i, ref) {
		if (ref->pin != pin)
			continue;
		reg = dpll_pin_registration_find(ref, ops, priv, cookie);
		if (reg)
			return -EEXIST;
		ref_exists = true;
		break;
	}

	if (!ref_exists) {
		ref = kzalloc_obj(*ref);
		if (!ref)
			return -ENOMEM;
		ref->pin = pin;
		INIT_LIST_HEAD(&ref->registration_list);
		ret = xa_insert(xa_pins, pin->pin_idx, ref, GFP_KERNEL);
		if (ret) {
			kfree(ref);
			return ret;
		}
		refcount_set(&ref->refcount, 1);
	}

	reg = kzalloc_obj(*reg);
	if (!reg) {
		if (!ref_exists) {
			xa_erase(xa_pins, pin->pin_idx);
			kfree(ref);
		}
		return -ENOMEM;
	}
	reg->ops = ops;
	reg->priv = priv;
	reg->cookie = cookie;
	__dpll_pin_hold(pin, &reg->tracker);
	if (ref_exists)
		refcount_inc(&ref->refcount);
	list_add_tail(&reg->list, &ref->registration_list);

	return 0;
}

static int dpll_xa_ref_pin_del(struct xarray *xa_pins, struct dpll_pin *pin,
			       const struct dpll_pin_ops *ops, void *priv,
			       void *cookie)
{
	struct dpll_pin_registration *reg;
	struct dpll_pin_ref *ref;
	unsigned long i;

	xa_for_each(xa_pins, i, ref) {
		if (ref->pin != pin)
			continue;
		reg = dpll_pin_registration_find(ref, ops, priv, cookie);
		if (WARN_ON(!reg))
			return -EINVAL;
		list_del(&reg->list);
		__dpll_pin_put(pin, &reg->tracker);
		kfree(reg);
		if (refcount_dec_and_test(&ref->refcount)) {
			xa_erase(xa_pins, i);
			WARN_ON(!list_empty(&ref->registration_list));
			kfree(ref);
		}
		return 0;
	}

	return -EINVAL;
}

static int
dpll_xa_ref_dpll_add(struct xarray *xa_dplls, struct dpll_device *dpll,
		     const struct dpll_pin_ops *ops, void *priv, void *cookie)
{
	struct dpll_pin_registration *reg;
	struct dpll_pin_ref *ref;
	bool ref_exists = false;
	unsigned long i;
	int ret;

	xa_for_each(xa_dplls, i, ref) {
		if (ref->dpll != dpll)
			continue;
		reg = dpll_pin_registration_find(ref, ops, priv, cookie);
		if (reg)
			return -EEXIST;
		ref_exists = true;
		break;
	}

	if (!ref_exists) {
		ref = kzalloc_obj(*ref);
		if (!ref)
			return -ENOMEM;
		ref->dpll = dpll;
		INIT_LIST_HEAD(&ref->registration_list);
		ret = xa_insert(xa_dplls, dpll->id, ref, GFP_KERNEL);
		if (ret) {
			kfree(ref);
			return ret;
		}
		refcount_set(&ref->refcount, 1);
	}

	reg = kzalloc_obj(*reg);
	if (!reg) {
		if (!ref_exists) {
			xa_erase(xa_dplls, dpll->id);
			kfree(ref);
		}
		return -ENOMEM;
	}
	reg->ops = ops;
	reg->priv = priv;
	reg->cookie = cookie;
	__dpll_device_hold(dpll, &reg->tracker);
	if (ref_exists)
		refcount_inc(&ref->refcount);
	list_add_tail(&reg->list, &ref->registration_list);

	return 0;
}

static void
dpll_xa_ref_dpll_del(struct xarray *xa_dplls, struct dpll_device *dpll,
		     const struct dpll_pin_ops *ops, void *priv, void *cookie)
{
	struct dpll_pin_registration *reg;
	struct dpll_pin_ref *ref;
	unsigned long i;

	xa_for_each(xa_dplls, i, ref) {
		if (ref->dpll != dpll)
			continue;
		reg = dpll_pin_registration_find(ref, ops, priv, cookie);
		if (WARN_ON(!reg))
			return;
		list_del(&reg->list);
		__dpll_device_put(dpll, &reg->tracker);
		kfree(reg);
		if (refcount_dec_and_test(&ref->refcount)) {
			xa_erase(xa_dplls, i);
			WARN_ON(!list_empty(&ref->registration_list));
			kfree(ref);
		}
		return;
	}
}

struct dpll_pin_ref *dpll_xa_ref_dpll_first(struct xarray *xa_refs)
{
	struct dpll_pin_ref *ref;
	unsigned long i = 0;

	ref = xa_find(xa_refs, &i, ULONG_MAX, XA_PRESENT);
	WARN_ON(!ref);
	return ref;
}

static struct dpll_device *
dpll_device_alloc(const u64 clock_id, u32 device_idx, struct module *module)
{
	struct dpll_device *dpll;
	int ret;

	dpll = kzalloc_obj(*dpll);
	if (!dpll)
		return ERR_PTR(-ENOMEM);
	refcount_set(&dpll->refcount, 1);
	INIT_LIST_HEAD(&dpll->registration_list);
	dpll->device_idx = device_idx;
	dpll->clock_id = clock_id;
	dpll->module = module;
	ret = xa_alloc_cyclic(&dpll_device_xa, &dpll->id, dpll, xa_limit_32b,
			      &dpll_device_xa_id, GFP_KERNEL);
	if (ret < 0) {
		kfree(dpll);
		return ERR_PTR(ret);
	}
	xa_init_flags(&dpll->pin_refs, XA_FLAGS_ALLOC);
	ref_tracker_dir_init(&dpll->refcnt_tracker, 128, "dpll_device");

	return dpll;
}

/**
 * dpll_device_get - find existing or create new dpll device
 * @clock_id: clock_id of creator
 * @device_idx: idx given by device driver
 * @module: reference to registering module
 * @tracker: tracking object for the acquired reference
 *
 * Get existing object of a dpll device, unique for given arguments.
 * Create new if doesn't exist yet.
 *
 * Context: Acquires a lock (dpll_lock)
 * Return:
 * * valid dpll_device struct pointer if succeeded
 * * ERR_PTR(X) - error
 */
struct dpll_device *
dpll_device_get(u64 clock_id, u32 device_idx, struct module *module,
		dpll_tracker *tracker)
{
	struct dpll_device *dpll, *ret = NULL;
	unsigned long index;

	mutex_lock(&dpll_lock);
	xa_for_each(&dpll_device_xa, index, dpll) {
		if (dpll->clock_id == clock_id &&
		    dpll->device_idx == device_idx &&
		    dpll->module == module) {
			__dpll_device_hold(dpll, tracker);
			ret = dpll;
			break;
		}
	}
	if (!ret) {
		ret = dpll_device_alloc(clock_id, device_idx, module);
		if (!IS_ERR(ret))
			dpll_device_tracker_alloc(ret, tracker);
	}

	mutex_unlock(&dpll_lock);

	return ret;
}
EXPORT_SYMBOL_GPL(dpll_device_get);

/**
 * dpll_device_put - decrease the refcount and free memory if possible
 * @dpll: dpll_device struct pointer
 * @tracker: tracking object for the acquired reference
 *
 * Context: Acquires a lock (dpll_lock)
 * Drop reference for a dpll device, if all references are gone, delete
 * dpll device object.
 */
void dpll_device_put(struct dpll_device *dpll, dpll_tracker *tracker)
{
	mutex_lock(&dpll_lock);
	__dpll_device_put(dpll, tracker);
	mutex_unlock(&dpll_lock);
}
EXPORT_SYMBOL_GPL(dpll_device_put);

static struct dpll_device_registration *
dpll_device_registration_find(struct dpll_device *dpll,
			      const struct dpll_device_ops *ops, void *priv)
{
	struct dpll_device_registration *reg;

	list_for_each_entry(reg, &dpll->registration_list, list) {
		if (reg->ops == ops && reg->priv == priv)
			return reg;
	}
	return NULL;
}

/**
 * dpll_device_register - register the dpll device in the subsystem
 * @dpll: pointer to a dpll
 * @type: type of a dpll
 * @ops: ops for a dpll device
 * @priv: pointer to private information of owner
 *
 * Make dpll device available for user space.
 *
 * Context: Acquires a lock (dpll_lock)
 * Return:
 * * 0 on success
 * * negative - error value
 */
int dpll_device_register(struct dpll_device *dpll, enum dpll_type type,
			 const struct dpll_device_ops *ops, void *priv)
{
	struct dpll_device_registration *reg;
	bool first_registration = false;

	if (WARN_ON(!ops))
		return -EINVAL;
	if (WARN_ON(!ops->mode_get))
		return -EINVAL;
	if (WARN_ON(!ops->lock_status_get))
		return -EINVAL;
	if (WARN_ON(type < DPLL_TYPE_PPS || type > DPLL_TYPE_MAX))
		return -EINVAL;

	mutex_lock(&dpll_lock);
	reg = dpll_device_registration_find(dpll, ops, priv);
	if (reg) {
		mutex_unlock(&dpll_lock);
		return -EEXIST;
	}

	reg = kzalloc_obj(*reg);
	if (!reg) {
		mutex_unlock(&dpll_lock);
		return -ENOMEM;
	}
	reg->ops = ops;
	reg->priv = priv;
	dpll->type = type;
	__dpll_device_hold(dpll, &reg->tracker);
	first_registration = list_empty(&dpll->registration_list);
	list_add_tail(&reg->list, &dpll->registration_list);
	if (!first_registration) {
		mutex_unlock(&dpll_lock);
		return 0;
	}

	xa_set_mark(&dpll_device_xa, dpll->id, DPLL_REGISTERED);
	dpll_device_create_ntf(dpll);
	mutex_unlock(&dpll_lock);

	return 0;
}
EXPORT_SYMBOL_GPL(dpll_device_register);

/**
 * dpll_device_unregister - unregister dpll device
 * @dpll: registered dpll pointer
 * @ops: ops for a dpll device
 * @priv: pointer to private information of owner
 *
 * Unregister device, make it unavailable for userspace.
 * Note: It does not free the memory
 * Context: Acquires a lock (dpll_lock)
 */
void dpll_device_unregister(struct dpll_device *dpll,
			    const struct dpll_device_ops *ops, void *priv)
{
	struct dpll_device_registration *reg;

	mutex_lock(&dpll_lock);
	ASSERT_DPLL_REGISTERED(dpll);
	dpll_device_delete_ntf(dpll);
	reg = dpll_device_registration_find(dpll, ops, priv);
	if (WARN_ON(!reg)) {
		mutex_unlock(&dpll_lock);
		return;
	}
	list_del(&reg->list);
	__dpll_device_put(dpll, &reg->tracker);
	kfree(reg);

	if (!list_empty(&dpll->registration_list)) {
		mutex_unlock(&dpll_lock);
		return;
	}
	xa_clear_mark(&dpll_device_xa, dpll->id, DPLL_REGISTERED);
	mutex_unlock(&dpll_lock);
}
EXPORT_SYMBOL_GPL(dpll_device_unregister);

static int dpll_pin_idx_alloc(u32 *pin_idx)
{
	int ret;

	if (!pin_idx)
		return -EINVAL;

	/* Alloc unique number from IDA. Number belongs to <0, INT_MAX> range */
	ret = ida_alloc(&dpll_pin_idx_ida, GFP_KERNEL);
	if (ret < 0)
		return ret;

	/* Map the value to dynamic pin index range <INT_MAX+1, U32_MAX> */
	*pin_idx = (u32)ret + INT_MAX + 1;

	return 0;
}

static void dpll_pin_idx_free(u32 pin_idx)
{
	if (pin_idx <= INT_MAX)
		return; /* Not a dynamic pin index */

	/* Map the index value from dynamic pin index range to IDA range and
	 * free it.
	 */
	pin_idx -= (u32)INT_MAX + 1;
	ida_free(&dpll_pin_idx_ida, pin_idx);
}

static void dpll_pin_prop_free(struct dpll_pin_properties *prop)
{
	kfree(prop->package_label);
	kfree(prop->panel_label);
	kfree(prop->board_label);
	kfree(prop->freq_supported);
}

static int dpll_pin_prop_dup(const struct dpll_pin_properties *src,
			     struct dpll_pin_properties *dst)
{
	if (WARN_ON(src->freq_supported && !src->freq_supported_num))
		return -EINVAL;

	memcpy(dst, src, sizeof(*dst));
	if (src->freq_supported) {
		size_t freq_size = src->freq_supported_num *
				   sizeof(*src->freq_supported);
		dst->freq_supported = kmemdup(src->freq_supported,
					      freq_size, GFP_KERNEL);
		if (!dst->freq_supported)
			return -ENOMEM;
	}
	if (src->board_label) {
		dst->board_label = kstrdup(src->board_label, GFP_KERNEL);
		if (!dst->board_label)
			goto err_board_label;
	}
	if (src->panel_label) {
		dst->panel_label = kstrdup(src->panel_label, GFP_KERNEL);
		if (!dst->panel_label)
			goto err_panel_label;
	}
	if (src->package_label) {
		dst->package_label = kstrdup(src->package_label, GFP_KERNEL);
		if (!dst->package_label)
			goto err_package_label;
	}

	return 0;

err_package_label:
	kfree(dst->panel_label);
err_panel_label:
	kfree(dst->board_label);
err_board_label:
	kfree(dst->freq_supported);
	return -ENOMEM;
}

static struct dpll_pin *
dpll_pin_alloc(u64 clock_id, u32 pin_idx, struct module *module,
	       const struct dpll_pin_properties *prop)
{
	struct dpll_pin *pin;
	int ret;

	if (pin_idx == DPLL_PIN_IDX_UNSPEC) {
		ret = dpll_pin_idx_alloc(&pin_idx);
		if (ret)
			return ERR_PTR(ret);
	} else if (pin_idx > INT_MAX) {
		return ERR_PTR(-EINVAL);
	}
	pin = kzalloc_obj(*pin);
	if (!pin) {
		ret = -ENOMEM;
		goto err_pin_alloc;
	}
	pin->pin_idx = pin_idx;
	pin->clock_id = clock_id;
	pin->module = module;
	strscpy(pin->module_name, module_name(module));
	if (WARN_ON(prop->type < DPLL_PIN_TYPE_MUX ||
		    prop->type > DPLL_PIN_TYPE_MAX)) {
		ret = -EINVAL;
		goto err_pin_prop;
	}
	ret = dpll_pin_prop_dup(prop, &pin->prop);
	if (ret)
		goto err_pin_prop;
	refcount_set(&pin->refcount, 1);
	xa_init_flags(&pin->dpll_refs, XA_FLAGS_ALLOC);
	xa_init_flags(&pin->parent_refs, XA_FLAGS_ALLOC);
	xa_init_flags(&pin->ref_sync_pins, XA_FLAGS_ALLOC);
	ret = xa_alloc_cyclic(&dpll_pin_xa, &pin->id, pin, xa_limit_32b,
			      &dpll_pin_xa_id, GFP_KERNEL);
	if (ret < 0)
		goto err_xa_alloc;
	ref_tracker_dir_init(&pin->refcnt_tracker, 128, "dpll_pin");
	return pin;
err_xa_alloc:
	xa_destroy(&pin->dpll_refs);
	xa_destroy(&pin->parent_refs);
	xa_destroy(&pin->ref_sync_pins);
	dpll_pin_prop_free(&pin->prop);
err_pin_prop:
	kfree(pin);
err_pin_alloc:
	dpll_pin_idx_free(pin_idx);
	return ERR_PTR(ret);
}

static void dpll_netdev_pin_assign(struct net_device *dev, struct dpll_pin *dpll_pin)
{
	rtnl_lock();
	rcu_assign_pointer(dev->dpll_pin, dpll_pin);
	rtnl_unlock();
}

void dpll_netdev_pin_set(struct net_device *dev, struct dpll_pin *dpll_pin)
{
	WARN_ON(!dpll_pin);
	dpll_netdev_pin_assign(dev, dpll_pin);
}
EXPORT_SYMBOL(dpll_netdev_pin_set);

void dpll_netdev_pin_clear(struct net_device *dev)
{
	dpll_netdev_pin_assign(dev, NULL);
}
EXPORT_SYMBOL(dpll_netdev_pin_clear);

int register_dpll_notifier(struct notifier_block *nb)
{
	int ret;

	mutex_lock(&dpll_lock);
	ret = raw_notifier_chain_register(&dpll_notifier_chain, nb);
	mutex_unlock(&dpll_lock);
	return ret;
}
EXPORT_SYMBOL_GPL(register_dpll_notifier);

int unregister_dpll_notifier(struct notifier_block *nb)
{
	int ret;

	mutex_lock(&dpll_lock);
	ret = raw_notifier_chain_unregister(&dpll_notifier_chain, nb);
	mutex_unlock(&dpll_lock);
	return ret;
}
EXPORT_SYMBOL_GPL(unregister_dpll_notifier);

/**
 * dpll_pin_get - find existing or create new dpll pin
 * @clock_id: clock_id of creator
 * @pin_idx: idx given by dev driver
 * @module: reference to registering module
 * @prop: dpll pin properties
 * @tracker: tracking object for the acquired reference
 *
 * Get existing object of a pin (unique for given arguments) or create new
 * if doesn't exist yet.
 *
 * Context: Acquires a lock (dpll_lock)
 * Return:
 * * valid allocated dpll_pin struct pointer if succeeded
 * * ERR_PTR(X) - error
 */
struct dpll_pin *
dpll_pin_get(u64 clock_id, u32 pin_idx, struct module *module,
	     const struct dpll_pin_properties *prop, dpll_tracker *tracker)
{
	struct dpll_pin *pos, *ret = NULL;
	unsigned long i;

	mutex_lock(&dpll_lock);
	xa_for_each(&dpll_pin_xa, i, pos) {
		if (pos->clock_id == clock_id &&
		    pos->pin_idx == pin_idx &&
		    pos->module == module) {
			__dpll_pin_hold(pos, tracker);
			ret = pos;
			break;
		}
	}
	if (!ret) {
		ret = dpll_pin_alloc(clock_id, pin_idx, module, prop);
		if (!IS_ERR(ret))
			dpll_pin_tracker_alloc(ret, tracker);
	}
	mutex_unlock(&dpll_lock);

	return ret;
}
EXPORT_SYMBOL_GPL(dpll_pin_get);

/**
 * dpll_pin_put - decrease the refcount and free memory if possible
 * @pin: pointer to a pin to be put
 * @tracker: tracking object for the acquired reference
 *
 * Drop reference for a pin, if all references are gone, delete pin object.
 *
 * Context: Acquires a lock (dpll_lock)
 */
void dpll_pin_put(struct dpll_pin *pin, dpll_tracker *tracker)
{
	mutex_lock(&dpll_lock);
	__dpll_pin_put(pin, tracker);
	mutex_unlock(&dpll_lock);
}
EXPORT_SYMBOL_GPL(dpll_pin_put);

/**
 * dpll_pin_fwnode_set - set dpll pin firmware node reference
 * @pin: pointer to a dpll pin
 * @fwnode: firmware node handle
 *
 * Set firmware node handle for the given dpll pin.
 */
void dpll_pin_fwnode_set(struct dpll_pin *pin, struct fwnode_handle *fwnode)
{
	mutex_lock(&dpll_lock);
	fwnode_handle_put(pin->fwnode); /* Drop fwnode previously set */
	pin->fwnode = fwnode_handle_get(fwnode);
	mutex_unlock(&dpll_lock);
}
EXPORT_SYMBOL_GPL(dpll_pin_fwnode_set);

/**
 * fwnode_dpll_pin_find - find dpll pin by firmware node reference
 * @fwnode: reference to firmware node
 * @tracker: tracking object for the acquired reference
 *
 * Get existing object of a pin that is associated with given firmware node
 * reference.
 *
 * Context: Acquires a lock (dpll_lock)
 * Return:
 * * valid dpll_pin pointer on success
 * * NULL when no such pin exists
 */
struct dpll_pin *fwnode_dpll_pin_find(struct fwnode_handle *fwnode,
				      dpll_tracker *tracker)
{
	struct dpll_pin *pin, *ret = NULL;
	unsigned long index;

	mutex_lock(&dpll_lock);
	xa_for_each(&dpll_pin_xa, index, pin) {
		if (pin->fwnode == fwnode) {
			__dpll_pin_hold(pin, tracker);
			ret = pin;
			break;
		}
	}
	mutex_unlock(&dpll_lock);

	return ret;
}
EXPORT_SYMBOL_GPL(fwnode_dpll_pin_find);

static int
__dpll_pin_register(struct dpll_device *dpll, struct dpll_pin *pin,
		    const struct dpll_pin_ops *ops, void *priv, void *cookie)
{
	int ret;

	ret = dpll_xa_ref_pin_add(&dpll->pin_refs, pin, ops, priv, cookie);
	if (ret)
		return ret;
	ret = dpll_xa_ref_dpll_add(&pin->dpll_refs, dpll, ops, priv, cookie);
	if (ret)
		goto ref_pin_del;
	xa_set_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED);
	dpll_pin_create_ntf(pin, dpll->clock_id);

	return ret;

ref_pin_del:
	dpll_xa_ref_pin_del(&dpll->pin_refs, pin, ops, priv, cookie);
	return ret;
}

/**
 * dpll_pin_register - register the dpll pin in the subsystem
 * @dpll: pointer to a dpll
 * @pin: pointer to a dpll pin
 * @ops: ops for a dpll pin ops
 * @priv: pointer to private information of owner
 *
 * Context: Acquires a lock (dpll_lock)
 * Return:
 * * 0 on success
 * * negative - error value
 */
int
dpll_pin_register(struct dpll_device *dpll, struct dpll_pin *pin,
		  const struct dpll_pin_ops *ops, void *priv)
{
	const struct dpll_device_ops *dev_ops;
	int ret;

	if (WARN_ON(!ops) ||
	    WARN_ON(!ops->state_on_dpll_get) ||
	    WARN_ON(!ops->direction_get) ||
	    WARN_ON(ops->supported_ffo && !ops->ffo_get) ||
	    WARN_ON((pin->prop.capabilities &
		     DPLL_PIN_CAPABILITIES_STATE_CONNECTED_OVERRIDE) &&
		    !(pin->prop.capabilities &
		      DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE)))
		return -EINVAL;

	mutex_lock(&dpll_lock);

	dev_ops = dpll_device_ops(dpll);
	if (WARN_ON(ops->measured_freq_get &&
		    (!dev_ops || !dev_ops->freq_monitor_get ||
		     !dev_ops->freq_monitor_set))) {
		ret = -EINVAL;
		goto out_unlock;
	}

	/*
	 * For pins identified via firmware (pin->fwnode), allow registration
	 * even if the pin's (module, clock_id) differs from the target DPLL.
	 * For non-fwnode pins, require a strict (module, clock_id) match.
	 */
	if (!pin->fwnode &&
	    WARN_ON_ONCE(dpll->module != pin->module ||
			 dpll->clock_id != pin->clock_id)) {
		ret = -EINVAL;
		goto out_unlock;
	}

	ret = __dpll_pin_register(dpll, pin, ops, priv, NULL);
out_unlock:
	mutex_unlock(&dpll_lock);

	return ret;
}
EXPORT_SYMBOL_GPL(dpll_pin_register);

static void dpll_pin_ref_sync_pair_del(u32 ref_sync_pin_id)
{
	struct dpll_pin *pin, *ref_sync_pin;
	unsigned long i;

	xa_for_each(&dpll_pin_xa, i, pin) {
		ref_sync_pin = xa_load(&pin->ref_sync_pins, ref_sync_pin_id);
		if (ref_sync_pin) {
			xa_erase(&pin->ref_sync_pins, ref_sync_pin_id);
			__dpll_pin_change_ntf(pin);
		}
	}
}

static void
__dpll_pin_unregister(struct dpll_device *dpll, struct dpll_pin *pin,
		      const struct dpll_pin_ops *ops, void *priv, void *cookie)
{
	ASSERT_DPLL_PIN_REGISTERED(pin);
	dpll_pin_delete_ntf(pin, dpll->clock_id);
	dpll_xa_ref_pin_del(&dpll->pin_refs, pin, ops, priv, cookie);
	dpll_xa_ref_dpll_del(&pin->dpll_refs, dpll, ops, priv, cookie);
	if (xa_empty(&pin->dpll_refs)) {
		dpll_pin_ref_sync_pair_del(pin->id);
		xa_clear_mark(&dpll_pin_xa, pin->id, DPLL_REGISTERED);
	}
}

/**
 * dpll_pin_unregister - unregister dpll pin from dpll device
 * @dpll: registered dpll pointer
 * @pin: pointer to a pin
 * @ops: ops for a dpll pin
 * @priv: pointer to private information of owner
 *
 * Note: It does not free the memory
 * Context: Acquires a lock (dpll_lock)
 */
void dpll_pin_unregister(struct dpll_device *dpll, struct dpll_pin *pin,
			 const struct dpll_pin_ops *ops, void *priv)
{
	if (WARN_ON(xa_empty(&dpll->pin_refs)))
		return;
	if (WARN_ON(!xa_empty(&pin->parent_refs)))
		return;

	mutex_lock(&dpll_lock);
	__dpll_pin_unregister(dpll, pin, ops, priv, NULL);
	mutex_unlock(&dpll_lock);
}
EXPORT_SYMBOL_GPL(dpll_pin_unregister);

/**
 * dpll_pin_on_pin_register - register a pin with a parent pin
 * @parent: pointer to a parent pin
 * @pin: pointer to a pin
 * @ops: ops for a dpll pin
 * @priv: pointer to private information of owner
 *
 * Register a pin with a parent pin, create references between them and
 * between newly registered pin and dplls connected with a parent pin.
 *
 * Context: Acquires a lock (dpll_lock)
 * Return:
 * * 0 on success
 * * negative - error value
 */
int dpll_pin_on_pin_register(struct dpll_pin *parent, struct dpll_pin *pin,
			     const struct dpll_pin_ops *ops, void *priv)
{
	struct dpll_pin_ref *ref;
	unsigned long i, stop;
	int ret;

	if (WARN_ON(parent->prop.type != DPLL_PIN_TYPE_MUX))
		return -EINVAL;

	if (WARN_ON(!ops) ||
	    WARN_ON(!ops->state_on_pin_get) ||
	    WARN_ON(!ops->direction_get))
		return -EINVAL;

	mutex_lock(&dpll_lock);
	ret = dpll_xa_ref_pin_add(&pin->parent_refs, parent, ops, priv, pin);
	if (ret)
		goto unlock;
	xa_for_each(&parent->dpll_refs, i, ref) {
		ret = __dpll_pin_register(ref->dpll, pin, ops, priv, parent);
		if (ret) {
			stop = i;
			goto dpll_unregister;
		}
		dpll_pin_create_ntf(pin, parent->clock_id);
	}
	mutex_unlock(&dpll_lock);

	return ret;

dpll_unregister:
	xa_for_each(&parent->dpll_refs, i, ref)
		if (i < stop) {
			dpll_pin_delete_ntf(pin, parent->clock_id);
			__dpll_pin_unregister(ref->dpll, pin, ops, priv,
					      parent);
		}
	dpll_xa_ref_pin_del(&pin->parent_refs, parent, ops, priv, pin);
unlock:
	mutex_unlock(&dpll_lock);
	return ret;
}
EXPORT_SYMBOL_GPL(dpll_pin_on_pin_register);

/**
 * dpll_pin_on_pin_unregister - unregister dpll pin from a parent pin
 * @parent: pointer to a parent pin
 * @pin: pointer to a pin
 * @ops: ops for a dpll pin
 * @priv: pointer to private information of owner
 *
 * Context: Acquires a lock (dpll_lock)
 * Note: It does not free the memory
 */
void dpll_pin_on_pin_unregister(struct dpll_pin *parent, struct dpll_pin *pin,
				const struct dpll_pin_ops *ops, void *priv)
{
	struct dpll_pin_registration *reg;
	struct dpll_pin_ref *ref;
	unsigned long i;

	mutex_lock(&dpll_lock);
	xa_for_each(&pin->dpll_refs, i, ref) {
		reg = dpll_pin_registration_find(ref, ops, priv, parent);
		if (!reg)
			continue;
		dpll_pin_delete_ntf(pin, parent->clock_id);
		__dpll_pin_unregister(ref->dpll, pin, ops, priv, parent);
	}
	dpll_xa_ref_pin_del(&pin->parent_refs, parent, ops, priv, pin);
	mutex_unlock(&dpll_lock);
}
EXPORT_SYMBOL_GPL(dpll_pin_on_pin_unregister);

/**
 * dpll_pin_ref_sync_pair_add - create a reference sync signal pin pair
 * @pin: pin which produces the base frequency
 * @ref_sync_pin: pin which produces the sync signal
 *
 * Once pins are paired, the user-space configuration of reference sync pair
 * is possible.
 * Context: Acquires a lock (dpll_lock)
 * Return:
 * * 0 on success
 * * negative - error value
 */
int dpll_pin_ref_sync_pair_add(struct dpll_pin *pin,
			       struct dpll_pin *ref_sync_pin)
{
	int ret;

	mutex_lock(&dpll_lock);
	ret = xa_insert(&pin->ref_sync_pins, ref_sync_pin->id,
			ref_sync_pin, GFP_KERNEL);
	__dpll_pin_change_ntf(pin);
	mutex_unlock(&dpll_lock);

	return ret;
}
EXPORT_SYMBOL_GPL(dpll_pin_ref_sync_pair_add);

static struct dpll_device_registration *
dpll_device_registration_first(struct dpll_device *dpll)
{
	return list_first_entry_or_null((struct list_head *)&dpll->registration_list,
					struct dpll_device_registration, list);
}

void *dpll_priv(struct dpll_device *dpll)
{
	struct dpll_device_registration *reg;

	reg = dpll_device_registration_first(dpll);
	if (!reg)
		return NULL;
	return reg->priv;
}

const struct dpll_device_ops *dpll_device_ops(struct dpll_device *dpll)
{
	struct dpll_device_registration *reg;

	reg = dpll_device_registration_first(dpll);
	if (!reg)
		return NULL;
	return reg->ops;
}

static struct dpll_pin_registration *
dpll_pin_registration_first(struct dpll_pin_ref *ref)
{
	struct dpll_pin_registration *reg;

	reg = list_first_entry_or_null(&ref->registration_list,
				       struct dpll_pin_registration, list);
	WARN_ON(!reg);
	return reg;
}

void *dpll_pin_on_dpll_priv(struct dpll_device *dpll,
			    struct dpll_pin *pin)
{
	struct dpll_pin_registration *reg;
	struct dpll_pin_ref *ref;

	ref = xa_load(&dpll->pin_refs, pin->pin_idx);
	if (!ref)
		return NULL;
	reg = dpll_pin_registration_first(ref);
	return reg->priv;
}

void *dpll_pin_on_pin_priv(struct dpll_pin *parent,
			   struct dpll_pin *pin)
{
	struct dpll_pin_registration *reg;
	struct dpll_pin_ref *ref;

	ref = xa_load(&pin->parent_refs, parent->pin_idx);
	if (!ref)
		return NULL;
	reg = dpll_pin_registration_first(ref);
	return reg->priv;
}

/**
 * dpll_pin_own_dpll_ref_first - find the first owner dpll ref of a pin
 * @pin: pointer to a dpll pin
 *
 * Search pin's dpll_refs for a ref whose dpll matches the pin's
 * (module, clock_id) tuple, i.e. the dpll registered by the driver
 * that created the pin. This ensures pin-level attributes are
 * reported and modified using the owner's ops even when the pin is
 * also registered with dplls from other drivers.
 *
 * Return: pointer to the owner's dpll_pin_ref, or NULL if no
 * owner ref is found.
 */
struct dpll_pin_ref *dpll_pin_own_dpll_ref_first(struct dpll_pin *pin)
{
	struct dpll_pin_ref *ref;
	unsigned long i;

	xa_for_each(&pin->dpll_refs, i, ref) {
		if (ref->dpll->module == pin->module &&
		    ref->dpll->clock_id == pin->clock_id)
			return ref;
	}

	return NULL;
}

const struct dpll_pin_ops *dpll_pin_ops(struct dpll_pin_ref *ref)
{
	struct dpll_pin_registration *reg;

	reg = dpll_pin_registration_first(ref);
	return reg->ops;
}

static int __init dpll_init(void)
{
	int ret;

	ret = genl_register_family(&dpll_nl_family);
	if (ret)
		goto error;

	return 0;

error:
	mutex_destroy(&dpll_lock);
	return ret;
}

static void __exit dpll_exit(void)
{
	genl_unregister_family(&dpll_nl_family);
	mutex_destroy(&dpll_lock);
}

subsys_initcall(dpll_init);
module_exit(dpll_exit);