blob: 797e1ebff2997d88c70ab7951b723e203a8184a1 [file] [log] [blame]
Dan Williams51cf7842017-07-12 17:58:21 -07001// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2017-2018 Intel Corporation. All rights reserved. */
Dan Williams89ec9f22018-10-29 15:52:42 -07003#include <linux/memremap.h>
Dan Williams51cf7842017-07-12 17:58:21 -07004#include <linux/device.h>
Dan Williamsd2007812018-11-07 15:31:23 -08005#include <linux/mutex.h>
6#include <linux/list.h>
Dan Williams51cf7842017-07-12 17:58:21 -07007#include <linux/slab.h>
8#include <linux/dax.h>
Dan Williamsfcffb6a2020-10-13 16:50:24 -07009#include <linux/io.h>
Dan Williams51cf7842017-07-12 17:58:21 -070010#include "dax-private.h"
11#include "bus.h"
12
Dan Williamsd2007812018-11-07 15:31:23 -080013static DEFINE_MUTEX(dax_bus_lock);
14
Vishal Vermac05ae9d2024-01-24 12:03:46 -080015/*
16 * All changes to the dax region configuration occur with this lock held
17 * for write.
18 */
19DECLARE_RWSEM(dax_region_rwsem);
20
21/*
22 * All changes to the dax device configuration occur with this lock held
23 * for write.
24 */
25DECLARE_RWSEM(dax_dev_rwsem);
26
Dan Williamsd2007812018-11-07 15:31:23 -080027#define DAX_NAME_LEN 30
28struct dax_id {
29 struct list_head list;
30 char dev_name[DAX_NAME_LEN];
31};
32
Greg Kroah-Hartman2a81ada2023-01-11 12:30:17 +010033static int dax_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
Dan Williams9567da02017-07-12 17:58:21 -070034{
35 /*
36 * We only ever expect to handle device-dax instances, i.e. the
37 * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
38 */
39 return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0);
40}
41
Dan Williamsd2007812018-11-07 15:31:23 -080042static struct dax_device_driver *to_dax_drv(struct device_driver *drv)
43{
44 return container_of(drv, struct dax_device_driver, drv);
45}
46
47static struct dax_id *__dax_match_id(struct dax_device_driver *dax_drv,
48 const char *dev_name)
49{
50 struct dax_id *dax_id;
51
52 lockdep_assert_held(&dax_bus_lock);
53
54 list_for_each_entry(dax_id, &dax_drv->ids, list)
55 if (sysfs_streq(dax_id->dev_name, dev_name))
56 return dax_id;
57 return NULL;
58}
59
60static int dax_match_id(struct dax_device_driver *dax_drv, struct device *dev)
61{
62 int match;
63
64 mutex_lock(&dax_bus_lock);
65 match = !!__dax_match_id(dax_drv, dev_name(dev));
66 mutex_unlock(&dax_bus_lock);
67
68 return match;
69}
70
Dan Williamse9ee9fe2023-02-10 01:07:13 -080071static int dax_match_type(struct dax_device_driver *dax_drv, struct device *dev)
72{
73 enum dax_driver_type type = DAXDRV_DEVICE_TYPE;
74 struct dev_dax *dev_dax = to_dev_dax(dev);
75
76 if (dev_dax->region->res.flags & IORESOURCE_DAX_KMEM)
77 type = DAXDRV_KMEM_TYPE;
78
79 if (dax_drv->type == type)
80 return 1;
81
82 /* default to device mode if dax_kmem is disabled */
83 if (dax_drv->type == DAXDRV_DEVICE_TYPE &&
84 !IS_ENABLED(CONFIG_DEV_DAX_KMEM))
85 return 1;
86
87 return 0;
88}
89
Dan Williams664525b2019-01-24 13:12:04 -080090enum id_action {
91 ID_REMOVE,
92 ID_ADD,
93};
94
Dan Williamsd2007812018-11-07 15:31:23 -080095static ssize_t do_id_store(struct device_driver *drv, const char *buf,
Dan Williams664525b2019-01-24 13:12:04 -080096 size_t count, enum id_action action)
Dan Williamsd2007812018-11-07 15:31:23 -080097{
98 struct dax_device_driver *dax_drv = to_dax_drv(drv);
99 unsigned int region_id, id;
100 char devname[DAX_NAME_LEN];
101 struct dax_id *dax_id;
102 ssize_t rc = count;
103 int fields;
104
105 fields = sscanf(buf, "dax%d.%d", &region_id, &id);
106 if (fields != 2)
107 return -EINVAL;
108 sprintf(devname, "dax%d.%d", region_id, id);
109 if (!sysfs_streq(buf, devname))
110 return -EINVAL;
111
112 mutex_lock(&dax_bus_lock);
113 dax_id = __dax_match_id(dax_drv, buf);
114 if (!dax_id) {
Dan Williams664525b2019-01-24 13:12:04 -0800115 if (action == ID_ADD) {
Dan Williamsd2007812018-11-07 15:31:23 -0800116 dax_id = kzalloc(sizeof(*dax_id), GFP_KERNEL);
117 if (dax_id) {
Justin Stitt75b3d5d2023-09-13 01:10:24 +0000118 strscpy(dax_id->dev_name, buf, DAX_NAME_LEN);
Dan Williamsd2007812018-11-07 15:31:23 -0800119 list_add(&dax_id->list, &dax_drv->ids);
120 } else
121 rc = -ENOMEM;
Arnd Bergmann9de82ca2021-03-22 12:44:58 +0100122 }
Dan Williams664525b2019-01-24 13:12:04 -0800123 } else if (action == ID_REMOVE) {
Dan Williamsd2007812018-11-07 15:31:23 -0800124 list_del(&dax_id->list);
125 kfree(dax_id);
Arnd Bergmann9de82ca2021-03-22 12:44:58 +0100126 }
Dan Williamsd2007812018-11-07 15:31:23 -0800127 mutex_unlock(&dax_bus_lock);
Dan Williams664525b2019-01-24 13:12:04 -0800128
129 if (rc < 0)
130 return rc;
131 if (action == ID_ADD)
132 rc = driver_attach(drv);
133 if (rc)
134 return rc;
135 return count;
Dan Williamsd2007812018-11-07 15:31:23 -0800136}
137
138static ssize_t new_id_store(struct device_driver *drv, const char *buf,
139 size_t count)
140{
Dan Williams664525b2019-01-24 13:12:04 -0800141 return do_id_store(drv, buf, count, ID_ADD);
Dan Williamsd2007812018-11-07 15:31:23 -0800142}
143static DRIVER_ATTR_WO(new_id);
144
145static ssize_t remove_id_store(struct device_driver *drv, const char *buf,
146 size_t count)
147{
Dan Williams664525b2019-01-24 13:12:04 -0800148 return do_id_store(drv, buf, count, ID_REMOVE);
Dan Williamsd2007812018-11-07 15:31:23 -0800149}
150static DRIVER_ATTR_WO(remove_id);
151
152static struct attribute *dax_drv_attrs[] = {
153 &driver_attr_new_id.attr,
154 &driver_attr_remove_id.attr,
155 NULL,
156};
157ATTRIBUTE_GROUPS(dax_drv);
158
Dan Williams9567da02017-07-12 17:58:21 -0700159static int dax_bus_match(struct device *dev, struct device_driver *drv);
160
Joao Martinsfc65c4e2022-01-14 14:04:33 -0800161/*
162 * Static dax regions are regions created by an external subsystem
163 * nvdimm where a single range is assigned. Its boundaries are by the external
164 * subsystem and are usually limited to one physical memory range. For example,
165 * for PMEM it is usually defined by NVDIMM Namespace boundaries (i.e. a
166 * single contiguous range)
167 *
168 * On dynamic dax regions, the assigned region can be partitioned by dax core
169 * into multiple subdivisions. A subdivision is represented into one
170 * /dev/daxN.M device composed by one or more potentially discontiguous ranges.
171 *
172 * When allocating a dax region, drivers must set whether it's static
173 * (IORESOURCE_DAX_STATIC). On static dax devices, the @pgmap is pre-assigned
174 * to dax core when calling devm_create_dev_dax(), whereas in dynamic dax
175 * devices it is NULL but afterwards allocated by dax core on device ->probe().
176 * Care is needed to make sure that dynamic dax devices are torn down with a
177 * cleared @pgmap field (see kill_dev_dax()).
178 */
Dan Williamsc2f30112020-10-13 16:50:03 -0700179static bool is_static(struct dax_region *dax_region)
180{
181 return (dax_region->res.flags & IORESOURCE_DAX_STATIC) != 0;
182}
183
Joao Martinsfc65c4e2022-01-14 14:04:33 -0800184bool static_dev_dax(struct dev_dax *dev_dax)
185{
186 return is_static(dev_dax->region);
187}
188EXPORT_SYMBOL_GPL(static_dev_dax);
189
Dan Williams60e93dc2020-10-13 16:50:39 -0700190static u64 dev_dax_size(struct dev_dax *dev_dax)
191{
192 u64 size = 0;
193 int i;
194
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800195 WARN_ON_ONCE(!rwsem_is_locked(&dax_dev_rwsem));
Dan Williams60e93dc2020-10-13 16:50:39 -0700196
197 for (i = 0; i < dev_dax->nr_range; i++)
198 size += range_len(&dev_dax->ranges[i].range);
199
200 return size;
201}
202
Dan Williamsf11cf812020-10-13 16:50:08 -0700203static int dax_bus_probe(struct device *dev)
204{
205 struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
206 struct dev_dax *dev_dax = to_dev_dax(dev);
Dan Williams0f3da142020-10-13 16:50:13 -0700207 struct dax_region *dax_region = dev_dax->region;
Dan Williams0f3da142020-10-13 16:50:13 -0700208 int rc;
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800209 u64 size;
Dan Williamsf11cf812020-10-13 16:50:08 -0700210
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800211 rc = down_read_interruptible(&dax_dev_rwsem);
212 if (rc)
213 return rc;
214 size = dev_dax_size(dev_dax);
215 up_read(&dax_dev_rwsem);
216
217 if (size == 0 || dev_dax->id < 0)
Dan Williams0f3da142020-10-13 16:50:13 -0700218 return -ENXIO;
219
220 rc = dax_drv->probe(dev_dax);
221
222 if (rc || is_static(dax_region))
223 return rc;
224
225 /*
226 * Track new seed creation only after successful probe of the
227 * previous seed.
228 */
229 if (dax_region->seed == dev)
230 dax_region->seed = NULL;
231
232 return 0;
Dan Williamsf11cf812020-10-13 16:50:08 -0700233}
234
Uwe Kleine-Königfc7a6202021-07-13 21:35:22 +0200235static void dax_bus_remove(struct device *dev)
Dan Williamsf11cf812020-10-13 16:50:08 -0700236{
237 struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
238 struct dev_dax *dev_dax = to_dev_dax(dev);
239
Uwe Kleine-König80299682021-02-05 23:28:39 +0100240 if (dax_drv->remove)
Uwe Kleine-König0d519e02021-02-05 23:28:42 +0100241 dax_drv->remove(dev_dax);
Dan Williamsf11cf812020-10-13 16:50:08 -0700242}
243
Ricardo B. Marlierebc223742024-02-04 13:07:11 -0300244static const struct bus_type dax_bus_type = {
Dan Williams9567da02017-07-12 17:58:21 -0700245 .name = "dax",
246 .uevent = dax_bus_uevent,
247 .match = dax_bus_match,
Dan Williamsf11cf812020-10-13 16:50:08 -0700248 .probe = dax_bus_probe,
249 .remove = dax_bus_remove,
Dan Williamsd2007812018-11-07 15:31:23 -0800250 .drv_groups = dax_drv_groups,
Dan Williams9567da02017-07-12 17:58:21 -0700251};
252
253static int dax_bus_match(struct device *dev, struct device_driver *drv)
254{
Dan Williamsd2007812018-11-07 15:31:23 -0800255 struct dax_device_driver *dax_drv = to_dax_drv(drv);
256
Dan Williamse9ee9fe2023-02-10 01:07:13 -0800257 if (dax_match_id(dax_drv, dev))
Dan Williamsd2007812018-11-07 15:31:23 -0800258 return 1;
Dan Williamse9ee9fe2023-02-10 01:07:13 -0800259 return dax_match_type(dax_drv, dev);
Dan Williams9567da02017-07-12 17:58:21 -0700260}
261
Dan Williams51cf7842017-07-12 17:58:21 -0700262/*
263 * Rely on the fact that drvdata is set before the attributes are
264 * registered, and that the attributes are unregistered before drvdata
265 * is cleared to assume that drvdata is always valid.
266 */
267static ssize_t id_show(struct device *dev,
268 struct device_attribute *attr, char *buf)
269{
270 struct dax_region *dax_region = dev_get_drvdata(dev);
271
Vishal Verma6ebed002024-01-24 12:03:47 -0800272 return sysfs_emit(buf, "%d\n", dax_region->id);
Dan Williams51cf7842017-07-12 17:58:21 -0700273}
274static DEVICE_ATTR_RO(id);
275
276static ssize_t region_size_show(struct device *dev,
277 struct device_attribute *attr, char *buf)
278{
279 struct dax_region *dax_region = dev_get_drvdata(dev);
280
Vishal Verma6ebed002024-01-24 12:03:47 -0800281 return sysfs_emit(buf, "%llu\n",
282 (unsigned long long)resource_size(&dax_region->res));
Dan Williams51cf7842017-07-12 17:58:21 -0700283}
284static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
285 region_size_show, NULL);
286
Dan Williams6d821202020-10-13 16:50:55 -0700287static ssize_t region_align_show(struct device *dev,
Dan Williams51cf7842017-07-12 17:58:21 -0700288 struct device_attribute *attr, char *buf)
289{
290 struct dax_region *dax_region = dev_get_drvdata(dev);
291
Vishal Verma6ebed002024-01-24 12:03:47 -0800292 return sysfs_emit(buf, "%u\n", dax_region->align);
Dan Williams51cf7842017-07-12 17:58:21 -0700293}
Dan Williams6d821202020-10-13 16:50:55 -0700294static struct device_attribute dev_attr_region_align =
295 __ATTR(align, 0400, region_align_show, NULL);
Dan Williams51cf7842017-07-12 17:58:21 -0700296
Dan Williamsc2f30112020-10-13 16:50:03 -0700297#define for_each_dax_region_resource(dax_region, res) \
298 for (res = (dax_region)->res.child; res; res = res->sibling)
299
300static unsigned long long dax_region_avail_size(struct dax_region *dax_region)
301{
302 resource_size_t size = resource_size(&dax_region->res);
303 struct resource *res;
304
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800305 WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem));
Dan Williamsc2f30112020-10-13 16:50:03 -0700306
307 for_each_dax_region_resource(dax_region, res)
308 size -= resource_size(res);
309 return size;
310}
311
312static ssize_t available_size_show(struct device *dev,
313 struct device_attribute *attr, char *buf)
314{
315 struct dax_region *dax_region = dev_get_drvdata(dev);
316 unsigned long long size;
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800317 int rc;
Dan Williamsc2f30112020-10-13 16:50:03 -0700318
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800319 rc = down_read_interruptible(&dax_region_rwsem);
320 if (rc)
321 return rc;
Dan Williamsc2f30112020-10-13 16:50:03 -0700322 size = dax_region_avail_size(dax_region);
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800323 up_read(&dax_region_rwsem);
Dan Williamsc2f30112020-10-13 16:50:03 -0700324
Vishal Verma6ebed002024-01-24 12:03:47 -0800325 return sysfs_emit(buf, "%llu\n", size);
Dan Williamsc2f30112020-10-13 16:50:03 -0700326}
327static DEVICE_ATTR_RO(available_size);
328
Dan Williams0f3da142020-10-13 16:50:13 -0700329static ssize_t seed_show(struct device *dev,
330 struct device_attribute *attr, char *buf)
331{
332 struct dax_region *dax_region = dev_get_drvdata(dev);
333 struct device *seed;
334 ssize_t rc;
335
336 if (is_static(dax_region))
337 return -EINVAL;
338
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800339 rc = down_read_interruptible(&dax_region_rwsem);
340 if (rc)
341 return rc;
Dan Williams0f3da142020-10-13 16:50:13 -0700342 seed = dax_region->seed;
Vishal Verma6ebed002024-01-24 12:03:47 -0800343 rc = sysfs_emit(buf, "%s\n", seed ? dev_name(seed) : "");
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800344 up_read(&dax_region_rwsem);
Dan Williams0f3da142020-10-13 16:50:13 -0700345
346 return rc;
347}
348static DEVICE_ATTR_RO(seed);
349
350static ssize_t create_show(struct device *dev,
351 struct device_attribute *attr, char *buf)
352{
353 struct dax_region *dax_region = dev_get_drvdata(dev);
354 struct device *youngest;
355 ssize_t rc;
356
357 if (is_static(dax_region))
358 return -EINVAL;
359
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800360 rc = down_read_interruptible(&dax_region_rwsem);
361 if (rc)
362 return rc;
Dan Williams0f3da142020-10-13 16:50:13 -0700363 youngest = dax_region->youngest;
Vishal Verma6ebed002024-01-24 12:03:47 -0800364 rc = sysfs_emit(buf, "%s\n", youngest ? dev_name(youngest) : "");
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800365 up_read(&dax_region_rwsem);
Dan Williams0f3da142020-10-13 16:50:13 -0700366
367 return rc;
368}
369
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800370static struct dev_dax *__devm_create_dev_dax(struct dev_dax_data *data);
371
Dan Williams0f3da142020-10-13 16:50:13 -0700372static ssize_t create_store(struct device *dev, struct device_attribute *attr,
373 const char *buf, size_t len)
374{
375 struct dax_region *dax_region = dev_get_drvdata(dev);
376 unsigned long long avail;
377 ssize_t rc;
378 int val;
379
380 if (is_static(dax_region))
381 return -EINVAL;
382
383 rc = kstrtoint(buf, 0, &val);
384 if (rc)
385 return rc;
386 if (val != 1)
387 return -EINVAL;
388
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800389 rc = down_write_killable(&dax_region_rwsem);
390 if (rc)
391 return rc;
Dan Williams0f3da142020-10-13 16:50:13 -0700392 avail = dax_region_avail_size(dax_region);
393 if (avail == 0)
394 rc = -ENOSPC;
395 else {
396 struct dev_dax_data data = {
397 .dax_region = dax_region,
398 .size = 0,
399 .id = -1,
Vishal Verma4eca0ef2023-11-07 00:22:43 -0700400 .memmap_on_memory = false,
Dan Williams0f3da142020-10-13 16:50:13 -0700401 };
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800402 struct dev_dax *dev_dax = __devm_create_dev_dax(&data);
Dan Williams0f3da142020-10-13 16:50:13 -0700403
404 if (IS_ERR(dev_dax))
405 rc = PTR_ERR(dev_dax);
406 else {
407 /*
408 * In support of crafting multiple new devices
409 * simultaneously multiple seeds can be created,
410 * but only the first one that has not been
411 * successfully bound is tracked as the region
412 * seed.
413 */
414 if (!dax_region->seed)
415 dax_region->seed = &dev_dax->dev;
416 dax_region->youngest = &dev_dax->dev;
417 rc = len;
418 }
419 }
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800420 up_write(&dax_region_rwsem);
Dan Williams0f3da142020-10-13 16:50:13 -0700421
422 return rc;
423}
424static DEVICE_ATTR_RW(create);
425
426void kill_dev_dax(struct dev_dax *dev_dax)
427{
428 struct dax_device *dax_dev = dev_dax->dax_dev;
429 struct inode *inode = dax_inode(dax_dev);
430
431 kill_dax(dax_dev);
432 unmap_mapping_range(inode->i_mapping, 0, 0, 1);
Joao Martinsfc65c4e2022-01-14 14:04:33 -0800433
434 /*
435 * Dynamic dax region have the pgmap allocated via dev_kzalloc()
436 * and thus freed by devm. Clear the pgmap to not have stale pgmap
437 * ranges on probe() from previous reconfigurations of region devices.
438 */
439 if (!static_dev_dax(dev_dax))
440 dev_dax->pgmap = NULL;
Dan Williams0f3da142020-10-13 16:50:13 -0700441}
442EXPORT_SYMBOL_GPL(kill_dev_dax);
443
Dan Williams6268d7d2020-12-18 18:41:41 -0800444static void trim_dev_dax_range(struct dev_dax *dev_dax)
Dan Williams0f3da142020-10-13 16:50:13 -0700445{
Dan Williams6268d7d2020-12-18 18:41:41 -0800446 int i = dev_dax->nr_range - 1;
447 struct range *range = &dev_dax->ranges[i].range;
Dan Williams0f3da142020-10-13 16:50:13 -0700448 struct dax_region *dax_region = dev_dax->region;
Dan Williams0f3da142020-10-13 16:50:13 -0700449
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800450 WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem));
Dan Williams6268d7d2020-12-18 18:41:41 -0800451 dev_dbg(&dev_dax->dev, "delete range[%d]: %#llx:%#llx\n", i,
452 (unsigned long long)range->start,
453 (unsigned long long)range->end);
Dan Williams60e93dc2020-10-13 16:50:39 -0700454
Dan Williams6268d7d2020-12-18 18:41:41 -0800455 __release_region(&dax_region->res, range->start, range_len(range));
456 if (--dev_dax->nr_range == 0) {
457 kfree(dev_dax->ranges);
458 dev_dax->ranges = NULL;
Dan Williams60e93dc2020-10-13 16:50:39 -0700459 }
Dan Williams6268d7d2020-12-18 18:41:41 -0800460}
461
462static void free_dev_dax_ranges(struct dev_dax *dev_dax)
463{
464 while (dev_dax->nr_range)
465 trim_dev_dax_range(dev_dax);
Dan Williams0f3da142020-10-13 16:50:13 -0700466}
467
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800468static void __unregister_dev_dax(void *dev)
Dan Williams0f3da142020-10-13 16:50:13 -0700469{
470 struct dev_dax *dev_dax = to_dev_dax(dev);
471
472 dev_dbg(dev, "%s\n", __func__);
473
474 kill_dev_dax(dev_dax);
Dan Williams0f3da142020-10-13 16:50:13 -0700475 device_del(dev);
Dan Williamse686c322023-02-16 00:36:02 -0800476 free_dev_dax_ranges(dev_dax);
Dan Williams0f3da142020-10-13 16:50:13 -0700477 put_device(dev);
478}
479
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800480static void unregister_dev_dax(void *dev)
481{
482 if (rwsem_is_locked(&dax_region_rwsem))
483 return __unregister_dev_dax(dev);
484
485 if (WARN_ON_ONCE(down_write_killable(&dax_region_rwsem) != 0))
486 return;
487 __unregister_dev_dax(dev);
488 up_write(&dax_region_rwsem);
489}
490
Dan Williams70aab282023-06-02 23:14:05 -0700491static void dax_region_free(struct kref *kref)
492{
493 struct dax_region *dax_region;
494
495 dax_region = container_of(kref, struct dax_region, kref);
496 kfree(dax_region);
497}
498
Dan Williams2532f412023-06-02 23:14:11 -0700499static void dax_region_put(struct dax_region *dax_region)
Dan Williams70aab282023-06-02 23:14:05 -0700500{
501 kref_put(&dax_region->kref, dax_region_free);
502}
Dan Williams70aab282023-06-02 23:14:05 -0700503
Dan Williams0f3da142020-10-13 16:50:13 -0700504/* a return value >= 0 indicates this invocation invalidated the id */
505static int __free_dev_dax_id(struct dev_dax *dev_dax)
506{
Dan Williams70aab282023-06-02 23:14:05 -0700507 struct dax_region *dax_region;
Dan Williams0f3da142020-10-13 16:50:13 -0700508 int rc = dev_dax->id;
509
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800510 WARN_ON_ONCE(!rwsem_is_locked(&dax_dev_rwsem));
Dan Williams0f3da142020-10-13 16:50:13 -0700511
Dan Williams70aab282023-06-02 23:14:05 -0700512 if (!dev_dax->dyn_id || dev_dax->id < 0)
Dan Williams0f3da142020-10-13 16:50:13 -0700513 return -1;
Dan Williams70aab282023-06-02 23:14:05 -0700514 dax_region = dev_dax->region;
Dan Williams0f3da142020-10-13 16:50:13 -0700515 ida_free(&dax_region->ida, dev_dax->id);
Dan Williams70aab282023-06-02 23:14:05 -0700516 dax_region_put(dax_region);
Dan Williams0f3da142020-10-13 16:50:13 -0700517 dev_dax->id = -1;
518 return rc;
519}
520
521static int free_dev_dax_id(struct dev_dax *dev_dax)
522{
Dan Williams0f3da142020-10-13 16:50:13 -0700523 int rc;
524
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800525 rc = down_write_killable(&dax_dev_rwsem);
526 if (rc)
527 return rc;
Dan Williams0f3da142020-10-13 16:50:13 -0700528 rc = __free_dev_dax_id(dev_dax);
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800529 up_write(&dax_dev_rwsem);
Dan Williams0f3da142020-10-13 16:50:13 -0700530 return rc;
531}
532
Dan Williams70aab282023-06-02 23:14:05 -0700533static int alloc_dev_dax_id(struct dev_dax *dev_dax)
534{
535 struct dax_region *dax_region = dev_dax->region;
536 int id;
537
538 id = ida_alloc(&dax_region->ida, GFP_KERNEL);
539 if (id < 0)
540 return id;
541 kref_get(&dax_region->kref);
542 dev_dax->dyn_id = true;
543 dev_dax->id = id;
544 return id;
545}
546
Dan Williams0f3da142020-10-13 16:50:13 -0700547static ssize_t delete_store(struct device *dev, struct device_attribute *attr,
548 const char *buf, size_t len)
549{
550 struct dax_region *dax_region = dev_get_drvdata(dev);
551 struct dev_dax *dev_dax;
552 struct device *victim;
553 bool do_del = false;
554 int rc;
555
556 if (is_static(dax_region))
557 return -EINVAL;
558
559 victim = device_find_child_by_name(dax_region->dev, buf);
560 if (!victim)
561 return -ENXIO;
562
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800563 rc = down_write_killable(&dax_region_rwsem);
564 if (rc)
565 return rc;
566 rc = down_write_killable(&dax_dev_rwsem);
567 if (rc) {
568 up_write(&dax_region_rwsem);
569 return rc;
570 }
Dan Williams0f3da142020-10-13 16:50:13 -0700571 dev_dax = to_dev_dax(victim);
Dan Williams60e93dc2020-10-13 16:50:39 -0700572 if (victim->driver || dev_dax_size(dev_dax))
Dan Williams0f3da142020-10-13 16:50:13 -0700573 rc = -EBUSY;
574 else {
575 /*
576 * Invalidate the device so it does not become active
577 * again, but always preserve device-id-0 so that
578 * /sys/bus/dax/ is guaranteed to be populated while any
579 * dax_region is registered.
580 */
581 if (dev_dax->id > 0) {
582 do_del = __free_dev_dax_id(dev_dax) >= 0;
583 rc = len;
584 if (dax_region->seed == victim)
585 dax_region->seed = NULL;
586 if (dax_region->youngest == victim)
587 dax_region->youngest = NULL;
588 } else
589 rc = -EBUSY;
590 }
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800591 up_write(&dax_dev_rwsem);
Dan Williams0f3da142020-10-13 16:50:13 -0700592
593 /* won the race to invalidate the device, clean it up */
594 if (do_del)
595 devm_release_action(dev, unregister_dev_dax, victim);
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800596 up_write(&dax_region_rwsem);
Dan Williams0f3da142020-10-13 16:50:13 -0700597 put_device(victim);
598
599 return rc;
600}
601static DEVICE_ATTR_WO(delete);
602
Dan Williamsc2f30112020-10-13 16:50:03 -0700603static umode_t dax_region_visible(struct kobject *kobj, struct attribute *a,
604 int n)
605{
606 struct device *dev = container_of(kobj, struct device, kobj);
607 struct dax_region *dax_region = dev_get_drvdata(dev);
608
Dan Williams0f3da142020-10-13 16:50:13 -0700609 if (is_static(dax_region))
610 if (a == &dev_attr_available_size.attr
611 || a == &dev_attr_create.attr
612 || a == &dev_attr_seed.attr
613 || a == &dev_attr_delete.attr)
614 return 0;
Dan Williamsc2f30112020-10-13 16:50:03 -0700615 return a->mode;
616}
617
Dan Williams51cf7842017-07-12 17:58:21 -0700618static struct attribute *dax_region_attributes[] = {
Dan Williamsc2f30112020-10-13 16:50:03 -0700619 &dev_attr_available_size.attr,
Dan Williams51cf7842017-07-12 17:58:21 -0700620 &dev_attr_region_size.attr,
Dan Williams6d821202020-10-13 16:50:55 -0700621 &dev_attr_region_align.attr,
Dan Williams0f3da142020-10-13 16:50:13 -0700622 &dev_attr_create.attr,
623 &dev_attr_seed.attr,
624 &dev_attr_delete.attr,
Dan Williams51cf7842017-07-12 17:58:21 -0700625 &dev_attr_id.attr,
626 NULL,
627};
628
629static const struct attribute_group dax_region_attribute_group = {
630 .name = "dax_region",
631 .attrs = dax_region_attributes,
Dan Williamsc2f30112020-10-13 16:50:03 -0700632 .is_visible = dax_region_visible,
Dan Williams51cf7842017-07-12 17:58:21 -0700633};
634
635static const struct attribute_group *dax_region_attribute_groups[] = {
636 &dax_region_attribute_group,
637 NULL,
638};
639
Dan Williams51cf7842017-07-12 17:58:21 -0700640static void dax_region_unregister(void *region)
641{
642 struct dax_region *dax_region = region;
643
644 sysfs_remove_groups(&dax_region->dev->kobj,
645 dax_region_attribute_groups);
646 dax_region_put(dax_region);
647}
648
649struct dax_region *alloc_dax_region(struct device *parent, int region_id,
Dan Williamsa4574f62020-10-13 16:50:29 -0700650 struct range *range, int target_node, unsigned int align,
Dan Williamsc2f30112020-10-13 16:50:03 -0700651 unsigned long flags)
Dan Williams51cf7842017-07-12 17:58:21 -0700652{
653 struct dax_region *dax_region;
654
655 /*
656 * The DAX core assumes that it can store its private data in
657 * parent->driver_data. This WARN is a reminder / safeguard for
658 * developers of device-dax drivers.
659 */
660 if (dev_get_drvdata(parent)) {
661 dev_WARN(parent, "dax core failed to setup private data\n");
662 return NULL;
663 }
664
Dan Williamsa4574f62020-10-13 16:50:29 -0700665 if (!IS_ALIGNED(range->start, align)
666 || !IS_ALIGNED(range_len(range), align))
Dan Williams51cf7842017-07-12 17:58:21 -0700667 return NULL;
668
669 dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
670 if (!dax_region)
671 return NULL;
672
673 dev_set_drvdata(parent, dax_region);
Dan Williams51cf7842017-07-12 17:58:21 -0700674 kref_init(&dax_region->kref);
675 dax_region->id = region_id;
676 dax_region->align = align;
677 dax_region->dev = parent;
Dan Williams8fc5c732018-11-09 12:43:07 -0800678 dax_region->target_node = target_node;
Dan Williams0f3da142020-10-13 16:50:13 -0700679 ida_init(&dax_region->ida);
Dan Williamsc2f30112020-10-13 16:50:03 -0700680 dax_region->res = (struct resource) {
Dan Williamsa4574f62020-10-13 16:50:29 -0700681 .start = range->start,
682 .end = range->end,
Dan Williamsc2f30112020-10-13 16:50:03 -0700683 .flags = IORESOURCE_MEM | flags,
684 };
685
Dan Williams51cf7842017-07-12 17:58:21 -0700686 if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
687 kfree(dax_region);
688 return NULL;
689 }
690
Dan Williams51cf7842017-07-12 17:58:21 -0700691 if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region))
692 return NULL;
693 return dax_region;
694}
695EXPORT_SYMBOL_GPL(alloc_dax_region);
696
Dan Williams0b07ce82020-10-13 16:50:45 -0700697static void dax_mapping_release(struct device *dev)
698{
699 struct dax_mapping *mapping = to_dax_mapping(dev);
Dan Williams6d24b172023-06-02 23:13:54 -0700700 struct device *parent = dev->parent;
701 struct dev_dax *dev_dax = to_dev_dax(parent);
Dan Williams0b07ce82020-10-13 16:50:45 -0700702
703 ida_free(&dev_dax->ida, mapping->id);
704 kfree(mapping);
Dan Williams6d24b172023-06-02 23:13:54 -0700705 put_device(parent);
Dan Williams0b07ce82020-10-13 16:50:45 -0700706}
707
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800708static void __unregister_dax_mapping(void *data)
Dan Williams0b07ce82020-10-13 16:50:45 -0700709{
710 struct device *dev = data;
711 struct dax_mapping *mapping = to_dax_mapping(dev);
712 struct dev_dax *dev_dax = to_dev_dax(dev->parent);
Dan Williams0b07ce82020-10-13 16:50:45 -0700713
714 dev_dbg(dev, "%s\n", __func__);
715
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800716 WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem));
Dan Williams0b07ce82020-10-13 16:50:45 -0700717
718 dev_dax->ranges[mapping->range_id].mapping = NULL;
719 mapping->range_id = -1;
720
Dan Williams82b4cee2023-06-02 23:13:59 -0700721 device_unregister(dev);
Dan Williams0b07ce82020-10-13 16:50:45 -0700722}
723
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800724static void unregister_dax_mapping(void *data)
725{
726 if (rwsem_is_locked(&dax_region_rwsem))
727 return __unregister_dax_mapping(data);
728
729 if (WARN_ON_ONCE(down_write_killable(&dax_region_rwsem) != 0))
730 return;
731 __unregister_dax_mapping(data);
732 up_write(&dax_region_rwsem);
733}
734
Dan Williams0b07ce82020-10-13 16:50:45 -0700735static struct dev_dax_range *get_dax_range(struct device *dev)
736{
737 struct dax_mapping *mapping = to_dax_mapping(dev);
738 struct dev_dax *dev_dax = to_dev_dax(dev->parent);
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800739 int rc;
Dan Williams0b07ce82020-10-13 16:50:45 -0700740
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800741 rc = down_write_killable(&dax_region_rwsem);
742 if (rc)
743 return NULL;
Dan Williams0b07ce82020-10-13 16:50:45 -0700744 if (mapping->range_id < 0) {
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800745 up_write(&dax_region_rwsem);
Dan Williams0b07ce82020-10-13 16:50:45 -0700746 return NULL;
747 }
748
749 return &dev_dax->ranges[mapping->range_id];
750}
751
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800752static void put_dax_range(void)
Dan Williams0b07ce82020-10-13 16:50:45 -0700753{
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800754 up_write(&dax_region_rwsem);
Dan Williams0b07ce82020-10-13 16:50:45 -0700755}
756
757static ssize_t start_show(struct device *dev,
758 struct device_attribute *attr, char *buf)
759{
760 struct dev_dax_range *dax_range;
761 ssize_t rc;
762
763 dax_range = get_dax_range(dev);
764 if (!dax_range)
765 return -ENXIO;
Vishal Verma6ebed002024-01-24 12:03:47 -0800766 rc = sysfs_emit(buf, "%#llx\n", dax_range->range.start);
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800767 put_dax_range();
Dan Williams0b07ce82020-10-13 16:50:45 -0700768
769 return rc;
770}
771static DEVICE_ATTR(start, 0400, start_show, NULL);
772
773static ssize_t end_show(struct device *dev,
774 struct device_attribute *attr, char *buf)
775{
776 struct dev_dax_range *dax_range;
777 ssize_t rc;
778
779 dax_range = get_dax_range(dev);
780 if (!dax_range)
781 return -ENXIO;
Vishal Verma6ebed002024-01-24 12:03:47 -0800782 rc = sysfs_emit(buf, "%#llx\n", dax_range->range.end);
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800783 put_dax_range();
Dan Williams0b07ce82020-10-13 16:50:45 -0700784
785 return rc;
786}
787static DEVICE_ATTR(end, 0400, end_show, NULL);
788
789static ssize_t pgoff_show(struct device *dev,
790 struct device_attribute *attr, char *buf)
791{
792 struct dev_dax_range *dax_range;
793 ssize_t rc;
794
795 dax_range = get_dax_range(dev);
796 if (!dax_range)
797 return -ENXIO;
Vishal Verma6ebed002024-01-24 12:03:47 -0800798 rc = sysfs_emit(buf, "%#lx\n", dax_range->pgoff);
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800799 put_dax_range();
Dan Williams0b07ce82020-10-13 16:50:45 -0700800
801 return rc;
802}
803static DEVICE_ATTR(page_offset, 0400, pgoff_show, NULL);
804
805static struct attribute *dax_mapping_attributes[] = {
806 &dev_attr_start.attr,
807 &dev_attr_end.attr,
808 &dev_attr_page_offset.attr,
809 NULL,
810};
811
812static const struct attribute_group dax_mapping_attribute_group = {
813 .attrs = dax_mapping_attributes,
814};
815
816static const struct attribute_group *dax_mapping_attribute_groups[] = {
817 &dax_mapping_attribute_group,
818 NULL,
819};
820
821static struct device_type dax_mapping_type = {
822 .release = dax_mapping_release,
823 .groups = dax_mapping_attribute_groups,
824};
825
826static int devm_register_dax_mapping(struct dev_dax *dev_dax, int range_id)
827{
828 struct dax_region *dax_region = dev_dax->region;
829 struct dax_mapping *mapping;
830 struct device *dev;
831 int rc;
832
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800833 WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem));
Dan Williams0b07ce82020-10-13 16:50:45 -0700834
835 if (dev_WARN_ONCE(&dev_dax->dev, !dax_region->dev->driver,
836 "region disabled\n"))
837 return -ENXIO;
838
839 mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
840 if (!mapping)
841 return -ENOMEM;
842 mapping->range_id = range_id;
843 mapping->id = ida_alloc(&dev_dax->ida, GFP_KERNEL);
844 if (mapping->id < 0) {
845 kfree(mapping);
846 return -ENOMEM;
847 }
848 dev_dax->ranges[range_id].mapping = mapping;
849 dev = &mapping->dev;
850 device_initialize(dev);
851 dev->parent = &dev_dax->dev;
Dan Williams6d24b172023-06-02 23:13:54 -0700852 get_device(dev->parent);
Dan Williams0b07ce82020-10-13 16:50:45 -0700853 dev->type = &dax_mapping_type;
854 dev_set_name(dev, "mapping%d", mapping->id);
855 rc = device_add(dev);
856 if (rc) {
857 put_device(dev);
858 return rc;
859 }
860
861 rc = devm_add_action_or_reset(dax_region->dev, unregister_dax_mapping,
862 dev);
863 if (rc)
864 return rc;
865 return 0;
866}
867
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700868static int alloc_dev_dax_range(struct dev_dax *dev_dax, u64 start,
869 resource_size_t size)
Dan Williamsc2f30112020-10-13 16:50:03 -0700870{
871 struct dax_region *dax_region = dev_dax->region;
872 struct resource *res = &dax_region->res;
873 struct device *dev = &dev_dax->dev;
Dan Williams60e93dc2020-10-13 16:50:39 -0700874 struct dev_dax_range *ranges;
875 unsigned long pgoff = 0;
Dan Williamsc2f30112020-10-13 16:50:03 -0700876 struct resource *alloc;
Dan Williams0b07ce82020-10-13 16:50:45 -0700877 int i, rc;
Dan Williamsc2f30112020-10-13 16:50:03 -0700878
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800879 WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem));
Dan Williamsc2f30112020-10-13 16:50:03 -0700880
Dan Williams0f3da142020-10-13 16:50:13 -0700881 /* handle the seed alloc special case */
882 if (!size) {
Dan Williams60e93dc2020-10-13 16:50:39 -0700883 if (dev_WARN_ONCE(dev, dev_dax->nr_range,
884 "0-size allocation must be first\n"))
885 return -EBUSY;
886 /* nr_range == 0 is elsewhere special cased as 0-size device */
Dan Williams0f3da142020-10-13 16:50:13 -0700887 return 0;
888 }
889
Zhen Leiff8da372020-12-19 16:18:40 +0800890 alloc = __request_region(res, start, size, dev_name(dev), 0);
891 if (!alloc)
Dan Williamsc2f30112020-10-13 16:50:03 -0700892 return -ENOMEM;
893
Zhen Leiff8da372020-12-19 16:18:40 +0800894 ranges = krealloc(dev_dax->ranges, sizeof(*ranges)
895 * (dev_dax->nr_range + 1), GFP_KERNEL);
896 if (!ranges) {
897 __release_region(res, alloc->start, resource_size(alloc));
Dan Williams60e93dc2020-10-13 16:50:39 -0700898 return -ENOMEM;
899 }
900
901 for (i = 0; i < dev_dax->nr_range; i++)
902 pgoff += PHYS_PFN(range_len(&ranges[i].range));
903 dev_dax->ranges = ranges;
904 ranges[dev_dax->nr_range++] = (struct dev_dax_range) {
905 .pgoff = pgoff,
906 .range = {
907 .start = alloc->start,
908 .end = alloc->end,
909 },
Dan Williamsc2f30112020-10-13 16:50:03 -0700910 };
911
Dan Williams60e93dc2020-10-13 16:50:39 -0700912 dev_dbg(dev, "alloc range[%d]: %pa:%pa\n", dev_dax->nr_range - 1,
913 &alloc->start, &alloc->end);
Dan Williams0b07ce82020-10-13 16:50:45 -0700914 /*
915 * A dev_dax instance must be registered before mapping device
916 * children can be added. Defer to devm_create_dev_dax() to add
917 * the initial mapping device.
918 */
919 if (!device_is_registered(&dev_dax->dev))
920 return 0;
921
922 rc = devm_register_dax_mapping(dev_dax, dev_dax->nr_range - 1);
Dan Williams6268d7d2020-12-18 18:41:41 -0800923 if (rc)
924 trim_dev_dax_range(dev_dax);
Dan Williams60e93dc2020-10-13 16:50:39 -0700925
Dan Williams6268d7d2020-12-18 18:41:41 -0800926 return rc;
Dan Williamsc2f30112020-10-13 16:50:03 -0700927}
928
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700929static int adjust_dev_dax_range(struct dev_dax *dev_dax, struct resource *res, resource_size_t size)
930{
Dan Williams60e93dc2020-10-13 16:50:39 -0700931 int last_range = dev_dax->nr_range - 1;
932 struct dev_dax_range *dax_range = &dev_dax->ranges[last_range];
Dan Williams60e93dc2020-10-13 16:50:39 -0700933 bool is_shrink = resource_size(res) > size;
934 struct range *range = &dax_range->range;
935 struct device *dev = &dev_dax->dev;
936 int rc;
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700937
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800938 WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem));
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700939
Dan Williams60e93dc2020-10-13 16:50:39 -0700940 if (dev_WARN_ONCE(dev, !size, "deletion is handled by dev_dax_shrink\n"))
941 return -EINVAL;
942
943 rc = adjust_resource(res, range->start, size);
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700944 if (rc)
945 return rc;
946
Dan Williams60e93dc2020-10-13 16:50:39 -0700947 *range = (struct range) {
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700948 .start = range->start,
949 .end = range->start + size - 1,
950 };
951
Dan Williams60e93dc2020-10-13 16:50:39 -0700952 dev_dbg(dev, "%s range[%d]: %#llx:%#llx\n", is_shrink ? "shrink" : "extend",
953 last_range, (unsigned long long) range->start,
954 (unsigned long long) range->end);
955
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700956 return 0;
957}
958
Dan Williams51cf7842017-07-12 17:58:21 -0700959static ssize_t size_show(struct device *dev,
960 struct device_attribute *attr, char *buf)
961{
962 struct dev_dax *dev_dax = to_dev_dax(dev);
Dan Williams60e93dc2020-10-13 16:50:39 -0700963 unsigned long long size;
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800964 int rc;
Dan Williams60e93dc2020-10-13 16:50:39 -0700965
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800966 rc = down_write_killable(&dax_dev_rwsem);
967 if (rc)
968 return rc;
Dan Williams60e93dc2020-10-13 16:50:39 -0700969 size = dev_dax_size(dev_dax);
Vishal Vermac05ae9d2024-01-24 12:03:46 -0800970 up_write(&dax_dev_rwsem);
Dan Williams51cf7842017-07-12 17:58:21 -0700971
Vishal Verma6ebed002024-01-24 12:03:47 -0800972 return sysfs_emit(buf, "%llu\n", size);
Dan Williams51cf7842017-07-12 17:58:21 -0700973}
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700974
Dan Williams6d821202020-10-13 16:50:55 -0700975static bool alloc_is_aligned(struct dev_dax *dev_dax, resource_size_t size)
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700976{
977 /*
978 * The minimum mapping granularity for a device instance is a
979 * single subsection, unless the arch says otherwise.
980 */
Dan Williams6d821202020-10-13 16:50:55 -0700981 return IS_ALIGNED(size, max_t(unsigned long, dev_dax->align, memremap_compat_align()));
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700982}
983
984static int dev_dax_shrink(struct dev_dax *dev_dax, resource_size_t size)
985{
Dan Williams60e93dc2020-10-13 16:50:39 -0700986 resource_size_t to_shrink = dev_dax_size(dev_dax) - size;
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700987 struct dax_region *dax_region = dev_dax->region;
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700988 struct device *dev = &dev_dax->dev;
Dan Williams60e93dc2020-10-13 16:50:39 -0700989 int i;
Dan Williamsfcffb6a2020-10-13 16:50:24 -0700990
Dan Williams60e93dc2020-10-13 16:50:39 -0700991 for (i = dev_dax->nr_range - 1; i >= 0; i--) {
992 struct range *range = &dev_dax->ranges[i].range;
Dan Williams0b07ce82020-10-13 16:50:45 -0700993 struct dax_mapping *mapping = dev_dax->ranges[i].mapping;
Dan Williams60e93dc2020-10-13 16:50:39 -0700994 struct resource *adjust = NULL, *res;
995 resource_size_t shrink;
996
997 shrink = min_t(u64, to_shrink, range_len(range));
998 if (shrink >= range_len(range)) {
Dan Williams0b07ce82020-10-13 16:50:45 -0700999 devm_release_action(dax_region->dev,
1000 unregister_dax_mapping, &mapping->dev);
Dan Williams6268d7d2020-12-18 18:41:41 -08001001 trim_dev_dax_range(dev_dax);
Dan Williams60e93dc2020-10-13 16:50:39 -07001002 to_shrink -= shrink;
1003 if (!to_shrink)
1004 break;
1005 continue;
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001006 }
1007
Dan Williams60e93dc2020-10-13 16:50:39 -07001008 for_each_dax_region_resource(dax_region, res)
1009 if (strcmp(res->name, dev_name(dev)) == 0
1010 && res->start == range->start) {
1011 adjust = res;
1012 break;
1013 }
1014
1015 if (dev_WARN_ONCE(dev, !adjust || i != dev_dax->nr_range - 1,
1016 "failed to find matching resource\n"))
1017 return -ENXIO;
1018 return adjust_dev_dax_range(dev_dax, adjust, range_len(range)
1019 - shrink);
1020 }
1021 return 0;
1022}
1023
1024/*
1025 * Only allow adjustments that preserve the relative pgoff of existing
1026 * allocations. I.e. the dev_dax->ranges array is ordered by increasing pgoff.
1027 */
1028static bool adjust_ok(struct dev_dax *dev_dax, struct resource *res)
1029{
1030 struct dev_dax_range *last;
1031 int i;
1032
1033 if (dev_dax->nr_range == 0)
1034 return false;
1035 if (strcmp(res->name, dev_name(&dev_dax->dev)) != 0)
1036 return false;
1037 last = &dev_dax->ranges[dev_dax->nr_range - 1];
1038 if (last->range.start != res->start || last->range.end != res->end)
1039 return false;
1040 for (i = 0; i < dev_dax->nr_range - 1; i++) {
1041 struct dev_dax_range *dax_range = &dev_dax->ranges[i];
1042
1043 if (dax_range->pgoff > last->pgoff)
1044 return false;
1045 }
1046
1047 return true;
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001048}
1049
1050static ssize_t dev_dax_resize(struct dax_region *dax_region,
1051 struct dev_dax *dev_dax, resource_size_t size)
1052{
1053 resource_size_t avail = dax_region_avail_size(dax_region), to_alloc;
Dan Williams60e93dc2020-10-13 16:50:39 -07001054 resource_size_t dev_size = dev_dax_size(dev_dax);
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001055 struct resource *region_res = &dax_region->res;
1056 struct device *dev = &dev_dax->dev;
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001057 struct resource *res, *first;
Dan Williams60e93dc2020-10-13 16:50:39 -07001058 resource_size_t alloc = 0;
1059 int rc;
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001060
1061 if (dev->driver)
1062 return -EBUSY;
1063 if (size == dev_size)
1064 return 0;
1065 if (size > dev_size && size - dev_size > avail)
1066 return -ENOSPC;
1067 if (size < dev_size)
1068 return dev_dax_shrink(dev_dax, size);
1069
1070 to_alloc = size - dev_size;
Dan Williams6d821202020-10-13 16:50:55 -07001071 if (dev_WARN_ONCE(dev, !alloc_is_aligned(dev_dax, to_alloc),
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001072 "resize of %pa misaligned\n", &to_alloc))
1073 return -ENXIO;
1074
1075 /*
1076 * Expand the device into the unused portion of the region. This
1077 * may involve adjusting the end of an existing resource, or
1078 * allocating a new resource.
1079 */
Dan Williams60e93dc2020-10-13 16:50:39 -07001080retry:
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001081 first = region_res->child;
1082 if (!first)
1083 return alloc_dev_dax_range(dev_dax, dax_region->res.start, to_alloc);
Dan Williams60e93dc2020-10-13 16:50:39 -07001084
1085 rc = -ENOSPC;
1086 for (res = first; res; res = res->sibling) {
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001087 struct resource *next = res->sibling;
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001088
1089 /* space at the beginning of the region */
Dan Williams60e93dc2020-10-13 16:50:39 -07001090 if (res == first && res->start > dax_region->res.start) {
1091 alloc = min(res->start - dax_region->res.start, to_alloc);
1092 rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, alloc);
1093 break;
1094 }
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001095
Dan Williams60e93dc2020-10-13 16:50:39 -07001096 alloc = 0;
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001097 /* space between allocations */
1098 if (next && next->start > res->end + 1)
Dan Williams60e93dc2020-10-13 16:50:39 -07001099 alloc = min(next->start - (res->end + 1), to_alloc);
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001100
1101 /* space at the end of the region */
Dan Williams60e93dc2020-10-13 16:50:39 -07001102 if (!alloc && !next && res->end < region_res->end)
1103 alloc = min(region_res->end - res->end, to_alloc);
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001104
Dan Williams60e93dc2020-10-13 16:50:39 -07001105 if (!alloc)
1106 continue;
1107
1108 if (adjust_ok(dev_dax, res)) {
1109 rc = adjust_dev_dax_range(dev_dax, res, resource_size(res) + alloc);
1110 break;
1111 }
1112 rc = alloc_dev_dax_range(dev_dax, res->end + 1, alloc);
1113 break;
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001114 }
Dan Williams60e93dc2020-10-13 16:50:39 -07001115 if (rc)
1116 return rc;
1117 to_alloc -= alloc;
1118 if (to_alloc)
1119 goto retry;
1120 return 0;
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001121}
1122
1123static ssize_t size_store(struct device *dev, struct device_attribute *attr,
1124 const char *buf, size_t len)
1125{
1126 ssize_t rc;
1127 unsigned long long val;
1128 struct dev_dax *dev_dax = to_dev_dax(dev);
1129 struct dax_region *dax_region = dev_dax->region;
1130
1131 rc = kstrtoull(buf, 0, &val);
1132 if (rc)
1133 return rc;
1134
Dan Williams6d821202020-10-13 16:50:55 -07001135 if (!alloc_is_aligned(dev_dax, val)) {
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001136 dev_dbg(dev, "%s: size: %lld misaligned\n", __func__, val);
1137 return -EINVAL;
1138 }
1139
Vishal Vermac05ae9d2024-01-24 12:03:46 -08001140 rc = down_write_killable(&dax_region_rwsem);
1141 if (rc)
1142 return rc;
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001143 if (!dax_region->dev->driver) {
Vishal Vermac05ae9d2024-01-24 12:03:46 -08001144 rc = -ENXIO;
1145 goto err_region;
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001146 }
Vishal Vermac05ae9d2024-01-24 12:03:46 -08001147 rc = down_write_killable(&dax_dev_rwsem);
1148 if (rc)
1149 goto err_dev;
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001150
Vishal Vermac05ae9d2024-01-24 12:03:46 -08001151 rc = dev_dax_resize(dax_region, dev_dax, val);
1152
1153err_dev:
1154 up_write(&dax_dev_rwsem);
1155err_region:
1156 up_write(&dax_region_rwsem);
1157
1158 if (rc == 0)
1159 return len;
1160 return rc;
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001161}
1162static DEVICE_ATTR_RW(size);
Dan Williams51cf7842017-07-12 17:58:21 -07001163
Joao Martins8490e2e2020-10-13 16:51:06 -07001164static ssize_t range_parse(const char *opt, size_t len, struct range *range)
1165{
1166 unsigned long long addr = 0;
1167 char *start, *end, *str;
Shiyang Ruan7323fb22021-01-26 10:13:31 +08001168 ssize_t rc = -EINVAL;
Joao Martins8490e2e2020-10-13 16:51:06 -07001169
1170 str = kstrdup(opt, GFP_KERNEL);
1171 if (!str)
1172 return rc;
1173
1174 end = str;
1175 start = strsep(&end, "-");
1176 if (!start || !end)
1177 goto err;
1178
1179 rc = kstrtoull(start, 16, &addr);
1180 if (rc)
1181 goto err;
1182 range->start = addr;
1183
1184 rc = kstrtoull(end, 16, &addr);
1185 if (rc)
1186 goto err;
1187 range->end = addr;
1188
1189err:
1190 kfree(str);
1191 return rc;
1192}
1193
1194static ssize_t mapping_store(struct device *dev, struct device_attribute *attr,
1195 const char *buf, size_t len)
1196{
1197 struct dev_dax *dev_dax = to_dev_dax(dev);
1198 struct dax_region *dax_region = dev_dax->region;
1199 size_t to_alloc;
1200 struct range r;
1201 ssize_t rc;
1202
1203 rc = range_parse(buf, len, &r);
1204 if (rc)
1205 return rc;
1206
1207 rc = -ENXIO;
Vishal Vermac05ae9d2024-01-24 12:03:46 -08001208 rc = down_write_killable(&dax_region_rwsem);
1209 if (rc)
1210 return rc;
Joao Martins8490e2e2020-10-13 16:51:06 -07001211 if (!dax_region->dev->driver) {
Vishal Vermac05ae9d2024-01-24 12:03:46 -08001212 up_write(&dax_region_rwsem);
Joao Martins8490e2e2020-10-13 16:51:06 -07001213 return rc;
1214 }
Vishal Vermac05ae9d2024-01-24 12:03:46 -08001215 rc = down_write_killable(&dax_dev_rwsem);
1216 if (rc) {
1217 up_write(&dax_region_rwsem);
1218 return rc;
1219 }
Joao Martins8490e2e2020-10-13 16:51:06 -07001220
1221 to_alloc = range_len(&r);
1222 if (alloc_is_aligned(dev_dax, to_alloc))
1223 rc = alloc_dev_dax_range(dev_dax, r.start, to_alloc);
Vishal Vermac05ae9d2024-01-24 12:03:46 -08001224 up_write(&dax_dev_rwsem);
1225 up_write(&dax_region_rwsem);
Joao Martins8490e2e2020-10-13 16:51:06 -07001226
1227 return rc == 0 ? len : rc;
1228}
1229static DEVICE_ATTR_WO(mapping);
1230
Dan Williams6d821202020-10-13 16:50:55 -07001231static ssize_t align_show(struct device *dev,
1232 struct device_attribute *attr, char *buf)
1233{
1234 struct dev_dax *dev_dax = to_dev_dax(dev);
1235
Vishal Verma6ebed002024-01-24 12:03:47 -08001236 return sysfs_emit(buf, "%d\n", dev_dax->align);
Dan Williams6d821202020-10-13 16:50:55 -07001237}
1238
1239static ssize_t dev_dax_validate_align(struct dev_dax *dev_dax)
1240{
Dan Williams6d821202020-10-13 16:50:55 -07001241 struct device *dev = &dev_dax->dev;
1242 int i;
1243
Dan Williams6d821202020-10-13 16:50:55 -07001244 for (i = 0; i < dev_dax->nr_range; i++) {
1245 size_t len = range_len(&dev_dax->ranges[i].range);
1246
1247 if (!alloc_is_aligned(dev_dax, len)) {
1248 dev_dbg(dev, "%s: align %u invalid for range %d\n",
1249 __func__, dev_dax->align, i);
1250 return -EINVAL;
1251 }
1252 }
1253
1254 return 0;
1255}
1256
1257static ssize_t align_store(struct device *dev, struct device_attribute *attr,
1258 const char *buf, size_t len)
1259{
1260 struct dev_dax *dev_dax = to_dev_dax(dev);
1261 struct dax_region *dax_region = dev_dax->region;
1262 unsigned long val, align_save;
1263 ssize_t rc;
1264
1265 rc = kstrtoul(buf, 0, &val);
1266 if (rc)
1267 return -ENXIO;
1268
1269 if (!dax_align_valid(val))
1270 return -EINVAL;
1271
Vishal Vermac05ae9d2024-01-24 12:03:46 -08001272 rc = down_write_killable(&dax_region_rwsem);
1273 if (rc)
1274 return rc;
Dan Williams6d821202020-10-13 16:50:55 -07001275 if (!dax_region->dev->driver) {
Vishal Vermac05ae9d2024-01-24 12:03:46 -08001276 up_write(&dax_region_rwsem);
Dan Williams6d821202020-10-13 16:50:55 -07001277 return -ENXIO;
1278 }
1279
Vishal Vermac05ae9d2024-01-24 12:03:46 -08001280 rc = down_write_killable(&dax_dev_rwsem);
1281 if (rc) {
1282 up_write(&dax_region_rwsem);
1283 return rc;
1284 }
Dan Williams6d821202020-10-13 16:50:55 -07001285 if (dev->driver) {
1286 rc = -EBUSY;
1287 goto out_unlock;
1288 }
1289
1290 align_save = dev_dax->align;
1291 dev_dax->align = val;
1292 rc = dev_dax_validate_align(dev_dax);
1293 if (rc)
1294 dev_dax->align = align_save;
1295out_unlock:
Vishal Vermac05ae9d2024-01-24 12:03:46 -08001296 up_write(&dax_dev_rwsem);
1297 up_write(&dax_region_rwsem);
Dan Williams6d821202020-10-13 16:50:55 -07001298 return rc == 0 ? len : rc;
1299}
1300static DEVICE_ATTR_RW(align);
1301
Dan Williams21c75762019-02-20 11:39:36 -08001302static int dev_dax_target_node(struct dev_dax *dev_dax)
1303{
1304 struct dax_region *dax_region = dev_dax->region;
1305
1306 return dax_region->target_node;
1307}
1308
1309static ssize_t target_node_show(struct device *dev,
1310 struct device_attribute *attr, char *buf)
1311{
1312 struct dev_dax *dev_dax = to_dev_dax(dev);
1313
Vishal Verma6ebed002024-01-24 12:03:47 -08001314 return sysfs_emit(buf, "%d\n", dev_dax_target_node(dev_dax));
Dan Williams21c75762019-02-20 11:39:36 -08001315}
1316static DEVICE_ATTR_RO(target_node);
1317
Vishal Verma40cdc602019-06-20 18:40:38 -06001318static ssize_t resource_show(struct device *dev,
1319 struct device_attribute *attr, char *buf)
1320{
1321 struct dev_dax *dev_dax = to_dev_dax(dev);
Dan Williams60e93dc2020-10-13 16:50:39 -07001322 struct dax_region *dax_region = dev_dax->region;
1323 unsigned long long start;
Vishal Verma40cdc602019-06-20 18:40:38 -06001324
Dan Williams60e93dc2020-10-13 16:50:39 -07001325 if (dev_dax->nr_range < 1)
1326 start = dax_region->res.start;
1327 else
1328 start = dev_dax->ranges[0].range.start;
1329
Vishal Verma6ebed002024-01-24 12:03:47 -08001330 return sysfs_emit(buf, "%#llx\n", start);
Vishal Verma40cdc602019-06-20 18:40:38 -06001331}
Dan Williams153dd282019-11-12 17:12:38 -08001332static DEVICE_ATTR(resource, 0400, resource_show, NULL);
Vishal Verma40cdc602019-06-20 18:40:38 -06001333
Vishal Vermac347bd72019-02-22 16:58:54 -07001334static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
1335 char *buf)
1336{
1337 /*
1338 * We only ever expect to handle device-dax instances, i.e. the
1339 * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
1340 */
Vishal Verma6ebed002024-01-24 12:03:47 -08001341 return sysfs_emit(buf, DAX_DEVICE_MODALIAS_FMT "\n", 0);
Vishal Vermac347bd72019-02-22 16:58:54 -07001342}
1343static DEVICE_ATTR_RO(modalias);
1344
Dan Williamscb4dd722019-11-12 17:13:31 -08001345static ssize_t numa_node_show(struct device *dev,
1346 struct device_attribute *attr, char *buf)
1347{
Vishal Verma6ebed002024-01-24 12:03:47 -08001348 return sysfs_emit(buf, "%d\n", dev_to_node(dev));
Dan Williamscb4dd722019-11-12 17:13:31 -08001349}
1350static DEVICE_ATTR_RO(numa_node);
1351
Vishal Verma73954d32024-01-24 12:03:50 -08001352static ssize_t memmap_on_memory_show(struct device *dev,
1353 struct device_attribute *attr, char *buf)
1354{
1355 struct dev_dax *dev_dax = to_dev_dax(dev);
1356
1357 return sysfs_emit(buf, "%d\n", dev_dax->memmap_on_memory);
1358}
1359
1360static ssize_t memmap_on_memory_store(struct device *dev,
1361 struct device_attribute *attr,
1362 const char *buf, size_t len)
1363{
1364 struct dev_dax *dev_dax = to_dev_dax(dev);
1365 bool val;
1366 int rc;
1367
1368 rc = kstrtobool(buf, &val);
1369 if (rc)
1370 return rc;
1371
1372 if (val == true && !mhp_supports_memmap_on_memory()) {
1373 dev_dbg(dev, "memmap_on_memory is not available\n");
1374 return -EOPNOTSUPP;
1375 }
1376
1377 rc = down_write_killable(&dax_dev_rwsem);
1378 if (rc)
1379 return rc;
1380
1381 if (dev_dax->memmap_on_memory != val && dev->driver &&
1382 to_dax_drv(dev->driver)->type == DAXDRV_KMEM_TYPE) {
1383 up_write(&dax_dev_rwsem);
1384 return -EBUSY;
1385 }
1386
1387 dev_dax->memmap_on_memory = val;
1388 up_write(&dax_dev_rwsem);
1389
1390 return len;
1391}
1392static DEVICE_ATTR_RW(memmap_on_memory);
1393
Dan Williams21c75762019-02-20 11:39:36 -08001394static umode_t dev_dax_visible(struct kobject *kobj, struct attribute *a, int n)
1395{
1396 struct device *dev = container_of(kobj, struct device, kobj);
1397 struct dev_dax *dev_dax = to_dev_dax(dev);
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001398 struct dax_region *dax_region = dev_dax->region;
Dan Williams21c75762019-02-20 11:39:36 -08001399
1400 if (a == &dev_attr_target_node.attr && dev_dax_target_node(dev_dax) < 0)
1401 return 0;
Dan Williamscb4dd722019-11-12 17:13:31 -08001402 if (a == &dev_attr_numa_node.attr && !IS_ENABLED(CONFIG_NUMA))
1403 return 0;
Joao Martins8490e2e2020-10-13 16:51:06 -07001404 if (a == &dev_attr_mapping.attr && is_static(dax_region))
1405 return 0;
Dan Williams6d821202020-10-13 16:50:55 -07001406 if ((a == &dev_attr_align.attr ||
1407 a == &dev_attr_size.attr) && is_static(dax_region))
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001408 return 0444;
Dan Williams21c75762019-02-20 11:39:36 -08001409 return a->mode;
1410}
1411
Dan Williams51cf7842017-07-12 17:58:21 -07001412static struct attribute *dev_dax_attributes[] = {
Vishal Vermac347bd72019-02-22 16:58:54 -07001413 &dev_attr_modalias.attr,
Dan Williams51cf7842017-07-12 17:58:21 -07001414 &dev_attr_size.attr,
Joao Martins8490e2e2020-10-13 16:51:06 -07001415 &dev_attr_mapping.attr,
Dan Williams21c75762019-02-20 11:39:36 -08001416 &dev_attr_target_node.attr,
Dan Williams6d821202020-10-13 16:50:55 -07001417 &dev_attr_align.attr,
Vishal Verma40cdc602019-06-20 18:40:38 -06001418 &dev_attr_resource.attr,
Dan Williamscb4dd722019-11-12 17:13:31 -08001419 &dev_attr_numa_node.attr,
Vishal Verma73954d32024-01-24 12:03:50 -08001420 &dev_attr_memmap_on_memory.attr,
Dan Williams51cf7842017-07-12 17:58:21 -07001421 NULL,
1422};
1423
1424static const struct attribute_group dev_dax_attribute_group = {
1425 .attrs = dev_dax_attributes,
Dan Williams21c75762019-02-20 11:39:36 -08001426 .is_visible = dev_dax_visible,
Dan Williams51cf7842017-07-12 17:58:21 -07001427};
1428
Dan Williams9567da02017-07-12 17:58:21 -07001429static const struct attribute_group *dax_attribute_groups[] = {
Dan Williams51cf7842017-07-12 17:58:21 -07001430 &dev_dax_attribute_group,
1431 NULL,
1432};
Dan Williams51cf7842017-07-12 17:58:21 -07001433
Dan Williams9567da02017-07-12 17:58:21 -07001434static void dev_dax_release(struct device *dev)
Dan Williams51cf7842017-07-12 17:58:21 -07001435{
1436 struct dev_dax *dev_dax = to_dev_dax(dev);
1437 struct dax_device *dax_dev = dev_dax->dax_dev;
Dan Williams51cf7842017-07-12 17:58:21 -07001438
Dan Williams9567da02017-07-12 17:58:21 -07001439 put_dax(dax_dev);
Dan Williams0f3da142020-10-13 16:50:13 -07001440 free_dev_dax_id(dev_dax);
Dan Williamsf5516ec2020-10-13 16:49:43 -07001441 kfree(dev_dax->pgmap);
Dan Williams9567da02017-07-12 17:58:21 -07001442 kfree(dev_dax);
1443}
1444
Dan Williams770619a2019-11-12 17:12:23 -08001445static const struct device_type dev_dax_type = {
1446 .release = dev_dax_release,
1447 .groups = dax_attribute_groups,
1448};
1449
Vishal Vermac05ae9d2024-01-24 12:03:46 -08001450static struct dev_dax *__devm_create_dev_dax(struct dev_dax_data *data)
Dan Williams9567da02017-07-12 17:58:21 -07001451{
Dan Williams174ebec2020-10-13 16:49:38 -07001452 struct dax_region *dax_region = data->dax_region;
Dan Williams9567da02017-07-12 17:58:21 -07001453 struct device *parent = dax_region->dev;
1454 struct dax_device *dax_dev;
1455 struct dev_dax *dev_dax;
1456 struct inode *inode;
1457 struct device *dev;
Dan Williamsc2f30112020-10-13 16:50:03 -07001458 int rc;
Dan Williams9567da02017-07-12 17:58:21 -07001459
Dan Williams9567da02017-07-12 17:58:21 -07001460 dev_dax = kzalloc(sizeof(*dev_dax), GFP_KERNEL);
1461 if (!dev_dax)
1462 return ERR_PTR(-ENOMEM);
1463
Dan Williams70aab282023-06-02 23:14:05 -07001464 dev_dax->region = dax_region;
Dan Williams0f3da142020-10-13 16:50:13 -07001465 if (is_static(dax_region)) {
1466 if (dev_WARN_ONCE(parent, data->id < 0,
1467 "dynamic id specified to static region\n")) {
1468 rc = -EINVAL;
1469 goto err_id;
1470 }
1471
1472 dev_dax->id = data->id;
1473 } else {
1474 if (dev_WARN_ONCE(parent, data->id >= 0,
1475 "static id specified to dynamic region\n")) {
1476 rc = -EINVAL;
1477 goto err_id;
1478 }
1479
Dan Williams70aab282023-06-02 23:14:05 -07001480 rc = alloc_dev_dax_id(dev_dax);
Dan Williams0f3da142020-10-13 16:50:13 -07001481 if (rc < 0)
1482 goto err_id;
Dan Williams0f3da142020-10-13 16:50:13 -07001483 }
1484
Dan Williamsc2f30112020-10-13 16:50:03 -07001485 dev = &dev_dax->dev;
1486 device_initialize(dev);
Dan Williams0f3da142020-10-13 16:50:13 -07001487 dev_set_name(dev, "dax%d.%d", dax_region->id, dev_dax->id);
Dan Williamsc2f30112020-10-13 16:50:03 -07001488
Dan Williamsfcffb6a2020-10-13 16:50:24 -07001489 rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, data->size);
Dan Williamsc2f30112020-10-13 16:50:03 -07001490 if (rc)
1491 goto err_range;
1492
Dan Williamsf5516ec2020-10-13 16:49:43 -07001493 if (data->pgmap) {
Dan Williamsc2f30112020-10-13 16:50:03 -07001494 dev_WARN_ONCE(parent, !is_static(dax_region),
1495 "custom dev_pagemap requires a static dax_region\n");
1496
Dan Williamsf5516ec2020-10-13 16:49:43 -07001497 dev_dax->pgmap = kmemdup(data->pgmap,
1498 sizeof(struct dev_pagemap), GFP_KERNEL);
Dan Williamsc2f30112020-10-13 16:50:03 -07001499 if (!dev_dax->pgmap) {
1500 rc = -ENOMEM;
Dan Williamsf5516ec2020-10-13 16:49:43 -07001501 goto err_pgmap;
Dan Williamsc2f30112020-10-13 16:50:03 -07001502 }
Dan Williamsf5516ec2020-10-13 16:49:43 -07001503 }
Dan Williams89ec9f22018-10-29 15:52:42 -07001504
Dan Williams9567da02017-07-12 17:58:21 -07001505 /*
Christoph Hellwigfb08a192021-11-29 11:21:38 +01001506 * No dax_operations since there is no access to this device outside of
1507 * mmap of the resulting character device.
Dan Williams9567da02017-07-12 17:58:21 -07001508 */
Christoph Hellwig30c68282021-12-15 09:45:07 +01001509 dax_dev = alloc_dax(dev_dax, NULL);
Vivek Goyal4e4ced92020-04-01 12:11:25 -04001510 if (IS_ERR(dax_dev)) {
1511 rc = PTR_ERR(dax_dev);
Dan Williamsf5516ec2020-10-13 16:49:43 -07001512 goto err_alloc_dax;
Vivek Goyal4e4ced92020-04-01 12:11:25 -04001513 }
Christoph Hellwig30c68282021-12-15 09:45:07 +01001514 set_dax_synchronous(dax_dev);
Christoph Hellwig7ac53602021-12-15 09:45:08 +01001515 set_dax_nocache(dax_dev);
1516 set_dax_nomc(dax_dev);
Dan Williams9567da02017-07-12 17:58:21 -07001517
1518 /* a device_dax instance is dead while the driver is not attached */
1519 kill_dax(dax_dev);
1520
Dan Williams9567da02017-07-12 17:58:21 -07001521 dev_dax->dax_dev = dax_dev;
Dan Williams8fc5c732018-11-09 12:43:07 -08001522 dev_dax->target_node = dax_region->target_node;
Joao Martins33cf94d2020-10-13 16:50:50 -07001523 dev_dax->align = dax_region->align;
Dan Williams0b07ce82020-10-13 16:50:45 -07001524 ida_init(&dev_dax->ida);
Dan Williams9567da02017-07-12 17:58:21 -07001525
Vishal Verma4eca0ef2023-11-07 00:22:43 -07001526 dev_dax->memmap_on_memory = data->memmap_on_memory;
1527
Dan Williams9567da02017-07-12 17:58:21 -07001528 inode = dax_inode(dax_dev);
1529 dev->devt = inode->i_rdev;
Dan Williams83762cb2021-11-15 13:20:57 -08001530 dev->bus = &dax_bus_type;
Dan Williams9567da02017-07-12 17:58:21 -07001531 dev->parent = parent;
Dan Williams770619a2019-11-12 17:12:23 -08001532 dev->type = &dev_dax_type;
Dan Williams9567da02017-07-12 17:58:21 -07001533
1534 rc = device_add(dev);
1535 if (rc) {
1536 kill_dev_dax(dev_dax);
1537 put_device(dev);
1538 return ERR_PTR(rc);
1539 }
1540
1541 rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev);
1542 if (rc)
1543 return ERR_PTR(rc);
1544
Dan Williams0b07ce82020-10-13 16:50:45 -07001545 /* register mapping device for the initial allocation range */
1546 if (dev_dax->nr_range && range_len(&dev_dax->ranges[0].range)) {
1547 rc = devm_register_dax_mapping(dev_dax, 0);
1548 if (rc)
1549 return ERR_PTR(rc);
1550 }
1551
Dan Williams9567da02017-07-12 17:58:21 -07001552 return dev_dax;
Dan Williamsc2f30112020-10-13 16:50:03 -07001553
Dan Williamsf5516ec2020-10-13 16:49:43 -07001554err_alloc_dax:
1555 kfree(dev_dax->pgmap);
1556err_pgmap:
Dan Williams60e93dc2020-10-13 16:50:39 -07001557 free_dev_dax_ranges(dev_dax);
Dan Williamsc2f30112020-10-13 16:50:03 -07001558err_range:
Dan Williams0f3da142020-10-13 16:50:13 -07001559 free_dev_dax_id(dev_dax);
1560err_id:
Dan Williams9567da02017-07-12 17:58:21 -07001561 kfree(dev_dax);
1562
1563 return ERR_PTR(rc);
1564}
Vishal Vermac05ae9d2024-01-24 12:03:46 -08001565
1566struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data)
1567{
1568 struct dev_dax *dev_dax;
1569 int rc;
1570
1571 rc = down_write_killable(&dax_region_rwsem);
1572 if (rc)
1573 return ERR_PTR(rc);
1574
1575 dev_dax = __devm_create_dev_dax(data);
1576 up_write(&dax_region_rwsem);
1577
1578 return dev_dax;
1579}
Dan Williams174ebec2020-10-13 16:49:38 -07001580EXPORT_SYMBOL_GPL(devm_create_dev_dax);
Dan Williams9567da02017-07-12 17:58:21 -07001581
Dan Williamsd2007812018-11-07 15:31:23 -08001582int __dax_driver_register(struct dax_device_driver *dax_drv,
Dan Williams9567da02017-07-12 17:58:21 -07001583 struct module *module, const char *mod_name)
1584{
Dan Williamsd2007812018-11-07 15:31:23 -08001585 struct device_driver *drv = &dax_drv->drv;
Dan Williamsd2007812018-11-07 15:31:23 -08001586
Uwe Kleine-König5b8e64f2021-02-05 23:28:38 +01001587 /*
1588 * dax_bus_probe() calls dax_drv->probe() unconditionally.
1589 * So better be safe than sorry and ensure it is provided.
1590 */
1591 if (!dax_drv->probe)
1592 return -EINVAL;
1593
Dan Williamsd2007812018-11-07 15:31:23 -08001594 INIT_LIST_HEAD(&dax_drv->ids);
Dan Williams9567da02017-07-12 17:58:21 -07001595 drv->owner = module;
1596 drv->name = mod_name;
1597 drv->mod_name = mod_name;
1598 drv->bus = &dax_bus_type;
Dan Williamsd2007812018-11-07 15:31:23 -08001599
Dan Williamse9ee9fe2023-02-10 01:07:13 -08001600 return driver_register(drv);
Dan Williams9567da02017-07-12 17:58:21 -07001601}
1602EXPORT_SYMBOL_GPL(__dax_driver_register);
1603
Dan Williamsd2007812018-11-07 15:31:23 -08001604void dax_driver_unregister(struct dax_device_driver *dax_drv)
1605{
Dan Williams730926c2017-07-16 13:51:53 -07001606 struct device_driver *drv = &dax_drv->drv;
Dan Williamsd2007812018-11-07 15:31:23 -08001607 struct dax_id *dax_id, *_id;
1608
1609 mutex_lock(&dax_bus_lock);
Dan Williamsd2007812018-11-07 15:31:23 -08001610 list_for_each_entry_safe(dax_id, _id, &dax_drv->ids, list) {
1611 list_del(&dax_id->list);
1612 kfree(dax_id);
1613 }
1614 mutex_unlock(&dax_bus_lock);
Dan Williams730926c2017-07-16 13:51:53 -07001615 driver_unregister(drv);
Dan Williamsd2007812018-11-07 15:31:23 -08001616}
1617EXPORT_SYMBOL_GPL(dax_driver_unregister);
1618
Dan Williams9567da02017-07-12 17:58:21 -07001619int __init dax_bus_init(void)
1620{
Dan Williams83762cb2021-11-15 13:20:57 -08001621 return bus_register(&dax_bus_type);
Dan Williams9567da02017-07-12 17:58:21 -07001622}
1623
1624void __exit dax_bus_exit(void)
1625{
1626 bus_unregister(&dax_bus_type);
1627}