blob: 3d8daaecb6d1a2570c1ad11ec765019da8e0dcd8 [file] [log] [blame]
Thomas Gleixner457c8992019-05-19 13:08:55 +01001// SPDX-License-Identifier: GPL-2.0-only
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * fs/dcache.c
4 *
5 * Complete reimplementation
6 * (C) 1997 Thomas Schoebel-Theuer,
7 * with heavy changes by Linus Torvalds
8 */
9
10/*
11 * Notes on the allocation strategy:
12 *
13 * The dcache is a master of the icache - whenever a dcache entry
14 * exists, the inode will always exist. "iput()" is done either when
15 * the dcache entry is deleted or garbage collected.
16 */
17
Al Viro7a5cf792018-03-05 19:15:50 -050018#include <linux/ratelimit.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/string.h>
20#include <linux/mm.h>
21#include <linux/fs.h>
Eric Biggers0bf3d5c12019-03-20 11:39:11 -070022#include <linux/fscrypt.h>
John McCutchan7a91bf72005-08-08 13:52:16 -040023#include <linux/fsnotify.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <linux/slab.h>
25#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <linux/hash.h>
27#include <linux/cache.h>
Paul Gortmaker630d9c42011-11-16 23:57:37 -050028#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <linux/security.h>
30#include <linux/seqlock.h>
Mike Rapoport57c8a662018-10-30 15:09:49 -070031#include <linux/memblock.h>
Nick Pigginceb5bdc2011-01-07 17:50:05 +110032#include <linux/bit_spinlock.h>
33#include <linux/rculist_bl.h>
Dave Chinnerf6041562013-08-28 10:18:00 +100034#include <linux/list_lru.h>
David Howells07f3f052006-09-30 20:52:18 +020035#include "internal.h"
Al Virob2dba1a2011-11-23 19:26:23 -050036#include "mount.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070037
Linus Torvaldse7829852024-06-04 12:30:02 -070038#include <asm/runtime-const.h>
39
Nick Piggin789680d2011-01-07 17:49:30 +110040/*
41 * Usage:
Nick Piggin873feea2011-01-07 17:50:06 +110042 * dcache->d_inode->i_lock protects:
Al Viro946e51f2014-10-26 19:19:16 -040043 * - i_dentry, d_u.d_alias, d_inode of aliases
Nick Pigginceb5bdc2011-01-07 17:50:05 +110044 * dcache_hash_bucket lock protects:
45 * - the dcache hash table
NeilBrownf1ee6162017-12-21 09:45:40 +110046 * s_roots bl list spinlock protects:
47 * - the s_roots list (see __d_drop)
Dave Chinner19156842013-08-28 10:17:55 +100048 * dentry->d_sb->s_dentry_lru_lock protects:
Nick Piggin23044502011-01-07 17:49:31 +110049 * - the dcache lru lists and counters
50 * d_lock protects:
51 * - d_flags
52 * - d_name
53 * - d_lru
Nick Pigginb7ab39f2011-01-07 17:49:32 +110054 * - d_count
Nick Pigginda502952011-01-07 17:49:33 +110055 * - d_unhashed()
Al Viroda549bd2023-11-07 02:00:39 -050056 * - d_parent and d_chilren
57 * - childrens' d_sib and d_parent
Al Viro946e51f2014-10-26 19:19:16 -040058 * - d_u.d_alias, d_inode
Nick Piggin789680d2011-01-07 17:49:30 +110059 *
60 * Ordering:
Nick Piggin873feea2011-01-07 17:50:06 +110061 * dentry->d_inode->i_lock
Nick Pigginb5c84bf2011-01-07 17:49:38 +110062 * dentry->d_lock
Dave Chinner19156842013-08-28 10:17:55 +100063 * dentry->d_sb->s_dentry_lru_lock
Nick Pigginceb5bdc2011-01-07 17:50:05 +110064 * dcache_hash_bucket lock
NeilBrownf1ee6162017-12-21 09:45:40 +110065 * s_roots lock
Nick Piggin789680d2011-01-07 17:49:30 +110066 *
Nick Pigginda502952011-01-07 17:49:33 +110067 * If there is an ancestor relationship:
68 * dentry->d_parent->...->d_parent->d_lock
69 * ...
70 * dentry->d_parent->d_lock
71 * dentry->d_lock
72 *
73 * If no ancestor relationship:
Al Viro076515f2018-03-10 23:15:52 -050074 * arbitrary, since it's serialized on rename_lock
Nick Piggin789680d2011-01-07 17:49:30 +110075 */
Eric Dumazetfa3536c2006-03-26 01:37:24 -080076int sysctl_vfs_cache_pressure __read_mostly = 100;
Linus Torvalds1da177e2005-04-16 15:20:36 -070077EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
78
Al Viro74c3cbe2007-07-22 08:04:18 -040079__cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070080
Nick Piggin949854d2011-01-07 17:49:37 +110081EXPORT_SYMBOL(rename_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070082
Alexey Dobriyan68279f92023-10-11 19:55:00 +030083static struct kmem_cache *dentry_cache __ro_after_init;
Linus Torvalds1da177e2005-04-16 15:20:36 -070084
David Howellscdf01222017-07-04 17:25:22 +010085const struct qstr empty_name = QSTR_INIT("", 0);
86EXPORT_SYMBOL(empty_name);
87const struct qstr slash_name = QSTR_INIT("/", 1);
88EXPORT_SYMBOL(slash_name);
Al Viro80e5d1ff52021-04-15 19:46:50 -040089const struct qstr dotdot_name = QSTR_INIT("..", 2);
90EXPORT_SYMBOL(dotdot_name);
David Howellscdf01222017-07-04 17:25:22 +010091
Linus Torvalds1da177e2005-04-16 15:20:36 -070092/*
93 * This is the single most critical data structure when it comes
94 * to the dcache: the hashtable for lookups. Somebody should try
95 * to make this good - I've just made it work.
96 *
97 * This hash-function tries to avoid losing too many bits of hash
98 * information, yet avoid using a prime hash-size or similar.
99 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100
Alexey Dobriyan68279f92023-10-11 19:55:00 +0300101static unsigned int d_hash_shift __ro_after_init;
Nick Pigginceb5bdc2011-01-07 17:50:05 +1100102
Alexey Dobriyan68279f92023-10-11 19:55:00 +0300103static struct hlist_bl_head *dentry_hashtable __ro_after_init;
Nick Pigginceb5bdc2011-01-07 17:50:05 +1100104
Linus Torvaldse60cc612024-06-10 11:47:06 -0700105static inline struct hlist_bl_head *d_hash(unsigned long hashlen)
Nick Pigginceb5bdc2011-01-07 17:50:05 +1100106{
Linus Torvaldse7829852024-06-04 12:30:02 -0700107 return runtime_const_ptr(dentry_hashtable) +
108 runtime_const_shift_right_32(hashlen, d_hash_shift);
Nick Pigginceb5bdc2011-01-07 17:50:05 +1100109}
110
Al Viro94bdd652016-04-15 02:42:04 -0400111#define IN_LOOKUP_SHIFT 10
112static struct hlist_bl_head in_lookup_hashtable[1 << IN_LOOKUP_SHIFT];
113
114static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent,
115 unsigned int hash)
116{
117 hash += (unsigned long) parent / L1_CACHE_BYTES;
118 return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT);
119}
120
Luis Chamberlainc8c0c232022-01-21 22:12:59 -0800121struct dentry_stat_t {
122 long nr_dentry;
123 long nr_unused;
124 long age_limit; /* age in seconds */
125 long want_pages; /* pages requested by system */
126 long nr_negative; /* # of unused negative dentries */
127 long dummy; /* Reserved for future use */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128};
129
Glauber Costa3942c072013-08-28 10:17:53 +1000130static DEFINE_PER_CPU(long, nr_dentry);
Dave Chinner62d36c72013-08-28 10:17:54 +1000131static DEFINE_PER_CPU(long, nr_dentry_unused);
Waiman Longaf0c9af2019-01-30 13:52:38 -0500132static DEFINE_PER_CPU(long, nr_dentry_negative);
Christoph Hellwig312d3ca2010-10-10 05:36:23 -0400133
134#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
Luis Chamberlainc8c0c232022-01-21 22:12:59 -0800135/* Statistics gathering. */
136static struct dentry_stat_t dentry_stat = {
137 .age_limit = 45,
138};
Dave Chinner62d36c72013-08-28 10:17:54 +1000139
140/*
141 * Here we resort to our own counters instead of using generic per-cpu counters
142 * for consistency with what the vfs inode code does. We are expected to harvest
143 * better code and performance by having our own specialized counters.
144 *
145 * Please note that the loop is done over all possible CPUs, not over all online
146 * CPUs. The reason for this is that we don't want to play games with CPUs going
147 * on and off. If one of them goes off, we will just keep their counters.
148 *
149 * glommer: See cffbc8a for details, and if you ever intend to change this,
150 * please update all vfs counters to match.
151 */
Glauber Costa3942c072013-08-28 10:17:53 +1000152static long get_nr_dentry(void)
Nick Piggin3e880fb2011-01-07 17:49:19 +1100153{
154 int i;
Glauber Costa3942c072013-08-28 10:17:53 +1000155 long sum = 0;
Nick Piggin3e880fb2011-01-07 17:49:19 +1100156 for_each_possible_cpu(i)
157 sum += per_cpu(nr_dentry, i);
158 return sum < 0 ? 0 : sum;
159}
160
Dave Chinner62d36c72013-08-28 10:17:54 +1000161static long get_nr_dentry_unused(void)
162{
163 int i;
164 long sum = 0;
165 for_each_possible_cpu(i)
166 sum += per_cpu(nr_dentry_unused, i);
167 return sum < 0 ? 0 : sum;
168}
169
Waiman Longaf0c9af2019-01-30 13:52:38 -0500170static long get_nr_dentry_negative(void)
171{
172 int i;
173 long sum = 0;
174
175 for_each_possible_cpu(i)
176 sum += per_cpu(nr_dentry_negative, i);
177 return sum < 0 ? 0 : sum;
178}
179
Joel Granados78eb4ea2024-07-24 20:59:29 +0200180static int proc_nr_dentry(const struct ctl_table *table, int write, void *buffer,
Luis Chamberlainc8c0c232022-01-21 22:12:59 -0800181 size_t *lenp, loff_t *ppos)
Christoph Hellwig312d3ca2010-10-10 05:36:23 -0400182{
Nick Piggin3e880fb2011-01-07 17:49:19 +1100183 dentry_stat.nr_dentry = get_nr_dentry();
Dave Chinner62d36c72013-08-28 10:17:54 +1000184 dentry_stat.nr_unused = get_nr_dentry_unused();
Waiman Longaf0c9af2019-01-30 13:52:38 -0500185 dentry_stat.nr_negative = get_nr_dentry_negative();
Glauber Costa3942c072013-08-28 10:17:53 +1000186 return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
Christoph Hellwig312d3ca2010-10-10 05:36:23 -0400187}
Luis Chamberlainc8c0c232022-01-21 22:12:59 -0800188
189static struct ctl_table fs_dcache_sysctls[] = {
190 {
191 .procname = "dentry-state",
192 .data = &dentry_stat,
193 .maxlen = 6*sizeof(long),
194 .mode = 0444,
195 .proc_handler = proc_nr_dentry,
196 },
Luis Chamberlainc8c0c232022-01-21 22:12:59 -0800197};
198
199static int __init init_fs_dcache_sysctls(void)
200{
201 register_sysctl_init("fs", fs_dcache_sysctls);
202 return 0;
203}
204fs_initcall(init_fs_dcache_sysctls);
Christoph Hellwig312d3ca2010-10-10 05:36:23 -0400205#endif
206
Linus Torvalds5483f182012-03-04 15:51:42 -0800207/*
208 * Compare 2 name strings, return 0 if they match, otherwise non-zero.
209 * The strings are both count bytes long, and count is non-zero.
210 */
Linus Torvaldse419b4c2012-05-03 10:16:43 -0700211#ifdef CONFIG_DCACHE_WORD_ACCESS
212
213#include <asm/word-at-a-time.h>
214/*
215 * NOTE! 'cs' and 'scount' come from a dentry, so it has a
216 * aligned allocation for this particular component. We don't
217 * strictly need the load_unaligned_zeropad() safety, but it
218 * doesn't hurt either.
219 *
220 * In contrast, 'ct' and 'tcount' can be from a pathname, and do
221 * need the careful unaligned handling.
222 */
Linus Torvalds94753db52012-05-10 12:19:19 -0700223static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
Linus Torvalds5483f182012-03-04 15:51:42 -0800224{
Linus Torvaldsbfcfaa72012-03-06 11:16:17 -0800225 unsigned long a,b,mask;
Linus Torvaldsbfcfaa72012-03-06 11:16:17 -0800226
227 for (;;) {
Andrey Ryabininbfe7aa62018-02-01 21:00:51 +0300228 a = read_word_at_a_time(cs);
Linus Torvaldse419b4c2012-05-03 10:16:43 -0700229 b = load_unaligned_zeropad(ct);
Linus Torvaldsbfcfaa72012-03-06 11:16:17 -0800230 if (tcount < sizeof(unsigned long))
231 break;
232 if (unlikely(a != b))
233 return 1;
234 cs += sizeof(unsigned long);
235 ct += sizeof(unsigned long);
236 tcount -= sizeof(unsigned long);
237 if (!tcount)
238 return 0;
239 }
Will Deacona5c21dc2013-12-12 17:40:21 +0000240 mask = bytemask_from_count(tcount);
Linus Torvaldsbfcfaa72012-03-06 11:16:17 -0800241 return unlikely(!!((a ^ b) & mask));
Linus Torvaldse419b4c2012-05-03 10:16:43 -0700242}
243
Linus Torvaldsbfcfaa72012-03-06 11:16:17 -0800244#else
Linus Torvaldse419b4c2012-05-03 10:16:43 -0700245
Linus Torvalds94753db52012-05-10 12:19:19 -0700246static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
Linus Torvaldse419b4c2012-05-03 10:16:43 -0700247{
Linus Torvalds5483f182012-03-04 15:51:42 -0800248 do {
249 if (*cs != *ct)
250 return 1;
251 cs++;
252 ct++;
253 tcount--;
254 } while (tcount);
255 return 0;
256}
257
Linus Torvaldse419b4c2012-05-03 10:16:43 -0700258#endif
259
Linus Torvalds94753db52012-05-10 12:19:19 -0700260static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)
261{
Linus Torvalds94753db52012-05-10 12:19:19 -0700262 /*
263 * Be careful about RCU walk racing with rename:
Will Deacon506458e2017-10-24 11:22:48 +0100264 * use 'READ_ONCE' to fetch the name pointer.
Linus Torvalds94753db52012-05-10 12:19:19 -0700265 *
266 * NOTE! Even if a rename will mean that the length
267 * was not loaded atomically, we don't care. The
268 * RCU walk will check the sequence count eventually,
269 * and catch it. And we won't overrun the buffer,
270 * because we're reading the name pointer atomically,
271 * and a dentry name is guaranteed to be properly
272 * terminated with a NUL byte.
273 *
274 * End result: even if 'len' is wrong, we'll exit
275 * early because the data cannot match (there can
276 * be no NUL in the ct/tcount data)
277 */
Will Deacon506458e2017-10-24 11:22:48 +0100278 const unsigned char *cs = READ_ONCE(dentry->d_name.name);
He Kuangae0a8432016-03-26 09:12:10 +0000279
Linus Torvalds6326c71f2012-05-21 16:14:04 -0700280 return dentry_string_cmp(cs, ct, tcount);
Linus Torvalds94753db52012-05-10 12:19:19 -0700281}
282
Al Viro8d85b482014-09-29 14:54:27 -0400283struct external_name {
284 union {
285 atomic_t count;
286 struct rcu_head head;
287 } u;
288 unsigned char name[];
289};
290
291static inline struct external_name *external_name(struct dentry *dentry)
292{
293 return container_of(dentry->d_name.name, struct external_name, name[0]);
294}
295
Christoph Hellwig9c82ab92010-10-10 05:36:22 -0400296static void __d_free(struct rcu_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297{
Christoph Hellwig9c82ab92010-10-10 05:36:22 -0400298 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
299
Al Viro8d85b482014-09-29 14:54:27 -0400300 kmem_cache_free(dentry_cache, dentry);
301}
302
303static void __d_free_external(struct rcu_head *head)
304{
305 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
Vlastimil Babka2e03b4b2018-10-26 15:05:41 -0700306 kfree(external_name(dentry));
Roman Gushchinf1782c92018-04-10 16:27:44 -0700307 kmem_cache_free(dentry_cache, dentry);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308}
309
Al Viro810bb172014-10-12 12:45:37 -0400310static inline int dname_external(const struct dentry *dentry)
311{
312 return dentry->d_name.name != dentry->d_iname;
313}
314
Al Viro49d31c22017-07-07 14:51:19 -0400315void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry)
316{
317 spin_lock(&dentry->d_lock);
Al Viro230c6402019-04-26 13:07:27 -0400318 name->name = dentry->d_name;
Al Viro49d31c22017-07-07 14:51:19 -0400319 if (unlikely(dname_external(dentry))) {
Al Viro230c6402019-04-26 13:07:27 -0400320 atomic_inc(&external_name(dentry)->u.count);
Al Viro49d31c22017-07-07 14:51:19 -0400321 } else {
Tetsuo Handa6cd00a02018-08-17 15:44:34 -0700322 memcpy(name->inline_name, dentry->d_iname,
323 dentry->d_name.len + 1);
Al Viro230c6402019-04-26 13:07:27 -0400324 name->name.name = name->inline_name;
Al Viro49d31c22017-07-07 14:51:19 -0400325 }
Al Viro230c6402019-04-26 13:07:27 -0400326 spin_unlock(&dentry->d_lock);
Al Viro49d31c22017-07-07 14:51:19 -0400327}
328EXPORT_SYMBOL(take_dentry_name_snapshot);
329
330void release_dentry_name_snapshot(struct name_snapshot *name)
331{
Al Viro230c6402019-04-26 13:07:27 -0400332 if (unlikely(name->name.name != name->inline_name)) {
Al Viro49d31c22017-07-07 14:51:19 -0400333 struct external_name *p;
Al Viro230c6402019-04-26 13:07:27 -0400334 p = container_of(name->name.name, struct external_name, name[0]);
Al Viro49d31c22017-07-07 14:51:19 -0400335 if (unlikely(atomic_dec_and_test(&p->u.count)))
Vlastimil Babka2e03b4b2018-10-26 15:05:41 -0700336 kfree_rcu(p, u.head);
Al Viro49d31c22017-07-07 14:51:19 -0400337 }
338}
339EXPORT_SYMBOL(release_dentry_name_snapshot);
340
David Howells4bf46a22015-03-05 14:09:22 +0000341static inline void __d_set_inode_and_type(struct dentry *dentry,
342 struct inode *inode,
343 unsigned type_flags)
344{
345 unsigned flags;
346
347 dentry->d_inode = inode;
David Howells4bf46a22015-03-05 14:09:22 +0000348 flags = READ_ONCE(dentry->d_flags);
Al Viro8219cb52023-11-10 14:32:05 -0500349 flags &= ~DCACHE_ENTRY_TYPE;
David Howells4bf46a22015-03-05 14:09:22 +0000350 flags |= type_flags;
Al Viro2fa6b1e2019-11-12 16:13:06 -0500351 smp_store_release(&dentry->d_flags, flags);
David Howells4bf46a22015-03-05 14:09:22 +0000352}
353
David Howells4bf46a22015-03-05 14:09:22 +0000354static inline void __d_clear_type_and_inode(struct dentry *dentry)
355{
356 unsigned flags = READ_ONCE(dentry->d_flags);
357
Al Viro8219cb52023-11-10 14:32:05 -0500358 flags &= ~DCACHE_ENTRY_TYPE;
David Howells4bf46a22015-03-05 14:09:22 +0000359 WRITE_ONCE(dentry->d_flags, flags);
David Howells4bf46a22015-03-05 14:09:22 +0000360 dentry->d_inode = NULL;
Brian Fosteraabfe572024-07-03 08:13:01 -0400361 /*
362 * The negative counter only tracks dentries on the LRU. Don't inc if
363 * d_lru is on another list.
364 */
365 if ((flags & (DCACHE_LRU_LIST|DCACHE_SHRINK_LIST)) == DCACHE_LRU_LIST)
Waiman Longaf0c9af2019-01-30 13:52:38 -0500366 this_cpu_inc(nr_dentry_negative);
David Howells4bf46a22015-03-05 14:09:22 +0000367}
368
Al Virob4f03542014-04-29 23:40:14 -0400369static void dentry_free(struct dentry *dentry)
370{
Al Viro946e51f2014-10-26 19:19:16 -0400371 WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias));
Al Viro8d85b482014-09-29 14:54:27 -0400372 if (unlikely(dname_external(dentry))) {
373 struct external_name *p = external_name(dentry);
374 if (likely(atomic_dec_and_test(&p->u.count))) {
375 call_rcu(&dentry->d_u.d_rcu, __d_free_external);
376 return;
377 }
378 }
Al Virob4f03542014-04-29 23:40:14 -0400379 /* if dentry was never visible to RCU, immediate free is OK */
Al Viro5467a682019-03-15 22:23:19 -0400380 if (dentry->d_flags & DCACHE_NORCU)
Al Virob4f03542014-04-29 23:40:14 -0400381 __d_free(&dentry->d_u.d_rcu);
382 else
383 call_rcu(&dentry->d_u.d_rcu, __d_free);
384}
385
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386/*
387 * Release the dentry's inode, using the filesystem
Al Viro550dce02016-05-29 20:13:30 -0400388 * d_iput() operation if defined.
Nick Piggin31e6b012011-01-07 17:49:52 +1100389 */
390static void dentry_unlink_inode(struct dentry * dentry)
391 __releases(dentry->d_lock)
Nick Piggin873feea2011-01-07 17:50:06 +1100392 __releases(dentry->d_inode->i_lock)
Nick Piggin31e6b012011-01-07 17:49:52 +1100393{
394 struct inode *inode = dentry->d_inode;
Al Viroa528aca2016-02-29 12:12:46 -0500395
Al Viro4c0d7cd2018-08-09 10:15:54 -0400396 raw_write_seqcount_begin(&dentry->d_seq);
David Howells4bf46a22015-03-05 14:09:22 +0000397 __d_clear_type_and_inode(dentry);
Al Viro946e51f2014-10-26 19:19:16 -0400398 hlist_del_init(&dentry->d_u.d_alias);
Al Viro4c0d7cd2018-08-09 10:15:54 -0400399 raw_write_seqcount_end(&dentry->d_seq);
Nick Piggin31e6b012011-01-07 17:49:52 +1100400 spin_unlock(&dentry->d_lock);
Nick Piggin873feea2011-01-07 17:50:06 +1100401 spin_unlock(&inode->i_lock);
Nick Piggin31e6b012011-01-07 17:49:52 +1100402 if (!inode->i_nlink)
403 fsnotify_inoderemove(inode);
404 if (dentry->d_op && dentry->d_op->d_iput)
405 dentry->d_op->d_iput(dentry, inode);
406 else
407 iput(inode);
408}
409
410/*
Linus Torvalds89dc77b2013-09-13 22:55:10 -0400411 * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry
412 * is in use - which includes both the "real" per-superblock
413 * LRU list _and_ the DCACHE_SHRINK_LIST use.
414 *
415 * The DCACHE_SHRINK_LIST bit is set whenever the dentry is
416 * on the shrink list (ie not on the superblock LRU list).
417 *
418 * The per-cpu "nr_dentry_unused" counters are updated with
419 * the DCACHE_LRU_LIST bit.
420 *
Waiman Longaf0c9af2019-01-30 13:52:38 -0500421 * The per-cpu "nr_dentry_negative" counters are only updated
422 * when deleted from or added to the per-superblock LRU list, not
423 * from/to the shrink list. That is to avoid an unneeded dec/inc
424 * pair when moving from LRU to shrink list in select_collect().
425 *
Linus Torvalds89dc77b2013-09-13 22:55:10 -0400426 * These helper functions make sure we always follow the
427 * rules. d_lock must be held by the caller.
428 */
429#define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
430static void d_lru_add(struct dentry *dentry)
431{
432 D_FLAG_VERIFY(dentry, 0);
433 dentry->d_flags |= DCACHE_LRU_LIST;
434 this_cpu_inc(nr_dentry_unused);
Waiman Longaf0c9af2019-01-30 13:52:38 -0500435 if (d_is_negative(dentry))
436 this_cpu_inc(nr_dentry_negative);
Nhat Pham0a97c012023-11-30 11:40:18 -0800437 WARN_ON_ONCE(!list_lru_add_obj(
438 &dentry->d_sb->s_dentry_lru, &dentry->d_lru));
Linus Torvalds89dc77b2013-09-13 22:55:10 -0400439}
440
441static void d_lru_del(struct dentry *dentry)
442{
443 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
444 dentry->d_flags &= ~DCACHE_LRU_LIST;
445 this_cpu_dec(nr_dentry_unused);
Waiman Longaf0c9af2019-01-30 13:52:38 -0500446 if (d_is_negative(dentry))
447 this_cpu_dec(nr_dentry_negative);
Nhat Pham0a97c012023-11-30 11:40:18 -0800448 WARN_ON_ONCE(!list_lru_del_obj(
449 &dentry->d_sb->s_dentry_lru, &dentry->d_lru));
Linus Torvalds89dc77b2013-09-13 22:55:10 -0400450}
451
452static void d_shrink_del(struct dentry *dentry)
453{
454 D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
455 list_del_init(&dentry->d_lru);
456 dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
457 this_cpu_dec(nr_dentry_unused);
458}
459
460static void d_shrink_add(struct dentry *dentry, struct list_head *list)
461{
462 D_FLAG_VERIFY(dentry, 0);
463 list_add(&dentry->d_lru, list);
464 dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;
465 this_cpu_inc(nr_dentry_unused);
466}
467
468/*
469 * These can only be called under the global LRU lock, ie during the
470 * callback for freeing the LRU list. "isolate" removes it from the
471 * LRU lists entirely, while shrink_move moves it to the indicated
472 * private list.
473 */
Vladimir Davydov3f97b162015-02-12 14:59:35 -0800474static void d_lru_isolate(struct list_lru_one *lru, struct dentry *dentry)
Linus Torvalds89dc77b2013-09-13 22:55:10 -0400475{
476 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
477 dentry->d_flags &= ~DCACHE_LRU_LIST;
478 this_cpu_dec(nr_dentry_unused);
Waiman Longaf0c9af2019-01-30 13:52:38 -0500479 if (d_is_negative(dentry))
480 this_cpu_dec(nr_dentry_negative);
Vladimir Davydov3f97b162015-02-12 14:59:35 -0800481 list_lru_isolate(lru, &dentry->d_lru);
Linus Torvalds89dc77b2013-09-13 22:55:10 -0400482}
483
Vladimir Davydov3f97b162015-02-12 14:59:35 -0800484static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry,
485 struct list_head *list)
Linus Torvalds89dc77b2013-09-13 22:55:10 -0400486{
487 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
488 dentry->d_flags |= DCACHE_SHRINK_LIST;
Waiman Longaf0c9af2019-01-30 13:52:38 -0500489 if (d_is_negative(dentry))
490 this_cpu_dec(nr_dentry_negative);
Vladimir Davydov3f97b162015-02-12 14:59:35 -0800491 list_lru_isolate_move(lru, &dentry->d_lru, list);
Linus Torvalds89dc77b2013-09-13 22:55:10 -0400492}
493
NeilBrown61647822017-11-10 15:45:41 +1100494static void ___d_drop(struct dentry *dentry)
Nick Piggin789680d2011-01-07 17:49:30 +1100495{
Al Viro0632a9a2018-03-07 00:49:10 -0500496 struct hlist_bl_head *b;
497 /*
498 * Hashed dentries are normally on the dentry hashtable,
499 * with the exception of those newly allocated by
500 * d_obtain_root, which are always IS_ROOT:
501 */
502 if (unlikely(IS_ROOT(dentry)))
503 b = &dentry->d_sb->s_roots;
504 else
505 b = d_hash(dentry->d_name.hash);
Al Virob61625d2013-10-04 11:09:01 -0400506
Al Viro0632a9a2018-03-07 00:49:10 -0500507 hlist_bl_lock(b);
508 __hlist_bl_del(&dentry->d_hash);
509 hlist_bl_unlock(b);
Nick Piggin789680d2011-01-07 17:49:30 +1100510}
NeilBrown61647822017-11-10 15:45:41 +1100511
512void __d_drop(struct dentry *dentry)
513{
Al Viro0632a9a2018-03-07 00:49:10 -0500514 if (!d_unhashed(dentry)) {
515 ___d_drop(dentry);
516 dentry->d_hash.pprev = NULL;
517 write_seqcount_invalidate(&dentry->d_seq);
518 }
NeilBrown61647822017-11-10 15:45:41 +1100519}
Nick Piggin789680d2011-01-07 17:49:30 +1100520EXPORT_SYMBOL(__d_drop);
521
Mauro Carvalho Chehab961f3c82021-01-14 09:04:39 +0100522/**
523 * d_drop - drop a dentry
524 * @dentry: dentry to drop
525 *
526 * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
527 * be found through a VFS lookup any more. Note that this is different from
528 * deleting the dentry - d_delete will try to mark the dentry negative if
529 * possible, giving a successful _negative_ lookup, while d_drop will
530 * just make the cache lookup fail.
531 *
532 * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
533 * reason (NFS timeouts or autofs deletes).
534 *
535 * __d_drop requires dentry->d_lock
536 *
537 * ___d_drop doesn't mark dentry as "unhashed"
538 * (dentry->d_hash.pprev will be LIST_POISON2, not NULL).
539 */
Nick Piggin789680d2011-01-07 17:49:30 +1100540void d_drop(struct dentry *dentry)
541{
Nick Piggin789680d2011-01-07 17:49:30 +1100542 spin_lock(&dentry->d_lock);
543 __d_drop(dentry);
544 spin_unlock(&dentry->d_lock);
Nick Piggin789680d2011-01-07 17:49:30 +1100545}
546EXPORT_SYMBOL(d_drop);
547
Al Viroda549bd2023-11-07 02:00:39 -0500548static inline void dentry_unlist(struct dentry *dentry)
Al Viroba65dc52016-06-10 11:32:47 -0400549{
550 struct dentry *next;
551 /*
552 * Inform d_walk() and shrink_dentry_list() that we are no longer
553 * attached to the dentry tree
554 */
555 dentry->d_flags |= DCACHE_DENTRY_KILLED;
Al Viroda549bd2023-11-07 02:00:39 -0500556 if (unlikely(hlist_unhashed(&dentry->d_sib)))
Al Viroba65dc52016-06-10 11:32:47 -0400557 return;
Al Viroda549bd2023-11-07 02:00:39 -0500558 __hlist_del(&dentry->d_sib);
Al Viroba65dc52016-06-10 11:32:47 -0400559 /*
560 * Cursors can move around the list of children. While we'd been
Al Viroda549bd2023-11-07 02:00:39 -0500561 * a normal list member, it didn't matter - ->d_sib.next would've
Al Viroba65dc52016-06-10 11:32:47 -0400562 * been updated. However, from now on it won't be and for the
563 * things like d_walk() it might end up with a nasty surprise.
564 * Normally d_walk() doesn't care about cursors moving around -
565 * ->d_lock on parent prevents that and since a cursor has no children
566 * of its own, we get through it without ever unlocking the parent.
567 * There is one exception, though - if we ascend from a child that
568 * gets killed as soon as we unlock it, the next sibling is found
Al Viroda549bd2023-11-07 02:00:39 -0500569 * using the value left in its ->d_sib.next. And if _that_
Al Viroba65dc52016-06-10 11:32:47 -0400570 * pointed to a cursor, and cursor got moved (e.g. by lseek())
571 * before d_walk() regains parent->d_lock, we'll end up skipping
572 * everything the cursor had been moved past.
573 *
Al Viroda549bd2023-11-07 02:00:39 -0500574 * Solution: make sure that the pointer left behind in ->d_sib.next
Al Viroba65dc52016-06-10 11:32:47 -0400575 * points to something that won't be moving around. I.e. skip the
576 * cursors.
577 */
Al Viroda549bd2023-11-07 02:00:39 -0500578 while (dentry->d_sib.next) {
579 next = hlist_entry(dentry->d_sib.next, struct dentry, d_sib);
Al Viroba65dc52016-06-10 11:32:47 -0400580 if (likely(!(next->d_flags & DCACHE_DENTRY_CURSOR)))
581 break;
Al Viroda549bd2023-11-07 02:00:39 -0500582 dentry->d_sib.next = next->d_sib.next;
Al Viroba65dc52016-06-10 11:32:47 -0400583 }
584}
585
Al Viro1c18edd2023-11-07 16:14:08 -0500586static struct dentry *__dentry_kill(struct dentry *dentry)
Nick Piggin77812a12011-01-07 17:49:48 +1100587{
Al Viro41edf2782014-05-01 10:30:00 -0400588 struct dentry *parent = NULL;
589 bool can_free = true;
Nick Piggin31e6b012011-01-07 17:49:52 +1100590
Linus Torvalds0d984392013-09-08 13:46:52 -0700591 /*
592 * The dentry is now unrecoverably dead to the world.
593 */
594 lockref_mark_dead(&dentry->d_lockref);
595
Sage Weilf0023bc2011-10-28 10:02:42 -0700596 /*
Sage Weilf0023bc2011-10-28 10:02:42 -0700597 * inform the fs via d_prune that this dentry is about to be
598 * unhashed and destroyed.
599 */
Al Viro29266202014-05-30 11:39:02 -0400600 if (dentry->d_flags & DCACHE_OP_PRUNE)
Yan, Zheng61572bb2013-04-15 14:13:21 +0800601 dentry->d_op->d_prune(dentry);
602
Al Viro01b60352014-04-29 23:42:52 -0400603 if (dentry->d_flags & DCACHE_LRU_LIST) {
604 if (!(dentry->d_flags & DCACHE_SHRINK_LIST))
605 d_lru_del(dentry);
Al Viro01b60352014-04-29 23:42:52 -0400606 }
Nick Piggin77812a12011-01-07 17:49:48 +1100607 /* if it was on the hash then remove it */
608 __d_drop(dentry);
Al Viro550dce02016-05-29 20:13:30 -0400609 if (dentry->d_inode)
610 dentry_unlink_inode(dentry);
611 else
612 spin_unlock(&dentry->d_lock);
Al Viro03b3b882014-04-29 15:45:28 -0400613 this_cpu_dec(nr_dentry);
614 if (dentry->d_op && dentry->d_op->d_release)
615 dentry->d_op->d_release(dentry);
616
Al Viro1c18edd2023-11-07 16:14:08 -0500617 cond_resched();
618 /* now that it's negative, ->d_parent is stable */
619 if (!IS_ROOT(dentry)) {
620 parent = dentry->d_parent;
621 spin_lock(&parent->d_lock);
Al Viro41edf2782014-05-01 10:30:00 -0400622 }
Al Viro1c18edd2023-11-07 16:14:08 -0500623 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
624 dentry_unlist(dentry);
Al Viro1b327b52023-11-10 14:07:43 -0500625 if (dentry->d_flags & DCACHE_SHRINK_LIST)
Al Viro41edf2782014-05-01 10:30:00 -0400626 can_free = false;
Al Viro41edf2782014-05-01 10:30:00 -0400627 spin_unlock(&dentry->d_lock);
Al Viro41edf2782014-05-01 10:30:00 -0400628 if (likely(can_free))
629 dentry_free(dentry);
Al Viro1c18edd2023-11-07 16:14:08 -0500630 if (parent && --parent->d_lockref.count) {
Al Viro046b9612014-05-29 08:54:52 -0400631 spin_unlock(&parent->d_lock);
Al Viro1c18edd2023-11-07 16:14:08 -0500632 return NULL;
Al Viro046b9612014-05-29 08:54:52 -0400633 }
Al Viro046b9612014-05-29 08:54:52 -0400634 return parent;
635}
636
Al Viro339e9e12023-10-31 01:23:47 -0400637/*
638 * Lock a dentry for feeding it to __dentry_kill().
639 * Called under rcu_read_lock() and dentry->d_lock; the former
640 * guarantees that nothing we access will be freed under us.
641 * Note that dentry is *not* protected from concurrent dentry_kill(),
642 * d_delete(), etc.
643 *
644 * Return false if dentry is busy. Otherwise, return true and have
Al Viro1c18edd2023-11-07 16:14:08 -0500645 * that dentry's inode locked.
Al Viro339e9e12023-10-31 01:23:47 -0400646 */
647
648static bool lock_for_kill(struct dentry *dentry)
Al Viro8b987a42018-02-23 22:11:34 -0500649{
Al Viro339e9e12023-10-31 01:23:47 -0400650 struct inode *inode = dentry->d_inode;
Al Viro339e9e12023-10-31 01:23:47 -0400651
652 if (unlikely(dentry->d_lockref.count))
653 return false;
654
Al Viro1c18edd2023-11-07 16:14:08 -0500655 if (!inode || likely(spin_trylock(&inode->i_lock)))
Al Viro339e9e12023-10-31 01:23:47 -0400656 return true;
657
Al Viro1c18edd2023-11-07 16:14:08 -0500658 do {
659 spin_unlock(&dentry->d_lock);
660 spin_lock(&inode->i_lock);
661 spin_lock(&dentry->d_lock);
Al Viro339e9e12023-10-31 01:23:47 -0400662 if (likely(inode == dentry->d_inode))
663 break;
Al Viro1c18edd2023-11-07 16:14:08 -0500664 spin_unlock(&inode->i_lock);
Al Viro339e9e12023-10-31 01:23:47 -0400665 inode = dentry->d_inode;
Al Viro1c18edd2023-11-07 16:14:08 -0500666 } while (inode);
Al Viro339e9e12023-10-31 01:23:47 -0400667 if (likely(!dentry->d_lockref.count))
668 return true;
669 if (inode)
670 spin_unlock(&inode->i_lock);
Al Viro339e9e12023-10-31 01:23:47 -0400671 return false;
Al Viro8b987a42018-02-23 22:11:34 -0500672}
673
Al Viro6367b492023-11-23 17:33:21 -0500674/*
675 * Decide if dentry is worth retaining. Usually this is called with dentry
676 * locked; if not locked, we are more limited and might not be able to tell
677 * without a lock. False in this case means "punt to locked path and recheck".
678 *
679 * In case we aren't locked, these predicates are not "stable". However, it is
680 * sufficient that at some point after we dropped the reference the dentry was
681 * hashed and the flags had the proper value. Other dentry users may have
682 * re-gotten a reference to the dentry and change that, but our work is done -
683 * we can leave the dentry around with a zero refcount.
684 */
685static inline bool retain_dentry(struct dentry *dentry, bool locked)
Al Viroa3385792018-02-23 21:07:24 -0500686{
Al Viro6367b492023-11-23 17:33:21 -0500687 unsigned int d_flags;
Al Viroa3385792018-02-23 21:07:24 -0500688
Al Viro6367b492023-11-23 17:33:21 -0500689 smp_rmb();
690 d_flags = READ_ONCE(dentry->d_flags);
691
692 // Unreachable? Nobody would be able to look it up, no point retaining
Al Viroa3385792018-02-23 21:07:24 -0500693 if (unlikely(d_unhashed(dentry)))
694 return false;
695
Al Viro6367b492023-11-23 17:33:21 -0500696 // Same if it's disconnected
697 if (unlikely(d_flags & DCACHE_DISCONNECTED))
Al Viroa3385792018-02-23 21:07:24 -0500698 return false;
699
Al Viro6367b492023-11-23 17:33:21 -0500700 // ->d_delete() might tell us not to bother, but that requires
701 // ->d_lock; can't decide without it
702 if (unlikely(d_flags & DCACHE_OP_DELETE)) {
703 if (!locked || dentry->d_op->d_delete(dentry))
Al Viroa3385792018-02-23 21:07:24 -0500704 return false;
705 }
Ira Weiny2c567af2020-04-30 07:41:37 -0700706
Al Viro6367b492023-11-23 17:33:21 -0500707 // Explicitly told not to bother
708 if (unlikely(d_flags & DCACHE_DONTCACHE))
Ira Weiny2c567af2020-04-30 07:41:37 -0700709 return false;
710
Al Viro6367b492023-11-23 17:33:21 -0500711 // At this point it looks like we ought to keep it. We also might
712 // need to do something - put it on LRU if it wasn't there already
713 // and mark it referenced if it was on LRU, but not marked yet.
714 // Unfortunately, both actions require ->d_lock, so in lockless
715 // case we'd have to punt rather than doing those.
716 if (unlikely(!(d_flags & DCACHE_LRU_LIST))) {
717 if (!locked)
718 return false;
Al Viro62d99562018-03-06 21:37:31 -0500719 d_lru_add(dentry);
Al Viro6367b492023-11-23 17:33:21 -0500720 } else if (unlikely(!(d_flags & DCACHE_REFERENCED))) {
721 if (!locked)
722 return false;
Al Viro62d99562018-03-06 21:37:31 -0500723 dentry->d_flags |= DCACHE_REFERENCED;
Al Viro6367b492023-11-23 17:33:21 -0500724 }
Al Viroa3385792018-02-23 21:07:24 -0500725 return true;
726}
727
Ira Weiny2c567af2020-04-30 07:41:37 -0700728void d_mark_dontcache(struct inode *inode)
729{
730 struct dentry *de;
731
732 spin_lock(&inode->i_lock);
733 hlist_for_each_entry(de, &inode->i_dentry, d_u.d_alias) {
734 spin_lock(&de->d_lock);
735 de->d_flags |= DCACHE_DONTCACHE;
736 spin_unlock(&de->d_lock);
737 }
738 inode->i_state |= I_DONTCACHE;
739 spin_unlock(&inode->i_lock);
740}
741EXPORT_SYMBOL(d_mark_dontcache);
742
Linus Torvalds360f5472015-01-09 15:19:03 -0800743/*
744 * Try to do a lockless dput(), and return whether that was successful.
745 *
746 * If unsuccessful, we return false, having already taken the dentry lock.
Al Virof05441c2023-10-31 00:45:40 -0400747 * In that case refcount is guaranteed to be zero and we have already
748 * decided that it's not worth keeping around.
Linus Torvalds360f5472015-01-09 15:19:03 -0800749 *
750 * The caller needs to hold the RCU read lock, so that the dentry is
751 * guaranteed to stay around even if the refcount goes down to zero!
752 */
753static inline bool fast_dput(struct dentry *dentry)
754{
755 int ret;
Linus Torvalds360f5472015-01-09 15:19:03 -0800756
757 /*
Al Viro15220fb2023-10-30 00:02:14 -0400758 * try to decrement the lockref optimistically.
Linus Torvalds360f5472015-01-09 15:19:03 -0800759 */
760 ret = lockref_put_return(&dentry->d_lockref);
761
762 /*
763 * If the lockref_put_return() failed due to the lock being held
764 * by somebody else, the fast path has failed. We will need to
765 * get the lock, and then check the count again.
766 */
767 if (unlikely(ret < 0)) {
768 spin_lock(&dentry->d_lock);
Al Viro504e08c2023-11-01 01:08:54 -0400769 if (WARN_ON_ONCE(dentry->d_lockref.count <= 0)) {
Linus Torvalds360f5472015-01-09 15:19:03 -0800770 spin_unlock(&dentry->d_lock);
Gustavo A. R. Silva79644102018-08-01 19:39:05 -0500771 return true;
Linus Torvalds360f5472015-01-09 15:19:03 -0800772 }
Al Viro504e08c2023-11-01 01:08:54 -0400773 dentry->d_lockref.count--;
774 goto locked;
Linus Torvalds360f5472015-01-09 15:19:03 -0800775 }
776
777 /*
778 * If we weren't the last ref, we're done.
779 */
780 if (ret)
Gustavo A. R. Silva79644102018-08-01 19:39:05 -0500781 return true;
Linus Torvalds360f5472015-01-09 15:19:03 -0800782
783 /*
Al Viro6367b492023-11-23 17:33:21 -0500784 * Can we decide that decrement of refcount is all we needed without
785 * taking the lock? There's a very common case when it's all we need -
786 * dentry looks like it ought to be retained and there's nothing else
787 * to do.
Linus Torvalds360f5472015-01-09 15:19:03 -0800788 */
Al Viro6367b492023-11-23 17:33:21 -0500789 if (retain_dentry(dentry, false))
Gustavo A. R. Silva79644102018-08-01 19:39:05 -0500790 return true;
Linus Torvalds360f5472015-01-09 15:19:03 -0800791
792 /*
Al Viro6367b492023-11-23 17:33:21 -0500793 * Either not worth retaining or we can't tell without the lock.
794 * Get the lock, then. We've already decremented the refcount to 0,
795 * but we'll need to re-check the situation after getting the lock.
Linus Torvalds360f5472015-01-09 15:19:03 -0800796 */
797 spin_lock(&dentry->d_lock);
798
799 /*
800 * Did somebody else grab a reference to it in the meantime, and
801 * we're no longer the last user after all? Alternatively, somebody
802 * else could have killed it and marked it dead. Either way, we
803 * don't need to do anything else.
804 */
Al Viro504e08c2023-11-01 01:08:54 -0400805locked:
Al Viro6367b492023-11-23 17:33:21 -0500806 if (dentry->d_lockref.count || retain_dentry(dentry, true)) {
Linus Torvalds360f5472015-01-09 15:19:03 -0800807 spin_unlock(&dentry->d_lock);
Gustavo A. R. Silva79644102018-08-01 19:39:05 -0500808 return true;
Linus Torvalds360f5472015-01-09 15:19:03 -0800809 }
Gustavo A. R. Silva79644102018-08-01 19:39:05 -0500810 return false;
Linus Torvalds360f5472015-01-09 15:19:03 -0800811}
812
813
Linus Torvalds1da177e2005-04-16 15:20:36 -0700814/*
815 * This is dput
816 *
817 * This is complicated by the fact that we do not want to put
818 * dentries that are no longer on any hash chain on the unused
819 * list: we'd much rather just get rid of them immediately.
820 *
821 * However, that implies that we have to traverse the dentry
822 * tree upwards to the parents which might _also_ now be
823 * scheduled for deletion (it may have been only waiting for
824 * its last child to go away).
825 *
826 * This tail recursion is done by hand as we don't want to depend
827 * on the compiler to always get this right (gcc generally doesn't).
828 * Real recursion would eat up our stack space.
829 */
830
831/*
832 * dput - release a dentry
833 * @dentry: dentry to release
834 *
835 * Release a dentry. This will drop the usage count and if appropriate
836 * call the dentry unlink method as well as removing it from the queues and
837 * releasing its resources. If the parent dentries were scheduled for release
838 * they too may now get deleted.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840void dput(struct dentry *dentry)
841{
Al Viro1c18edd2023-11-07 16:14:08 -0500842 if (!dentry)
843 return;
844 might_sleep();
845 rcu_read_lock();
846 if (likely(fast_dput(dentry))) {
Linus Torvalds360f5472015-01-09 15:19:03 -0800847 rcu_read_unlock();
Al Viro1c18edd2023-11-07 16:14:08 -0500848 return;
849 }
850 while (lock_for_kill(dentry)) {
851 rcu_read_unlock();
852 dentry = __dentry_kill(dentry);
853 if (!dentry)
Linus Torvalds8aab6a22013-09-08 13:26:18 -0700854 return;
Al Viro6367b492023-11-23 17:33:21 -0500855 if (retain_dentry(dentry, true)) {
Al Viro1088a642018-04-15 18:31:03 -0400856 spin_unlock(&dentry->d_lock);
857 return;
858 }
Al Viro1c18edd2023-11-07 16:14:08 -0500859 rcu_read_lock();
Wei Fang47be6182016-07-06 11:32:20 +0800860 }
Al Viro1c18edd2023-11-07 16:14:08 -0500861 rcu_read_unlock();
862 spin_unlock(&dentry->d_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863}
H Hartley Sweetenec4f8602010-01-05 13:45:18 -0700864EXPORT_SYMBOL(dput);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865
Al Viro6511f6b2023-10-30 00:43:48 -0400866static void to_shrink_list(struct dentry *dentry, struct list_head *list)
Al Viro9bdebc22019-06-29 18:31:24 -0400867__must_hold(&dentry->d_lock)
868{
Al Viro6511f6b2023-10-30 00:43:48 -0400869 if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
Al Viro9bdebc22019-06-29 18:31:24 -0400870 if (dentry->d_flags & DCACHE_LRU_LIST)
871 d_lru_del(dentry);
Al Viroc2e5e29f2023-10-30 01:06:06 -0400872 d_shrink_add(dentry, list);
Al Viro9bdebc22019-06-29 18:31:24 -0400873 }
874}
875
876void dput_to_list(struct dentry *dentry, struct list_head *list)
877{
878 rcu_read_lock();
879 if (likely(fast_dput(dentry))) {
880 rcu_read_unlock();
881 return;
882 }
883 rcu_read_unlock();
Al Virof05441c2023-10-31 00:45:40 -0400884 to_shrink_list(dentry, list);
Al Viro9bdebc22019-06-29 18:31:24 -0400885 spin_unlock(&dentry->d_lock);
886}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887
Nick Pigginb7ab39f2011-01-07 17:49:32 +1100888struct dentry *dget_parent(struct dentry *dentry)
889{
Waiman Longdf3d0bb2013-09-02 11:29:22 -0700890 int gotref;
Nick Pigginb7ab39f2011-01-07 17:49:32 +1100891 struct dentry *ret;
Al Viroe8400932019-10-31 01:43:31 -0400892 unsigned seq;
Nick Pigginb7ab39f2011-01-07 17:49:32 +1100893
Waiman Longdf3d0bb2013-09-02 11:29:22 -0700894 /*
895 * Do optimistic parent lookup without any
896 * locking.
897 */
898 rcu_read_lock();
Al Viroe8400932019-10-31 01:43:31 -0400899 seq = raw_seqcount_begin(&dentry->d_seq);
Mark Rutland66702eb2017-10-23 14:07:14 -0700900 ret = READ_ONCE(dentry->d_parent);
Waiman Longdf3d0bb2013-09-02 11:29:22 -0700901 gotref = lockref_get_not_zero(&ret->d_lockref);
902 rcu_read_unlock();
903 if (likely(gotref)) {
Al Viroe8400932019-10-31 01:43:31 -0400904 if (!read_seqcount_retry(&dentry->d_seq, seq))
Waiman Longdf3d0bb2013-09-02 11:29:22 -0700905 return ret;
906 dput(ret);
907 }
908
Nick Pigginb7ab39f2011-01-07 17:49:32 +1100909repeat:
Nick Piggina734eb42011-01-07 17:49:44 +1100910 /*
911 * Don't need rcu_dereference because we re-check it was correct under
912 * the lock.
913 */
914 rcu_read_lock();
Nick Pigginb7ab39f2011-01-07 17:49:32 +1100915 ret = dentry->d_parent;
Nick Piggina734eb42011-01-07 17:49:44 +1100916 spin_lock(&ret->d_lock);
917 if (unlikely(ret != dentry->d_parent)) {
918 spin_unlock(&ret->d_lock);
919 rcu_read_unlock();
Nick Pigginb7ab39f2011-01-07 17:49:32 +1100920 goto repeat;
921 }
Nick Piggina734eb42011-01-07 17:49:44 +1100922 rcu_read_unlock();
Waiman Long98474232013-08-28 18:24:59 -0700923 BUG_ON(!ret->d_lockref.count);
924 ret->d_lockref.count++;
Nick Pigginb7ab39f2011-01-07 17:49:32 +1100925 spin_unlock(&ret->d_lock);
Nick Pigginb7ab39f2011-01-07 17:49:32 +1100926 return ret;
927}
928EXPORT_SYMBOL(dget_parent);
929
Al Viro61fec492018-04-25 10:52:25 -0400930static struct dentry * __d_find_any_alias(struct inode *inode)
931{
932 struct dentry *alias;
933
934 if (hlist_empty(&inode->i_dentry))
935 return NULL;
936 alias = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
Al Viro6d73c9c2023-10-29 13:41:06 -0400937 lockref_get(&alias->d_lockref);
Al Viro61fec492018-04-25 10:52:25 -0400938 return alias;
939}
940
941/**
942 * d_find_any_alias - find any alias for a given inode
943 * @inode: inode to find an alias for
944 *
945 * If any aliases exist for the given inode, take and return a
946 * reference for one of them. If no aliases exist, return %NULL.
947 */
948struct dentry *d_find_any_alias(struct inode *inode)
949{
950 struct dentry *de;
951
952 spin_lock(&inode->i_lock);
953 de = __d_find_any_alias(inode);
954 spin_unlock(&inode->i_lock);
955 return de;
956}
957EXPORT_SYMBOL(d_find_any_alias);
958
J. Bruce Fields52ed46f2014-01-16 11:15:51 -0500959static struct dentry *__d_find_alias(struct inode *inode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960{
Al Viro61fec492018-04-25 10:52:25 -0400961 struct dentry *alias;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962
Al Viro61fec492018-04-25 10:52:25 -0400963 if (S_ISDIR(inode->i_mode))
964 return __d_find_any_alias(inode);
965
Al Viro946e51f2014-10-26 19:19:16 -0400966 hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
Nick Pigginda502952011-01-07 17:49:33 +1100967 spin_lock(&alias->d_lock);
Al Viro61fec492018-04-25 10:52:25 -0400968 if (!d_unhashed(alias)) {
Vegard Nossum1b6ae9f2023-11-06 14:44:17 +0100969 dget_dlock(alias);
J. Bruce Fields8d80d7d2014-01-16 17:17:31 -0500970 spin_unlock(&alias->d_lock);
971 return alias;
Nick Pigginda502952011-01-07 17:49:33 +1100972 }
973 spin_unlock(&alias->d_lock);
Nick Pigginda502952011-01-07 17:49:33 +1100974 }
975 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976}
977
Mauro Carvalho Chehab961f3c82021-01-14 09:04:39 +0100978/**
979 * d_find_alias - grab a hashed alias of inode
980 * @inode: inode in question
981 *
982 * If inode has a hashed alias, or is a directory and has any alias,
983 * acquire the reference to alias and return it. Otherwise return NULL.
984 * Notice that if inode is a directory there can be only one alias and
985 * it can be unhashed only if it has no children, or if it is the root
986 * of a filesystem, or if the directory was renamed and d_revalidate
987 * was the first vfs operation to notice.
988 *
989 * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer
990 * any other hashed alias over that one.
991 */
Nick Pigginda502952011-01-07 17:49:33 +1100992struct dentry *d_find_alias(struct inode *inode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993{
David Howells214fda12006-03-25 03:06:36 -0800994 struct dentry *de = NULL;
995
Al Virob3d9b7a2012-06-09 13:51:19 -0400996 if (!hlist_empty(&inode->i_dentry)) {
Nick Piggin873feea2011-01-07 17:50:06 +1100997 spin_lock(&inode->i_lock);
J. Bruce Fields52ed46f2014-01-16 11:15:51 -0500998 de = __d_find_alias(inode);
Nick Piggin873feea2011-01-07 17:50:06 +1100999 spin_unlock(&inode->i_lock);
David Howells214fda12006-03-25 03:06:36 -08001000 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001 return de;
1002}
H Hartley Sweetenec4f8602010-01-05 13:45:18 -07001003EXPORT_SYMBOL(d_find_alias);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004
1005/*
Al Virobca585d2021-01-05 14:13:52 -05001006 * Caller MUST be holding rcu_read_lock() and be guaranteed
1007 * that inode won't get freed until rcu_read_unlock().
1008 */
1009struct dentry *d_find_alias_rcu(struct inode *inode)
1010{
1011 struct hlist_head *l = &inode->i_dentry;
1012 struct dentry *de = NULL;
1013
1014 spin_lock(&inode->i_lock);
1015 // ->i_dentry and ->i_rcu are colocated, but the latter won't be
1016 // used without having I_FREEING set, which means no aliases left
1017 if (likely(!(inode->i_state & I_FREEING) && !hlist_empty(l))) {
1018 if (S_ISDIR(inode->i_mode)) {
1019 de = hlist_entry(l->first, struct dentry, d_u.d_alias);
1020 } else {
1021 hlist_for_each_entry(de, l, d_u.d_alias)
1022 if (!d_unhashed(de))
1023 break;
1024 }
1025 }
1026 spin_unlock(&inode->i_lock);
1027 return de;
1028}
1029
1030/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031 * Try to kill dentries associated with this inode.
1032 * WARNING: you must own a reference to inode.
1033 */
1034void d_prune_aliases(struct inode *inode)
1035{
Al Virob4cc0732023-11-03 14:46:14 -04001036 LIST_HEAD(dispose);
Domen Puncer0cdca3f2005-09-10 00:27:07 -07001037 struct dentry *dentry;
Al Virob4cc0732023-11-03 14:46:14 -04001038
Nick Piggin873feea2011-01-07 17:50:06 +11001039 spin_lock(&inode->i_lock);
Al Viro946e51f2014-10-26 19:19:16 -04001040 hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041 spin_lock(&dentry->d_lock);
Al Virob4cc0732023-11-03 14:46:14 -04001042 if (!dentry->d_lockref.count)
1043 to_shrink_list(dentry, &dispose);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001044 spin_unlock(&dentry->d_lock);
1045 }
Nick Piggin873feea2011-01-07 17:50:06 +11001046 spin_unlock(&inode->i_lock);
Al Virob4cc0732023-11-03 14:46:14 -04001047 shrink_dentry_list(&dispose);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048}
H Hartley Sweetenec4f8602010-01-05 13:45:18 -07001049EXPORT_SYMBOL(d_prune_aliases);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001050
Al Viro1c18edd2023-11-07 16:14:08 -05001051static inline void shrink_kill(struct dentry *victim)
Al Viro3b3f09f2018-02-23 21:54:18 -05001052{
Al Viro1c18edd2023-11-07 16:14:08 -05001053 do {
1054 rcu_read_unlock();
1055 victim = __dentry_kill(victim);
1056 rcu_read_lock();
1057 } while (victim && lock_for_kill(victim));
1058 rcu_read_unlock();
1059 if (victim)
1060 spin_unlock(&victim->d_lock);
Al Viro3b3f09f2018-02-23 21:54:18 -05001061}
1062
Al Viro9bdebc22019-06-29 18:31:24 -04001063void shrink_dentry_list(struct list_head *list)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001064{
Miklos Szeredi60942f2f2014-05-02 15:38:39 -04001065 while (!list_empty(list)) {
Al Viro3fcf5352023-11-07 15:21:33 -05001066 struct dentry *dentry;
Al Viro3b3f09f2018-02-23 21:54:18 -05001067
Miklos Szeredi60942f2f2014-05-02 15:38:39 -04001068 dentry = list_entry(list->prev, struct dentry, d_lru);
Nick Pigginec336792011-01-07 17:49:47 +11001069 spin_lock(&dentry->d_lock);
John Ogness8f04da22018-02-23 00:50:24 +01001070 rcu_read_lock();
Al Viro339e9e12023-10-31 01:23:47 -04001071 if (!lock_for_kill(dentry)) {
Al Virocd9f84f2023-10-30 13:57:12 -04001072 bool can_free;
John Ogness8f04da22018-02-23 00:50:24 +01001073 rcu_read_unlock();
Al Viro3b3f09f2018-02-23 21:54:18 -05001074 d_shrink_del(dentry);
Al Viro1b327b52023-11-10 14:07:43 -05001075 can_free = dentry->d_flags & DCACHE_DENTRY_KILLED;
Kentaro Makitada3bbdd2008-07-23 21:27:13 -07001076 spin_unlock(&dentry->d_lock);
Al Viro64fd72e2014-05-28 09:48:44 -04001077 if (can_free)
1078 dentry_free(dentry);
1079 continue;
1080 }
Al Viro3b3f09f2018-02-23 21:54:18 -05001081 d_shrink_del(dentry);
Al Viro1c18edd2023-11-07 16:14:08 -05001082 shrink_kill(dentry);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083 }
Christoph Hellwig3049cfe2010-10-10 05:36:25 -04001084}
1085
Vladimir Davydov3f97b162015-02-12 14:59:35 -08001086static enum lru_status dentry_lru_isolate(struct list_head *item,
1087 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
Dave Chinnerf6041562013-08-28 10:18:00 +10001088{
1089 struct list_head *freeable = arg;
1090 struct dentry *dentry = container_of(item, struct dentry, d_lru);
1091
1092
1093 /*
1094 * we are inverting the lru lock/dentry->d_lock here,
1095 * so use a trylock. If we fail to get the lock, just skip
1096 * it
1097 */
1098 if (!spin_trylock(&dentry->d_lock))
1099 return LRU_SKIP;
1100
1101 /*
1102 * Referenced dentries are still in use. If they have active
1103 * counts, just remove them from the LRU. Otherwise give them
1104 * another pass through the LRU.
1105 */
1106 if (dentry->d_lockref.count) {
Vladimir Davydov3f97b162015-02-12 14:59:35 -08001107 d_lru_isolate(lru, dentry);
Dave Chinnerf6041562013-08-28 10:18:00 +10001108 spin_unlock(&dentry->d_lock);
1109 return LRU_REMOVED;
1110 }
1111
1112 if (dentry->d_flags & DCACHE_REFERENCED) {
1113 dentry->d_flags &= ~DCACHE_REFERENCED;
1114 spin_unlock(&dentry->d_lock);
1115
1116 /*
1117 * The list move itself will be made by the common LRU code. At
1118 * this point, we've dropped the dentry->d_lock but keep the
1119 * lru lock. This is safe to do, since every list movement is
1120 * protected by the lru lock even if both locks are held.
1121 *
1122 * This is guaranteed by the fact that all LRU management
1123 * functions are intermediated by the LRU API calls like
Nhat Pham0a97c012023-11-30 11:40:18 -08001124 * list_lru_add_obj and list_lru_del_obj. List movement in this file
Dave Chinnerf6041562013-08-28 10:18:00 +10001125 * only ever occur through this functions or through callbacks
1126 * like this one, that are called from the LRU API.
1127 *
1128 * The only exceptions to this are functions like
1129 * shrink_dentry_list, and code that first checks for the
1130 * DCACHE_SHRINK_LIST flag. Those are guaranteed to be
1131 * operating only with stack provided lists after they are
1132 * properly isolated from the main list. It is thus, always a
1133 * local access.
1134 */
1135 return LRU_ROTATE;
1136 }
1137
Vladimir Davydov3f97b162015-02-12 14:59:35 -08001138 d_lru_shrink_move(lru, dentry, freeable);
Dave Chinnerf6041562013-08-28 10:18:00 +10001139 spin_unlock(&dentry->d_lock);
1140
1141 return LRU_REMOVED;
1142}
1143
Christoph Hellwig3049cfe2010-10-10 05:36:25 -04001144/**
Dave Chinnerb48f03b32011-08-23 18:56:24 +10001145 * prune_dcache_sb - shrink the dcache
1146 * @sb: superblock
Vladimir Davydov503c3582015-02-12 14:58:47 -08001147 * @sc: shrink control, passed to list_lru_shrink_walk()
Christoph Hellwig3049cfe2010-10-10 05:36:25 -04001148 *
Vladimir Davydov503c3582015-02-12 14:58:47 -08001149 * Attempt to shrink the superblock dcache LRU by @sc->nr_to_scan entries. This
1150 * is done when we need more memory and called from the superblock shrinker
Dave Chinnerb48f03b32011-08-23 18:56:24 +10001151 * function.
1152 *
1153 * This function may fail to free any resources if all the dentries are in
1154 * use.
Christoph Hellwig3049cfe2010-10-10 05:36:25 -04001155 */
Vladimir Davydov503c3582015-02-12 14:58:47 -08001156long prune_dcache_sb(struct super_block *sb, struct shrink_control *sc)
Christoph Hellwig3049cfe2010-10-10 05:36:25 -04001157{
Dave Chinnerf6041562013-08-28 10:18:00 +10001158 LIST_HEAD(dispose);
1159 long freed;
Christoph Hellwig3049cfe2010-10-10 05:36:25 -04001160
Vladimir Davydov503c3582015-02-12 14:58:47 -08001161 freed = list_lru_shrink_walk(&sb->s_dentry_lru, sc,
1162 dentry_lru_isolate, &dispose);
Dave Chinnerf6041562013-08-28 10:18:00 +10001163 shrink_dentry_list(&dispose);
Dave Chinner0a234c62013-08-28 10:17:57 +10001164 return freed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001165}
1166
Glauber Costa4e717f52013-08-28 10:18:03 +10001167static enum lru_status dentry_lru_isolate_shrink(struct list_head *item,
Vladimir Davydov3f97b162015-02-12 14:59:35 -08001168 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
Dave Chinnerdd1f6b22013-08-28 10:17:55 +10001169{
Glauber Costa4e717f52013-08-28 10:18:03 +10001170 struct list_head *freeable = arg;
1171 struct dentry *dentry = container_of(item, struct dentry, d_lru);
Dave Chinnerdd1f6b22013-08-28 10:17:55 +10001172
Glauber Costa4e717f52013-08-28 10:18:03 +10001173 /*
1174 * we are inverting the lru lock/dentry->d_lock here,
1175 * so use a trylock. If we fail to get the lock, just skip
1176 * it
1177 */
1178 if (!spin_trylock(&dentry->d_lock))
1179 return LRU_SKIP;
1180
Vladimir Davydov3f97b162015-02-12 14:59:35 -08001181 d_lru_shrink_move(lru, dentry, freeable);
Glauber Costa4e717f52013-08-28 10:18:03 +10001182 spin_unlock(&dentry->d_lock);
1183
1184 return LRU_REMOVED;
Dave Chinnerdd1f6b22013-08-28 10:17:55 +10001185}
1186
Glauber Costa4e717f52013-08-28 10:18:03 +10001187
Kentaro Makitada3bbdd2008-07-23 21:27:13 -07001188/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189 * shrink_dcache_sb - shrink dcache for a superblock
1190 * @sb: superblock
1191 *
Christoph Hellwig3049cfe2010-10-10 05:36:25 -04001192 * Shrink the dcache for the specified super block. This is used to free
1193 * the dcache before unmounting a file system.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001194 */
Christoph Hellwig3049cfe2010-10-10 05:36:25 -04001195void shrink_dcache_sb(struct super_block *sb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196{
Glauber Costa4e717f52013-08-28 10:18:03 +10001197 do {
1198 LIST_HEAD(dispose);
1199
Waiman Long1dbd4492019-01-30 13:52:36 -05001200 list_lru_walk(&sb->s_dentry_lru,
Sahitya Tummalab17c0702017-07-10 15:50:00 -07001201 dentry_lru_isolate_shrink, &dispose, 1024);
Glauber Costa4e717f52013-08-28 10:18:03 +10001202 shrink_dentry_list(&dispose);
Sahitya Tummalab17c0702017-07-10 15:50:00 -07001203 } while (list_lru_count(&sb->s_dentry_lru) > 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001204}
H Hartley Sweetenec4f8602010-01-05 13:45:18 -07001205EXPORT_SYMBOL(shrink_dcache_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001206
Linus Torvalds1da177e2005-04-16 15:20:36 -07001207/**
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001208 * enum d_walk_ret - action to talke during tree walk
1209 * @D_WALK_CONTINUE: contrinue walk
1210 * @D_WALK_QUIT: quit walk
1211 * @D_WALK_NORETRY: quit when retry is needed
1212 * @D_WALK_SKIP: skip this dentry and its children
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213 */
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001214enum d_walk_ret {
1215 D_WALK_CONTINUE,
1216 D_WALK_QUIT,
1217 D_WALK_NORETRY,
1218 D_WALK_SKIP,
1219};
1220
1221/**
1222 * d_walk - walk the dentry tree
1223 * @parent: start of walk
1224 * @data: data passed to @enter() and @finish()
1225 * @enter: callback when first entering the dentry
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001226 *
Al Viro3a8e3612018-04-15 18:27:23 -04001227 * The @enter() callbacks are called with d_lock held.
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001228 */
1229static void d_walk(struct dentry *parent, void *data,
Al Viro3a8e3612018-04-15 18:27:23 -04001230 enum d_walk_ret (*enter)(void *, struct dentry *))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001231{
Al Viroda549bd2023-11-07 02:00:39 -05001232 struct dentry *this_parent, *dentry;
Al Viro48f5ec22013-09-09 15:22:25 -04001233 unsigned seq = 0;
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001234 enum d_walk_ret ret;
1235 bool retry = true;
Nick Piggin949854d2011-01-07 17:49:37 +11001236
Nick Piggin58db63d2011-01-07 17:49:39 +11001237again:
Al Viro48f5ec22013-09-09 15:22:25 -04001238 read_seqbegin_or_lock(&rename_lock, &seq);
Nick Piggin58db63d2011-01-07 17:49:39 +11001239 this_parent = parent;
Nick Piggin2fd6b7f2011-01-07 17:49:34 +11001240 spin_lock(&this_parent->d_lock);
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001241
1242 ret = enter(data, this_parent);
1243 switch (ret) {
1244 case D_WALK_CONTINUE:
1245 break;
1246 case D_WALK_QUIT:
1247 case D_WALK_SKIP:
1248 goto out_unlock;
1249 case D_WALK_NORETRY:
1250 retry = false;
1251 break;
1252 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001253repeat:
Al Viroda549bd2023-11-07 02:00:39 -05001254 dentry = d_first_child(this_parent);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001255resume:
Al Viroda549bd2023-11-07 02:00:39 -05001256 hlist_for_each_entry_from(dentry, d_sib) {
Al Viroba65dc52016-06-10 11:32:47 -04001257 if (unlikely(dentry->d_flags & DCACHE_DENTRY_CURSOR))
1258 continue;
1259
Nick Piggin2fd6b7f2011-01-07 17:49:34 +11001260 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001261
1262 ret = enter(data, dentry);
1263 switch (ret) {
1264 case D_WALK_CONTINUE:
1265 break;
1266 case D_WALK_QUIT:
Nick Piggin2fd6b7f2011-01-07 17:49:34 +11001267 spin_unlock(&dentry->d_lock);
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001268 goto out_unlock;
1269 case D_WALK_NORETRY:
1270 retry = false;
1271 break;
1272 case D_WALK_SKIP:
1273 spin_unlock(&dentry->d_lock);
1274 continue;
Nick Piggin2fd6b7f2011-01-07 17:49:34 +11001275 }
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001276
Al Viroda549bd2023-11-07 02:00:39 -05001277 if (!hlist_empty(&dentry->d_children)) {
Nick Piggin2fd6b7f2011-01-07 17:49:34 +11001278 spin_unlock(&this_parent->d_lock);
Qian Cai5facae42019-09-19 12:09:40 -04001279 spin_release(&dentry->d_lock.dep_map, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001280 this_parent = dentry;
Nick Piggin2fd6b7f2011-01-07 17:49:34 +11001281 spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001282 goto repeat;
1283 }
Nick Piggin2fd6b7f2011-01-07 17:49:34 +11001284 spin_unlock(&dentry->d_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001285 }
1286 /*
1287 * All done at this level ... ascend and resume the search.
1288 */
Al Viroca5358e2014-10-26 19:31:10 -04001289 rcu_read_lock();
1290ascend:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291 if (this_parent != parent) {
Al Viroda549bd2023-11-07 02:00:39 -05001292 dentry = this_parent;
1293 this_parent = dentry->d_parent;
Al Viro31dec132013-10-25 17:04:27 -04001294
Al Viroda549bd2023-11-07 02:00:39 -05001295 spin_unlock(&dentry->d_lock);
Al Viro31dec132013-10-25 17:04:27 -04001296 spin_lock(&this_parent->d_lock);
1297
Al Viroca5358e2014-10-26 19:31:10 -04001298 /* might go back up the wrong parent if we have had a rename. */
1299 if (need_seqretry(&rename_lock, seq))
Nick Piggin949854d2011-01-07 17:49:37 +11001300 goto rename_retry;
Al Viro21591842015-05-28 23:09:19 -04001301 /* go into the first sibling still alive */
Al Viroda549bd2023-11-07 02:00:39 -05001302 hlist_for_each_entry_continue(dentry, d_sib) {
1303 if (likely(!(dentry->d_flags & DCACHE_DENTRY_KILLED))) {
1304 rcu_read_unlock();
1305 goto resume;
1306 }
1307 }
1308 goto ascend;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309 }
Al Viroca5358e2014-10-26 19:31:10 -04001310 if (need_seqretry(&rename_lock, seq))
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001311 goto rename_retry;
Al Viroca5358e2014-10-26 19:31:10 -04001312 rcu_read_unlock();
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001313
1314out_unlock:
Nick Piggin2fd6b7f2011-01-07 17:49:34 +11001315 spin_unlock(&this_parent->d_lock);
Al Viro48f5ec22013-09-09 15:22:25 -04001316 done_seqretry(&rename_lock, seq);
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001317 return;
Nick Piggin58db63d2011-01-07 17:49:39 +11001318
1319rename_retry:
Al Viroca5358e2014-10-26 19:31:10 -04001320 spin_unlock(&this_parent->d_lock);
1321 rcu_read_unlock();
1322 BUG_ON(seq & 1);
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001323 if (!retry)
1324 return;
Al Viro48f5ec22013-09-09 15:22:25 -04001325 seq = 1;
Nick Piggin58db63d2011-01-07 17:49:39 +11001326 goto again;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001327}
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001328
Ian Kent01619492016-11-24 08:03:41 +11001329struct check_mount {
1330 struct vfsmount *mnt;
1331 unsigned int mounted;
1332};
1333
1334static enum d_walk_ret path_check_mount(void *data, struct dentry *dentry)
1335{
1336 struct check_mount *info = data;
1337 struct path path = { .mnt = info->mnt, .dentry = dentry };
1338
1339 if (likely(!d_mountpoint(dentry)))
1340 return D_WALK_CONTINUE;
1341 if (__path_is_mountpoint(&path)) {
1342 info->mounted = 1;
1343 return D_WALK_QUIT;
1344 }
1345 return D_WALK_CONTINUE;
1346}
1347
1348/**
1349 * path_has_submounts - check for mounts over a dentry in the
1350 * current namespace.
1351 * @parent: path to check.
1352 *
1353 * Return true if the parent or its subdirectories contain
1354 * a mount point in the current namespace.
1355 */
1356int path_has_submounts(const struct path *parent)
1357{
1358 struct check_mount data = { .mnt = parent->mnt, .mounted = 0 };
1359
1360 read_seqlock_excl(&mount_lock);
Al Viro3a8e3612018-04-15 18:27:23 -04001361 d_walk(parent->dentry, &data, path_check_mount);
Ian Kent01619492016-11-24 08:03:41 +11001362 read_sequnlock_excl(&mount_lock);
1363
1364 return data.mounted;
1365}
1366EXPORT_SYMBOL(path_has_submounts);
1367
Linus Torvalds1da177e2005-04-16 15:20:36 -07001368/*
Miklos Szeredieed81002013-09-05 14:39:11 +02001369 * Called by mount code to set a mountpoint and check if the mountpoint is
1370 * reachable (e.g. NFS can unhash a directory dentry and then the complete
1371 * subtree can become unreachable).
1372 *
Eric W. Biederman1ffe46d2014-02-13 09:39:37 -08001373 * Only one of d_invalidate() and d_set_mounted() must succeed. For
Miklos Szeredieed81002013-09-05 14:39:11 +02001374 * this reason take rename_lock and d_lock on dentry and ancestors.
1375 */
1376int d_set_mounted(struct dentry *dentry)
1377{
1378 struct dentry *p;
1379 int ret = -ENOENT;
1380 write_seqlock(&rename_lock);
1381 for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
Eric W. Biederman1ffe46d2014-02-13 09:39:37 -08001382 /* Need exclusion wrt. d_invalidate() */
Miklos Szeredieed81002013-09-05 14:39:11 +02001383 spin_lock(&p->d_lock);
1384 if (unlikely(d_unhashed(p))) {
1385 spin_unlock(&p->d_lock);
1386 goto out;
1387 }
1388 spin_unlock(&p->d_lock);
1389 }
1390 spin_lock(&dentry->d_lock);
1391 if (!d_unlinked(dentry)) {
Eric W. Biederman3895dbf2017-01-03 14:18:43 +13001392 ret = -EBUSY;
1393 if (!d_mountpoint(dentry)) {
1394 dentry->d_flags |= DCACHE_MOUNTED;
1395 ret = 0;
1396 }
Miklos Szeredieed81002013-09-05 14:39:11 +02001397 }
1398 spin_unlock(&dentry->d_lock);
1399out:
1400 write_sequnlock(&rename_lock);
1401 return ret;
1402}
1403
1404/*
J. Bruce Fieldsfd517902012-09-18 16:35:51 -04001405 * Search the dentry child list of the specified parent,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001406 * and move any unused dentries to the end of the unused
1407 * list for prune_dcache(). We descend to the next level
Al Viroda549bd2023-11-07 02:00:39 -05001408 * whenever the d_children list is non-empty and continue
Linus Torvalds1da177e2005-04-16 15:20:36 -07001409 * searching.
1410 *
1411 * It returns zero iff there are no unused children,
1412 * otherwise it returns the number of children moved to
1413 * the end of the unused list. This may not be the total
1414 * number of unused children, because select_parent can
1415 * drop the lock and return early due to latency
1416 * constraints.
1417 */
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001418
1419struct select_data {
1420 struct dentry *start;
Al Viro9bdebc22019-06-29 18:31:24 -04001421 union {
1422 long found;
1423 struct dentry *victim;
1424 };
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001425 struct list_head dispose;
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001426};
1427
1428static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001429{
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001430 struct select_data *data = _data;
1431 enum d_walk_ret ret = D_WALK_CONTINUE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001432
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001433 if (data->start == dentry)
1434 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001435
Al Virofe915222014-05-03 00:02:25 -04001436 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001437 data->found++;
Al Virof5c8a8a2023-10-30 21:07:34 -04001438 } else if (!dentry->d_lockref.count) {
1439 to_shrink_list(dentry, &data->dispose);
1440 data->found++;
Al Viro1c18edd2023-11-07 16:14:08 -05001441 } else if (dentry->d_lockref.count < 0) {
1442 data->found++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001443 }
1444 /*
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001445 * We can return to the caller if we have found some (this
1446 * ensures forward progress). We'll be coming back to find
1447 * the rest.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001448 */
Al Virofe915222014-05-03 00:02:25 -04001449 if (!list_empty(&data->dispose))
1450 ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001451out:
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001452 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001453}
1454
Al Viro9bdebc22019-06-29 18:31:24 -04001455static enum d_walk_ret select_collect2(void *_data, struct dentry *dentry)
1456{
1457 struct select_data *data = _data;
1458 enum d_walk_ret ret = D_WALK_CONTINUE;
1459
1460 if (data->start == dentry)
1461 goto out;
1462
Al Virof5c8a8a2023-10-30 21:07:34 -04001463 if (!dentry->d_lockref.count) {
1464 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
Al Viro9bdebc22019-06-29 18:31:24 -04001465 rcu_read_lock();
1466 data->victim = dentry;
1467 return D_WALK_QUIT;
1468 }
Al Virof5c8a8a2023-10-30 21:07:34 -04001469 to_shrink_list(dentry, &data->dispose);
Al Viro9bdebc22019-06-29 18:31:24 -04001470 }
1471 /*
1472 * We can return to the caller if we have found some (this
1473 * ensures forward progress). We'll be coming back to find
1474 * the rest.
1475 */
1476 if (!list_empty(&data->dispose))
1477 ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
1478out:
1479 return ret;
1480}
1481
Linus Torvalds1da177e2005-04-16 15:20:36 -07001482/**
1483 * shrink_dcache_parent - prune dcache
1484 * @parent: parent of entries to prune
1485 *
1486 * Prune the dcache to remove unused children of the parent dentry.
1487 */
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001488void shrink_dcache_parent(struct dentry *parent)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489{
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001490 for (;;) {
Al Viro9bdebc22019-06-29 18:31:24 -04001491 struct select_data data = {.start = parent};
Linus Torvalds1da177e2005-04-16 15:20:36 -07001492
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001493 INIT_LIST_HEAD(&data.dispose);
Al Viro3a8e3612018-04-15 18:27:23 -04001494 d_walk(parent, &data, select_collect);
Al Viro4fb48872018-04-19 23:58:48 -04001495
1496 if (!list_empty(&data.dispose)) {
1497 shrink_dentry_list(&data.dispose);
1498 continue;
1499 }
1500
1501 cond_resched();
Miklos Szeredidb14fc32013-09-05 11:44:35 +02001502 if (!data.found)
1503 break;
Al Viro9bdebc22019-06-29 18:31:24 -04001504 data.victim = NULL;
1505 d_walk(parent, &data, select_collect2);
1506 if (data.victim) {
Al Viro9bdebc22019-06-29 18:31:24 -04001507 spin_lock(&data.victim->d_lock);
Al Viro339e9e12023-10-31 01:23:47 -04001508 if (!lock_for_kill(data.victim)) {
Al Viro9bdebc22019-06-29 18:31:24 -04001509 spin_unlock(&data.victim->d_lock);
1510 rcu_read_unlock();
1511 } else {
Al Viro1c18edd2023-11-07 16:14:08 -05001512 shrink_kill(data.victim);
Al Viro9bdebc22019-06-29 18:31:24 -04001513 }
1514 }
1515 if (!list_empty(&data.dispose))
1516 shrink_dentry_list(&data.dispose);
Greg Thelen421348f2013-04-30 15:26:48 -07001517 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001518}
H Hartley Sweetenec4f8602010-01-05 13:45:18 -07001519EXPORT_SYMBOL(shrink_dcache_parent);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520
Al Viro9c8c10e2014-05-02 20:36:10 -04001521static enum d_walk_ret umount_check(void *_data, struct dentry *dentry)
Al Viro42c32602013-11-08 12:31:16 -05001522{
Al Viro9c8c10e2014-05-02 20:36:10 -04001523 /* it has busy descendents; complain about those instead */
Al Viroda549bd2023-11-07 02:00:39 -05001524 if (!hlist_empty(&dentry->d_children))
Al Viro9c8c10e2014-05-02 20:36:10 -04001525 return D_WALK_CONTINUE;
Al Viro42c32602013-11-08 12:31:16 -05001526
Al Viro9c8c10e2014-05-02 20:36:10 -04001527 /* root with refcount 1 is fine */
1528 if (dentry == _data && dentry->d_lockref.count == 1)
1529 return D_WALK_CONTINUE;
1530
Anh Tuan Phan8c8e7db2023-08-17 23:31:42 +07001531 WARN(1, "BUG: Dentry %p{i=%lx,n=%pd} "
Al Viro9c8c10e2014-05-02 20:36:10 -04001532 " still in use (%d) [unmount of %s %s]\n",
Al Viro42c32602013-11-08 12:31:16 -05001533 dentry,
1534 dentry->d_inode ?
1535 dentry->d_inode->i_ino : 0UL,
Al Viro9c8c10e2014-05-02 20:36:10 -04001536 dentry,
Al Viro42c32602013-11-08 12:31:16 -05001537 dentry->d_lockref.count,
1538 dentry->d_sb->s_type->name,
1539 dentry->d_sb->s_id);
Al Viro9c8c10e2014-05-02 20:36:10 -04001540 return D_WALK_CONTINUE;
1541}
1542
1543static void do_one_tree(struct dentry *dentry)
1544{
1545 shrink_dcache_parent(dentry);
Al Viro3a8e3612018-04-15 18:27:23 -04001546 d_walk(dentry, dentry, umount_check);
Al Viro9c8c10e2014-05-02 20:36:10 -04001547 d_drop(dentry);
1548 dput(dentry);
Al Viro42c32602013-11-08 12:31:16 -05001549}
1550
1551/*
1552 * destroy the dentries attached to a superblock on unmounting
1553 */
1554void shrink_dcache_for_umount(struct super_block *sb)
1555{
1556 struct dentry *dentry;
1557
Mateusz Guzik54018132024-06-02 14:37:19 +02001558 rwsem_assert_held_write(&sb->s_umount);
Al Viro42c32602013-11-08 12:31:16 -05001559
1560 dentry = sb->s_root;
1561 sb->s_root = NULL;
Al Viro9c8c10e2014-05-02 20:36:10 -04001562 do_one_tree(dentry);
Al Viro42c32602013-11-08 12:31:16 -05001563
NeilBrownf1ee6162017-12-21 09:45:40 +11001564 while (!hlist_bl_empty(&sb->s_roots)) {
1565 dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_roots), struct dentry, d_hash));
Al Viro9c8c10e2014-05-02 20:36:10 -04001566 do_one_tree(dentry);
Al Viro42c32602013-11-08 12:31:16 -05001567 }
1568}
1569
Al Viroff17fa52018-04-15 18:21:47 -04001570static enum d_walk_ret find_submount(void *_data, struct dentry *dentry)
Miklos Szeredi848ac1142013-09-05 11:44:36 +02001571{
Al Viroff17fa52018-04-15 18:21:47 -04001572 struct dentry **victim = _data;
Miklos Szeredi848ac1142013-09-05 11:44:36 +02001573 if (d_mountpoint(dentry)) {
Vegard Nossum1b6ae9f2023-11-06 14:44:17 +01001574 *victim = dget_dlock(dentry);
Miklos Szeredi848ac1142013-09-05 11:44:36 +02001575 return D_WALK_QUIT;
1576 }
Al Viroff17fa52018-04-15 18:21:47 -04001577 return D_WALK_CONTINUE;
Miklos Szeredi848ac1142013-09-05 11:44:36 +02001578}
1579
1580/**
Eric W. Biederman1ffe46d2014-02-13 09:39:37 -08001581 * d_invalidate - detach submounts, prune dcache, and drop
1582 * @dentry: dentry to invalidate (aka detach, prune and drop)
Miklos Szeredi848ac1142013-09-05 11:44:36 +02001583 */
Eric W. Biederman5542aa22014-02-13 09:46:25 -08001584void d_invalidate(struct dentry *dentry)
Miklos Szeredi848ac1142013-09-05 11:44:36 +02001585{
Al Viroff17fa52018-04-15 18:21:47 -04001586 bool had_submounts = false;
Eric W. Biederman1ffe46d2014-02-13 09:39:37 -08001587 spin_lock(&dentry->d_lock);
1588 if (d_unhashed(dentry)) {
1589 spin_unlock(&dentry->d_lock);
Eric W. Biederman5542aa22014-02-13 09:46:25 -08001590 return;
Eric W. Biederman1ffe46d2014-02-13 09:39:37 -08001591 }
Al Viroff17fa52018-04-15 18:21:47 -04001592 __d_drop(dentry);
Eric W. Biederman1ffe46d2014-02-13 09:39:37 -08001593 spin_unlock(&dentry->d_lock);
1594
Miklos Szeredi848ac1142013-09-05 11:44:36 +02001595 /* Negative dentries can be dropped without further checks */
Al Viroff17fa52018-04-15 18:21:47 -04001596 if (!dentry->d_inode)
Eric W. Biederman5542aa22014-02-13 09:46:25 -08001597 return;
Miklos Szeredi848ac1142013-09-05 11:44:36 +02001598
Al Viroff17fa52018-04-15 18:21:47 -04001599 shrink_dcache_parent(dentry);
Miklos Szeredi848ac1142013-09-05 11:44:36 +02001600 for (;;) {
Al Viroff17fa52018-04-15 18:21:47 -04001601 struct dentry *victim = NULL;
Al Viro3a8e3612018-04-15 18:27:23 -04001602 d_walk(dentry, &victim, find_submount);
Al Viroff17fa52018-04-15 18:21:47 -04001603 if (!victim) {
1604 if (had_submounts)
1605 shrink_dcache_parent(dentry);
Al Viro81be24d2017-06-03 07:20:09 +01001606 return;
Eric W. Biederman8ed936b2013-10-01 18:33:48 -07001607 }
Al Viroff17fa52018-04-15 18:21:47 -04001608 had_submounts = true;
1609 detach_mounts(victim);
1610 dput(victim);
Miklos Szeredi848ac1142013-09-05 11:44:36 +02001611 }
Miklos Szeredi848ac1142013-09-05 11:44:36 +02001612}
Eric W. Biederman1ffe46d2014-02-13 09:39:37 -08001613EXPORT_SYMBOL(d_invalidate);
Miklos Szeredi848ac1142013-09-05 11:44:36 +02001614
Linus Torvalds1da177e2005-04-16 15:20:36 -07001615/**
Al Viroa4464db2011-07-07 15:03:58 -04001616 * __d_alloc - allocate a dcache entry
1617 * @sb: filesystem it will belong to
Linus Torvalds1da177e2005-04-16 15:20:36 -07001618 * @name: qstr of the name
1619 *
1620 * Allocates a dentry. It returns %NULL if there is insufficient memory
1621 * available. On a success the dentry is returned. The name passed in is
1622 * copied and the copy passed in may be reused after this call.
1623 */
1624
Al Viro5c8b0df2019-10-25 14:08:24 -04001625static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001626{
1627 struct dentry *dentry;
1628 char *dname;
Miklos Szeredi285b1022016-06-28 11:47:32 +02001629 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001630
Muchun Songf53bf7112022-03-22 14:41:09 -07001631 dentry = kmem_cache_alloc_lru(dentry_cache, &sb->s_dentry_lru,
1632 GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001633 if (!dentry)
1634 return NULL;
1635
Linus Torvalds6326c71f2012-05-21 16:14:04 -07001636 /*
1637 * We guarantee that the inline name is always NUL-terminated.
1638 * This way the memcpy() done by the name switching in rename
1639 * will still always have a NUL at the end, even if we might
1640 * be overwriting an internal NUL character
1641 */
1642 dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
Al Viro798434b2016-03-24 20:38:43 -04001643 if (unlikely(!name)) {
David Howellscdf01222017-07-04 17:25:22 +01001644 name = &slash_name;
Al Viro798434b2016-03-24 20:38:43 -04001645 dname = dentry->d_iname;
1646 } else if (name->len > DNAME_INLINE_LEN-1) {
Al Viro8d85b482014-09-29 14:54:27 -04001647 size_t size = offsetof(struct external_name, name[1]);
Vlastimil Babka2e03b4b2018-10-26 15:05:41 -07001648 struct external_name *p = kmalloc(size + name->len,
1649 GFP_KERNEL_ACCOUNT |
1650 __GFP_RECLAIMABLE);
1651 if (!p) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001652 kmem_cache_free(dentry_cache, dentry);
1653 return NULL;
1654 }
Vlastimil Babka2e03b4b2018-10-26 15:05:41 -07001655 atomic_set(&p->u.count, 1);
1656 dname = p->name;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657 } else {
1658 dname = dentry->d_iname;
1659 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001660
1661 dentry->d_name.len = name->len;
1662 dentry->d_name.hash = name->hash;
1663 memcpy(dname, name->name, name->len);
1664 dname[name->len] = 0;
1665
Linus Torvalds6326c71f2012-05-21 16:14:04 -07001666 /* Make sure we always see the terminating NUL character */
Paul E. McKenney7088efa2017-10-09 10:04:27 -07001667 smp_store_release(&dentry->d_name.name, dname); /* ^^^ */
Linus Torvalds6326c71f2012-05-21 16:14:04 -07001668
Waiman Long98474232013-08-28 18:24:59 -07001669 dentry->d_lockref.count = 1;
Linus Torvaldsdea36672011-04-24 07:58:46 -07001670 dentry->d_flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001671 spin_lock_init(&dentry->d_lock);
Ahmed S. Darwish26475372020-07-20 17:55:24 +02001672 seqcount_spinlock_init(&dentry->d_seq, &dentry->d_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001673 dentry->d_inode = NULL;
Al Viroa4464db2011-07-07 15:03:58 -04001674 dentry->d_parent = dentry;
1675 dentry->d_sb = sb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001676 dentry->d_op = NULL;
1677 dentry->d_fsdata = NULL;
Nick Pigginceb5bdc2011-01-07 17:50:05 +11001678 INIT_HLIST_BL_NODE(&dentry->d_hash);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001679 INIT_LIST_HEAD(&dentry->d_lru);
Al Viroda549bd2023-11-07 02:00:39 -05001680 INIT_HLIST_HEAD(&dentry->d_children);
Al Viro946e51f2014-10-26 19:19:16 -04001681 INIT_HLIST_NODE(&dentry->d_u.d_alias);
Al Viroda549bd2023-11-07 02:00:39 -05001682 INIT_HLIST_NODE(&dentry->d_sib);
Al Viroa4464db2011-07-07 15:03:58 -04001683 d_set_d_op(dentry, dentry->d_sb->s_d_op);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001684
Miklos Szeredi285b1022016-06-28 11:47:32 +02001685 if (dentry->d_op && dentry->d_op->d_init) {
1686 err = dentry->d_op->d_init(dentry);
1687 if (err) {
1688 if (dname_external(dentry))
1689 kfree(external_name(dentry));
1690 kmem_cache_free(dentry_cache, dentry);
1691 return NULL;
1692 }
1693 }
1694
Nick Piggin3e880fb2011-01-07 17:49:19 +11001695 this_cpu_inc(nr_dentry);
Christoph Hellwig312d3ca2010-10-10 05:36:23 -04001696
Linus Torvalds1da177e2005-04-16 15:20:36 -07001697 return dentry;
1698}
Al Viroa4464db2011-07-07 15:03:58 -04001699
1700/**
1701 * d_alloc - allocate a dcache entry
1702 * @parent: parent of entry to allocate
1703 * @name: qstr of the name
1704 *
1705 * Allocates a dentry. It returns %NULL if there is insufficient memory
1706 * available. On a success the dentry is returned. The name passed in is
1707 * copied and the copy passed in may be reused after this call.
1708 */
1709struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
1710{
1711 struct dentry *dentry = __d_alloc(parent->d_sb, name);
1712 if (!dentry)
1713 return NULL;
Al Viroa4464db2011-07-07 15:03:58 -04001714 spin_lock(&parent->d_lock);
1715 /*
1716 * don't need child lock because it is not subject
1717 * to concurrency here
1718 */
Vegard Nossum1b6ae9f2023-11-06 14:44:17 +01001719 dentry->d_parent = dget_dlock(parent);
Al Viroda549bd2023-11-07 02:00:39 -05001720 hlist_add_head(&dentry->d_sib, &parent->d_children);
Al Viroa4464db2011-07-07 15:03:58 -04001721 spin_unlock(&parent->d_lock);
1722
1723 return dentry;
1724}
H Hartley Sweetenec4f8602010-01-05 13:45:18 -07001725EXPORT_SYMBOL(d_alloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001726
Miklos Szeredif9c34672018-01-19 11:39:52 +01001727struct dentry *d_alloc_anon(struct super_block *sb)
1728{
1729 return __d_alloc(sb, NULL);
1730}
1731EXPORT_SYMBOL(d_alloc_anon);
1732
Al Viroba65dc52016-06-10 11:32:47 -04001733struct dentry *d_alloc_cursor(struct dentry * parent)
1734{
Miklos Szeredif9c34672018-01-19 11:39:52 +01001735 struct dentry *dentry = d_alloc_anon(parent->d_sb);
Al Viroba65dc52016-06-10 11:32:47 -04001736 if (dentry) {
Al Viro5467a682019-03-15 22:23:19 -04001737 dentry->d_flags |= DCACHE_DENTRY_CURSOR;
Al Viroba65dc52016-06-10 11:32:47 -04001738 dentry->d_parent = dget(parent);
1739 }
1740 return dentry;
1741}
1742
J. Bruce Fieldse1a24bb2012-06-29 16:20:47 -04001743/**
1744 * d_alloc_pseudo - allocate a dentry (for lookup-less filesystems)
1745 * @sb: the superblock
1746 * @name: qstr of the name
1747 *
1748 * For a filesystem that just pins its dentries in memory and never
1749 * performs lookups at all, return an unhashed IS_ROOT dentry.
Al Viro5467a682019-03-15 22:23:19 -04001750 * This is used for pipes, sockets et.al. - the stuff that should
1751 * never be anyone's children or parents. Unlike all other
1752 * dentries, these will not have RCU delay between dropping the
1753 * last reference and freeing them.
Al Viroab1152d2019-03-15 22:58:11 -04001754 *
1755 * The only user is alloc_file_pseudo() and that's what should
1756 * be considered a public interface. Don't use directly.
J. Bruce Fieldse1a24bb2012-06-29 16:20:47 -04001757 */
Nick Piggin4b936882011-01-07 17:50:07 +11001758struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
1759{
Al Viro9024b4c2023-11-12 00:04:46 -05001760 static const struct dentry_operations anon_ops = {
1761 .d_dname = simple_dname
1762 };
Al Viro5467a682019-03-15 22:23:19 -04001763 struct dentry *dentry = __d_alloc(sb, name);
Al Viro9024b4c2023-11-12 00:04:46 -05001764 if (likely(dentry)) {
Al Viro5467a682019-03-15 22:23:19 -04001765 dentry->d_flags |= DCACHE_NORCU;
Al Viro9024b4c2023-11-12 00:04:46 -05001766 if (!sb->s_d_op)
1767 d_set_d_op(dentry, &anon_ops);
1768 }
Al Viro5467a682019-03-15 22:23:19 -04001769 return dentry;
Nick Piggin4b936882011-01-07 17:50:07 +11001770}
Nick Piggin4b936882011-01-07 17:50:07 +11001771
Linus Torvalds1da177e2005-04-16 15:20:36 -07001772struct dentry *d_alloc_name(struct dentry *parent, const char *name)
1773{
1774 struct qstr q;
1775
1776 q.name = name;
Linus Torvalds8387ff22016-06-10 07:51:30 -07001777 q.hash_len = hashlen_string(parent, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001778 return d_alloc(parent, &q);
1779}
H Hartley Sweetenef26ca92009-09-29 20:09:42 -04001780EXPORT_SYMBOL(d_alloc_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001781
Nick Pigginfb045ad2011-01-07 17:49:55 +11001782void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
1783{
Linus Torvalds6f7f7ca2011-01-14 13:26:18 -08001784 WARN_ON_ONCE(dentry->d_op);
1785 WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH |
Nick Pigginfb045ad2011-01-07 17:49:55 +11001786 DCACHE_OP_COMPARE |
1787 DCACHE_OP_REVALIDATE |
Jeff Laytonecf3d1f2013-02-20 11:19:05 -05001788 DCACHE_OP_WEAK_REVALIDATE |
David Howells4bacc9c2015-06-18 14:32:31 +01001789 DCACHE_OP_DELETE |
Miklos Szeredid101a122016-03-26 16:14:37 -04001790 DCACHE_OP_REAL));
Nick Pigginfb045ad2011-01-07 17:49:55 +11001791 dentry->d_op = op;
1792 if (!op)
1793 return;
1794 if (op->d_hash)
1795 dentry->d_flags |= DCACHE_OP_HASH;
1796 if (op->d_compare)
1797 dentry->d_flags |= DCACHE_OP_COMPARE;
1798 if (op->d_revalidate)
1799 dentry->d_flags |= DCACHE_OP_REVALIDATE;
Jeff Laytonecf3d1f2013-02-20 11:19:05 -05001800 if (op->d_weak_revalidate)
1801 dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE;
Nick Pigginfb045ad2011-01-07 17:49:55 +11001802 if (op->d_delete)
1803 dentry->d_flags |= DCACHE_OP_DELETE;
Sage Weilf0023bc2011-10-28 10:02:42 -07001804 if (op->d_prune)
1805 dentry->d_flags |= DCACHE_OP_PRUNE;
Miklos Szeredid101a122016-03-26 16:14:37 -04001806 if (op->d_real)
1807 dentry->d_flags |= DCACHE_OP_REAL;
Nick Pigginfb045ad2011-01-07 17:49:55 +11001808
1809}
1810EXPORT_SYMBOL(d_set_d_op);
1811
David Howellsb18825a2013-09-12 19:22:53 +01001812static unsigned d_flags_for_inode(struct inode *inode)
1813{
David Howells44bdb5e2015-01-29 12:02:29 +00001814 unsigned add_flags = DCACHE_REGULAR_TYPE;
David Howellsb18825a2013-09-12 19:22:53 +01001815
1816 if (!inode)
1817 return DCACHE_MISS_TYPE;
1818
1819 if (S_ISDIR(inode->i_mode)) {
1820 add_flags = DCACHE_DIRECTORY_TYPE;
1821 if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) {
1822 if (unlikely(!inode->i_op->lookup))
1823 add_flags = DCACHE_AUTODIR_TYPE;
1824 else
1825 inode->i_opflags |= IOP_LOOKUP;
1826 }
David Howells44bdb5e2015-01-29 12:02:29 +00001827 goto type_determined;
David Howellsb18825a2013-09-12 19:22:53 +01001828 }
1829
David Howells44bdb5e2015-01-29 12:02:29 +00001830 if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) {
Al Viro6b255392015-11-17 10:20:54 -05001831 if (unlikely(inode->i_op->get_link)) {
David Howells44bdb5e2015-01-29 12:02:29 +00001832 add_flags = DCACHE_SYMLINK_TYPE;
1833 goto type_determined;
1834 }
1835 inode->i_opflags |= IOP_NOFOLLOW;
1836 }
1837
1838 if (unlikely(!S_ISREG(inode->i_mode)))
1839 add_flags = DCACHE_SPECIAL_TYPE;
1840
1841type_determined:
David Howellsb18825a2013-09-12 19:22:53 +01001842 if (unlikely(IS_AUTOMOUNT(inode)))
1843 add_flags |= DCACHE_NEED_AUTOMOUNT;
1844 return add_flags;
1845}
1846
OGAWA Hirofumi360da902008-10-16 07:50:28 +09001847static void __d_instantiate(struct dentry *dentry, struct inode *inode)
1848{
David Howellsb18825a2013-09-12 19:22:53 +01001849 unsigned add_flags = d_flags_for_inode(inode);
Al Viro85c7f812016-04-14 19:52:13 -04001850 WARN_ON(d_in_lookup(dentry));
David Howellsb18825a2013-09-12 19:22:53 +01001851
Nick Pigginb23fb0a2011-01-07 17:49:35 +11001852 spin_lock(&dentry->d_lock);
Waiman Longaf0c9af2019-01-30 13:52:38 -05001853 /*
Brian Fosteraabfe572024-07-03 08:13:01 -04001854 * The negative counter only tracks dentries on the LRU. Don't dec if
1855 * d_lru is on another list.
Waiman Longaf0c9af2019-01-30 13:52:38 -05001856 */
Brian Fosteraabfe572024-07-03 08:13:01 -04001857 if ((dentry->d_flags &
1858 (DCACHE_LRU_LIST|DCACHE_SHRINK_LIST)) == DCACHE_LRU_LIST)
Waiman Longaf0c9af2019-01-30 13:52:38 -05001859 this_cpu_dec(nr_dentry_negative);
Al Virode689f52016-03-09 18:05:42 -05001860 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
Al Viroa528aca2016-02-29 12:12:46 -05001861 raw_write_seqcount_begin(&dentry->d_seq);
David Howells4bf46a22015-03-05 14:09:22 +00001862 __d_set_inode_and_type(dentry, inode, add_flags);
Al Viroa528aca2016-02-29 12:12:46 -05001863 raw_write_seqcount_end(&dentry->d_seq);
Al Viroaffda482016-05-29 18:35:12 -04001864 fsnotify_update_flags(dentry);
Nick Pigginb23fb0a2011-01-07 17:49:35 +11001865 spin_unlock(&dentry->d_lock);
OGAWA Hirofumi360da902008-10-16 07:50:28 +09001866}
1867
Linus Torvalds1da177e2005-04-16 15:20:36 -07001868/**
1869 * d_instantiate - fill in inode information for a dentry
1870 * @entry: dentry to complete
1871 * @inode: inode to attach to this dentry
1872 *
1873 * Fill in inode information in the entry.
1874 *
1875 * This turns negative dentries into productive full members
1876 * of society.
1877 *
1878 * NOTE! This assumes that the inode count has been incremented
1879 * (or otherwise set) by the caller to indicate that it is now
1880 * in use by the dcache.
1881 */
1882
1883void d_instantiate(struct dentry *entry, struct inode * inode)
1884{
Al Viro946e51f2014-10-26 19:19:16 -04001885 BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
Al Virode689f52016-03-09 18:05:42 -05001886 if (inode) {
Al Virob9680912016-04-11 00:53:26 -04001887 security_d_instantiate(entry, inode);
Nick Piggin873feea2011-01-07 17:50:06 +11001888 spin_lock(&inode->i_lock);
Al Virode689f52016-03-09 18:05:42 -05001889 __d_instantiate(entry, inode);
Nick Piggin873feea2011-01-07 17:50:06 +11001890 spin_unlock(&inode->i_lock);
Al Virode689f52016-03-09 18:05:42 -05001891 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001892}
H Hartley Sweetenec4f8602010-01-05 13:45:18 -07001893EXPORT_SYMBOL(d_instantiate);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001894
Al Viro1e2e5472018-05-04 08:23:01 -04001895/*
1896 * This should be equivalent to d_instantiate() + unlock_new_inode(),
1897 * with lockdep-related part of unlock_new_inode() done before
1898 * anything else. Use that instead of open-coding d_instantiate()/
1899 * unlock_new_inode() combinations.
1900 */
1901void d_instantiate_new(struct dentry *entry, struct inode *inode)
1902{
1903 BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
1904 BUG_ON(!inode);
1905 lockdep_annotate_inode_mutex_key(inode);
1906 security_d_instantiate(entry, inode);
1907 spin_lock(&inode->i_lock);
1908 __d_instantiate(entry, inode);
1909 WARN_ON(!(inode->i_state & I_NEW));
Al Viroc2b6d622018-06-28 15:53:17 -04001910 inode->i_state &= ~I_NEW & ~I_CREATING;
Al Viro1e2e5472018-05-04 08:23:01 -04001911 smp_mb();
1912 wake_up_bit(&inode->i_state, __I_NEW);
1913 spin_unlock(&inode->i_lock);
1914}
1915EXPORT_SYMBOL(d_instantiate_new);
1916
Al Viroadc0e91a2012-01-08 16:49:21 -05001917struct dentry *d_make_root(struct inode *root_inode)
1918{
1919 struct dentry *res = NULL;
1920
1921 if (root_inode) {
Miklos Szeredif9c34672018-01-19 11:39:52 +01001922 res = d_alloc_anon(root_inode->i_sb);
Al Viro5467a682019-03-15 22:23:19 -04001923 if (res)
Al Viroadc0e91a2012-01-08 16:49:21 -05001924 d_instantiate(res, root_inode);
Al Viro5467a682019-03-15 22:23:19 -04001925 else
Al Viroadc0e91a2012-01-08 16:49:21 -05001926 iput(root_inode);
1927 }
1928 return res;
1929}
1930EXPORT_SYMBOL(d_make_root);
1931
Miklos Szeredif9c34672018-01-19 11:39:52 +01001932static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected)
Christoph Hellwig4ea3ada2008-08-11 15:48:57 +02001933{
Al Virof2824db2023-11-18 16:42:43 -05001934 struct super_block *sb;
1935 struct dentry *new, *res;
Christoph Hellwig4ea3ada2008-08-11 15:48:57 +02001936
1937 if (!inode)
Christoph Hellwig44003722008-08-11 15:49:04 +02001938 return ERR_PTR(-ESTALE);
Christoph Hellwig4ea3ada2008-08-11 15:48:57 +02001939 if (IS_ERR(inode))
1940 return ERR_CAST(inode);
1941
Al Virof2824db2023-11-18 16:42:43 -05001942 sb = inode->i_sb;
Christoph Hellwig9308a612008-08-11 15:49:12 +02001943
Al Virof2824db2023-11-18 16:42:43 -05001944 res = d_find_any_alias(inode); /* existing alias? */
1945 if (res)
1946 goto out;
1947
1948 new = d_alloc_anon(sb);
1949 if (!new) {
Christoph Hellwig9308a612008-08-11 15:49:12 +02001950 res = ERR_PTR(-ENOMEM);
Al Virof2824db2023-11-18 16:42:43 -05001951 goto out;
Christoph Hellwig4ea3ada2008-08-11 15:48:57 +02001952 }
Nick Pigginb5c84bf2011-01-07 17:49:38 +11001953
Al Virof2824db2023-11-18 16:42:43 -05001954 security_d_instantiate(new, inode);
1955 spin_lock(&inode->i_lock);
1956 res = __d_find_any_alias(inode); /* recheck under lock */
1957 if (likely(!res)) { /* still no alias, attach a disconnected dentry */
1958 unsigned add_flags = d_flags_for_inode(inode);
Christoph Hellwig9308a612008-08-11 15:49:12 +02001959
Al Virof2824db2023-11-18 16:42:43 -05001960 if (disconnected)
1961 add_flags |= DCACHE_DISCONNECTED;
1962
1963 spin_lock(&new->d_lock);
1964 __d_set_inode_and_type(new, inode, add_flags);
1965 hlist_add_head(&new->d_u.d_alias, &inode->i_dentry);
1966 if (!disconnected) {
1967 hlist_bl_lock(&sb->s_roots);
1968 hlist_bl_add_head(&new->d_hash, &sb->s_roots);
1969 hlist_bl_unlock(&sb->s_roots);
1970 }
1971 spin_unlock(&new->d_lock);
1972 spin_unlock(&inode->i_lock);
1973 inode = NULL; /* consumed by new->d_inode */
1974 res = new;
1975 } else {
1976 spin_unlock(&inode->i_lock);
1977 dput(new);
1978 }
1979
1980 out:
Christoph Hellwig9308a612008-08-11 15:49:12 +02001981 iput(inode);
1982 return res;
Christoph Hellwig4ea3ada2008-08-11 15:48:57 +02001983}
J. Bruce Fields1a0a3972014-02-14 17:35:37 -05001984
1985/**
1986 * d_obtain_alias - find or allocate a DISCONNECTED dentry for a given inode
1987 * @inode: inode to allocate the dentry for
1988 *
1989 * Obtain a dentry for an inode resulting from NFS filehandle conversion or
1990 * similar open by handle operations. The returned dentry may be anonymous,
1991 * or may have a full name (if the inode was already in the cache).
1992 *
1993 * When called on a directory inode, we must ensure that the inode only ever
1994 * has one dentry. If a dentry is found, that is returned instead of
1995 * allocating a new one.
1996 *
1997 * On successful return, the reference to the inode has been transferred
1998 * to the dentry. In case of an error the reference on the inode is released.
1999 * To make it easier to use in export operations a %NULL or IS_ERR inode may
2000 * be passed in and the error will be propagated to the return value,
2001 * with a %NULL @inode replaced by ERR_PTR(-ESTALE).
2002 */
2003struct dentry *d_obtain_alias(struct inode *inode)
2004{
Miklos Szeredif9c34672018-01-19 11:39:52 +01002005 return __d_obtain_alias(inode, true);
J. Bruce Fields1a0a3972014-02-14 17:35:37 -05002006}
Benny Halevyadc48722009-02-27 14:02:59 -08002007EXPORT_SYMBOL(d_obtain_alias);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002008
2009/**
J. Bruce Fields1a0a3972014-02-14 17:35:37 -05002010 * d_obtain_root - find or allocate a dentry for a given inode
2011 * @inode: inode to allocate the dentry for
2012 *
2013 * Obtain an IS_ROOT dentry for the root of a filesystem.
2014 *
2015 * We must ensure that directory inodes only ever have one dentry. If a
2016 * dentry is found, that is returned instead of allocating a new one.
2017 *
2018 * On successful return, the reference to the inode has been transferred
2019 * to the dentry. In case of an error the reference on the inode is
2020 * released. A %NULL or IS_ERR inode may be passed in and will be the
2021 * error will be propagate to the return value, with a %NULL @inode
2022 * replaced by ERR_PTR(-ESTALE).
2023 */
2024struct dentry *d_obtain_root(struct inode *inode)
2025{
Miklos Szeredif9c34672018-01-19 11:39:52 +01002026 return __d_obtain_alias(inode, false);
J. Bruce Fields1a0a3972014-02-14 17:35:37 -05002027}
2028EXPORT_SYMBOL(d_obtain_root);
2029
2030/**
Barry Naujok94035402008-05-21 16:50:46 +10002031 * d_add_ci - lookup or allocate new dentry with case-exact name
2032 * @inode: the inode case-insensitive lookup has found
2033 * @dentry: the negative dentry that was passed to the parent's lookup func
2034 * @name: the case-exact name to be associated with the returned dentry
2035 *
2036 * This is to avoid filling the dcache with case-insensitive names to the
2037 * same inode, only the actual correct case is stored in the dcache for
2038 * case-insensitive filesystems.
2039 *
Randy Dunlap3d742d42021-02-24 12:00:48 -08002040 * For a case-insensitive lookup match and if the case-exact dentry
2041 * already exists in the dcache, use it and return it.
Barry Naujok94035402008-05-21 16:50:46 +10002042 *
2043 * If no entry exists with the exact case name, allocate new dentry with
2044 * the exact case, and return the spliced entry.
2045 */
Christoph Hellwige45b5902008-08-07 23:49:07 +02002046struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
Barry Naujok94035402008-05-21 16:50:46 +10002047 struct qstr *name)
2048{
Al Virod9171b92016-04-15 03:33:13 -04002049 struct dentry *found, *res;
Barry Naujok94035402008-05-21 16:50:46 +10002050
Christoph Hellwigb6520c82009-01-05 19:10:37 +01002051 /*
2052 * First check if a dentry matching the name already exists,
2053 * if not go ahead and create it now.
2054 */
Barry Naujok94035402008-05-21 16:50:46 +10002055 found = d_hash_and_lookup(dentry->d_parent, name);
Al Virod9171b92016-04-15 03:33:13 -04002056 if (found) {
2057 iput(inode);
2058 return found;
Barry Naujok94035402008-05-21 16:50:46 +10002059 }
Al Virod9171b92016-04-15 03:33:13 -04002060 if (d_in_lookup(dentry)) {
2061 found = d_alloc_parallel(dentry->d_parent, name,
2062 dentry->d_wait);
2063 if (IS_ERR(found) || !d_in_lookup(found)) {
2064 iput(inode);
2065 return found;
2066 }
2067 } else {
2068 found = d_alloc(dentry->d_parent, name);
2069 if (!found) {
2070 iput(inode);
2071 return ERR_PTR(-ENOMEM);
2072 }
2073 }
2074 res = d_splice_alias(inode, found);
2075 if (res) {
Al Viro40a3cb02022-07-30 00:29:05 -04002076 d_lookup_done(found);
Al Virod9171b92016-04-15 03:33:13 -04002077 dput(found);
2078 return res;
2079 }
Al Viro4f522a22013-02-11 23:20:37 -05002080 return found;
Barry Naujok94035402008-05-21 16:50:46 +10002081}
H Hartley Sweetenec4f8602010-01-05 13:45:18 -07002082EXPORT_SYMBOL(d_add_ci);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002083
Xiubo Li4f48d5d2022-05-16 11:23:19 +08002084/**
2085 * d_same_name - compare dentry name with case-exact name
2086 * @parent: parent dentry
2087 * @dentry: the negative dentry that was passed to the parent's lookup func
2088 * @name: the case-exact name to be associated with the returned dentry
2089 *
2090 * Return: true if names are same, or false
2091 */
2092bool d_same_name(const struct dentry *dentry, const struct dentry *parent,
2093 const struct qstr *name)
Linus Torvalds12f8ad42012-05-04 14:59:14 -07002094{
Al Virod4c91a82016-06-25 23:33:49 -04002095 if (likely(!(parent->d_flags & DCACHE_OP_COMPARE))) {
2096 if (dentry->d_name.len != name->len)
2097 return false;
2098 return dentry_cmp(dentry, name->name, name->len) == 0;
Linus Torvalds12f8ad42012-05-04 14:59:14 -07002099 }
Al Viro6fa67e72016-07-31 16:37:25 -04002100 return parent->d_op->d_compare(dentry,
Al Virod4c91a82016-06-25 23:33:49 -04002101 dentry->d_name.len, dentry->d_name.name,
2102 name) == 0;
Linus Torvalds12f8ad42012-05-04 14:59:14 -07002103}
Xiubo Li4f48d5d2022-05-16 11:23:19 +08002104EXPORT_SYMBOL_GPL(d_same_name);
Linus Torvalds12f8ad42012-05-04 14:59:14 -07002105
Linus Torvaldsae2a8232022-08-11 15:29:46 -07002106/*
2107 * This is __d_lookup_rcu() when the parent dentry has
2108 * DCACHE_OP_COMPARE, which makes things much nastier.
2109 */
2110static noinline struct dentry *__d_lookup_rcu_op_compare(
2111 const struct dentry *parent,
2112 const struct qstr *name,
2113 unsigned *seqp)
2114{
2115 u64 hashlen = name->hash_len;
Linus Torvaldse60cc612024-06-10 11:47:06 -07002116 struct hlist_bl_head *b = d_hash(hashlen);
Linus Torvaldsae2a8232022-08-11 15:29:46 -07002117 struct hlist_bl_node *node;
2118 struct dentry *dentry;
2119
2120 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2121 int tlen;
2122 const char *tname;
2123 unsigned seq;
2124
2125seqretry:
2126 seq = raw_seqcount_begin(&dentry->d_seq);
2127 if (dentry->d_parent != parent)
2128 continue;
2129 if (d_unhashed(dentry))
2130 continue;
2131 if (dentry->d_name.hash != hashlen_hash(hashlen))
2132 continue;
2133 tlen = dentry->d_name.len;
2134 tname = dentry->d_name.name;
2135 /* we want a consistent (name,len) pair */
2136 if (read_seqcount_retry(&dentry->d_seq, seq)) {
2137 cpu_relax();
2138 goto seqretry;
2139 }
2140 if (parent->d_op->d_compare(dentry, tlen, tname, name) != 0)
2141 continue;
2142 *seqp = seq;
2143 return dentry;
2144 }
2145 return NULL;
2146}
2147
Linus Torvalds1da177e2005-04-16 15:20:36 -07002148/**
Nick Piggin31e6b012011-01-07 17:49:52 +11002149 * __d_lookup_rcu - search for a dentry (racy, store-free)
2150 * @parent: parent dentry
2151 * @name: qstr of name we wish to find
Randy Dunlap1f1e6e52012-03-18 21:23:05 -07002152 * @seqp: returns d_seq value at the point where the dentry was found
Nick Piggin31e6b012011-01-07 17:49:52 +11002153 * Returns: dentry, or NULL
2154 *
2155 * __d_lookup_rcu is the dcache lookup function for rcu-walk name
2156 * resolution (store-free path walking) design described in
2157 * Documentation/filesystems/path-lookup.txt.
2158 *
2159 * This is not to be used outside core vfs.
2160 *
2161 * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock
2162 * held, and rcu_read_lock held. The returned dentry must not be stored into
2163 * without taking d_lock and checking d_seq sequence count against @seq
2164 * returned here.
2165 *
Linus Torvalds15570082013-09-02 11:38:06 -07002166 * A refcount may be taken on the found dentry with the d_rcu_to_refcount
Nick Piggin31e6b012011-01-07 17:49:52 +11002167 * function.
2168 *
2169 * Alternatively, __d_lookup_rcu may be called again to look up the child of
2170 * the returned dentry, so long as its parent's seqlock is checked after the
2171 * child is looked up. Thus, an interlocking stepping of sequence lock checks
2172 * is formed, giving integrity down the path walk.
Linus Torvalds12f8ad42012-05-04 14:59:14 -07002173 *
2174 * NOTE! The caller *has* to check the resulting dentry against the sequence
2175 * number we've returned before using any of the resulting dentry state!
Nick Piggin31e6b012011-01-07 17:49:52 +11002176 */
Linus Torvalds8966be92012-03-02 14:23:30 -08002177struct dentry *__d_lookup_rcu(const struct dentry *parent,
2178 const struct qstr *name,
Linus Torvaldsda53be12013-05-21 15:22:44 -07002179 unsigned *seqp)
Nick Piggin31e6b012011-01-07 17:49:52 +11002180{
Linus Torvalds26fe5752012-05-10 13:14:12 -07002181 u64 hashlen = name->hash_len;
Nick Piggin31e6b012011-01-07 17:49:52 +11002182 const unsigned char *str = name->name;
Linus Torvaldse60cc612024-06-10 11:47:06 -07002183 struct hlist_bl_head *b = d_hash(hashlen);
Nick Pigginceb5bdc2011-01-07 17:50:05 +11002184 struct hlist_bl_node *node;
Nick Piggin31e6b012011-01-07 17:49:52 +11002185 struct dentry *dentry;
2186
2187 /*
2188 * Note: There is significant duplication with __d_lookup_rcu which is
2189 * required to prevent single threaded performance regressions
2190 * especially on architectures where smp_rmb (in seqcounts) are costly.
2191 * Keep the two functions in sync.
2192 */
2193
Linus Torvaldsae2a8232022-08-11 15:29:46 -07002194 if (unlikely(parent->d_flags & DCACHE_OP_COMPARE))
2195 return __d_lookup_rcu_op_compare(parent, name, seqp);
2196
Nick Piggin31e6b012011-01-07 17:49:52 +11002197 /*
2198 * The hash list is protected using RCU.
2199 *
2200 * Carefully use d_seq when comparing a candidate dentry, to avoid
2201 * races with d_move().
2202 *
2203 * It is possible that concurrent renames can mess up our list
2204 * walk here and result in missing our dentry, resulting in the
2205 * false-negative result. d_lookup() protects against concurrent
2206 * renames using rename_lock seqlock.
2207 *
Namhyung Kimb0a4bb82011-01-22 15:31:32 +09002208 * See Documentation/filesystems/path-lookup.txt for more details.
Nick Piggin31e6b012011-01-07 17:49:52 +11002209 */
Linus Torvaldsb07ad992011-04-23 22:32:03 -07002210 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
Linus Torvalds8966be92012-03-02 14:23:30 -08002211 unsigned seq;
Nick Piggin31e6b012011-01-07 17:49:52 +11002212
Linus Torvalds12f8ad42012-05-04 14:59:14 -07002213 /*
2214 * The dentry sequence count protects us from concurrent
Linus Torvaldsda53be12013-05-21 15:22:44 -07002215 * renames, and thus protects parent and name fields.
Linus Torvalds12f8ad42012-05-04 14:59:14 -07002216 *
2217 * The caller must perform a seqcount check in order
Linus Torvaldsda53be12013-05-21 15:22:44 -07002218 * to do anything useful with the returned dentry.
Linus Torvalds12f8ad42012-05-04 14:59:14 -07002219 *
2220 * NOTE! We do a "raw" seqcount_begin here. That means that
2221 * we don't wait for the sequence count to stabilize if it
2222 * is in the middle of a sequence change. If we do the slow
2223 * dentry compare, we will do seqretries until it is stable,
2224 * and if we end up with a successful lookup, we actually
2225 * want to exit RCU lookup anyway.
Al Virod4c91a82016-06-25 23:33:49 -04002226 *
2227 * Note that raw_seqcount_begin still *does* smp_rmb(), so
2228 * we are still guaranteed NUL-termination of ->d_name.name.
Linus Torvalds12f8ad42012-05-04 14:59:14 -07002229 */
2230 seq = raw_seqcount_begin(&dentry->d_seq);
Nick Piggin31e6b012011-01-07 17:49:52 +11002231 if (dentry->d_parent != parent)
2232 continue;
Linus Torvalds2e321802012-05-21 18:48:10 -07002233 if (d_unhashed(dentry))
2234 continue;
Linus Torvaldsae2a8232022-08-11 15:29:46 -07002235 if (dentry->d_name.hash_len != hashlen)
2236 continue;
2237 if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0)
2238 continue;
Linus Torvaldsda53be12013-05-21 15:22:44 -07002239 *seqp = seq;
Al Virod4c91a82016-06-25 23:33:49 -04002240 return dentry;
Nick Piggin31e6b012011-01-07 17:49:52 +11002241 }
2242 return NULL;
2243}
2244
2245/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07002246 * d_lookup - search for a dentry
2247 * @parent: parent dentry
2248 * @name: qstr of name we wish to find
Nick Pigginb04f7842010-08-18 04:37:34 +10002249 * Returns: dentry, or NULL
Linus Torvalds1da177e2005-04-16 15:20:36 -07002250 *
Nick Pigginb04f7842010-08-18 04:37:34 +10002251 * d_lookup searches the children of the parent dentry for the name in
2252 * question. If the dentry is found its reference count is incremented and the
2253 * dentry is returned. The caller must use dput to free the entry when it has
2254 * finished using it. %NULL is returned if the dentry does not exist.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002255 */
Al Viroda2d8452013-01-24 18:29:34 -05002256struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002257{
Nick Piggin31e6b012011-01-07 17:49:52 +11002258 struct dentry *dentry;
Nick Piggin949854d2011-01-07 17:49:37 +11002259 unsigned seq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002260
Daeseok Younb8314f92014-08-11 11:46:53 +09002261 do {
2262 seq = read_seqbegin(&rename_lock);
2263 dentry = __d_lookup(parent, name);
2264 if (dentry)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002265 break;
2266 } while (read_seqretry(&rename_lock, seq));
2267 return dentry;
2268}
H Hartley Sweetenec4f8602010-01-05 13:45:18 -07002269EXPORT_SYMBOL(d_lookup);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002270
Nick Piggin31e6b012011-01-07 17:49:52 +11002271/**
Nick Pigginb04f7842010-08-18 04:37:34 +10002272 * __d_lookup - search for a dentry (racy)
2273 * @parent: parent dentry
2274 * @name: qstr of name we wish to find
2275 * Returns: dentry, or NULL
2276 *
2277 * __d_lookup is like d_lookup, however it may (rarely) return a
2278 * false-negative result due to unrelated rename activity.
2279 *
2280 * __d_lookup is slightly faster by avoiding rename_lock read seqlock,
2281 * however it must be used carefully, eg. with a following d_lookup in
2282 * the case of failure.
2283 *
2284 * __d_lookup callers must be commented.
2285 */
Al Viroa713ca2a2013-01-24 18:27:00 -05002286struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002287{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002288 unsigned int hash = name->hash;
Linus Torvalds8387ff22016-06-10 07:51:30 -07002289 struct hlist_bl_head *b = d_hash(hash);
Nick Pigginceb5bdc2011-01-07 17:50:05 +11002290 struct hlist_bl_node *node;
Nick Piggin31e6b012011-01-07 17:49:52 +11002291 struct dentry *found = NULL;
Paul E. McKenney665a7582005-11-07 00:59:17 -08002292 struct dentry *dentry;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002293
Nick Pigginb04f7842010-08-18 04:37:34 +10002294 /*
Nick Piggin31e6b012011-01-07 17:49:52 +11002295 * Note: There is significant duplication with __d_lookup_rcu which is
2296 * required to prevent single threaded performance regressions
2297 * especially on architectures where smp_rmb (in seqcounts) are costly.
2298 * Keep the two functions in sync.
2299 */
2300
2301 /*
Nick Pigginb04f7842010-08-18 04:37:34 +10002302 * The hash list is protected using RCU.
2303 *
2304 * Take d_lock when comparing a candidate dentry, to avoid races
2305 * with d_move().
2306 *
2307 * It is possible that concurrent renames can mess up our list
2308 * walk here and result in missing our dentry, resulting in the
2309 * false-negative result. d_lookup() protects against concurrent
2310 * renames using rename_lock seqlock.
2311 *
Namhyung Kimb0a4bb82011-01-22 15:31:32 +09002312 * See Documentation/filesystems/path-lookup.txt for more details.
Nick Pigginb04f7842010-08-18 04:37:34 +10002313 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002314 rcu_read_lock();
2315
Linus Torvaldsb07ad992011-04-23 22:32:03 -07002316 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002317
Linus Torvalds1da177e2005-04-16 15:20:36 -07002318 if (dentry->d_name.hash != hash)
2319 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002320
2321 spin_lock(&dentry->d_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002322 if (dentry->d_parent != parent)
2323 goto next;
Linus Torvaldsd0185c02008-09-29 07:42:57 -07002324 if (d_unhashed(dentry))
2325 goto next;
2326
Al Virod4c91a82016-06-25 23:33:49 -04002327 if (!d_same_name(dentry, parent, name))
2328 goto next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002329
Waiman Long98474232013-08-28 18:24:59 -07002330 dentry->d_lockref.count++;
Linus Torvaldsd0185c02008-09-29 07:42:57 -07002331 found = dentry;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002332 spin_unlock(&dentry->d_lock);
2333 break;
2334next:
2335 spin_unlock(&dentry->d_lock);
2336 }
2337 rcu_read_unlock();
2338
2339 return found;
2340}
2341
2342/**
Eric W. Biederman3e7e2412006-03-31 02:31:43 -08002343 * d_hash_and_lookup - hash the qstr then search for a dentry
2344 * @dir: Directory to search in
2345 * @name: qstr of name we wish to find
2346 *
Al Viro4f522a22013-02-11 23:20:37 -05002347 * On lookup failure NULL is returned; on bad name - ERR_PTR(-error)
Eric W. Biederman3e7e2412006-03-31 02:31:43 -08002348 */
2349struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
2350{
Eric W. Biederman3e7e2412006-03-31 02:31:43 -08002351 /*
2352 * Check for a fs-specific hash function. Note that we must
2353 * calculate the standard hash first, as the d_op->d_hash()
2354 * routine may choose to leave the hash value unchanged.
2355 */
Linus Torvalds8387ff22016-06-10 07:51:30 -07002356 name->hash = full_name_hash(dir, name->name, name->len);
Nick Pigginfb045ad2011-01-07 17:49:55 +11002357 if (dir->d_flags & DCACHE_OP_HASH) {
Linus Torvaldsda53be12013-05-21 15:22:44 -07002358 int err = dir->d_op->d_hash(dir, name);
Al Viro4f522a22013-02-11 23:20:37 -05002359 if (unlikely(err < 0))
2360 return ERR_PTR(err);
Eric W. Biederman3e7e2412006-03-31 02:31:43 -08002361 }
Al Viro4f522a22013-02-11 23:20:37 -05002362 return d_lookup(dir, name);
Eric W. Biederman3e7e2412006-03-31 02:31:43 -08002363}
Al Viro4f522a22013-02-11 23:20:37 -05002364EXPORT_SYMBOL(d_hash_and_lookup);
Eric W. Biederman3e7e2412006-03-31 02:31:43 -08002365
Linus Torvalds1da177e2005-04-16 15:20:36 -07002366/*
2367 * When a file is deleted, we have two options:
2368 * - turn this dentry into a negative dentry
2369 * - unhash this dentry and free it.
2370 *
2371 * Usually, we want to just turn this into
Linus Torvalds4a4be1a2024-05-29 09:39:34 -07002372 * a negative dentry, but if anybody else is
2373 * currently using the dentry or the inode
2374 * we can't do that and we fall back on removing
2375 * it from the hash queues and waiting for
2376 * it to be deleted later when it has no users
Linus Torvalds1da177e2005-04-16 15:20:36 -07002377 */
2378
2379/**
2380 * d_delete - delete a dentry
2381 * @dentry: The dentry to delete
2382 *
Linus Torvalds4a4be1a2024-05-29 09:39:34 -07002383 * Turn the dentry into a negative dentry if possible, otherwise
2384 * remove it from the hash queues so it can be deleted later
Linus Torvalds1da177e2005-04-16 15:20:36 -07002385 */
2386
2387void d_delete(struct dentry * dentry)
2388{
Al Viroc19457f2018-02-23 21:02:31 -05002389 struct inode *inode = dentry->d_inode;
Al Viroc19457f2018-02-23 21:02:31 -05002390
2391 spin_lock(&inode->i_lock);
2392 spin_lock(&dentry->d_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002393 /*
2394 * Are we the only user?
2395 */
Waiman Long98474232013-08-28 18:24:59 -07002396 if (dentry->d_lockref.count == 1) {
Al Viro13e3c5e2010-05-21 16:11:04 -04002397 dentry->d_flags &= ~DCACHE_CANT_MOUNT;
Nick Piggin31e6b012011-01-07 17:49:52 +11002398 dentry_unlink_inode(dentry);
Al Viroc19457f2018-02-23 21:02:31 -05002399 } else {
Linus Torvalds4a4be1a2024-05-29 09:39:34 -07002400 __d_drop(dentry);
Al Viroc19457f2018-02-23 21:02:31 -05002401 spin_unlock(&dentry->d_lock);
2402 spin_unlock(&inode->i_lock);
2403 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002404}
H Hartley Sweetenec4f8602010-01-05 13:45:18 -07002405EXPORT_SYMBOL(d_delete);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002406
Al Viro15d3c582016-07-29 17:45:21 -04002407static void __d_rehash(struct dentry *entry)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002408{
Al Viro15d3c582016-07-29 17:45:21 -04002409 struct hlist_bl_head *b = d_hash(entry->d_name.hash);
NeilBrown61647822017-11-10 15:45:41 +11002410
Christoph Hellwig1879fd62011-04-25 14:01:36 -04002411 hlist_bl_lock(b);
Linus Torvaldsb07ad992011-04-23 22:32:03 -07002412 hlist_bl_add_head_rcu(&entry->d_hash, b);
Christoph Hellwig1879fd62011-04-25 14:01:36 -04002413 hlist_bl_unlock(b);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002414}
2415
2416/**
2417 * d_rehash - add an entry back to the hash
2418 * @entry: dentry to add to the hash
2419 *
2420 * Adds a dentry to the hash according to its name.
2421 */
2422
2423void d_rehash(struct dentry * entry)
2424{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002425 spin_lock(&entry->d_lock);
Al Viro15d3c582016-07-29 17:45:21 -04002426 __d_rehash(entry);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002427 spin_unlock(&entry->d_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002428}
H Hartley Sweetenec4f8602010-01-05 13:45:18 -07002429EXPORT_SYMBOL(d_rehash);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002430
Al Viro84e710d2016-04-15 00:58:55 -04002431static inline unsigned start_dir_add(struct inode *dir)
2432{
Thomas Gleixner93f6d4e2022-08-25 18:41:25 +02002433 preempt_disable_nested();
Al Viro84e710d2016-04-15 00:58:55 -04002434 for (;;) {
2435 unsigned n = dir->i_dir_seq;
2436 if (!(n & 1) && cmpxchg(&dir->i_dir_seq, n, n + 1) == n)
2437 return n;
2438 cpu_relax();
2439 }
2440}
2441
Sebastian Andrzej Siewior50417d22022-07-27 13:49:04 +02002442static inline void end_dir_add(struct inode *dir, unsigned int n,
2443 wait_queue_head_t *d_wait)
Al Viro84e710d2016-04-15 00:58:55 -04002444{
2445 smp_store_release(&dir->i_dir_seq, n + 2);
Thomas Gleixner93f6d4e2022-08-25 18:41:25 +02002446 preempt_enable_nested();
Sebastian Andrzej Siewior50417d22022-07-27 13:49:04 +02002447 wake_up_all(d_wait);
Al Viro84e710d2016-04-15 00:58:55 -04002448}
2449
Al Virod9171b92016-04-15 03:33:13 -04002450static void d_wait_lookup(struct dentry *dentry)
2451{
2452 if (d_in_lookup(dentry)) {
2453 DECLARE_WAITQUEUE(wait, current);
2454 add_wait_queue(dentry->d_wait, &wait);
2455 do {
2456 set_current_state(TASK_UNINTERRUPTIBLE);
2457 spin_unlock(&dentry->d_lock);
2458 schedule();
2459 spin_lock(&dentry->d_lock);
2460 } while (d_in_lookup(dentry));
2461 }
2462}
2463
Al Viro94bdd652016-04-15 02:42:04 -04002464struct dentry *d_alloc_parallel(struct dentry *parent,
Al Virod9171b92016-04-15 03:33:13 -04002465 const struct qstr *name,
2466 wait_queue_head_t *wq)
Al Viro94bdd652016-04-15 02:42:04 -04002467{
Al Viro94bdd652016-04-15 02:42:04 -04002468 unsigned int hash = name->hash;
Al Viro94bdd652016-04-15 02:42:04 -04002469 struct hlist_bl_head *b = in_lookup_hash(parent, hash);
2470 struct hlist_bl_node *node;
2471 struct dentry *new = d_alloc(parent, name);
2472 struct dentry *dentry;
2473 unsigned seq, r_seq, d_seq;
2474
2475 if (unlikely(!new))
2476 return ERR_PTR(-ENOMEM);
2477
2478retry:
2479 rcu_read_lock();
Will Deacon015555f2018-02-19 14:55:54 +00002480 seq = smp_load_acquire(&parent->d_inode->i_dir_seq);
Al Viro94bdd652016-04-15 02:42:04 -04002481 r_seq = read_seqbegin(&rename_lock);
2482 dentry = __d_lookup_rcu(parent, name, &d_seq);
2483 if (unlikely(dentry)) {
2484 if (!lockref_get_not_dead(&dentry->d_lockref)) {
2485 rcu_read_unlock();
2486 goto retry;
2487 }
2488 if (read_seqcount_retry(&dentry->d_seq, d_seq)) {
2489 rcu_read_unlock();
2490 dput(dentry);
2491 goto retry;
2492 }
2493 rcu_read_unlock();
2494 dput(new);
2495 return dentry;
2496 }
2497 if (unlikely(read_seqretry(&rename_lock, r_seq))) {
2498 rcu_read_unlock();
2499 goto retry;
2500 }
Will Deacon015555f2018-02-19 14:55:54 +00002501
2502 if (unlikely(seq & 1)) {
2503 rcu_read_unlock();
2504 goto retry;
2505 }
2506
Al Viro94bdd652016-04-15 02:42:04 -04002507 hlist_bl_lock(b);
Will Deacon8cc07c82018-02-19 14:55:55 +00002508 if (unlikely(READ_ONCE(parent->d_inode->i_dir_seq) != seq)) {
Al Viro94bdd652016-04-15 02:42:04 -04002509 hlist_bl_unlock(b);
2510 rcu_read_unlock();
2511 goto retry;
2512 }
Al Viro94bdd652016-04-15 02:42:04 -04002513 /*
2514 * No changes for the parent since the beginning of d_lookup().
2515 * Since all removals from the chain happen with hlist_bl_lock(),
2516 * any potential in-lookup matches are going to stay here until
2517 * we unlock the chain. All fields are stable in everything
2518 * we encounter.
2519 */
2520 hlist_bl_for_each_entry(dentry, node, b, d_u.d_in_lookup_hash) {
2521 if (dentry->d_name.hash != hash)
2522 continue;
2523 if (dentry->d_parent != parent)
2524 continue;
Al Virod4c91a82016-06-25 23:33:49 -04002525 if (!d_same_name(dentry, parent, name))
2526 continue;
Al Viro94bdd652016-04-15 02:42:04 -04002527 hlist_bl_unlock(b);
Al Viroe7d6ef92016-06-20 01:35:59 -04002528 /* now we can try to grab a reference */
2529 if (!lockref_get_not_dead(&dentry->d_lockref)) {
2530 rcu_read_unlock();
2531 goto retry;
2532 }
2533
2534 rcu_read_unlock();
2535 /*
2536 * somebody is likely to be still doing lookup for it;
2537 * wait for them to finish
2538 */
Al Virod9171b92016-04-15 03:33:13 -04002539 spin_lock(&dentry->d_lock);
2540 d_wait_lookup(dentry);
2541 /*
2542 * it's not in-lookup anymore; in principle we should repeat
2543 * everything from dcache lookup, but it's likely to be what
2544 * d_lookup() would've found anyway. If it is, just return it;
2545 * otherwise we really have to repeat the whole thing.
2546 */
2547 if (unlikely(dentry->d_name.hash != hash))
2548 goto mismatch;
2549 if (unlikely(dentry->d_parent != parent))
2550 goto mismatch;
2551 if (unlikely(d_unhashed(dentry)))
2552 goto mismatch;
Al Virod4c91a82016-06-25 23:33:49 -04002553 if (unlikely(!d_same_name(dentry, parent, name)))
2554 goto mismatch;
Al Virod9171b92016-04-15 03:33:13 -04002555 /* OK, it *is* a hashed match; return it */
2556 spin_unlock(&dentry->d_lock);
Al Viro94bdd652016-04-15 02:42:04 -04002557 dput(new);
2558 return dentry;
2559 }
Al Viroe7d6ef92016-06-20 01:35:59 -04002560 rcu_read_unlock();
Al Viro94bdd652016-04-15 02:42:04 -04002561 /* we can't take ->d_lock here; it's OK, though. */
2562 new->d_flags |= DCACHE_PAR_LOOKUP;
Al Virod9171b92016-04-15 03:33:13 -04002563 new->d_wait = wq;
Al Virof9f677c2023-11-12 00:38:02 -05002564 hlist_bl_add_head(&new->d_u.d_in_lookup_hash, b);
Al Viro94bdd652016-04-15 02:42:04 -04002565 hlist_bl_unlock(b);
2566 return new;
Al Virod9171b92016-04-15 03:33:13 -04002567mismatch:
2568 spin_unlock(&dentry->d_lock);
2569 dput(dentry);
2570 goto retry;
Al Viro94bdd652016-04-15 02:42:04 -04002571}
2572EXPORT_SYMBOL(d_alloc_parallel);
2573
Sebastian Andrzej Siewior45f78b02022-07-27 13:49:03 +02002574/*
2575 * - Unhash the dentry
2576 * - Retrieve and clear the waitqueue head in dentry
2577 * - Return the waitqueue head
2578 */
2579static wait_queue_head_t *__d_lookup_unhash(struct dentry *dentry)
Al Viro85c7f812016-04-14 19:52:13 -04002580{
Sebastian Andrzej Siewior45f78b02022-07-27 13:49:03 +02002581 wait_queue_head_t *d_wait;
2582 struct hlist_bl_head *b;
2583
2584 lockdep_assert_held(&dentry->d_lock);
2585
2586 b = in_lookup_hash(dentry->d_parent, dentry->d_name.hash);
Al Viro94bdd652016-04-15 02:42:04 -04002587 hlist_bl_lock(b);
Al Viro85c7f812016-04-14 19:52:13 -04002588 dentry->d_flags &= ~DCACHE_PAR_LOOKUP;
Al Viro94bdd652016-04-15 02:42:04 -04002589 __hlist_bl_del(&dentry->d_u.d_in_lookup_hash);
Sebastian Andrzej Siewior45f78b02022-07-27 13:49:03 +02002590 d_wait = dentry->d_wait;
Al Virod9171b92016-04-15 03:33:13 -04002591 dentry->d_wait = NULL;
Al Viro94bdd652016-04-15 02:42:04 -04002592 hlist_bl_unlock(b);
2593 INIT_HLIST_NODE(&dentry->d_u.d_alias);
Al Virod9171b92016-04-15 03:33:13 -04002594 INIT_LIST_HEAD(&dentry->d_lru);
Sebastian Andrzej Siewior45f78b02022-07-27 13:49:03 +02002595 return d_wait;
Al Viro85c7f812016-04-14 19:52:13 -04002596}
Sebastian Andrzej Siewior45f78b02022-07-27 13:49:03 +02002597
2598void __d_lookup_unhash_wake(struct dentry *dentry)
2599{
2600 spin_lock(&dentry->d_lock);
2601 wake_up_all(__d_lookup_unhash(dentry));
2602 spin_unlock(&dentry->d_lock);
2603}
2604EXPORT_SYMBOL(__d_lookup_unhash_wake);
Al Viroed782b52016-03-09 19:52:39 -05002605
2606/* inode->i_lock held if inode is non-NULL */
2607
2608static inline void __d_add(struct dentry *dentry, struct inode *inode)
2609{
Sebastian Andrzej Siewior45f78b02022-07-27 13:49:03 +02002610 wait_queue_head_t *d_wait;
Al Viro84e710d2016-04-15 00:58:55 -04002611 struct inode *dir = NULL;
2612 unsigned n;
Al Viro0568d702016-04-14 19:40:56 -04002613 spin_lock(&dentry->d_lock);
Al Viro84e710d2016-04-15 00:58:55 -04002614 if (unlikely(d_in_lookup(dentry))) {
2615 dir = dentry->d_parent->d_inode;
2616 n = start_dir_add(dir);
Sebastian Andrzej Siewior45f78b02022-07-27 13:49:03 +02002617 d_wait = __d_lookup_unhash(dentry);
Al Viro84e710d2016-04-15 00:58:55 -04002618 }
Al Viroed782b52016-03-09 19:52:39 -05002619 if (inode) {
Al Viro0568d702016-04-14 19:40:56 -04002620 unsigned add_flags = d_flags_for_inode(inode);
2621 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
2622 raw_write_seqcount_begin(&dentry->d_seq);
2623 __d_set_inode_and_type(dentry, inode, add_flags);
2624 raw_write_seqcount_end(&dentry->d_seq);
Al Viroaffda482016-05-29 18:35:12 -04002625 fsnotify_update_flags(dentry);
Al Viroed782b52016-03-09 19:52:39 -05002626 }
Al Viro15d3c582016-07-29 17:45:21 -04002627 __d_rehash(dentry);
Al Viro84e710d2016-04-15 00:58:55 -04002628 if (dir)
Sebastian Andrzej Siewior50417d22022-07-27 13:49:04 +02002629 end_dir_add(dir, n, d_wait);
Al Viro0568d702016-04-14 19:40:56 -04002630 spin_unlock(&dentry->d_lock);
2631 if (inode)
2632 spin_unlock(&inode->i_lock);
Al Viroed782b52016-03-09 19:52:39 -05002633}
2634
Nick Pigginfb2d5b82011-01-07 17:49:26 +11002635/**
Al Viro34d0d192016-03-08 21:01:03 -05002636 * d_add - add dentry to hash queues
2637 * @entry: dentry to add
2638 * @inode: The inode to attach to this dentry
2639 *
2640 * This adds the entry to the hash queues and initializes @inode.
2641 * The entry was actually filled in earlier during d_alloc().
2642 */
2643
2644void d_add(struct dentry *entry, struct inode *inode)
2645{
Al Virob9680912016-04-11 00:53:26 -04002646 if (inode) {
2647 security_d_instantiate(entry, inode);
Al Viroed782b52016-03-09 19:52:39 -05002648 spin_lock(&inode->i_lock);
Al Virob9680912016-04-11 00:53:26 -04002649 }
Al Viroed782b52016-03-09 19:52:39 -05002650 __d_add(entry, inode);
Al Viro34d0d192016-03-08 21:01:03 -05002651}
2652EXPORT_SYMBOL(d_add);
2653
2654/**
Al Viro668d0cd2016-03-08 12:44:17 -05002655 * d_exact_alias - find and hash an exact unhashed alias
2656 * @entry: dentry to add
2657 * @inode: The inode to go with this dentry
2658 *
2659 * If an unhashed dentry with the same name/parent and desired
2660 * inode already exists, hash and return it. Otherwise, return
2661 * NULL.
2662 *
2663 * Parent directory should be locked.
2664 */
2665struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode)
2666{
2667 struct dentry *alias;
Al Viro668d0cd2016-03-08 12:44:17 -05002668 unsigned int hash = entry->d_name.hash;
2669
2670 spin_lock(&inode->i_lock);
2671 hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
2672 /*
2673 * Don't need alias->d_lock here, because aliases with
2674 * d_parent == entry->d_parent are not subject to name or
2675 * parent changes, because the parent inode i_mutex is held.
2676 */
2677 if (alias->d_name.hash != hash)
2678 continue;
2679 if (alias->d_parent != entry->d_parent)
2680 continue;
Al Virod4c91a82016-06-25 23:33:49 -04002681 if (!d_same_name(alias, entry->d_parent, &entry->d_name))
Al Viro668d0cd2016-03-08 12:44:17 -05002682 continue;
2683 spin_lock(&alias->d_lock);
2684 if (!d_unhashed(alias)) {
2685 spin_unlock(&alias->d_lock);
2686 alias = NULL;
2687 } else {
Vegard Nossum1b6ae9f2023-11-06 14:44:17 +01002688 dget_dlock(alias);
Al Viro15d3c582016-07-29 17:45:21 -04002689 __d_rehash(alias);
Al Viro668d0cd2016-03-08 12:44:17 -05002690 spin_unlock(&alias->d_lock);
2691 }
2692 spin_unlock(&inode->i_lock);
2693 return alias;
2694 }
2695 spin_unlock(&inode->i_lock);
2696 return NULL;
2697}
2698EXPORT_SYMBOL(d_exact_alias);
2699
Al Viro8d85b482014-09-29 14:54:27 -04002700static void swap_names(struct dentry *dentry, struct dentry *target)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002701{
Al Viro8d85b482014-09-29 14:54:27 -04002702 if (unlikely(dname_external(target))) {
2703 if (unlikely(dname_external(dentry))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002704 /*
2705 * Both external: swap the pointers
2706 */
Wu Fengguang9a8d5bb2009-01-07 18:09:14 -08002707 swap(target->d_name.name, dentry->d_name.name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002708 } else {
2709 /*
2710 * dentry:internal, target:external. Steal target's
2711 * storage and make target internal.
2712 */
J. Bruce Fields321bcf92007-10-21 16:41:38 -07002713 memcpy(target->d_iname, dentry->d_name.name,
2714 dentry->d_name.len + 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002715 dentry->d_name.name = target->d_name.name;
2716 target->d_name.name = target->d_iname;
2717 }
2718 } else {
Al Viro8d85b482014-09-29 14:54:27 -04002719 if (unlikely(dname_external(dentry))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002720 /*
2721 * dentry:external, target:internal. Give dentry's
2722 * storage to target and make dentry internal
2723 */
2724 memcpy(dentry->d_iname, target->d_name.name,
2725 target->d_name.len + 1);
2726 target->d_name.name = dentry->d_name.name;
2727 dentry->d_name.name = dentry->d_iname;
2728 } else {
2729 /*
Miklos Szeredida1ce062014-04-01 17:08:43 +02002730 * Both are internal.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002731 */
Miklos Szeredida1ce062014-04-01 17:08:43 +02002732 unsigned int i;
2733 BUILD_BUG_ON(!IS_ALIGNED(DNAME_INLINE_LEN, sizeof(long)));
2734 for (i = 0; i < DNAME_INLINE_LEN / sizeof(long); i++) {
2735 swap(((long *) &dentry->d_iname)[i],
2736 ((long *) &target->d_iname)[i]);
2737 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002738 }
2739 }
Linus Torvaldsa28ddb82014-09-24 12:27:39 -07002740 swap(dentry->d_name.hash_len, target->d_name.hash_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002741}
2742
Al Viro8d85b482014-09-29 14:54:27 -04002743static void copy_name(struct dentry *dentry, struct dentry *target)
2744{
2745 struct external_name *old_name = NULL;
2746 if (unlikely(dname_external(dentry)))
2747 old_name = external_name(dentry);
2748 if (unlikely(dname_external(target))) {
2749 atomic_inc(&external_name(target)->u.count);
2750 dentry->d_name = target->d_name;
2751 } else {
2752 memcpy(dentry->d_iname, target->d_name.name,
2753 target->d_name.len + 1);
2754 dentry->d_name.name = dentry->d_iname;
2755 dentry->d_name.hash_len = target->d_name.hash_len;
2756 }
2757 if (old_name && likely(atomic_dec_and_test(&old_name->u.count)))
Vlastimil Babka2e03b4b2018-10-26 15:05:41 -07002758 kfree_rcu(old_name, u.head);
Al Viro8d85b482014-09-29 14:54:27 -04002759}
2760
Trond Myklebust9eaef272006-10-21 10:24:20 -07002761/*
Al Viro18367502011-07-12 21:42:24 -04002762 * __d_move - move a dentry
Linus Torvalds1da177e2005-04-16 15:20:36 -07002763 * @dentry: entry to move
2764 * @target: new dentry
Miklos Szeredida1ce062014-04-01 17:08:43 +02002765 * @exchange: exchange the two dentries
Linus Torvalds1da177e2005-04-16 15:20:36 -07002766 *
2767 * Update the dcache to reflect the move of a file name. Negative
Jeff Laytonc46c8872011-07-26 13:33:16 -04002768 * dcache entries should not be moved in this way. Caller must hold
2769 * rename_lock, the i_mutex of the source and target directories,
2770 * and the sb->s_vfs_rename_mutex if they differ. See lock_rename().
Linus Torvalds1da177e2005-04-16 15:20:36 -07002771 */
Miklos Szeredida1ce062014-04-01 17:08:43 +02002772static void __d_move(struct dentry *dentry, struct dentry *target,
2773 bool exchange)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002774{
Al Viro42177002018-03-11 15:15:46 -04002775 struct dentry *old_parent, *p;
Sebastian Andrzej Siewior45f78b02022-07-27 13:49:03 +02002776 wait_queue_head_t *d_wait;
Al Viro84e710d2016-04-15 00:58:55 -04002777 struct inode *dir = NULL;
2778 unsigned n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002779
Al Viro42177002018-03-11 15:15:46 -04002780 WARN_ON(!dentry->d_inode);
2781 if (WARN_ON(dentry == target))
2782 return;
2783
Nick Piggin2fd6b7f2011-01-07 17:49:34 +11002784 BUG_ON(d_ancestor(target, dentry));
Al Viro42177002018-03-11 15:15:46 -04002785 old_parent = dentry->d_parent;
2786 p = d_ancestor(old_parent, target);
2787 if (IS_ROOT(dentry)) {
2788 BUG_ON(p);
2789 spin_lock(&target->d_parent->d_lock);
2790 } else if (!p) {
2791 /* target is not a descendent of dentry->d_parent */
2792 spin_lock(&target->d_parent->d_lock);
2793 spin_lock_nested(&old_parent->d_lock, DENTRY_D_LOCK_NESTED);
2794 } else {
2795 BUG_ON(p == dentry);
2796 spin_lock(&old_parent->d_lock);
2797 if (p != target)
2798 spin_lock_nested(&target->d_parent->d_lock,
2799 DENTRY_D_LOCK_NESTED);
2800 }
2801 spin_lock_nested(&dentry->d_lock, 2);
2802 spin_lock_nested(&target->d_lock, 3);
Nick Piggin2fd6b7f2011-01-07 17:49:34 +11002803
Al Viro84e710d2016-04-15 00:58:55 -04002804 if (unlikely(d_in_lookup(target))) {
2805 dir = target->d_parent->d_inode;
2806 n = start_dir_add(dir);
Sebastian Andrzej Siewior45f78b02022-07-27 13:49:03 +02002807 d_wait = __d_lookup_unhash(target);
Al Viro84e710d2016-04-15 00:58:55 -04002808 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002809
Nick Piggin31e6b012011-01-07 17:49:52 +11002810 write_seqcount_begin(&dentry->d_seq);
John Stultz1ca7d672013-10-07 15:51:59 -07002811 write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
Nick Piggin31e6b012011-01-07 17:49:52 +11002812
Al Viro15d3c582016-07-29 17:45:21 -04002813 /* unhash both */
Al Viro0632a9a2018-03-07 00:49:10 -05002814 if (!d_unhashed(dentry))
2815 ___d_drop(dentry);
2816 if (!d_unhashed(target))
2817 ___d_drop(target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002818
Al Viro63cf4272014-09-26 23:06:14 -04002819 /* ... and switch them in the tree */
Al Viro076515f2018-03-10 23:15:52 -05002820 dentry->d_parent = target->d_parent;
2821 if (!exchange) {
2822 copy_name(dentry, target);
2823 target->d_hash.pprev = NULL;
2824 dentry->d_parent->d_lockref.count++;
Al Viro5467a682019-03-15 22:23:19 -04002825 if (dentry != old_parent) /* wasn't IS_ROOT */
Al Viro076515f2018-03-10 23:15:52 -05002826 WARN_ON(!--old_parent->d_lockref.count);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002827 } else {
Al Viro076515f2018-03-10 23:15:52 -05002828 target->d_parent = old_parent;
2829 swap_names(dentry, target);
Al Viroda549bd2023-11-07 02:00:39 -05002830 if (!hlist_unhashed(&target->d_sib))
2831 __hlist_del(&target->d_sib);
2832 hlist_add_head(&target->d_sib, &target->d_parent->d_children);
Al Viro076515f2018-03-10 23:15:52 -05002833 __d_rehash(target);
2834 fsnotify_update_flags(target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002835 }
Al Viroda549bd2023-11-07 02:00:39 -05002836 if (!hlist_unhashed(&dentry->d_sib))
2837 __hlist_del(&dentry->d_sib);
2838 hlist_add_head(&dentry->d_sib, &dentry->d_parent->d_children);
Al Viro076515f2018-03-10 23:15:52 -05002839 __d_rehash(dentry);
2840 fsnotify_update_flags(dentry);
Eric Biggers0bf3d5c12019-03-20 11:39:11 -07002841 fscrypt_handle_d_move(dentry);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002842
Nick Piggin31e6b012011-01-07 17:49:52 +11002843 write_seqcount_end(&target->d_seq);
2844 write_seqcount_end(&dentry->d_seq);
2845
Al Viro84e710d2016-04-15 00:58:55 -04002846 if (dir)
Sebastian Andrzej Siewior50417d22022-07-27 13:49:04 +02002847 end_dir_add(dir, n, d_wait);
Al Viro076515f2018-03-10 23:15:52 -05002848
2849 if (dentry->d_parent != old_parent)
2850 spin_unlock(&dentry->d_parent->d_lock);
2851 if (dentry != old_parent)
2852 spin_unlock(&old_parent->d_lock);
2853 spin_unlock(&target->d_lock);
2854 spin_unlock(&dentry->d_lock);
Al Viro18367502011-07-12 21:42:24 -04002855}
2856
2857/*
2858 * d_move - move a dentry
2859 * @dentry: entry to move
2860 * @target: new dentry
2861 *
2862 * Update the dcache to reflect the move of a file name. Negative
Jeff Laytonc46c8872011-07-26 13:33:16 -04002863 * dcache entries should not be moved in this way. See the locking
2864 * requirements for __d_move.
Al Viro18367502011-07-12 21:42:24 -04002865 */
2866void d_move(struct dentry *dentry, struct dentry *target)
2867{
2868 write_seqlock(&rename_lock);
Miklos Szeredida1ce062014-04-01 17:08:43 +02002869 __d_move(dentry, target, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002870 write_sequnlock(&rename_lock);
Trond Myklebust9eaef272006-10-21 10:24:20 -07002871}
H Hartley Sweetenec4f8602010-01-05 13:45:18 -07002872EXPORT_SYMBOL(d_move);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002873
Miklos Szeredida1ce062014-04-01 17:08:43 +02002874/*
2875 * d_exchange - exchange two dentries
2876 * @dentry1: first dentry
2877 * @dentry2: second dentry
2878 */
2879void d_exchange(struct dentry *dentry1, struct dentry *dentry2)
2880{
2881 write_seqlock(&rename_lock);
2882
2883 WARN_ON(!dentry1->d_inode);
2884 WARN_ON(!dentry2->d_inode);
2885 WARN_ON(IS_ROOT(dentry1));
2886 WARN_ON(IS_ROOT(dentry2));
2887
2888 __d_move(dentry1, dentry2, true);
2889
2890 write_sequnlock(&rename_lock);
2891}
2892
OGAWA Hirofumie2761a12008-10-16 07:50:28 +09002893/**
2894 * d_ancestor - search for an ancestor
2895 * @p1: ancestor dentry
2896 * @p2: child dentry
2897 *
2898 * Returns the ancestor dentry of p2 which is a child of p1, if p1 is
2899 * an ancestor of p2, else NULL.
Trond Myklebust9eaef272006-10-21 10:24:20 -07002900 */
OGAWA Hirofumie2761a12008-10-16 07:50:28 +09002901struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
Trond Myklebust9eaef272006-10-21 10:24:20 -07002902{
2903 struct dentry *p;
2904
OGAWA Hirofumi871c0062008-10-16 07:50:27 +09002905 for (p = p2; !IS_ROOT(p); p = p->d_parent) {
Trond Myklebust9eaef272006-10-21 10:24:20 -07002906 if (p->d_parent == p1)
OGAWA Hirofumie2761a12008-10-16 07:50:28 +09002907 return p;
Trond Myklebust9eaef272006-10-21 10:24:20 -07002908 }
OGAWA Hirofumie2761a12008-10-16 07:50:28 +09002909 return NULL;
Trond Myklebust9eaef272006-10-21 10:24:20 -07002910}
2911
2912/*
2913 * This helper attempts to cope with remotely renamed directories
2914 *
2915 * It assumes that the caller is already holding
Eric W. Biedermana03e2832015-08-15 13:36:41 -05002916 * dentry->d_parent->d_inode->i_mutex, and rename_lock
Trond Myklebust9eaef272006-10-21 10:24:20 -07002917 *
2918 * Note: If ever the locking in lock_rename() changes, then please
2919 * remember to update this too...
Trond Myklebust9eaef272006-10-21 10:24:20 -07002920 */
Al Viroef69f052023-11-23 18:11:00 -05002921static int __d_unalias(struct dentry *dentry, struct dentry *alias)
Trond Myklebust9eaef272006-10-21 10:24:20 -07002922{
Al Viro9902af72016-04-15 15:08:36 -04002923 struct mutex *m1 = NULL;
2924 struct rw_semaphore *m2 = NULL;
J. Bruce Fields3d330dc2015-02-10 10:55:53 -05002925 int ret = -ESTALE;
Trond Myklebust9eaef272006-10-21 10:24:20 -07002926
2927 /* If alias and dentry share a parent, then no extra locks required */
2928 if (alias->d_parent == dentry->d_parent)
2929 goto out_unalias;
2930
Trond Myklebust9eaef272006-10-21 10:24:20 -07002931 /* See lock_rename() */
Trond Myklebust9eaef272006-10-21 10:24:20 -07002932 if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
2933 goto out_err;
2934 m1 = &dentry->d_sb->s_vfs_rename_mutex;
Al Viro9902af72016-04-15 15:08:36 -04002935 if (!inode_trylock_shared(alias->d_parent->d_inode))
Trond Myklebust9eaef272006-10-21 10:24:20 -07002936 goto out_err;
Al Viro9902af72016-04-15 15:08:36 -04002937 m2 = &alias->d_parent->d_inode->i_rwsem;
Trond Myklebust9eaef272006-10-21 10:24:20 -07002938out_unalias:
Eric W. Biederman8ed936b2013-10-01 18:33:48 -07002939 __d_move(alias, dentry, false);
Al Virob5ae6b12014-10-12 22:16:02 -04002940 ret = 0;
Trond Myklebust9eaef272006-10-21 10:24:20 -07002941out_err:
Trond Myklebust9eaef272006-10-21 10:24:20 -07002942 if (m2)
Al Viro9902af72016-04-15 15:08:36 -04002943 up_read(m2);
Trond Myklebust9eaef272006-10-21 10:24:20 -07002944 if (m1)
2945 mutex_unlock(m1);
2946 return ret;
2947}
2948
David Howells770bfad2006-08-22 20:06:07 -04002949/**
J. Bruce Fields3f70bd52014-02-18 14:11:26 -05002950 * d_splice_alias - splice a disconnected dentry into the tree if one exists
2951 * @inode: the inode which may have a disconnected dentry
2952 * @dentry: a negative dentry which we want to point to the inode.
2953 *
J. Bruce Fieldsda093a92014-02-17 18:03:57 -05002954 * If inode is a directory and has an IS_ROOT alias, then d_move that in
2955 * place of the given dentry and return it, else simply d_add the inode
2956 * to the dentry and return NULL.
J. Bruce Fields3f70bd52014-02-18 14:11:26 -05002957 *
J. Bruce Fields908790f2014-02-17 17:58:42 -05002958 * If a non-IS_ROOT directory is found, the filesystem is corrupt, and
2959 * we should error out: directories can't have multiple aliases.
2960 *
J. Bruce Fields3f70bd52014-02-18 14:11:26 -05002961 * This is needed in the lookup routine of any filesystem that is exportable
2962 * (via knfsd) so that we can build dcache paths to directories effectively.
2963 *
2964 * If a dentry was found and moved, then it is returned. Otherwise NULL
2965 * is returned. This matches the expected return value of ->lookup.
2966 *
2967 * Cluster filesystems may call this function with a negative, hashed dentry.
2968 * In that case, we know that the inode will be a regular file, and also this
2969 * will only occur during atomic_open. So we need to check for the dentry
2970 * being already hashed only in the final case.
2971 */
2972struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
2973{
J. Bruce Fields3f70bd52014-02-18 14:11:26 -05002974 if (IS_ERR(inode))
2975 return ERR_CAST(inode);
2976
David Howells770bfad2006-08-22 20:06:07 -04002977 BUG_ON(!d_unhashed(dentry));
2978
Al Virode689f52016-03-09 18:05:42 -05002979 if (!inode)
Al Virob5ae6b12014-10-12 22:16:02 -04002980 goto out;
Al Virode689f52016-03-09 18:05:42 -05002981
Al Virob9680912016-04-11 00:53:26 -04002982 security_d_instantiate(dentry, inode);
Nick Piggin873feea2011-01-07 17:50:06 +11002983 spin_lock(&inode->i_lock);
Trond Myklebust9eaef272006-10-21 10:24:20 -07002984 if (S_ISDIR(inode->i_mode)) {
Al Virob5ae6b12014-10-12 22:16:02 -04002985 struct dentry *new = __d_find_any_alias(inode);
2986 if (unlikely(new)) {
Eric W. Biedermana03e2832015-08-15 13:36:41 -05002987 /* The reference to new ensures it remains an alias */
2988 spin_unlock(&inode->i_lock);
Al Viro18367502011-07-12 21:42:24 -04002989 write_seqlock(&rename_lock);
Al Virob5ae6b12014-10-12 22:16:02 -04002990 if (unlikely(d_ancestor(new, dentry))) {
Al Viro18367502011-07-12 21:42:24 -04002991 write_sequnlock(&rename_lock);
Al Virob5ae6b12014-10-12 22:16:02 -04002992 dput(new);
2993 new = ERR_PTR(-ELOOP);
2994 pr_warn_ratelimited(
2995 "VFS: Lookup of '%s' in %s %s"
2996 " would have caused loop\n",
2997 dentry->d_name.name,
2998 inode->i_sb->s_type->name,
2999 inode->i_sb->s_id);
3000 } else if (!IS_ROOT(new)) {
Al Viro076515f2018-03-10 23:15:52 -05003001 struct dentry *old_parent = dget(new->d_parent);
Al Viroef69f052023-11-23 18:11:00 -05003002 int err = __d_unalias(dentry, new);
Al Virob5ae6b12014-10-12 22:16:02 -04003003 write_sequnlock(&rename_lock);
3004 if (err) {
3005 dput(new);
3006 new = ERR_PTR(err);
3007 }
Al Viro076515f2018-03-10 23:15:52 -05003008 dput(old_parent);
Al Viro18367502011-07-12 21:42:24 -04003009 } else {
Al Virob5ae6b12014-10-12 22:16:02 -04003010 __d_move(new, dentry, false);
3011 write_sequnlock(&rename_lock);
Trond Myklebust9eaef272006-10-21 10:24:20 -07003012 }
Al Virob5ae6b12014-10-12 22:16:02 -04003013 iput(inode);
3014 return new;
Trond Myklebust9eaef272006-10-21 10:24:20 -07003015 }
David Howells770bfad2006-08-22 20:06:07 -04003016 }
Al Virob5ae6b12014-10-12 22:16:02 -04003017out:
Al Viroed782b52016-03-09 19:52:39 -05003018 __d_add(dentry, inode);
Al Virob5ae6b12014-10-12 22:16:02 -04003019 return NULL;
David Howells770bfad2006-08-22 20:06:07 -04003020}
Al Virob5ae6b12014-10-12 22:16:02 -04003021EXPORT_SYMBOL(d_splice_alias);
David Howells770bfad2006-08-22 20:06:07 -04003022
Linus Torvalds1da177e2005-04-16 15:20:36 -07003023/*
3024 * Test whether new_dentry is a subdirectory of old_dentry.
3025 *
3026 * Trivially implemented using the dcache structure
3027 */
3028
3029/**
3030 * is_subdir - is new dentry a subdirectory of old_dentry
3031 * @new_dentry: new dentry
3032 * @old_dentry: old dentry
3033 *
Yaowei Baia6e57872015-11-17 14:40:11 +08003034 * Returns true if new_dentry is a subdirectory of the parent (at any depth).
3035 * Returns false otherwise.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003036 * Caller must ensure that "new_dentry" is pinned before calling is_subdir()
3037 */
3038
Yaowei Baia6e57872015-11-17 14:40:11 +08003039bool is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003040{
Christian Brauner391b59b2024-07-02 21:03:26 +02003041 bool subdir;
Nick Piggin949854d2011-01-07 17:49:37 +11003042 unsigned seq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003043
OGAWA Hirofumie2761a12008-10-16 07:50:28 +09003044 if (new_dentry == old_dentry)
Yaowei Baia6e57872015-11-17 14:40:11 +08003045 return true;
OGAWA Hirofumie2761a12008-10-16 07:50:28 +09003046
Christian Brauner391b59b2024-07-02 21:03:26 +02003047 /* Access d_parent under rcu as d_move() may change it. */
3048 rcu_read_lock();
3049 seq = read_seqbegin(&rename_lock);
3050 subdir = d_ancestor(old_dentry, new_dentry);
3051 /* Try lockless once... */
3052 if (read_seqretry(&rename_lock, seq)) {
3053 /* ...else acquire lock for progress even on deep chains. */
3054 read_seqlock_excl(&rename_lock);
3055 subdir = d_ancestor(old_dentry, new_dentry);
3056 read_sequnlock_excl(&rename_lock);
3057 }
3058 rcu_read_unlock();
3059 return subdir;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003060}
Amir Goldsteine8f9e5b2018-01-11 11:33:24 +02003061EXPORT_SYMBOL(is_subdir);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003062
Miklos Szeredidb14fc32013-09-05 11:44:35 +02003063static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003064{
Miklos Szeredidb14fc32013-09-05 11:44:35 +02003065 struct dentry *root = data;
3066 if (dentry != root) {
3067 if (d_unhashed(dentry) || !dentry->d_inode)
3068 return D_WALK_SKIP;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003069
Al Viro7e4a2052024-02-03 23:53:05 -05003070 if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
3071 dentry->d_flags |= DCACHE_GENOCIDE;
3072 dentry->d_lockref.count--;
3073 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003074 }
Miklos Szeredidb14fc32013-09-05 11:44:35 +02003075 return D_WALK_CONTINUE;
3076}
Nick Piggin58db63d2011-01-07 17:49:39 +11003077
Miklos Szeredidb14fc32013-09-05 11:44:35 +02003078void d_genocide(struct dentry *parent)
3079{
Al Viro3a8e3612018-04-15 18:27:23 -04003080 d_walk(parent, parent, d_genocide_kill);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003081}
3082
Kent Overstreet771eb4f2018-07-09 23:27:33 -04003083void d_mark_tmpfile(struct file *file, struct inode *inode)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003084{
Miklos Szeredi863f1442022-09-24 07:00:00 +02003085 struct dentry *dentry = file->f_path.dentry;
3086
Al Viro60545d02013-06-07 01:20:27 -04003087 BUG_ON(dentry->d_name.name != dentry->d_iname ||
Al Viro946e51f2014-10-26 19:19:16 -04003088 !hlist_unhashed(&dentry->d_u.d_alias) ||
Al Viro60545d02013-06-07 01:20:27 -04003089 !d_unlinked(dentry));
3090 spin_lock(&dentry->d_parent->d_lock);
3091 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
3092 dentry->d_name.len = sprintf(dentry->d_iname, "#%llu",
3093 (unsigned long long)inode->i_ino);
3094 spin_unlock(&dentry->d_lock);
3095 spin_unlock(&dentry->d_parent->d_lock);
Kent Overstreet771eb4f2018-07-09 23:27:33 -04003096}
3097EXPORT_SYMBOL(d_mark_tmpfile);
3098
3099void d_tmpfile(struct file *file, struct inode *inode)
3100{
3101 struct dentry *dentry = file->f_path.dentry;
3102
3103 inode_dec_link_count(inode);
3104 d_mark_tmpfile(file, inode);
Al Viro60545d02013-06-07 01:20:27 -04003105 d_instantiate(dentry, inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003106}
Al Viro60545d02013-06-07 01:20:27 -04003107EXPORT_SYMBOL(d_tmpfile);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003108
Mateusz Guzikf378ec42024-06-27 18:11:52 +02003109/*
3110 * Obtain inode number of the parent dentry.
3111 */
3112ino_t d_parent_ino(struct dentry *dentry)
3113{
3114 struct dentry *parent;
3115 struct inode *iparent;
3116 unsigned seq;
3117 ino_t ret;
3118
3119 scoped_guard(rcu) {
3120 seq = raw_seqcount_begin(&dentry->d_seq);
3121 parent = READ_ONCE(dentry->d_parent);
3122 iparent = d_inode_rcu(parent);
3123 if (likely(iparent)) {
3124 ret = iparent->i_ino;
3125 if (!read_seqcount_retry(&dentry->d_seq, seq))
3126 return ret;
3127 }
3128 }
3129
3130 spin_lock(&dentry->d_lock);
3131 ret = dentry->d_parent->d_inode->i_ino;
3132 spin_unlock(&dentry->d_lock);
3133 return ret;
3134}
3135EXPORT_SYMBOL(d_parent_ino);
3136
Linus Torvalds1da177e2005-04-16 15:20:36 -07003137static __initdata unsigned long dhash_entries;
3138static int __init set_dhash_entries(char *str)
3139{
3140 if (!str)
3141 return 0;
3142 dhash_entries = simple_strtoul(str, &str, 0);
3143 return 1;
3144}
3145__setup("dhash_entries=", set_dhash_entries);
3146
3147static void __init dcache_init_early(void)
3148{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003149 /* If hashes are distributed across NUMA nodes, defer
3150 * hash allocation until vmalloc space is available.
3151 */
3152 if (hashdist)
3153 return;
3154
3155 dentry_hashtable =
3156 alloc_large_system_hash("Dentry cache",
Linus Torvaldsb07ad992011-04-23 22:32:03 -07003157 sizeof(struct hlist_bl_head),
Linus Torvalds1da177e2005-04-16 15:20:36 -07003158 dhash_entries,
3159 13,
Pavel Tatashin3d375d72017-07-06 15:39:11 -07003160 HASH_EARLY | HASH_ZERO,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003161 &d_hash_shift,
Alexey Dobriyanb35d7862017-11-20 18:05:52 +03003162 NULL,
Tim Bird31fe62b2012-05-23 13:33:35 +00003163 0,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003164 0);
Alexey Dobriyan854d3e62017-11-20 18:05:07 +03003165 d_hash_shift = 32 - d_hash_shift;
Linus Torvaldse7829852024-06-04 12:30:02 -07003166
3167 runtime_const_init(shift, d_hash_shift);
3168 runtime_const_init(ptr, dentry_hashtable);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003169}
3170
Denis Cheng74bf17c2007-10-16 23:26:30 -07003171static void __init dcache_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003172{
Pavel Tatashin3d375d72017-07-06 15:39:11 -07003173 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07003174 * A constructor could be added for stable state like the lists,
3175 * but it is probably not worth it because of the cache nature
Pavel Tatashin3d375d72017-07-06 15:39:11 -07003176 * of the dcache.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003177 */
David Windsor80344262017-06-10 22:50:44 -04003178 dentry_cache = KMEM_CACHE_USERCOPY(dentry,
Chengming Zhouc997d682024-02-24 13:53:15 +00003179 SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_ACCOUNT,
David Windsor80344262017-06-10 22:50:44 -04003180 d_iname);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003181
3182 /* Hash may have been set up in dcache_init_early */
3183 if (!hashdist)
3184 return;
3185
3186 dentry_hashtable =
3187 alloc_large_system_hash("Dentry cache",
Linus Torvaldsb07ad992011-04-23 22:32:03 -07003188 sizeof(struct hlist_bl_head),
Linus Torvalds1da177e2005-04-16 15:20:36 -07003189 dhash_entries,
3190 13,
Pavel Tatashin3d375d72017-07-06 15:39:11 -07003191 HASH_ZERO,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003192 &d_hash_shift,
Alexey Dobriyanb35d7862017-11-20 18:05:52 +03003193 NULL,
Tim Bird31fe62b2012-05-23 13:33:35 +00003194 0,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003195 0);
Alexey Dobriyan854d3e62017-11-20 18:05:07 +03003196 d_hash_shift = 32 - d_hash_shift;
Linus Torvaldse7829852024-06-04 12:30:02 -07003197
3198 runtime_const_init(shift, d_hash_shift);
3199 runtime_const_init(ptr, dentry_hashtable);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003200}
3201
3202/* SLAB cache for __getname() consumers */
Alexey Dobriyan68279f92023-10-11 19:55:00 +03003203struct kmem_cache *names_cachep __ro_after_init;
H Hartley Sweetenec4f8602010-01-05 13:45:18 -07003204EXPORT_SYMBOL(names_cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003205
Linus Torvalds1da177e2005-04-16 15:20:36 -07003206void __init vfs_caches_init_early(void)
3207{
Sebastian Andrzej Siewior69163632017-06-27 18:19:11 +02003208 int i;
3209
3210 for (i = 0; i < ARRAY_SIZE(in_lookup_hashtable); i++)
3211 INIT_HLIST_BL_HEAD(&in_lookup_hashtable[i]);
3212
Linus Torvalds1da177e2005-04-16 15:20:36 -07003213 dcache_init_early();
3214 inode_init_early();
3215}
3216
Mel Gorman4248b0d2015-08-06 15:46:20 -07003217void __init vfs_caches_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003218{
David Windsor6a9b8822017-06-10 22:50:30 -04003219 names_cachep = kmem_cache_create_usercopy("names_cache", PATH_MAX, 0,
3220 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 0, PATH_MAX, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003221
Denis Cheng74bf17c2007-10-16 23:26:30 -07003222 dcache_init();
3223 inode_init();
Mel Gorman4248b0d2015-08-06 15:46:20 -07003224 files_init();
3225 files_maxfiles_init();
Denis Cheng74bf17c2007-10-16 23:26:30 -07003226 mnt_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003227 bdev_cache_init();
3228 chrdev_init();
3229}