Merge tag 'sound-7.3-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound
Pull sound fixes from Takashi Iwai:
"A collection of small fixes. Most of them are device-specific fixes
while there are a few core fixes. The continued flux, but not too
scaring yet. Some highlights below.
ALSA Core:
- Fix potential UAF after asynchronous card release
- Fix a race condition in PCM timer initialization order
USB-Audio:
- Hardening fixes for issues reported by fuzzer for 6fire, bcd2000,
and implicit FB packets
- Fix double list addition in implicit FB handling
- Quirks for AVerMedia GC553Pro and Behringer FCA1616
HD-Audio:
- Quirks / fixes for HP OmniBook 7, OMEN 15, and Victus 15 laptops
ASoC:
- Support for DAI link codec channel mask to avoid mismatches
- Fix HDMI-codec channel status change report
- Fixes for various codecs and platforms: Realtek rt712/rt721
(calibration, reset fixes), Cirrus Logic (empty EFI variable
validation, capture channel fixup), AMD ACP SoundWire (bounds
checks, refactorings), ADAU1977 (OF match table support, SPI
cleanups), ES8336 (Huawei Matebook B3-420 quirk), UX500 (macro
fix)"
* tag 'sound-7.3-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound: (33 commits)
ASoC: adau1977-i2c: add OF match table for I2C
ASoC: adau1977-spi: drop __maybe_unused and of_match_ptr()
ASoC: adau1977: make the Kconfig symbols user selectable
ASoC: amd: acp: fix card name length warning in SOF SoundWire machine driver
ASoC: amd: acp: fix ffs() operator precedence for SoundWire link ID
ASoC: amd: acp: refactor codec config count in SOF SoundWire machine driver
ASoC: amd: acp: bounds-check SoundWire link ID in machine drivers
ASoC: cs-amp-lib: Prevent NULL pointer if efi variable is zero length
ASoC: codecs: rt712-sdca-dmic: fix uninitialized stream_config->type
ASoC: hdmi-codec: Report a change when the channel status moves
ASoC: ux500: Parenthesize MSP_{RX,TX}_CLKPOL_BIT() arguments
ASoC: rt721: Reset codec to fix abnormal sound
ALSA: usb-audio: fix list_add double-add in push_back_to_ready_list
ALSA: hda: trace PCM open only after assigning a stream
ALSA: usb-audio: skip the broken mute control on AVerMedia GC553Pro
ALSA: hda/realtek: Enable mute LEDs on HP OmniBook 7 17-dc0xxx
ALSA: 6fire: fix OOB write from device-reported iso length
ALSA: usb-audio: Add capture quirk for Behringer FCA1616
ALSA: hda/realtek: Add mute LED quirk for HP OMEN 15-ax
ASoC: Intel: sof_es8336: Add a quirk for Huawei Matebook B3-420
...
diff --git a/.mailmap b/.mailmap
index 29c556c..1f5540b 100644
--- a/.mailmap
+++ b/.mailmap
@@ -211,7 +211,8 @@
Christophe Leroy <chleroy@kernel.org> <christophe.leroy@csgroup.eu>
Christophe Leroy <chleroy@kernel.org> <christophe.leroy2@cs-soprasteria.com>
Christophe Ricard <christophe.ricard@gmail.com>
-Christopher Obbard <christopher.obbard@linaro.org> <chris.obbard@collabora.com>
+Christopher Obbard <chris.obbard@oss.qualcomm.com> <chris.obbard@collabora.com>
+Christopher Obbard <chris.obbard@oss.qualcomm.com> <christopher.obbard@linaro.org>
Christoph Hellwig <hch@lst.de>
Christoph Manszewski <c.manszewski@gmail.com> <christoph.manszewski@intel.com>
Christoph Paasch <cpaasch@openai.com> <christoph.paasch@gmail.com>
@@ -222,6 +223,7 @@
Chuck Lever <cel@kernel.org> <cel@netapp.com>
Chuck Lever <cel@kernel.org> <cel@citi.umich.edu>
Claudiu Beznea <claudiu.beznea@tuxon.dev> <claudiu.beznea@microchip.com>
+Coiby Xu <coiby.xu@gmail.com> <coxu@redhat.com>
Colin Ian King <colin.i.king@gmail.com> <colin.king@canonical.com>
Corey Minyard <minyard@acm.org>
Damian Hobson-Garcia <dhobsong@igel.co.jp>
diff --git a/MAINTAINERS b/MAINTAINERS
index c241444..3b2eb2a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -17031,8 +17031,8 @@
M: Mike Rapoport <rppt@kernel.org>
L: linux-mm@kvack.org
S: Maintained
-T: git git://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git for-next
-T: git git://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git fixes
+T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/memblock.git for-next
+T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/memblock.git fixes
F: Documentation/core-api/boot-time-mm.rst
F: include/linux/kho/abi/memblock.h
F: include/linux/memblock.h
diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c
index ee1b5cb..1ae8384 100644
--- a/arch/powerpc/kernel/iommu.c
+++ b/arch/powerpc/kernel/iommu.c
@@ -1076,7 +1076,7 @@ int iommu_tce_check_ioba(unsigned long page_shift,
if (ioba < offset)
return -EINVAL;
- if ((ioba + 1) > (offset + size))
+ if ((ioba + npages < ioba) || (ioba - offset + npages > size))
return -EINVAL;
return 0;
diff --git a/arch/powerpc/kvm/book3s_hv_nested.c b/arch/powerpc/kvm/book3s_hv_nested.c
index 22e6166..a6ff42d 100644
--- a/arch/powerpc/kvm/book3s_hv_nested.c
+++ b/arch/powerpc/kvm/book3s_hv_nested.c
@@ -1204,8 +1204,10 @@ static void kvmhv_emulate_tlbie_all_lpid(struct kvm_vcpu *vcpu, int ric)
spin_lock(&kvm->mmu_lock);
idr_for_each_entry(&kvm->arch.kvm_nested_guest_idr, gp, lpid) {
+ ++gp->refcnt;
spin_unlock(&kvm->mmu_lock);
kvmhv_emulate_tlbie_lpid(vcpu, gp, ric);
+ kvmhv_put_nested(gp);
spin_lock(&kvm->mmu_lock);
}
spin_unlock(&kvm->mmu_lock);
diff --git a/arch/powerpc/kvm/book3s_hv_uvmem.c b/arch/powerpc/kvm/book3s_hv_uvmem.c
index 5fbb95d..463aef8 100644
--- a/arch/powerpc/kvm/book3s_hv_uvmem.c
+++ b/arch/powerpc/kvm/book3s_hv_uvmem.c
@@ -779,8 +779,11 @@ static int kvmppc_svm_page_in(struct vm_area_struct *vma,
if (spage) {
ret = uv_page_in(kvm->arch.lpid, pfn << page_shift,
gpa, 0, page_shift);
- if (ret)
+ if (ret) {
+ unlock_page(dpage);
+ put_page(dpage);
goto out_finalize;
+ }
}
}
diff --git a/arch/x86/include/asm/div64.h b/arch/x86/include/asm/div64.h
index 30fd06e..8a2d343 100644
--- a/arch/x86/include/asm/div64.h
+++ b/arch/x86/include/asm/div64.h
@@ -111,7 +111,7 @@ static inline u64 mul_u64_add_u64_div_u64(u64 rax, u64 mul, u64 add, u64 div)
if (!statically_true(!add))
asm ("addq %[add], %[lo]; adcq $0, %[hi]" :
- [lo] "+r" (rax), [hi] "+r" (rdx) : [add] "irm" (add));
+ [lo] "+r" (rax), [hi] "+r" (rdx) : [add] "erm" (add));
asm ("divq %[div]" : "+a" (rax), "+d" (rdx) : [div] "rm" (div));
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index d5f4917..d551120 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -806,7 +806,7 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
pmdval_t val = pmd_val(pmd), oldval = val;
pmd_t pmd_result;
- val &= (_HPAGE_CHG_MASK & ~_PAGE_DIRTY);
+ val &= _HPAGE_CHG_MASK;
val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK;
val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK);
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 91b1cdd..741d876 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -6,6 +6,9 @@
#include <linux/vmalloc.h>
#include <linux/memory.h>
#include <linux/execmem.h>
+#include <linux/cleanup.h>
+#include <linux/kgdb.h>
+#include <linux/mmap_lock.h>
#include <asm/text-patching.h>
#include <asm/insn.h>
@@ -1198,6 +1201,41 @@ static bool cfi_debug __ro_after_init;
bool cfi_bhi __ro_after_init = false;
#endif
+#ifdef CONFIG_FINEIBT
+/*
+ * <fineibt_preamble_start>:
+ * 0: f3 0f 1e fa endbr64
+ * 4: 2d 78 56 34 12 sub $0x12345678, %eax
+ * 9: 2e 0f 85 03 00 00 00 jne,pn 13 <fineibt_preamble_start+0x13>
+ * 10: 0f 1f 40 d6 nopl -0x2a(%rax)
+ *
+ * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as
+ * UDB on x86_64 and raises #UD.
+ */
+asm( ".pushsection .rodata \n"
+ "fineibt_preamble_start: \n"
+ " endbr64 \n"
+ " subl $0x12345678, %eax \n"
+ "fineibt_preamble_bhi: \n"
+ " cs jne.d32 fineibt_preamble_start+0x13 \n"
+ "#fineibt_func: \n"
+ " nopl -42(%rax) \n"
+ "fineibt_preamble_end: \n"
+ ".popsection\n"
+);
+
+extern u8 fineibt_preamble_start[];
+extern u8 fineibt_preamble_bhi[];
+extern u8 fineibt_preamble_end[];
+
+#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
+#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start)
+#define fineibt_preamble_ud 0x13
+#define fineibt_preamble_hash 5
+
+#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE)
+#endif /* CONFIG_FINEIBT */
+
#ifdef CONFIG_CFI
u32 cfi_get_func_hash(void *func)
{
@@ -1205,9 +1243,11 @@ u32 cfi_get_func_hash(void *func)
func -= cfi_get_offset();
switch (cfi_mode) {
+#ifdef CONFIG_FINEIBT
case CFI_FINEIBT:
- func += 7;
+ func += fineibt_preamble_hash;
break;
+#endif
case CFI_KCFI:
func += 1;
break;
@@ -1364,39 +1404,6 @@ early_param("cfi", cfi_parse_cmdline);
*/
/*
- * <fineibt_preamble_start>:
- * 0: f3 0f 1e fa endbr64
- * 4: 2d 78 56 34 12 sub $0x12345678, %eax
- * 9: 2e 0f 85 03 00 00 00 jne,pn 13 <fineibt_preamble_start+0x13>
- * 10: 0f 1f 40 d6 nopl -0x2a(%rax)
- *
- * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as
- * UDB on x86_64 and raises #UD.
- */
-asm( ".pushsection .rodata \n"
- "fineibt_preamble_start: \n"
- " endbr64 \n"
- " subl $0x12345678, %eax \n"
- "fineibt_preamble_bhi: \n"
- " cs jne.d32 fineibt_preamble_start+0x13 \n"
- "#fineibt_func: \n"
- " nopl -42(%rax) \n"
- "fineibt_preamble_end: \n"
- ".popsection\n"
-);
-
-extern u8 fineibt_preamble_start[];
-extern u8 fineibt_preamble_bhi[];
-extern u8 fineibt_preamble_end[];
-
-#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
-#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start)
-#define fineibt_preamble_ud 0x13
-#define fineibt_preamble_hash 5
-
-#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE)
-
-/*
* <fineibt_caller_start>:
* 0: b8 78 56 34 12 mov $0x12345678, %eax
* 5: 4d 8d 5b f0 lea -0x10(%r11), %r11
@@ -2372,6 +2379,38 @@ static void text_poke_memset(void *dst, const void *src, size_t len)
typedef void text_poke_f(void *dst, const void *src, size_t len);
+static void __poke_vmalloc_pages(struct page **pages, void *addr,
+ bool cross_page_boundary)
+{
+ pages[0] = vmalloc_to_page(addr);
+ if (cross_page_boundary)
+ pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
+}
+
+static void poke_vmalloc_pages(struct page **pages, void *addr,
+ bool cross_page_boundary)
+{
+ if (in_dbg_master()) {
+ /*
+ * If called from kgdb cannot sleep, but all other CPUs stopped
+ * anyway so safe to proceed without locks
+ */
+ __poke_vmalloc_pages(pages, addr, cross_page_boundary);
+ } else {
+ /*
+ * execmem ROX ranges are shared between modules and can be
+ * collapsed to huge PMD entries, and this collapse can happen
+ * concurrently with a racing set_memory_rox().
+ *
+ * Prevent vmalloc_to_page() from racing by acquiring an
+ * init_mm read lock which pairs with the init_mm write lock in
+ * cpa_collapse_large_pages().
+ */
+ guard(mmap_read_lock)(&init_mm);
+ __poke_vmalloc_pages(pages, addr, cross_page_boundary);
+ }
+}
+
static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len)
{
bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE;
@@ -2389,9 +2428,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l
BUG_ON(!after_bootmem);
if (!core_kernel_text((unsigned long)addr)) {
- pages[0] = vmalloc_to_page(addr);
- if (cross_page_boundary)
- pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
+ poke_vmalloc_pages(pages, addr, cross_page_boundary);
} else {
pages[0] = virt_to_page(addr);
WARN_ON(!PageReserved(pages[0]));
diff --git a/arch/x86/kernel/amd_node.c b/arch/x86/kernel/amd_node.c
index 7625857..b7926ba 100644
--- a/arch/x86/kernel/amd_node.c
+++ b/arch/x86/kernel/amd_node.c
@@ -251,7 +251,7 @@ __setup("amd_smn_debugfs_enable", amd_smn_enable_dfs);
static int __init amd_smn_init(void)
{
u16 count, num_roots, roots_per_node, node, num_nodes;
- struct pci_dev *root;
+ struct pci_dev *root __free(pci_dev_put) = NULL;
if (!cpu_feature_enabled(X86_FEATURE_ZEN))
return 0;
@@ -262,7 +262,6 @@ static int __init amd_smn_init(void)
return 0;
num_roots = 0;
- root = NULL;
while ((root = get_next_root(root))) {
pci_dbg(root, "Reserving PCI config space\n");
@@ -299,14 +298,13 @@ static int __init amd_smn_init(void)
count = 0;
node = 0;
- root = NULL;
while (node < num_nodes && (root = get_next_root(root))) {
/* Use one root for each node and skip the rest. */
if (count++ % roots_per_node)
continue;
pci_dbg(root, "is root for AMD node %u\n", node);
- amd_roots[node++] = root;
+ amd_roots[node++] = pci_dev_get(root);
}
if (enable_dfs) {
diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c
index 95bae74..3aefdef5 100644
--- a/arch/x86/mm/mem_encrypt.c
+++ b/arch/x86/mm/mem_encrypt.c
@@ -13,6 +13,7 @@
#include <linux/cc_platform.h>
#include <linux/mem_encrypt.h>
#include <linux/virtio_anchor.h>
+#include <linux/iommu-dma.h>
#include <asm/sev.h>
@@ -30,7 +31,7 @@ bool force_dma_unencrypted(struct device *dev)
* device does not support DMA to addresses that include the
* encryption mask.
*/
- if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
+ if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT) && !use_dma_iommu(dev)) {
u64 dma_enc_mask = DMA_BIT_MASK(__ffs64(sme_me_mask));
u64 dma_dev_mask = min_not_zero(dev->coherent_dma_mask,
dev->bus_dma_limit);
diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c
index c38faf3..4652487 100644
--- a/arch/x86/mm/pat/set_memory.c
+++ b/arch/x86/mm/pat/set_memory.c
@@ -22,6 +22,7 @@
#include <linux/cc_platform.h>
#include <linux/set_memory.h>
#include <linux/memregion.h>
+#include <linux/cleanup.h>
#include <asm/e820/api.h>
#include <asm/processor.h>
@@ -49,7 +50,8 @@ struct cpa_data {
unsigned int flags;
unsigned int force_split : 1,
force_static_prot : 1,
- force_flush_all : 1;
+ force_flush_all : 1,
+ init_mm_read_locked : 1;
struct page **pages;
};
@@ -409,7 +411,7 @@ static void __cpa_flush_tlb(void *data)
static int collapse_large_pages(unsigned long addr, struct list_head *pgtables);
-static void cpa_collapse_large_pages(struct cpa_data *cpa)
+static void __cpa_collapse_large_pages(struct cpa_data *cpa)
{
unsigned long start, addr, end;
struct ptdesc *ptdesc, *tmp;
@@ -439,10 +441,30 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa)
list_for_each_entry_safe(ptdesc, tmp, &pgtables, pt_list) {
list_del(&ptdesc->pt_list);
- pagetable_free(ptdesc);
+ /*
+ * Only early alloc'd direct map should not be flagged PG_table
+ * here and those shouldn't be collapsed. However be abundantly
+ * cautious and handle the !PG_table case too.
+ */
+ if (PageTable((ptdesc_page(ptdesc))))
+ pagetable_dtor_free(ptdesc);
+ else
+ pagetable_free(ptdesc);
}
}
+static void cpa_collapse_large_pages(struct cpa_data *cpa)
+{
+ /*
+ * Take the mmap write lock on init_mm to:
+ * - Avoid a use-after-free if raced by ptdump (which takes its own
+ * write lock on init_mm).
+ * - Serialise concurrent CPA walkers.
+ */
+ scoped_guard(mmap_write_lock, &init_mm)
+ __cpa_collapse_large_pages(cpa);
+}
+
static void cpa_flush(struct cpa_data *cpa, int cache)
{
unsigned int i;
@@ -1120,11 +1142,10 @@ static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn,
static int
__split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
- struct ptdesc *ptdesc)
+ pte_t *pbase)
{
unsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1;
- struct page *base = ptdesc_page(ptdesc);
- pte_t *pbase = (pte_t *)page_address(base);
+ struct page *base = virt_to_page(pbase);
unsigned int i, level;
pgprot_t ref_prot;
bool nx, rw;
@@ -1224,16 +1245,20 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
unsigned long address)
{
- struct ptdesc *ptdesc;
+ pte_t *pte;
spin_unlock(&cpa_lock);
- ptdesc = pagetable_alloc(GFP_KERNEL, 0);
+ if (cpa->init_mm_read_locked)
+ mmap_read_unlock(&init_mm);
+ pte = pte_alloc_one_kernel(&init_mm);
+ if (cpa->init_mm_read_locked)
+ mmap_read_lock(&init_mm);
spin_lock(&cpa_lock);
- if (!ptdesc)
+ if (!pte)
return -ENOMEM;
- if (__split_large_page(cpa, kpte, address, ptdesc))
- pagetable_free(ptdesc);
+ if (__split_large_page(cpa, kpte, address, pte))
+ pte_free_kernel(&init_mm, pte);
return 0;
}
@@ -2121,7 +2146,11 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
cpa.curpage = 0;
cpa.force_split = force_split;
- ret = __change_page_attr_set_clr(&cpa, 1);
+ /* Avoid race with concurrent CPA collapse. */
+ cpa.init_mm_read_locked = true;
+ scoped_guard(mmap_read_lock, &init_mm)
+ ret = __change_page_attr_set_clr(&cpa, 1);
+ cpa.init_mm_read_locked = false;
/*
* Check whether we really changed something:
diff --git a/drivers/gpio/gpio-virtuser.c b/drivers/gpio/gpio-virtuser.c
index 449fb1a..70ddccc 100644
--- a/drivers/gpio/gpio-virtuser.c
+++ b/drivers/gpio/gpio-virtuser.c
@@ -692,7 +692,8 @@ static int gpio_virtuser_interrupts_set(void *data, u64 val)
atomic_set(&ld->irq, irq);
} else {
irq = atomic_xchg(&ld->irq, 0);
- free_irq(irq, ld);
+ if (irq)
+ free_irq(irq, ld);
}
return 0;
diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c
index 940b5669..f36e4b1 100644
--- a/drivers/gpio/gpiolib-of.c
+++ b/drivers/gpio/gpiolib-of.c
@@ -788,13 +788,13 @@ static int of_gpio_notify(struct notifier_block *nb, unsigned long action,
if (!of_property_read_bool(rd->dn, "gpio-hog"))
return NOTIFY_DONE; /* not for us */
- if (of_node_test_and_set_flag(rd->dn, OF_POPULATED))
- return NOTIFY_DONE;
-
gdev = of_find_gpio_device_by_node(rd->dn->parent);
if (!gdev)
return NOTIFY_DONE; /* not for us */
+ if (of_node_test_and_set_flag(rd->dn, OF_POPULATED))
+ return NOTIFY_DONE;
+
ret = gpiochip_add_hog(gpio_device_get_chip(gdev), of_fwnode_handle(rd->dn));
if (ret < 0) {
pr_err("%s: failed to add hogs for %pOF\n", __func__,
diff --git a/drivers/gpio/gpiolib-shared.c b/drivers/gpio/gpiolib-shared.c
index 495bd3d..5f9623e 100644
--- a/drivers/gpio/gpiolib-shared.c
+++ b/drivers/gpio/gpiolib-shared.c
@@ -261,10 +261,13 @@ static int gpio_shared_of_traverse(struct device_node *curr)
con_id[con_id_len - suffix_len] = '\0';
}
- ref = gpio_shared_make_ref(fwnode_handle_get(of_fwnode_handle(curr)),
- con_id, args.args[1]);
- if (!ref)
+ struct fwnode_handle *curr_fwnode =
+ fwnode_handle_get(of_fwnode_handle(curr));
+ ref = gpio_shared_make_ref(curr_fwnode, con_id, args.args[1]);
+ if (!ref) {
+ fwnode_handle_put(curr_fwnode);
return -ENOMEM;
+ }
if (!list_empty(&entry->refs))
pr_debug("GPIO %u at %s is shared by multiple firmware nodes\n",
diff --git a/drivers/infiniband/core/iwpm_util.c b/drivers/infiniband/core/iwpm_util.c
index 990cf92..51af8c1 100644
--- a/drivers/infiniband/core/iwpm_util.c
+++ b/drivers/infiniband/core/iwpm_util.c
@@ -314,10 +314,6 @@ struct iwpm_nlmsg_request *iwpm_get_nlmsg_request(__u32 nlmsg_seq,
if (!nlmsg_request)
return NULL;
- spin_lock_irqsave(&iwpm_nlmsg_req_lock, flags);
- list_add_tail(&nlmsg_request->inprocess_list, &iwpm_nlmsg_req_list);
- spin_unlock_irqrestore(&iwpm_nlmsg_req_lock, flags);
-
kref_init(&nlmsg_request->kref);
kref_get(&nlmsg_request->kref);
nlmsg_request->nlmsg_seq = nlmsg_seq;
@@ -326,6 +322,11 @@ struct iwpm_nlmsg_request *iwpm_get_nlmsg_request(__u32 nlmsg_seq,
nlmsg_request->err_code = 0;
sema_init(&nlmsg_request->sem, 1);
down(&nlmsg_request->sem);
+
+ spin_lock_irqsave(&iwpm_nlmsg_req_lock, flags);
+ list_add_tail(&nlmsg_request->inprocess_list, &iwpm_nlmsg_req_list);
+ spin_unlock_irqrestore(&iwpm_nlmsg_req_lock, flags);
+
return nlmsg_request;
}
diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
index e0b3b36..3c91f00 100644
--- a/drivers/infiniband/core/mad.c
+++ b/drivers/infiniband/core/mad.c
@@ -2059,6 +2059,8 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
int ret;
INIT_LIST_HEAD(&mad_recv_wc->rmpp_list);
+ list_add(&mad_recv_wc->recv_buf.list, &mad_recv_wc->rmpp_list);
+
ret = ib_mad_enforce_security(mad_agent_priv,
mad_recv_wc->wc->pkey_index);
if (ret) {
@@ -2067,7 +2069,6 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
return;
}
- list_add(&mad_recv_wc->recv_buf.list, &mad_recv_wc->rmpp_list);
if (is_kernel_rmpp_data_response(mad_agent_priv, mad_recv_wc)) {
spin_lock_irqsave(&mad_agent_priv->lock, flags);
mad_send_wr = ib_find_send_mad(mad_agent_priv, mad_recv_wc);
diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c
index fd5651c..a7cbe64 100644
--- a/drivers/infiniband/core/rdma_core.c
+++ b/drivers/infiniband/core/rdma_core.c
@@ -69,7 +69,6 @@ void ib_uverbs_release_file(struct kref *ref)
if (file->disassociate_page)
__free_pages(file->disassociate_page, 0);
- mutex_destroy(&file->disassociation_lock);
mutex_destroy(&file->umap_lock);
mutex_destroy(&file->ucontext_lock);
kfree(file);
diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c
index 4929636..a15182f 100644
--- a/drivers/infiniband/core/ucma.c
+++ b/drivers/infiniband/core/ucma.c
@@ -1556,9 +1556,10 @@ static ssize_t ucma_process_join(struct ucma_file *file,
mutex_lock(&ctx->mutex);
ret = rdma_join_multicast(ctx->cm_id, (struct sockaddr *)&mc->addr,
join_state, mc);
- mutex_unlock(&ctx->mutex);
- if (ret)
+ if (ret) {
+ mutex_unlock(&ctx->mutex);
goto err_xa_erase;
+ }
resp.id = mc->id;
if (copy_to_user(u64_to_user_ptr(cmd->response),
@@ -1566,6 +1567,7 @@ static ssize_t ucma_process_join(struct ucma_file *file,
ret = -EFAULT;
goto err_leave_multicast;
}
+ mutex_unlock(&ctx->mutex);
xa_store(&multicast_table, mc->id, mc, 0);
@@ -1573,7 +1575,6 @@ static ssize_t ucma_process_join(struct ucma_file *file,
return 0;
err_leave_multicast:
- mutex_lock(&ctx->mutex);
rdma_leave_multicast(ctx->cm_id, (struct sockaddr *) &mc->addr);
mutex_unlock(&ctx->mutex);
ucma_cleanup_mc_events(mc);
diff --git a/drivers/infiniband/core/uverbs_flow.c b/drivers/infiniband/core/uverbs_flow.c
index 1528a29..de5a276 100644
--- a/drivers/infiniband/core/uverbs_flow.c
+++ b/drivers/infiniband/core/uverbs_flow.c
@@ -26,6 +26,7 @@ struct ib_uflow_resources *flow_resources_alloc(size_t num_specs)
return resources;
err:
+ kfree(resources->collection);
kfree(resources->counters);
kfree(resources);
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index 0d88b2e..2a046c88 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -644,12 +644,15 @@ static int ib_uverbs_mmap(struct file *filp, struct vm_area_struct *vma)
goto out;
}
- mutex_lock(&file->disassociation_lock);
+ if (!down_read_trylock(&file->hw_destroy_rwsem)) {
+ ret = -EIO;
+ goto out;
+ }
vma->vm_ops = &rdma_umap_ops;
ret = ucontext->device->ops.mmap(ucontext, vma);
- mutex_unlock(&file->disassociation_lock);
+ up_read(&file->hw_destroy_rwsem);
out:
srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
return ret;
@@ -671,7 +674,6 @@ static void rdma_umap_open(struct vm_area_struct *vma)
/* We are racing with disassociation */
if (!down_read_trylock(&ufile->hw_destroy_rwsem))
goto out_zap;
- mutex_lock(&ufile->disassociation_lock);
/*
* Disassociation already completed, the VMA should already be zapped.
@@ -684,12 +686,10 @@ static void rdma_umap_open(struct vm_area_struct *vma)
goto out_unlock;
rdma_umap_priv_init(priv, vma, opriv->entry);
- mutex_unlock(&ufile->disassociation_lock);
up_read(&ufile->hw_destroy_rwsem);
return;
out_unlock:
- mutex_unlock(&ufile->disassociation_lock);
up_read(&ufile->hw_destroy_rwsem);
out_zap:
/*
@@ -773,7 +773,7 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)
{
struct rdma_umap_priv *priv, *next_priv;
- mutex_lock(&ufile->disassociation_lock);
+ lockdep_assert_held_write(&ufile->hw_destroy_rwsem);
while (1) {
struct mm_struct *mm = NULL;
@@ -799,10 +799,8 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)
break;
}
mutex_unlock(&ufile->umap_lock);
- if (!mm) {
- mutex_unlock(&ufile->disassociation_lock);
+ if (!mm)
return;
- }
/*
* The umap_lock is nested under mmap_lock since it used within
@@ -832,8 +830,6 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)
mmap_read_unlock(mm);
mmput(mm);
}
-
- mutex_unlock(&ufile->disassociation_lock);
}
/**
@@ -851,8 +847,11 @@ void rdma_user_mmap_disassociate(struct ib_device *device)
mutex_lock(&uverbs_dev->lists_mutex);
list_for_each_entry(ufile, &uverbs_dev->uverbs_file_list, list) {
- if (ufile->ucontext)
+ if (ufile->ucontext) {
+ down_write(&ufile->hw_destroy_rwsem);
uverbs_user_mmap_disassociate(ufile);
+ up_write(&ufile->hw_destroy_rwsem);
+ }
}
mutex_unlock(&uverbs_dev->lists_mutex);
}
@@ -927,8 +926,6 @@ static int ib_uverbs_open(struct inode *inode, struct file *filp)
mutex_init(&file->umap_lock);
INIT_LIST_HEAD(&file->umaps);
- mutex_init(&file->disassociation_lock);
-
filp->private_data = file;
list_add_tail(&file->list, &dev->uverbs_file_list);
mutex_unlock(&dev->lists_mutex);
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 04abc80..c43e25d 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -2058,11 +2058,13 @@ int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u16 *speed, u8 *width)
return -ENODEV;
rtnl_lock();
+ if (READ_ONCE(netdev->reg_state) != NETREG_REGISTERED) {
+ dev_put(netdev);
+ rtnl_unlock();
+ return -ENODEV;
+ }
+
rc = __ethtool_get_link_ksettings(netdev, &lksettings);
- rtnl_unlock();
-
- dev_put(netdev);
-
if (!rc && lksettings.base.speed != (u32)SPEED_UNKNOWN) {
netdev_speed = lksettings.base.speed;
} else {
@@ -2071,6 +2073,8 @@ int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u16 *speed, u8 *width)
pr_warn("%s speed is unknown, defaulting to %u\n",
netdev->name, netdev_speed);
}
+ dev_put(netdev);
+ rtnl_unlock();
ib_get_width_and_speed(netdev_speed, lksettings.lanes,
speed, width);
diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index ce72db1..17654a9 100644
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -356,9 +356,13 @@ static int bnxt_re_update_qp1_tos_dscp(struct bnxt_re_dev *rdev)
return bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
}
-static void bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev)
+static int bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev)
{
rdev->dcb_wq = create_singlethread_workqueue("bnxt_re_dcb_wq");
+ if (!rdev->dcb_wq)
+ return -ENOMEM;
+
+ return 0;
}
static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev)
@@ -2339,7 +2343,9 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type)
}
bnxt_re_debugfs_add_pdev(rdev);
- bnxt_re_init_dcb_wq(rdev);
+ rc = bnxt_re_init_dcb_wq(rdev);
+ if (rc)
+ goto fail;
bnxt_re_net_register_async_event(rdev);
if (!rdev->is_virtfn)
diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c
index feaf986..a407c6b 100644
--- a/drivers/infiniband/hw/bnxt_re/uapi.c
+++ b/drivers/infiniband/hw/bnxt_re/uapi.c
@@ -462,7 +462,6 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_DBR_ALLOC)(struct uverbs_attr_bundle *a
uobj->object = obj;
uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_DBR_HANDLE);
- dbr.umdbr = dpi->umdbr;
dbr.dpi = dpi->dpi;
ret = uverbs_copy_to_struct_or_zero(attrs, BNXT_RE_ALLOC_DBR_ATTR,
&dbr, sizeof(dbr));
@@ -525,7 +524,6 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_DEFAULT_DBR)(struct uverbs_attr_bun
return PTR_ERR(ib_uctx);
uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
- dpi.umdbr = uctx->dpi.umdbr;
dpi.dpi = uctx->dpi.dpi;
ret = uverbs_copy_to_struct_or_zero(attrs, BNXT_RE_DEFAULT_DBR_ATTR,
@@ -543,7 +541,7 @@ DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_DBR_ALLOC,
UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_DBR_ATTR,
UVERBS_ATTR_STRUCT(struct bnxt_re_db_region,
- umdbr),
+ reserved2),
UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_DBR_OFFSET,
UVERBS_ATTR_TYPE(u64),
@@ -563,7 +561,7 @@ DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_DBR,
DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR,
UVERBS_ATTR_PTR_OUT(BNXT_RE_DEFAULT_DBR_ATTR,
UVERBS_ATTR_STRUCT(struct bnxt_re_db_region,
- umdbr),
+ reserved2),
UA_MANDATORY));
DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_DEFAULT_DBR,
diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c
index 583b1cf..04c6d63 100644
--- a/drivers/infiniband/hw/efa/efa_com.c
+++ b/drivers/infiniband/hw/efa/efa_com.c
@@ -850,7 +850,7 @@ int efa_com_admin_init(struct efa_com_dev *edev,
aq->dmadev = edev->dmadev;
aq->efa_dev = edev->efa_dev;
- set_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state);
+ efa_com_set_admin_polling_mode(edev, true);
sema_init(&aq->avail_cmds, aq->depth);
@@ -868,8 +868,6 @@ int efa_com_admin_init(struct efa_com_dev *edev,
if (err)
goto err_destroy_sq;
- efa_com_set_admin_polling_mode(edev, false);
-
err = efa_com_admin_init_aenq(edev, aenq_handlers);
if (err)
goto err_destroy_cq;
@@ -1254,7 +1252,7 @@ static void efa_com_destroy_eq(struct efa_com_dev *edev,
err);
}
-static void efa_com_arm_eq(struct efa_com_dev *edev, struct efa_com_eq *eeq)
+void efa_com_arm_eq(struct efa_com_dev *edev, struct efa_com_eq *eeq)
{
u32 val = 0;
@@ -1343,7 +1341,6 @@ int efa_com_eq_init(struct efa_com_dev *edev, struct efa_com_eq *eeq,
eeq->phase = 1;
eeq->depth = params.depth;
eeq->cb = cb;
- efa_com_arm_eq(edev, eeq);
return 0;
diff --git a/drivers/infiniband/hw/efa/efa_com.h b/drivers/infiniband/hw/efa/efa_com.h
index 0341704..98fb6a4 100644
--- a/drivers/infiniband/hw/efa/efa_com.h
+++ b/drivers/infiniband/hw/efa/efa_com.h
@@ -169,6 +169,7 @@ int efa_com_admin_init(struct efa_com_dev *edev,
void efa_com_admin_destroy(struct efa_com_dev *edev);
int efa_com_eq_init(struct efa_com_dev *edev, struct efa_com_eq *eeq,
efa_eqe_handler cb, u16 depth, u8 msix_vec);
+void efa_com_arm_eq(struct efa_com_dev *edev, struct efa_com_eq *eeq);
void efa_com_eq_destroy(struct efa_com_dev *edev, struct efa_com_eq *eeq);
int efa_com_dev_reset(struct efa_com_dev *edev,
enum efa_regs_reset_reason_types reset_reason);
diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c
index 4cd8072..753ed84 100644
--- a/drivers/infiniband/hw/efa/efa_main.c
+++ b/drivers/infiniband/hw/efa/efa_main.c
@@ -302,28 +302,30 @@ static void efa_set_host_info(struct efa_dev *dev)
static void efa_destroy_eq(struct efa_dev *dev, struct efa_eq *eq)
{
- efa_com_eq_destroy(&dev->edev, &eq->eeq);
efa_free_irq(dev, &eq->irq);
+ efa_com_eq_destroy(&dev->edev, &eq->eeq);
}
static int efa_create_eq(struct efa_dev *dev, struct efa_eq *eq, u32 msix_vec)
{
int err;
- efa_setup_comp_irq(dev, eq, msix_vec);
- err = efa_request_irq(dev, &eq->irq);
- if (err)
- return err;
-
err = efa_com_eq_init(&dev->edev, &eq->eeq, efa_process_eqe,
dev->dev_attr.max_eq_depth, msix_vec);
if (err)
- goto err_free_comp_irq;
+ return err;
+
+ efa_setup_comp_irq(dev, eq, msix_vec);
+ err = efa_request_irq(dev, &eq->irq);
+ if (err)
+ goto err_destroy_eq;
+
+ efa_com_arm_eq(&dev->edev, &eq->eeq);
return 0;
-err_free_comp_irq:
- efa_free_irq(dev, &eq->irq);
+err_destroy_eq:
+ efa_com_eq_destroy(&dev->edev, &eq->eeq);
return err;
}
@@ -619,18 +621,21 @@ static struct efa_dev *efa_probe_device(struct pci_dev *pdev)
edev->aq.msix_vector_idx = dev->admin_msix_vector_idx;
edev->aenq.msix_vector_idx = dev->admin_msix_vector_idx;
- err = efa_set_mgmnt_irq(dev);
+ err = efa_com_admin_init(edev, &aenq_handlers);
if (err)
goto err_disable_msix;
- err = efa_com_admin_init(edev, &aenq_handlers);
+ err = efa_set_mgmnt_irq(dev);
if (err)
- goto err_free_mgmnt_irq;
+ goto err_destroy_admin;
+
+ efa_com_set_admin_polling_mode(edev, false);
return dev;
-err_free_mgmnt_irq:
- efa_free_irq(dev, &dev->admin_irq);
+err_destroy_admin:
+ efa_com_dev_reset(edev, EFA_REGS_RESET_INIT_ERR);
+ efa_com_admin_destroy(edev);
err_disable_msix:
efa_disable_msix(dev);
err_reg_read_destroy:
@@ -654,8 +659,8 @@ static void efa_remove_device(struct pci_dev *pdev,
edev = &dev->edev;
efa_com_dev_reset(edev, reset_reason);
- efa_com_admin_destroy(edev);
efa_free_irq(dev, &dev->admin_irq);
+ efa_com_admin_destroy(edev);
efa_disable_msix(dev);
efa_com_mmio_reg_read_destroy(edev);
devm_iounmap(&pdev->dev, edev->reg_bar);
diff --git a/drivers/infiniband/hw/erdma/erdma_main.c b/drivers/infiniband/hw/erdma/erdma_main.c
index 7e87a81..445182c 100644
--- a/drivers/infiniband/hw/erdma/erdma_main.c
+++ b/drivers/infiniband/hw/erdma/erdma_main.c
@@ -572,8 +572,8 @@ static int erdma_ib_device_add(struct pci_dev *pdev)
INIT_LIST_HEAD(&dev->cep_list);
spin_lock_init(&dev->lock);
- xa_init_flags(&dev->qp_xa, XA_FLAGS_ALLOC1);
- xa_init_flags(&dev->cq_xa, XA_FLAGS_ALLOC1);
+ xa_init_flags(&dev->qp_xa, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_IRQ);
+ xa_init_flags(&dev->cq_xa, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_IRQ);
dev->next_alloc_cqn = 1;
dev->next_alloc_qpn = 1;
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index 65b1af1..f18b88b 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -1021,15 +1021,15 @@ int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
init_completion(&qp->safe_free);
if (qp->ibqp.qp_type == IB_QPT_GSI) {
- old_entry = xa_store(&dev->qp_xa, 1, qp, GFP_KERNEL);
+ old_entry = xa_store_irq(&dev->qp_xa, 1, qp, GFP_KERNEL);
if (xa_is_err(old_entry))
ret = xa_err(old_entry);
else
qp->ibqp.qp_num = 1;
} else {
- ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp,
- XA_LIMIT(1, dev->attrs.max_qp - 1),
- &dev->next_alloc_qpn, GFP_KERNEL);
+ ret = xa_alloc_cyclic_irq(&dev->qp_xa, &qp->ibqp.qp_num, qp,
+ XA_LIMIT(1, dev->attrs.max_qp - 1),
+ &dev->next_alloc_qpn, GFP_KERNEL);
}
if (ret < 0) {
@@ -1089,7 +1089,7 @@ int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
else
free_kernel_qp(qp);
err_out_xa:
- xa_erase(&dev->qp_xa, QP_ID(qp));
+ xa_erase_irq(&dev->qp_xa, QP_ID(qp));
err_out:
return ret;
}
@@ -1993,9 +1993,9 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
refcount_set(&cq->refcount, 1);
init_completion(&cq->free);
- ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq,
- XA_LIMIT(1, dev->attrs.max_cq - 1),
- &dev->next_alloc_cqn, GFP_KERNEL);
+ ret = xa_alloc_cyclic_irq(&dev->cq_xa, &cq->cqn, cq,
+ XA_LIMIT(1, dev->attrs.max_cq - 1),
+ &dev->next_alloc_cqn, GFP_KERNEL);
if (ret < 0)
return ret;
@@ -2041,7 +2041,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
}
err_out_xa:
- xa_erase(&dev->cq_xa, cq->cqn);
+ xa_erase_irq(&dev->cq_xa, cq->cqn);
return ret;
}
diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c
index dc548e6..1a36f99 100644
--- a/drivers/infiniband/hw/hfi1/file_ops.c
+++ b/drivers/infiniband/hw/hfi1/file_ops.c
@@ -326,6 +326,7 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
void *memvirt = NULL;
dma_addr_t memdma = 0;
u8 subctxt, mapio = 0, vmf = 0, type;
+ size_t memdmalen = 0;
ssize_t memlen = 0;
int ret = 0;
u16 ctxt;
@@ -371,7 +372,9 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
mapio = 1;
break;
case PIO_CRED: {
+ struct credit_return_base *cr = &dd->cr_base[uctxt->sc->node];
u64 cr_page_offset;
+
if (flags & VM_WRITE) {
ret = -EPERM;
goto done;
@@ -381,11 +384,18 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
* second or third page allocated for credit returns (if number
* of enabled contexts > 64 and 128 respectively).
*/
- cr_page_offset = ((u64)uctxt->sc->hw_free -
- (u64)dd->cr_base[uctxt->numa_id].va) &
- PAGE_MASK;
- memvirt = dd->cr_base[uctxt->numa_id].va + cr_page_offset;
- memdma = dd->cr_base[uctxt->numa_id].dma + cr_page_offset;
+ cr_page_offset = ((u64)uctxt->sc->hw_free - (u64)cr->va) &
+ PAGE_MASK;
+ /*
+ * dma_mmap_coherent() describes the whole coherent buffer and
+ * selects the page within it with vma->vm_pgoff, so pass the
+ * base of the allocation and its length and let vm_pgoff pick
+ * the page.
+ */
+ vma->vm_pgoff = cr_page_offset >> PAGE_SHIFT;
+ memvirt = cr->va;
+ memdma = cr->dma;
+ memdmalen = TXE_NUM_CONTEXTS * sizeof(struct credit_return);
memlen = PAGE_SIZE;
flags &= ~VM_MAYWRITE;
flags |= VM_DONTCOPY | VM_DONTEXPAND;
@@ -567,7 +577,8 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
ret = 0;
} else if (memdma) {
ret = dma_mmap_coherent(&dd->pcidev->dev, vma,
- memvirt, memdma, memlen);
+ memvirt, memdma,
+ memdmalen ? memdmalen : memlen);
} else if (mapio) {
ret = io_remap_pfn_range(vma, vma->vm_start,
PFN_DOWN(memaddr),
diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c
index 9cfd84d..5d71300d 100644
--- a/drivers/infiniband/hw/irdma/verbs.c
+++ b/drivers/infiniband/hw/irdma/verbs.c
@@ -4281,7 +4281,7 @@ static int irdma_post_send(struct ib_qp *ibqp,
stag_info.total_len = iwmr->ibmr.length;
stag_info.reg_addr_pa = *palloc->level1.addr;
stag_info.first_pm_pbl_index = palloc->level1.idx;
- stag_info.local_fence = ib_wr->send_flags & IB_SEND_FENCE;
+ stag_info.local_fence = true;
if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR)
stag_info.chunk_size = 1;
err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info,
diff --git a/drivers/infiniband/hw/mlx4/sysfs.c b/drivers/infiniband/hw/mlx4/sysfs.c
index e688ad6..5438224 100644
--- a/drivers/infiniband/hw/mlx4/sysfs.c
+++ b/drivers/infiniband/hw/mlx4/sysfs.c
@@ -751,11 +751,13 @@ static int register_one_pkey_tree(struct mlx4_ib_dev *dev, int slave)
kobject_put(p);
}
kobject_put(dev->dev_ports_parent[slave]);
+ dev->dev_ports_parent[slave] = NULL;
err_ports:
kobject_put(dev->pkeys.device_parent[slave]);
/* extra put for the device_parent create_and_add */
kobject_put(dev->pkeys.device_parent[slave]);
+ dev->pkeys.device_parent[slave] = NULL;
fail_dev:
kobject_put(dev->iov_parent);
@@ -785,6 +787,8 @@ static void unregister_pkey_tree(struct mlx4_ib_dev *device)
return;
for (slave = device->dev->persist->num_vfs; slave >= 0; --slave) {
+ if (!device->pkeys.device_parent[slave])
+ continue;
list_for_each_entry_safe(p, t,
&device->pkeys.pkey_port_list[slave],
entry) {
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 373ee1f..a457647 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -1683,11 +1683,8 @@ static int mlx5_ib_query_port_speed_rep(struct mlx5_ib_dev *dev, u32 port_num,
struct mlx5_core_dev *mdev;
u16 op_mod;
- if (!dev->port[port_num - 1].rep) {
- mlx5_ib_warn(dev, "Representor doesn't exist for port %u\n",
- port_num);
- return -EINVAL;
- }
+ if (!dev->port[port_num - 1].rep)
+ return -ENODEV;
rep = dev->port[port_num - 1].rep;
mdev = mlx5_eswitch_get_core_dev(rep->esw);
diff --git a/drivers/infiniband/sw/rxe/rxe_mcast.c b/drivers/infiniband/sw/rxe/rxe_mcast.c
index acd03bd..5ca9211 100644
--- a/drivers/infiniband/sw/rxe/rxe_mcast.c
+++ b/drivers/infiniband/sw/rxe/rxe_mcast.c
@@ -175,7 +175,9 @@ struct rxe_mcg *rxe_lookup_mcg(struct rxe_dev *rxe, union ib_gid *mgid)
* @mgid: multicast address as a gid
* @mcg: new mcg object
*
- * Context: caller should hold rxe->mcg lock
+ * Initializes the mcg fields. The mcg is private and not yet visible in
+ * mcg_tree, so this may run without rxe->mcg_lock; __rxe_publish_mcg()
+ * makes it visible under the lock once it is ready.
*/
static void __rxe_init_mcg(struct rxe_dev *rxe, union ib_gid *mgid,
struct rxe_mcg *mcg)
@@ -184,13 +186,22 @@ static void __rxe_init_mcg(struct rxe_dev *rxe, union ib_gid *mgid,
memcpy(&mcg->mgid, mgid, sizeof(mcg->mgid));
INIT_LIST_HEAD(&mcg->qp_list);
mcg->rxe = rxe;
+}
- /* caller holds a ref on mcg but that will be
- * dropped when mcg goes out of scope. We need to take a ref
- * on the pointer that will be saved in the red-black tree
- * by __rxe_insert_mcg and used to lookup mcg from mgid later.
- * Inserting mcg makes it visible to outside so this should
- * be done last after the object is ready.
+/**
+ * __rxe_publish_mcg - make a fully initialized mcg visible in mcg_tree
+ * @mcg: the mcg object
+ *
+ * Context: caller must hold rxe->mcg_lock and a reference on mcg
+ */
+static void __rxe_publish_mcg(struct rxe_mcg *mcg)
+{
+ /* caller holds a ref on mcg but that will be dropped when mcg goes
+ * out of scope. We need to take a ref on the pointer that will be
+ * saved in the red-black tree by __rxe_insert_mcg and used to lookup
+ * mcg from mgid later. Inserting mcg makes it visible to outside so
+ * this is done last after the object is ready and the multicast
+ * address has been programmed.
*/
kref_get(&mcg->ref_cnt);
__rxe_insert_mcg(mcg);
@@ -228,26 +239,37 @@ static struct rxe_mcg *rxe_get_mcg(struct rxe_dev *rxe, union ib_gid *mgid)
err = -ENOMEM;
goto err_dec;
}
+ __rxe_init_mcg(rxe, mgid, mcg);
+
+ /* program the multicast address while mcg is still private, before
+ * it is inserted into mcg_tree. dev_mc_add() may sleep so this must
+ * run outside mcg_lock. On failure mcg was never published, so a
+ * plain free is correct and the tree is untouched.
+ */
+ err = rxe_mcast_add(rxe, mgid);
+ if (err) {
+ kfree(mcg);
+ goto err_dec;
+ }
spin_lock_bh(&rxe->mcg_lock);
- /* re-check to see if someone else just added it */
+ /* re-check to see if someone else just added it while we were adding
+ * the multicast address; if so use theirs and drop ours
+ */
tmp = __rxe_lookup_mcg(rxe, mgid);
if (tmp) {
spin_unlock_bh(&rxe->mcg_lock);
+ rxe_mcast_del(rxe, mgid);
atomic_dec(&rxe->mcg_num);
kfree(mcg);
return tmp;
}
- __rxe_init_mcg(rxe, mgid, mcg);
+ __rxe_publish_mcg(mcg);
spin_unlock_bh(&rxe->mcg_lock);
- /* add mcast address outside of lock */
- err = rxe_mcast_add(rxe, mgid);
- if (!err)
- return mcg;
+ return mcg;
- kfree(mcg);
err_dec:
atomic_dec(&rxe->mcg_num);
return ERR_PTR(err);
diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
index 875eceb..71d9ea4 100644
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -33,7 +33,8 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
case IB_MR_TYPE_USER:
case IB_MR_TYPE_MEM_REG:
if (iova < mr->ibmr.iova ||
- iova + length > mr->ibmr.iova + mr->ibmr.length) {
+ length > mr->ibmr.length ||
+ iova - mr->ibmr.iova > mr->ibmr.length - length) {
rxe_dbg_mr(mr, "iova/length out of range\n");
return -EINVAL;
}
diff --git a/drivers/infiniband/sw/rxe/rxe_odp.c b/drivers/infiniband/sw/rxe/rxe_odp.c
index e870efa..ab21b62 100644
--- a/drivers/infiniband/sw/rxe/rxe_odp.c
+++ b/drivers/infiniband/sw/rxe/rxe_odp.c
@@ -120,19 +120,23 @@ int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length,
}
static inline bool rxe_check_pagefault(struct ib_umem_odp *umem_odp, u64 iova,
- int length)
+ int length, bool write)
{
bool need_fault = false;
+ u64 access = HMM_PFN_VALID;
u64 addr;
int idx;
+ if (write)
+ access |= HMM_PFN_WRITE;
+
addr = iova & (~(BIT(umem_odp->page_shift) - 1));
/* Skim through all pages that are to be accessed. */
while (addr < iova + length) {
idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift;
- if (!(umem_odp->map.pfn_list[idx] & HMM_PFN_VALID)) {
+ if ((umem_odp->map.pfn_list[idx] & access) != access) {
need_fault = true;
break;
}
@@ -155,6 +159,7 @@ static unsigned long rxe_odp_iova_to_page_offset(struct ib_umem_odp *umem_odp, u
static int rxe_odp_map_range_and_lock(struct rxe_mr *mr, u64 iova, int length, u32 flags)
{
struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem);
+ bool write = !(flags & RXE_PAGEFAULT_RDONLY);
bool need_fault;
int err;
@@ -163,7 +168,7 @@ static int rxe_odp_map_range_and_lock(struct rxe_mr *mr, u64 iova, int length, u
mutex_lock(&umem_odp->umem_mutex);
- need_fault = rxe_check_pagefault(umem_odp, iova, length);
+ need_fault = rxe_check_pagefault(umem_odp, iova, length, write);
if (need_fault) {
mutex_unlock(&umem_odp->umem_mutex);
@@ -173,7 +178,7 @@ static int rxe_odp_map_range_and_lock(struct rxe_mr *mr, u64 iova, int length, u
if (err < 0)
return err;
- need_fault = rxe_check_pagefault(umem_odp, iova, length);
+ need_fault = rxe_check_pagefault(umem_odp, iova, length, write);
if (need_fault) {
mutex_unlock(&umem_odp->umem_mutex);
return -EFAULT;
@@ -335,8 +340,9 @@ int rxe_odp_flush_pmem_iova(struct rxe_mr *mr, u64 iova,
int err;
u8 *va;
+ /* A flush never modifies memory; read-only access suffices. */
err = rxe_odp_map_range_and_lock(mr, iova, length,
- RXE_PAGEFAULT_DEFAULT);
+ RXE_PAGEFAULT_RDONLY);
if (err)
return err;
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 96c7716..3864284 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -1331,19 +1331,20 @@ static struct ib_mr *rxe_rereg_user_mr(struct ib_mr *ibmr, int flags,
if (err)
return ERR_PTR(err);
+ if ((flags & IB_MR_REREG_ACCESS) &&
+ (access & ~RXE_ACCESS_SUPPORTED_MR)) {
+ rxe_err_mr(mr, "access = %#x not supported\n", access);
+ return ERR_PTR(-EOPNOTSUPP);
+ }
+
if (flags & IB_MR_REREG_PD) {
rxe_put(old_pd);
rxe_get(pd);
mr->ibmr.pd = ibpd;
}
- if (flags & IB_MR_REREG_ACCESS) {
- if (access & ~RXE_ACCESS_SUPPORTED_MR) {
- rxe_err_mr(mr, "access = %#x not supported\n", access);
- return ERR_PTR(-EOPNOTSUPP);
- }
+ if (flags & IB_MR_REREG_ACCESS)
mr->access = access;
- }
return NULL;
}
diff --git a/drivers/infiniband/sw/siw/siw_cm.c b/drivers/infiniband/sw/siw/siw_cm.c
index 0245b25..ed49818 100644
--- a/drivers/infiniband/sw/siw/siw_cm.c
+++ b/drivers/infiniband/sw/siw/siw_cm.c
@@ -1719,9 +1719,12 @@ int siw_accept(struct iw_cm_id *id, struct iw_cm_conn_param *params)
SIW_QP_ATTR_STATE | SIW_QP_ATTR_LLP_HANDLE |
SIW_QP_ATTR_ORD | SIW_QP_ATTR_IRD |
SIW_QP_ATTR_MPA);
+ if (rv) {
+ qp->cep = NULL;
+ siw_cep_put(cep);
+ goto error_unlock;
+ }
up_write(&qp->state_lock);
- if (rv)
- goto error;
siw_dbg_cep(cep, "[QP %u]: send mpa reply, %d byte pdata\n",
qp_id(qp), params->private_data_len);
diff --git a/drivers/infiniband/sw/siw/siw_qp_rx.c b/drivers/infiniband/sw/siw/siw_qp_rx.c
index b566d16..e5b641c 100644
--- a/drivers/infiniband/sw/siw/siw_qp_rx.c
+++ b/drivers/infiniband/sw/siw/siw_qp_rx.c
@@ -1079,7 +1079,7 @@ static int siw_get_hdr(struct siw_rx_stream *srx)
if (iwarp_pktinfo[opcode].hdr_len > sizeof(struct iwarp_ctrl_tagged)) {
int hdrlen = iwarp_pktinfo[opcode].hdr_len;
- bytes = min_t(int, hdrlen - MIN_DDP_HDR, srx->skb_new);
+ bytes = min_t(int, hdrlen - srx->fpdu_part_rcvd, srx->skb_new);
skb_copy_bits(skb, srx->skb_offset,
(char *)c_hdr + srx->fpdu_part_rcvd, bytes);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h
index 91f866e..143e03b 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib.h
+++ b/drivers/infiniband/ulp/ipoib/ipoib.h
@@ -87,6 +87,7 @@ enum {
IPOIB_FLAG_INITIALIZED = 1,
IPOIB_FLAG_ADMIN_UP = 2,
IPOIB_PKEY_ASSIGNED = 3,
+ IPOIB_FLAG_MCAST_FLUSH = 4,
IPOIB_FLAG_SUBINTERFACE = 5,
IPOIB_STOP_REAPER = 7,
IPOIB_FLAG_ADMIN_CM = 9,
@@ -414,6 +415,12 @@ struct ipoib_dev_priv {
const struct net_device_ops *rn_ops;
};
+static inline bool ipoib_mcast_allowed(struct ipoib_dev_priv *priv)
+{
+ return test_bit(IPOIB_FLAG_OPER_UP, &priv->flags) &&
+ !test_bit(IPOIB_FLAG_MCAST_FLUSH, &priv->flags);
+}
+
struct ipoib_ah {
struct net_device *dev;
struct ib_ah *ah;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
index 5061d52..81bbb3f 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
@@ -1227,17 +1227,19 @@ static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv,
}
if (level == IPOIB_FLUSH_LIGHT) {
- int oper_up;
ipoib_mark_paths_invalid(dev);
- /* Set IPoIB operation as down to prevent races between:
+ /* Set MCAST_FLUSH to prevent races between:
* the flush flow which leaves MCG and on the fly joins
* which can happen during that time. mcast restart task
* should deal with join requests we missed.
+ *
+ * Do not clear OPER_UP for this; restoring it races with
+ * ipoib_ib_dev_down() and can leave OPER_UP set after the
+ * device is down.
*/
- oper_up = test_and_clear_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
+ set_bit(IPOIB_FLAG_MCAST_FLUSH, &priv->flags);
ipoib_mcast_dev_flush(dev);
- if (oper_up)
- set_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
+ clear_bit(IPOIB_FLAG_MCAST_FLUSH, &priv->flags);
ipoib_reap_dead_ahs(priv);
}
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
index 6401af2f..379b783 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
@@ -74,7 +74,7 @@ static void __ipoib_mcast_schedule_join_thread(struct ipoib_dev_priv *priv,
struct ipoib_mcast *mcast,
bool delay)
{
- if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags))
+ if (!ipoib_mcast_allowed(priv))
return;
/*
@@ -469,7 +469,7 @@ static int ipoib_mcast_join(struct net_device *dev, struct ipoib_mcast *mcast)
int ret = 0;
if (!priv->broadcast ||
- !test_bit(IPOIB_FLAG_OPER_UP, &priv->flags))
+ !ipoib_mcast_allowed(priv))
return -EINVAL;
init_completion(&mcast->done);
@@ -555,7 +555,7 @@ void ipoib_mcast_join_task(struct work_struct *work)
unsigned long delay_until = 0;
struct ipoib_mcast *mcast = NULL;
- if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags))
+ if (!ipoib_mcast_allowed(priv))
return;
if (ib_query_port(priv->ca, priv->port, &port_attr)) {
@@ -577,7 +577,7 @@ void ipoib_mcast_join_task(struct work_struct *work)
netif_addr_unlock_bh(dev);
spin_lock_irq(&priv->lock);
- if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags))
+ if (!ipoib_mcast_allowed(priv))
goto out;
if (!priv->broadcast) {
@@ -749,7 +749,7 @@ void ipoib_mcast_send(struct net_device *dev, u8 *daddr, struct sk_buff *skb)
spin_lock_irqsave(&priv->lock, flags);
- if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags) ||
+ if (!ipoib_mcast_allowed(priv) ||
!priv->broadcast ||
!test_bit(IPOIB_MCAST_FLAG_ATTACHED, &priv->broadcast->flags)) {
++dev->stats.tx_dropped;
@@ -871,7 +871,7 @@ void ipoib_mcast_restart_task(struct work_struct *work)
LIST_HEAD(remove_list);
struct ib_sa_mcmember_rec rec;
- if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags))
+ if (!ipoib_mcast_allowed(priv))
/*
* shortcut...on shutdown flush is called next, just
* let it do all the work
@@ -965,9 +965,9 @@ void ipoib_mcast_restart_task(struct work_struct *work)
ipoib_mcast_remove_list(&remove_list);
/*
- * Double check that we are still up
+ * Double check that we are still up and not flushing
*/
- if (test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) {
+ if (ipoib_mcast_allowed(priv)) {
spin_lock_irq(&priv->lock);
__ipoib_mcast_schedule_join_thread(priv, NULL, 0);
spin_unlock_irq(&priv->lock);
diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c
index 12a2d12..7ea6888 100644
--- a/drivers/infiniband/ulp/iser/iser_initiator.c
+++ b/drivers/infiniband/ulp/iser/iser_initiator.c
@@ -598,11 +598,8 @@ static int iser_check_remote_inv(struct iser_conn *iser_conn, struct ib_wc *wc,
iser_dbg("conn %p: remote invalidation for rkey %#x\n",
iser_conn, rkey);
- if (unlikely(!iser_conn->snd_w_inv)) {
- iser_err("conn %p: unexpected remote invalidation, terminating connection\n",
- iser_conn);
- return -EPROTO;
- }
+ if (unlikely(!iser_conn->snd_w_inv))
+ goto bad_inv;
task = iscsi_itt_to_ctask(iser_conn->iscsi_conn, hdr->itt);
if (likely(task)) {
@@ -611,12 +608,16 @@ static int iser_check_remote_inv(struct iser_conn *iser_conn, struct ib_wc *wc,
if (iser_task->dir[ISER_DIR_IN]) {
desc = iser_task->rdma_reg[ISER_DIR_IN].desc;
+ if (unlikely(!desc))
+ goto bad_inv;
if (unlikely(iser_inv_desc(desc, rkey)))
return -EINVAL;
}
if (iser_task->dir[ISER_DIR_OUT]) {
desc = iser_task->rdma_reg[ISER_DIR_OUT].desc;
+ if (unlikely(!desc))
+ goto bad_inv;
if (unlikely(iser_inv_desc(desc, rkey)))
return -EINVAL;
}
@@ -627,6 +628,11 @@ static int iser_check_remote_inv(struct iser_conn *iser_conn, struct ib_wc *wc,
}
return 0;
+
+bad_inv:
+ iser_err("conn %p: unexpected remote invalidation, terminating connection\n",
+ iser_conn);
+ return -EPROTO;
}
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index 5087ea9..e69db43 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -21,6 +21,7 @@
#include <target/target_core_fabric.h>
#include <target/iscsi/iscsi_transport.h>
#include <linux/semaphore.h>
+#include <linux/wait_bit.h>
#include "ib_isert.h"
@@ -310,6 +311,7 @@ isert_init_conn(struct isert_conn *isert_conn)
init_completion(&isert_conn->login_req_comp);
init_waitqueue_head(&isert_conn->rem_wait);
kref_init(&isert_conn->kref);
+ atomic_set(&isert_conn->ctrl_comp_cnt, 0);
mutex_init(&isert_conn->mutex);
INIT_WORK(&isert_conn->release_work, isert_release_work);
}
@@ -1694,6 +1696,8 @@ isert_do_control_comp(struct work_struct *work)
struct isert_conn *isert_conn = isert_cmd->conn;
struct ib_device *ib_dev = isert_conn->cm_id->device;
struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd;
+ /* The switch below may free isert_cmd. */
+ bool counted = isert_cmd->ctrl_counted;
isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state);
@@ -1715,6 +1719,14 @@ isert_do_control_comp(struct work_struct *work)
dump_stack();
break;
}
+
+ /*
+ * The count is what keeps isert_conn alive, so drop it last. The wait
+ * queue lives in the global hash table, not in isert_conn, so this is
+ * safe even if the waiter has already freed the connection.
+ */
+ if (counted && atomic_dec_and_test(&isert_conn->ctrl_comp_cnt))
+ wake_up_var(&isert_conn->ctrl_comp_cnt);
}
static void
@@ -1758,6 +1770,12 @@ isert_send_done(struct ib_cq *cq, struct ib_wc *wc)
case ISTATE_SEND_TEXTRSP:
isert_unmap_tx_desc(tx_desc, ib_dev);
+ /* Paired with the wait in isert_wait_conn(). */
+ isert_cmd->ctrl_counted =
+ isert_cmd->iscsit_cmd->i_state != ISTATE_SEND_LOGOUTRSP;
+ if (isert_cmd->ctrl_counted)
+ atomic_inc(&isert_conn->ctrl_comp_cnt);
+
INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);
queue_work(isert_comp_wq, &isert_cmd->comp_work);
return;
@@ -2602,6 +2620,10 @@ static void isert_wait_conn(struct iscsit_conn *conn)
isert_wait4cmds(conn);
isert_wait4logout(isert_conn);
+ /* Paired with the count taken in isert_send_done(). */
+ wait_var_event(&isert_conn->ctrl_comp_cnt,
+ !atomic_read(&isert_conn->ctrl_comp_cnt));
+
queue_work(isert_release_wq, &isert_conn->release_work);
}
diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h
index 0bac5aa..519b17e 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.h
+++ b/drivers/infiniband/ulp/isert/ib_isert.h
@@ -153,6 +153,7 @@ struct isert_cmd {
struct work_struct comp_work;
struct scatterlist sg;
bool ctx_init_done;
+ bool ctrl_counted;
};
static inline struct isert_cmd *tx_desc_to_cmd(struct iser_tx_desc *desc)
@@ -187,6 +188,7 @@ struct isert_conn {
struct mutex mutex;
struct kref kref;
struct work_struct release_work;
+ atomic_t ctrl_comp_cnt;
bool logout_posted;
bool snd_w_inv;
wait_queue_head_t rem_wait;
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h
index 7738e26..29e2340 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h
@@ -55,7 +55,7 @@ DECLARE_EVENT_CLASS(rtrs_clt_conn_class,
__entry->max_reconnect_attempts = clt->max_reconnect_attempts;
__entry->fail_cnt = clt_path->stats->reconnects.fail_cnt;
__entry->success_cnt = clt_path->stats->reconnects.successful_cnt;
- memcpy(__entry->sessname, kobject_name(&clt_path->kobj), NAME_MAX);
+ strscpy(__entry->sessname, kobject_name(&clt_path->kobj) ?: "", NAME_MAX);
),
TP_printk("RTRS-CLT: sess='%s' state=%s attempts='%d' max-attempts='%d' fail='%d' success='%d'",
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
index 7b2c51a..eac38b5 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
@@ -1732,6 +1732,8 @@ static void destroy_con_cq_qp(struct rtrs_clt_con *con)
/*
* Be careful here: destroy_con_cq_qp() can be called even
* create_con_cq_qp() failed, see comments there.
+ * Caller must set con->destroyed under this lock first so a
+ * racing ADDR_RESOLVED cannot ib_cq_pool_get() after we PUT/SKIP.
*/
lockdep_assert_held(&con->con_mutex);
rtrs_cq_qp_destroy(&con->c);
@@ -1766,6 +1768,10 @@ static int rtrs_rdma_addr_resolved(struct rtrs_clt_con *con)
int err;
mutex_lock(&con->con_mutex);
+ if (con->destroyed) {
+ mutex_unlock(&con->con_mutex);
+ return -ECONNABORTED;
+ }
err = create_con_cq_qp(con);
mutex_unlock(&con->con_mutex);
if (err) {
@@ -2221,6 +2227,7 @@ static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_path *clt_path)
break;
con = to_clt_con(clt_path->s.con[cid]);
mutex_lock(&con->con_mutex);
+ con->destroyed = true;
destroy_con_cq_qp(con);
mutex_unlock(&con->con_mutex);
destroy_cm(con);
@@ -2387,6 +2394,7 @@ static int init_conns(struct rtrs_clt_path *clt_path)
if (con->c.cm_id) {
stop_cm(con);
mutex_lock(&con->con_mutex);
+ con->destroyed = true;
destroy_con_cq_qp(con);
mutex_unlock(&con->con_mutex);
destroy_cm(con);
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.h b/drivers/infiniband/ulp/rtrs/rtrs-clt.h
index 1305601..ad64f45 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.h
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.h
@@ -75,6 +75,8 @@ struct rtrs_clt_con {
unsigned int cpu;
struct mutex con_mutex;
int cm_err;
+ /* Set under con_mutex before CQ/QP teardown. */
+ bool destroyed;
};
/**
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h b/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h
index 587d3e0..a7d7b97 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h
@@ -61,7 +61,7 @@ TRACE_EVENT(send_io_resp_imm,
__entry->msg_id = id->msg_id;
__entry->wr_cnt = atomic_read(&con->c.wr_cnt);
__entry->signal_interval = s->signal_interval;
- memcpy(__entry->sessname, kobject_name(&srv_path->kobj), NAME_MAX);
+ strscpy(__entry->sessname, kobject_name(&srv_path->kobj) ?: "", NAME_MAX);
),
TP_printk("sess='%s' state='%s' dir=%s err='%d' inval='%d' glob-inval='%d' msgid='%u' wrcnt='%d' sig-interval='%u'",
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index 6b429ef..955f36e 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -1038,15 +1038,20 @@ static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
static void srp_remove_target(struct srp_target_port *target)
{
+ struct scsi_device *sdev;
struct srp_rdma_ch *ch;
int i;
WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
srp_del_scsi_host_attr(target->scsi_host);
- srp_rport_get(target->rport);
- srp_remove_host(target->scsi_host);
- scsi_remove_host(target->scsi_host);
+ /*
+ * Remove all logical units. This must happen before the
+ * srp_disconnect_target() call because scsi_remove_device() may trigger
+ * submission of SCSI commands. See also sd_shutdown().
+ */
+ shost_for_each_device(sdev, target->scsi_host)
+ scsi_remove_device(sdev);
srp_stop_rport_timers(target->rport);
srp_disconnect_target(target);
kobj_ns_drop(KOBJ_NS_TYPE_NET, to_ns_common(target->net));
@@ -1055,7 +1060,8 @@ static void srp_remove_target(struct srp_target_port *target)
srp_free_ch_ib(target, ch);
}
cancel_work_sync(&target->tl_err_work);
- srp_rport_put(target->rport);
+ srp_remove_host(target->scsi_host);
+ scsi_remove_host(target->scsi_host);
kfree(target->ch);
target->ch = NULL;
diff --git a/drivers/power/sequencing/Kconfig b/drivers/power/sequencing/Kconfig
index 1c5f582..226c627 100644
--- a/drivers/power/sequencing/Kconfig
+++ b/drivers/power/sequencing/Kconfig
@@ -29,7 +29,8 @@
config POWER_SEQUENCING_TH1520_GPU
tristate "T-HEAD TH1520 GPU power sequencing driver"
- depends on (ARCH_THEAD && AUXILIARY_BUS) || COMPILE_TEST
+ depends on ARCH_THEAD || COMPILE_TEST
+ select AUXILIARY_BUS
help
Say Y here to enable the power sequencing driver for the TH1520 SoC
GPU. This driver handles the complex clock and reset sequence
diff --git a/drivers/power/sequencing/core.c b/drivers/power/sequencing/core.c
index 0cb71ef..3076b38 100644
--- a/drivers/power/sequencing/core.c
+++ b/drivers/power/sequencing/core.c
@@ -101,6 +101,7 @@ static struct pwrseq_unit *pwrseq_unit_new(const struct pwrseq_unit_data *data)
}
kref_init(&unit->ref);
+ INIT_LIST_HEAD(&unit->list);
INIT_LIST_HEAD(&unit->deps);
unit->enable = data->enable;
unit->disable = data->disable;
@@ -504,10 +505,6 @@ pwrseq_device_register(const struct pwrseq_config *config)
*/
device_initialize(&pwrseq->dev);
- ret = dev_set_name(&pwrseq->dev, "pwrseq.%d", pwrseq->id);
- if (ret)
- goto err_put_pwrseq;
-
pwrseq->owner = config->owner ?: THIS_MODULE;
pwrseq->match = config->match;
@@ -516,6 +513,10 @@ pwrseq_device_register(const struct pwrseq_config *config)
INIT_LIST_HEAD(&pwrseq->targets);
INIT_LIST_HEAD(&pwrseq->units);
+ ret = dev_set_name(&pwrseq->dev, "pwrseq.%d", pwrseq->id);
+ if (ret)
+ goto err_put_pwrseq;
+
ret = pwrseq_setup_targets(config->targets, pwrseq);
if (ret)
goto err_put_pwrseq;
@@ -912,6 +913,8 @@ int pwrseq_enable(struct pwrseq_desc *desc)
if (!ret)
desc->powered_on = true;
}
+ if (ret)
+ return ret;
if (target->post_enable) {
ret = target->post_enable(pwrseq);
diff --git a/fs/9p/vfs_addr.c b/fs/9p/vfs_addr.c
index 1ac0b3d..13cf87a 100644
--- a/fs/9p/vfs_addr.c
+++ b/fs/9p/vfs_addr.c
@@ -54,11 +54,37 @@ static void v9fs_begin_writeback(struct netfs_io_request *wreq)
static void v9fs_issue_write(struct netfs_io_subrequest *subreq)
{
struct p9_fid *fid = subreq->rreq->netfs_priv;
+ struct inode *inode = subreq->rreq->inode;
+ struct netfs_inode *ictx = netfs_inode(inode);
int err, len;
len = p9_client_write(fid, subreq->start, &subreq->io_iter, &err);
- if (len > 0)
+ if (len > 0) {
+ uoff_t end = subreq->start + len, i_size, remote, zp;
+ bool set = false;
+
+ spin_lock(&inode->i_lock);
+
+ /* We can read the sizes directly as we hold i_lock. */
+ i_size = inode->i_size;
+ remote = ictx->_remote_i_size;
+ zp = ictx->_zero_point;
+
+ if (end > i_size) {
+ i_size = end;
+ set = true;
+ }
+ if (end > remote) {
+ remote = end;
+ set = true;
+ }
+
+ if (set)
+ netfs_write_sizes(inode, i_size, remote, zp);
+ spin_unlock(&inode->i_lock);
+
__set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
+ }
netfs_write_subrequest_terminated(subreq, len ?: err);
}
diff --git a/fs/dax.c b/fs/dax.c
index 6ba5014..1fbba0d 100644
--- a/fs/dax.c
+++ b/fs/dax.c
@@ -480,11 +480,12 @@ static void dax_associate_entry(void *entry, struct address_space *mapping,
unsigned long address, bool shared)
{
unsigned long size = dax_entry_size(entry), index;
- struct folio *folio = dax_to_folio(entry);
+ struct folio *folio;
if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry))
return;
+ folio = dax_to_folio(entry);
index = linear_page_index(vma, address & ~(size - 1));
if (shared && (folio->mapping || dax_folio_is_shared(folio))) {
if (folio->mapping)
@@ -505,21 +506,23 @@ static void dax_associate_entry(void *entry, struct address_space *mapping,
static void dax_disassociate_entry(void *entry, struct address_space *mapping,
bool trunc)
{
- struct folio *folio = dax_to_folio(entry);
+ struct folio *folio;
if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry))
return;
+ folio = dax_to_folio(entry);
dax_folio_put(folio);
}
static struct page *dax_busy_page(void *entry)
{
- struct folio *folio = dax_to_folio(entry);
+ struct folio *folio;
if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry))
return NULL;
+ folio = dax_to_folio(entry);
if (folio_ref_count(folio) - folio_mapcount(folio))
return &folio->page;
else
diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c
index a7ebce5..b6e0c54 100644
--- a/fs/nfsd/export.c
+++ b/fs/nfsd/export.c
@@ -1005,7 +1005,8 @@ static int nfsd_nl_parse_one_export(struct cache_detail *cd,
goto out_uuid;
err = 0;
- nfsd4_setup_layout_type(&exp);
+ if (exp.ex_flags & NFSEXP_PNFS)
+ nfsd4_setup_layout_type(&exp);
}
expp = svc_export_lookup(&exp);
diff --git a/fs/super.c b/fs/super.c
index 01db612..9d40252 100644
--- a/fs/super.c
+++ b/fs/super.c
@@ -172,19 +172,6 @@ static void super_wake(struct super_block *sb, unsigned int flag)
}
/*
- * The s_op->nr_cached_objects hooks (used for example by btrfs and xfs)
- * operate on filesystem-global state and ignore sc->memcg. Driving them
- * from per-memcg shrink_slab_memcg() invocations only burns CPU walking
- * per-cpu counters and queueing duplicate work: the actual reclaim happens on
- * the global path (kswapd or root direct reclaim) regardless. Restrict them
- * to that path.
- */
-static inline bool super_fs_objects_eligible(struct shrink_control *sc)
-{
- return !sc->memcg || mem_cgroup_is_root(sc->memcg);
-}
-
-/*
* One thing we have to be careful of with a per-sb shrinker is that we don't
* drop the last active reference to the superblock from within the shrinker.
* If that happens we could trigger unregistering the shrinker from within the
@@ -213,7 +200,7 @@ static unsigned long super_cache_scan(struct shrinker *shrink,
if (!super_trylock_shared(sb))
return SHRINK_STOP;
- if (sb->s_op->nr_cached_objects && super_fs_objects_eligible(sc))
+ if (sb->s_op->nr_cached_objects)
fs_objects = sb->s_op->nr_cached_objects(sb, sc);
inodes = list_lru_shrink_count(&sb->s_inode_lru, sc);
@@ -274,8 +261,7 @@ static unsigned long super_cache_count(struct shrinker *shrink,
return 0;
smp_rmb();
- if (sb->s_op && sb->s_op->nr_cached_objects &&
- super_fs_objects_eligible(sc))
+ if (sb->s_op && sb->s_op->nr_cached_objects)
total_objects = sb->s_op->nr_cached_objects(sb, sc);
total_objects += list_lru_shrink_count(&sb->s_dentry_lru, sc);
diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h
index bec0eab..b8a13b8 100644
--- a/include/linux/mmap_lock.h
+++ b/include/linux/mmap_lock.h
@@ -630,6 +630,8 @@ static inline void mmap_read_unlock(struct mm_struct *mm)
DEFINE_GUARD(mmap_read_lock, struct mm_struct *,
mmap_read_lock(_T), mmap_read_unlock(_T))
DEFINE_GUARD_COND(mmap_read_lock, _try, mmap_read_trylock(_T))
+DEFINE_GUARD(mmap_write_lock, struct mm_struct *,
+ mmap_write_lock(_T), mmap_write_unlock(_T))
static inline void mmap_read_unlock_non_owner(struct mm_struct *mm)
{
diff --git a/include/rdma/uverbs_types.h b/include/rdma/uverbs_types.h
index 5a07f9a6..6f36228 100644
--- a/include/rdma/uverbs_types.h
+++ b/include/rdma/uverbs_types.h
@@ -180,8 +180,6 @@ struct ib_uverbs_file {
struct page *disassociate_page;
struct xarray idr;
-
- struct mutex disassociation_lock;
};
extern const struct uverbs_obj_type_class uverbs_idr_class;
diff --git a/include/trace/events/dma.h b/include/trace/events/dma.h
index 9df02c1..b06d8f9 100644
--- a/include/trace/events/dma.h
+++ b/include/trace/events/dma.h
@@ -134,7 +134,7 @@ DECLARE_EVENT_CLASS(dma_alloc_class,
TP_fast_assign(
__assign_str(device);
__entry->virt_addr = virt_addr;
- __entry->dma_addr = dma_addr;
+ __entry->dma_addr = virt_addr ? dma_addr : 0;
__entry->size = size;
__entry->flags = flags;
__entry->dir = dir;
diff --git a/include/uapi/rdma/bnxt_re-abi.h b/include/uapi/rdma/bnxt_re-abi.h
index 856a1b3..15ed2a5 100644
--- a/include/uapi/rdma/bnxt_re-abi.h
+++ b/include/uapi/rdma/bnxt_re-abi.h
@@ -250,7 +250,7 @@ struct bnxt_re_query_device_ex_resp {
struct bnxt_re_db_region {
__u32 dpi;
__u32 reserved;
- __aligned_u64 umdbr;
+ __aligned_u64 reserved2;
};
enum bnxt_re_obj_dbr_alloc_attrs {
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 2d532bf..227d097 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -5303,10 +5303,13 @@ struct task_struct *css_task_iter_next(struct css_task_iter *it)
if (it->flags & CSS_TASK_ITER_SKIPPED)
css_task_iter_advance(it);
- if (it->task_pos) {
+ while (it->task_pos && !it->cur_task) {
it->cur_task = list_entry(it->task_pos, struct task_struct,
cg_list);
- get_task_struct(it->cur_task);
+ /* a task on dying_tasks with zero refcount is only valid for
+ * RCU readers, not even interesting for
+ * CSS_TASK_ITER_WITH_DEAD, find another one */
+ it->cur_task = tryget_task_struct(it->cur_task);
css_task_iter_advance(it);
}
diff --git a/kernel/dma/coherent.c b/kernel/dma/coherent.c
index 45bbae9..4d026689 100644
--- a/kernel/dma/coherent.c
+++ b/kernel/dma/coherent.c
@@ -352,8 +352,7 @@ static int rmem_dma_device_init(struct reserved_mem *rmem, struct device *dev)
min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit))
dev_warn(dev, "reserved memory is beyond device's set DMA address range\n");
- dma_assign_coherent_memory(dev, mem);
- return 0;
+ return dma_assign_coherent_memory(dev, mem);
}
static void rmem_dma_device_release(struct reserved_mem *rmem,
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index ded7016a..aa2f1c4 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -1019,7 +1019,6 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size
int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
phys_addr_t orig_addr = mem->slots[index].orig_addr;
size_t alloc_size = mem->slots[index].alloc_size;
- unsigned long pfn = PFN_DOWN(orig_addr);
unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start;
int tlb_offset;
@@ -1052,7 +1051,8 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size
size = alloc_size;
}
- if (PageHighMem(pfn_to_page(pfn))) {
+ if (PhysHighMem(orig_addr)) {
+ unsigned long pfn = PFN_DOWN(orig_addr);
unsigned int offset = orig_addr & ~PAGE_MASK;
struct page *page;
unsigned int sz = 0;
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 51de1d8..3219f0d 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -2919,7 +2919,7 @@ static inline void maybe_queue_balance_callback(struct rq *rq)
static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev)
{
- struct scx_sched *sch = scx_root_protected_live();
+ struct scx_sched *root_sch = scx_root_protected_live();
enum scx_dsp_verdict verdict;
s32 cpu = cpu_of(rq);
@@ -2928,7 +2928,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
scx_process_sync_ecaps(rq, prev);
- if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) &&
+ if ((root_sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) &&
unlikely(rq->scx.cpu_released)) {
/*
* If the previous sched_class for the current CPU was not SCX,
@@ -2936,8 +2936,8 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
* core. This callback complements ->cpu_release(), which is
* emitted in switch_class().
*/
- if (sch->ops.cpu_acquire)
- SCX_CALL_OP(sch, cpu_acquire, rq, cpu, NULL);
+ if (root_sch->ops.cpu_acquire)
+ SCX_CALL_OP(root_sch, cpu_acquire, rq, cpu, NULL);
rq->scx.cpu_released = false;
}
@@ -2955,7 +2955,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
* test.
*/
if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice &&
- !scx_bypassing(sch, cpu)) {
+ !scx_bypassing(scx_task_sched(prev), cpu)) {
verdict = SCX_DSP_PREV;
goto has_tasks;
}
@@ -2967,20 +2967,25 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
goto has_tasks;
}
- verdict = scx_dispatch_sched(sch, rq, prev, false);
+ verdict = scx_dispatch_sched(root_sch, rq, prev, false);
if (verdict != SCX_DSP_NONE)
goto has_tasks;
/*
- * Didn't find another task to run. Keep running @prev unless
- * %SCX_OPS_ENQ_LAST is in effect.
+ * Didn't find another task to run. Keep running @prev unless its own
+ * scheduler set %SCX_OPS_ENQ_LAST and takes the enqueue instead, see
+ * put_prev_task_scx(). Read the scheduler here as the dispatch above
+ * may have dropped the rq lock while @prev changed class or scheduler.
*/
- if ((prev->scx.flags & SCX_TASK_QUEUED) &&
- (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu)) &&
- scx_task_can_stay_on_cpu(rq, prev)) {
- __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1);
- verdict = SCX_DSP_PREV;
- goto has_tasks;
+ if (prev->scx.flags & SCX_TASK_QUEUED) {
+ struct scx_sched *prev_sch = scx_task_sched(prev);
+
+ if ((!(prev_sch->ops.flags & SCX_OPS_ENQ_LAST) ||
+ scx_bypassing(prev_sch, cpu)) && scx_task_can_stay_on_cpu(rq, prev)) {
+ __scx_add_event(prev_sch, SCX_EV_DISPATCH_KEEP_LAST, 1);
+ verdict = SCX_DSP_PREV;
+ goto has_tasks;
+ }
}
rq->scx.flags &= ~SCX_RQ_IN_DISPATCH;
return SCX_DSP_NONE;
@@ -3665,8 +3670,20 @@ static void handle_hotplug(struct rq *rq, bool online)
s16 *tbl = rcu_dereference_check(scx_cpu_to_cid_tbl,
lockdep_is_cpus_held());
- if (tbl)
+ if (tbl) {
+ struct scx_sched *pos;
+
cpu_or_cid = tbl[cpu];
+
+ guard(raw_spinlock_irqsave)(&scx_sched_lock);
+ list_for_each_entry(pos, &scx_sched_all, all) {
+ struct scx_cmask *mask = pos->online_cmask;
+
+ if (mask)
+ __assign_bit(cpu_or_cid, (unsigned long *)mask->bits,
+ online);
+ }
+ }
}
if (online && SCX_HAS_OP(sch, cpu_online))
@@ -4766,7 +4783,8 @@ int scx_tg_online(struct task_group *tg)
{ .weight = tg->scx.weight,
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
- .bw_burst_us = tg->scx.bw_burst_us };
+ .bw_burst_us = tg->scx.bw_burst_us,
+ .sched_idle = tg->scx.idle };
ret = SCX_CALL_OP_RET(sch, cgroup_init,
NULL, tg->css.cgroup, &args);
@@ -4932,7 +4950,8 @@ void scx_group_set_idle(struct task_group *tg, bool idle)
percpu_down_read(&scx_cgroup_ops_rwsem);
sch = scx_tg_knob_sched(tg);
- if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle))
+ if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle) &&
+ tg->scx.idle != idle)
SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle);
/* Update the task group's idle state */
@@ -5187,6 +5206,7 @@ static int scx_cgroup_init(struct scx_sched *sch)
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
.bw_burst_us = tg->scx.bw_burst_us,
+ .sched_idle = tg->scx.idle,
};
ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args);
@@ -5272,12 +5292,17 @@ static void free_exit_info(struct scx_exit_info *ei);
static const char *scx_exit_reason(enum scx_exit_kind kind);
static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind);
-s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch)
+s32 scx_alloc_kern_arena_objs(struct scx_sched *sch)
{
size_t size = struct_size_t(struct scx_cmask, bits,
SCX_CMASK_NR_WORDS(num_possible_cpus()));
+ struct scx_cmask *online;
+ struct scx_cmask_ref ref;
int cpu;
+ /* hotplug stays excluded until the online mask is published */
+ lockdep_assert_cpus_held();
+
if (!sch->is_cid_type || !sch->arena_pool)
return 0;
@@ -5293,15 +5318,28 @@ s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch)
return -ENOMEM;
scx_cmask_init(*slot, 0, num_possible_cpus());
}
+
+ /* pack the online mask alongside the scratch masks */
+ online = scx_arena_alloc(sch, size);
+ if (!online)
+ return -ENOMEM;
+
+ scoped_guard(rcu) {
+ scx_cmask_ref_init_kern(sch, online, 0, num_possible_cpus(), &ref);
+ scx_cmask_ref_from_cpumask(&ref, cpu_active_mask);
+ }
+ sch->online_cmask = online;
+
return 0;
}
-static void scx_set_cmask_scratch_free(struct scx_sched *sch)
+static void scx_free_kern_arena_objs(struct scx_sched *sch)
{
size_t size = struct_size_t(struct scx_cmask, bits,
SCX_CMASK_NR_WORDS(num_possible_cpus()));
int cpu;
+ scx_arena_free(sch, sch->online_cmask, size);
if (!sch->set_cmask_scratch)
return;
@@ -5388,7 +5426,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL);
free_exit_info(sch->exit_info);
- scx_set_cmask_scratch_free(sch);
+ scx_free_kern_arena_objs(sch);
scx_arena_pool_destroy(sch);
if (sch->arena_map)
bpf_map_put(sch->arena_map);
@@ -7508,22 +7546,24 @@ static void scx_root_enable_workfn(struct kthread_work *work)
#ifdef CONFIG_EXT_SUB_SCHED
cgroup_get(cgrp);
#endif
+ /*
+ * Transition to ENABLING to arm the disable path. Allocation failure
+ * still unwinds locally. Full disabling on failure applies only after
+ * scx_alloc_and_add_sched() succeeds.
+ */
+ WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED);
+ WARN_ON_ONCE(scx_root);
+
sch = scx_alloc_and_add_sched(cmd, cgrp, NULL);
if (IS_ERR(sch)) {
ret = PTR_ERR(sch);
+ WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_ENABLING);
goto err_free_tid_hash;
}
if (sch->is_cid_type)
static_branch_enable(&__scx_is_cid_type);
- /*
- * Transition to ENABLING and clear exit info to arm the disable path.
- * Failure triggers full disabling from here on.
- */
- WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED);
- WARN_ON_ONCE(scx_root);
-
atomic_long_set(&scx_nr_rejected, 0);
for_each_possible_cpu(cpu) {
@@ -7591,7 +7631,7 @@ static void scx_root_enable_workfn(struct kthread_work *work)
goto err_disable;
}
- ret = scx_set_cmask_scratch_alloc(sch);
+ ret = scx_alloc_kern_arena_objs(sch);
if (ret) {
cpus_read_unlock();
goto err_disable;
@@ -8946,10 +8986,17 @@ __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id,
#ifdef CONFIG_EXT_SUB_SCHED
/*
* Disallow if any sub-scheds are attached. There is no way to tell
- * which scheduler called us, just error out @p's scheduler.
+ * which scheduler called us, so error out @p's scheduler -- read it
+ * under RCU as @p's locks aren't necessarily held here. @p may be a
+ * task past sched_ext_dead() or an idle task, in which case its
+ * scheduler can't be determined and there is nothing obviously wrong
+ * to report; just refuse the call.
*/
if (unlikely(!list_empty(&sch->children))) {
- scx_error(scx_task_sched(p), "__scx_bpf_dsq_insert_vtime() must be used");
+ struct scx_sched *tsch = scx_task_sched_rcu(p);
+
+ if (tsch)
+ scx_error(tsch, "__scx_bpf_dsq_insert_vtime() must be used");
return;
}
#endif
@@ -10321,7 +10368,8 @@ __bpf_kfunc u32 scx_bpf_nr_cids(void)
* hotplug, which lets schedulers treat [0, nr_online_cids) as the online
* range. Schedulers that prefer to handle hotplug without a restart should
* install a custom mapping via scx_bpf_cid_override() and track onlining
- * through the ops.cid_online / ops.cid_offline callbacks.
+ * through the ops.cid_online / ops.cid_offline callbacks, starting from the
+ * mask scx_bpf_online_cmask() returns.
*/
__bpf_kfunc u32 scx_bpf_nr_online_cids(void)
{
@@ -10329,6 +10377,37 @@ __bpf_kfunc u32 scx_bpf_nr_online_cids(void)
}
/**
+ * scx_bpf_online_cmask - Return the online cid mask in the scheduler arena
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Return a kernel-maintained cmask covering [0, scx_bpf_nr_cids()), or NULL if
+ * the calling program is not associated with a live cid-form scheduler or the
+ * mask is not allocated yet, as in ops.init_cids(). Treat the mask as read-only
+ * even though arena memory stays writable by the BPF scheduler. The mask
+ * follows the SCX hotplug notifications: a cid's bit is updated before
+ * ops.cid_online/offline() runs for it. The pointer is valid from ops.init()
+ * through ops.exit(). Root ops.init() runs with hotplug excluded. Other
+ * contexts can observe concurrent updates.
+ */
+__bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux)
+{
+ struct scx_sched *sch;
+ struct scx_cmask *online;
+
+ guard(rcu)();
+
+ sch = scx_prog_sched(aux);
+ if (unlikely(!sch))
+ return NULL;
+ online = sch->online_cmask;
+ if (unlikely(!online))
+ return NULL;
+
+ /* BPF rebases by the low 32 bits, like __arena callback args */
+ return (void *)((unsigned long)online - sch->arena_kern_base);
+}
+
+/**
* scx_bpf_this_cid - Return the cid of the CPU this program is running on
*
* cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs.
@@ -10691,6 +10770,7 @@ BTF_ID_FLAGS(func, scx_bpf_nr_node_ids)
BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids)
BTF_ID_FLAGS(func, scx_bpf_nr_cids)
BTF_ID_FLAGS(func, scx_bpf_nr_online_cids)
+BTF_ID_FLAGS(func, scx_bpf_online_cmask, KF_IMPLICIT_ARGS | KF_ARENA_RET)
BTF_ID_FLAGS(func, scx_bpf_this_cid)
BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)
BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)
diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c
index d2973fb3..aa9fb6d 100644
--- a/kernel/sched/ext/idle.c
+++ b/kernel/sched/ext/idle.c
@@ -1142,10 +1142,17 @@ __bpf_kfunc s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64
#ifdef CONFIG_EXT_SUB_SCHED
/*
* Disallow if any sub-scheds are attached. There is no way to tell
- * which scheduler called us, just error out @p's scheduler.
+ * which scheduler called us, so error out @p's scheduler -- read it
+ * under RCU as @p's locks aren't necessarily held here. @p may be a
+ * task past sched_ext_dead() or an idle task, in which case its
+ * scheduler can't be determined and there is nothing obviously wrong
+ * to report; just refuse the call.
*/
if (unlikely(!list_empty(&sch->children))) {
- scx_error(scx_task_sched(p), "__scx_bpf_select_cpu_and() must be used");
+ struct scx_sched *tsch = scx_task_sched_rcu(p);
+
+ if (tsch)
+ scx_error(tsch, "__scx_bpf_select_cpu_and() must be used");
return -EINVAL;
}
#endif
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 0967b99..3464e0f 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -259,6 +259,9 @@ struct scx_cgroup_init_args {
u64 bw_period_us;
u64 bw_quota_us;
u64 bw_burst_us;
+
+ /* whether the cgroup is configured SCHED_IDLE via cpu.idle */
+ bool sched_idle;
};
enum scx_cpu_preempt_reason {
@@ -569,6 +572,12 @@ struct sched_ext_ops {
*
* Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle
* tracking.
+ *
+ * Only actual transitions are reported. A CPU that is claimed with an
+ * idle pick and kicked but dispatches no task returns to idle without a
+ * transition. A scheduler tracking idle CPUs itself must restore the
+ * idle state from ops.dispatch() when it returns without the next task
+ * to run.
*/
void (*update_idle)(s32 cpu, bool idle);
@@ -1552,6 +1561,7 @@ struct scx_sched {
* and passes it to the callback's __arena argument.
*/
struct scx_cmask * __percpu *set_cmask_scratch;
+ struct scx_cmask *online_cmask;
DECLARE_BITMAP(has_op, SCX_OPI_END);
@@ -2078,7 +2088,7 @@ void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p);
void scx_cgroup_lock(void);
void scx_cgroup_unlock(void);
#endif
-s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch);
+s32 scx_alloc_kern_arena_objs(struct scx_sched *sch);
void scx_disable_bypass_dsp(struct scx_sched *sch);
void scx_bypass(struct scx_sched *sch, bool bypass);
s32 scx_link_sched(struct scx_sched *sch);
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 9e70404..34e642a 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -1361,6 +1361,7 @@ static s32 scx_cgroup_claim_subtree(struct scx_sched *sch)
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
.bw_burst_us = tg->scx.bw_burst_us,
+ .sched_idle = tg->scx.idle,
};
if (tg->scx.sched != parent ||
@@ -1464,6 +1465,7 @@ static void scx_cgroup_return_subtree(struct scx_sched *sch)
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
.bw_burst_us = tg->scx.bw_burst_us,
+ .sched_idle = tg->scx.idle,
};
/* the first pass must have transferred everything */
@@ -1803,6 +1805,12 @@ void scx_sub_enable_workfn(struct kthread_work *work)
goto err_disable;
}
+ scoped_guard(cpus_read_lock) {
+ ret = scx_alloc_kern_arena_objs(sch);
+ if (ret)
+ goto err_disable;
+ }
+
if (sch->ops.init) {
ret = SCX_CALL_OP_RET(sch, init, NULL);
if (ret) {
@@ -1813,10 +1821,6 @@ void scx_sub_enable_workfn(struct kthread_work *work)
sch->exit_info->flags |= SCX_EFLAG_INITIALIZED;
}
- ret = scx_set_cmask_scratch_alloc(sch);
- if (ret)
- goto err_disable;
-
struct scx_sub_attach_args sub_attach_args = {
.ops = &sch->ops,
.cgroup_path = sch->cgrp_path,
diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c
index 213ae1d..80c3548 100644
--- a/kernel/time/jiffies.c
+++ b/kernel/time/jiffies.c
@@ -136,6 +136,8 @@ static int sysctl_k2u_int_conv_userhz(bool *negp, ulong *u_ptr, const int *k_ptr
static ulong sysctl_msecs_to_jiffies(const ulong val)
{
+ if (val > jiffies_to_msecs(MAX_JIFFY_OFFSET))
+ return MAX_JIFFY_OFFSET;
return msecs_to_jiffies(val);
}
@@ -181,7 +183,7 @@ static int do_proc_int_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr,
int *k_ptr, int dir,
const struct ctl_table *tbl)
{
- return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, false,
+ return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, true,
sysctl_u2k_int_conv_ms, sysctl_k2u_int_conv_ms);
}
@@ -195,10 +197,10 @@ static int sysctl_k2u_ulong_conv_ms(ulong *u_ptr, const ulong *k_ptr)
return proc_ulong_k2u_conv_kop(u_ptr, k_ptr, sysctl_jiffies_to_msecs);
}
-static int do_proc_ulong_conv_ms_jiffies(bool *negp, ulong *u_ptr, ulong *k_ptr,
- int dir, const struct ctl_table *tbl)
+static int do_proc_ulong_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr, ulong *k_ptr,
+ int dir, const struct ctl_table *tbl)
{
- return proc_ulong_conv(u_ptr, k_ptr, dir, tbl, false,
+ return proc_ulong_conv(u_ptr, k_ptr, dir, tbl, true,
sysctl_u2k_ulong_conv_ms, sysctl_k2u_ulong_conv_ms);
}
@@ -229,8 +231,8 @@ static int do_proc_int_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr,
return -ENOSYS;
}
-static int do_proc_ulong_conv_ms_jiffies(bool *negp, ulong *u_ptr, ulong *k_ptr,
- int dir, const struct ctl_table *tbl)
+static int do_proc_ulong_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr, ulong *k_ptr,
+ int dir, const struct ctl_table *tbl)
{
return -ENOSYS;
}
@@ -333,7 +335,7 @@ int proc_doulongvec_ms_jiffies_minmax(const struct ctl_table *table, int dir,
void *buffer, size_t *lenp, loff_t *ppos)
{
return proc_doulongvec_conv(table, dir, buffer, lenp, ppos,
- do_proc_ulong_conv_ms_jiffies);
+ do_proc_ulong_conv_ms_jiffies_minmax);
}
EXPORT_SYMBOL(proc_doulongvec_ms_jiffies_minmax);
diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c
deleted file mode 100644
index e5b2181..0000000
--- a/lib/alloc_tag.c
+++ /dev/null
@@ -1,1029 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-#include <linux/alloc_tag.h>
-#include <linux/execmem.h>
-#include <linux/fs.h>
-#include <linux/gfp.h>
-#include <linux/kallsyms.h>
-#include <linux/module.h>
-#include <linux/page_ext.h>
-#include <linux/pgalloc_tag.h>
-#include <linux/proc_fs.h>
-#include <linux/rcupdate.h>
-#include <linux/seq_buf.h>
-#include <linux/seq_file.h>
-#include <linux/string_choices.h>
-#include <linux/vmalloc.h>
-#include <linux/kmemleak.h>
-
-#define ALLOCINFO_FILE_NAME "allocinfo"
-#define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag))
-#define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME)
-#define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME)
-
-#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
-static bool mem_profiling_support = true;
-#else
-static bool mem_profiling_support;
-#endif
-
-/*
- * Memory allocation profiling is permanently disabled and cannot be enabled.
- * Must be called after setup_early_mem_profiling().
- */
-bool mem_alloc_profiling_permanently_disabled(void)
-{
- return !mem_profiling_support;
-}
-
-static struct codetag_type *alloc_tag_cttype;
-
-#ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU
-DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
-EXPORT_SYMBOL(_shared_alloc_tag);
-#endif
-
-DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
- mem_alloc_profiling_key);
-EXPORT_SYMBOL(mem_alloc_profiling_key);
-
-DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed);
-
-struct alloc_tag_kernel_section kernel_tags = { NULL, 0 };
-unsigned long alloc_tag_ref_mask;
-int alloc_tag_ref_offs;
-
-struct allocinfo_private {
- struct codetag_iterator iter;
- struct codetag_iterator reported_iter;
- bool print_header;
-};
-
-static void *allocinfo_start(struct seq_file *m, loff_t *pos)
-{
- struct allocinfo_private *priv;
- loff_t node = *pos;
-
- priv = (struct allocinfo_private *)m->private;
- codetag_lock_module_list(alloc_tag_cttype);
- if (node == 0) {
- priv->print_header = true;
- priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
- } else {
- priv->iter = priv->reported_iter;
- }
- codetag_next_ct(&priv->iter);
- return priv->iter.ct ? priv : NULL;
-}
-
-static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
-{
- struct allocinfo_private *priv = (struct allocinfo_private *)arg;
- struct codetag *ct;
-
- priv->reported_iter = priv->iter;
- ct = codetag_next_ct(&priv->iter);
- (*pos)++;
- if (!ct)
- return NULL;
-
- return priv;
-}
-
-static void allocinfo_stop(struct seq_file *m, void *arg)
-{
- codetag_unlock_module_list(alloc_tag_cttype);
-}
-
-static void print_allocinfo_header(struct seq_buf *buf)
-{
- /* Output format version, so we can change it. */
- seq_buf_printf(buf, "allocinfo - version: 2.0\n");
- seq_buf_printf(buf, "# <size> <calls> <tag info>\n");
-}
-
-static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
-{
- struct alloc_tag *tag = ct_to_alloc_tag(ct);
- struct alloc_tag_counters counter = alloc_tag_read(tag);
- s64 bytes = counter.bytes;
-
- seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
- codetag_to_text(out, ct);
- if (unlikely(alloc_tag_is_inaccurate(tag)))
- seq_buf_printf(out, " accurate:no");
- seq_buf_putc(out, ' ');
- seq_buf_putc(out, '\n');
-}
-
-static int allocinfo_show(struct seq_file *m, void *arg)
-{
- struct allocinfo_private *priv = (struct allocinfo_private *)arg;
- char *bufp;
- size_t n = seq_get_buf(m, &bufp);
- struct seq_buf buf;
-
- seq_buf_init(&buf, bufp, n);
- if (priv->print_header) {
- print_allocinfo_header(&buf);
- priv->print_header = false;
- }
- alloc_tag_to_text(&buf, priv->iter.ct);
- seq_commit(m, seq_buf_used(&buf));
- return 0;
-}
-
-static const struct seq_operations allocinfo_seq_op = {
- .start = allocinfo_start,
- .next = allocinfo_next,
- .stop = allocinfo_stop,
- .show = allocinfo_show,
-};
-
-size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
-{
- struct codetag_iterator iter;
- struct codetag *ct;
- struct codetag_bytes n;
- unsigned int i, nr = 0;
-
- if (IS_ERR_OR_NULL(alloc_tag_cttype))
- return 0;
-
- if (can_sleep)
- codetag_lock_module_list(alloc_tag_cttype);
- else if (!codetag_trylock_module_list(alloc_tag_cttype))
- return 0;
-
- iter = codetag_get_ct_iter(alloc_tag_cttype);
- while ((ct = codetag_next_ct(&iter))) {
- struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
-
- n.ct = ct;
- n.bytes = counter.bytes;
-
- for (i = 0; i < nr; i++)
- if (n.bytes > tags[i].bytes)
- break;
-
- if (i < count) {
- nr -= nr == count;
- memmove(&tags[i + 1],
- &tags[i],
- sizeof(tags[0]) * (nr - i));
- nr++;
- tags[i] = n;
- }
- }
-
- codetag_unlock_module_list(alloc_tag_cttype);
-
- return nr;
-}
-
-void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
-{
- int i;
- struct alloc_tag *tag;
- unsigned int nr_pages = 1 << new_order;
-
- if (!mem_alloc_profiling_enabled())
- return;
-
- tag = __pgalloc_tag_get(&folio->page);
- if (!tag)
- return;
-
- for (i = nr_pages; i < (1 << old_order); i += nr_pages) {
- union pgtag_ref_handle handle;
- union codetag_ref ref;
-
- if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) {
- /* Set new reference to point to the original tag */
- alloc_tag_ref_set(&ref, tag);
- update_page_tag_ref(handle, &ref);
- put_page_tag_ref(handle);
- }
- }
-}
-
-void pgalloc_tag_swap(struct folio *new, struct folio *old)
-{
- union pgtag_ref_handle handle_old, handle_new;
- union codetag_ref ref_old, ref_new;
- struct alloc_tag *tag_old, *tag_new;
-
- if (!mem_alloc_profiling_enabled())
- return;
-
- tag_old = __pgalloc_tag_get(&old->page);
- if (!tag_old)
- return;
- tag_new = __pgalloc_tag_get(&new->page);
- if (!tag_new)
- return;
-
- if (!get_page_tag_ref(&old->page, &ref_old, &handle_old))
- return;
- if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) {
- put_page_tag_ref(handle_old);
- return;
- }
-
- /*
- * Clear tag references to avoid debug warning when using
- * __alloc_tag_ref_set() with non-empty reference.
- */
- set_codetag_empty(&ref_old);
- set_codetag_empty(&ref_new);
-
- /* swap tags */
- __alloc_tag_ref_set(&ref_old, tag_new);
- update_page_tag_ref(handle_old, &ref_old);
- __alloc_tag_ref_set(&ref_new, tag_old);
- update_page_tag_ref(handle_new, &ref_new);
-
- put_page_tag_ref(handle_old);
- put_page_tag_ref(handle_new);
-}
-
-static void shutdown_mem_profiling(bool remove_file)
-{
- if (mem_alloc_profiling_enabled())
- static_branch_disable(&mem_alloc_profiling_key);
-
- if (!mem_profiling_support)
- return;
-
- if (remove_file)
- remove_proc_entry(ALLOCINFO_FILE_NAME, NULL);
- mem_profiling_support = false;
-}
-
-void __init alloc_tag_sec_init(void)
-{
- struct alloc_tag *last_codetag;
-
- if (!mem_profiling_support)
- return;
-
- if (!static_key_enabled(&mem_profiling_compressed))
- return;
-
- kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name(
- SECTION_START(ALLOC_TAG_SECTION_NAME));
- last_codetag = (struct alloc_tag *)kallsyms_lookup_name(
- SECTION_STOP(ALLOC_TAG_SECTION_NAME));
- kernel_tags.count = last_codetag - kernel_tags.first_tag;
-
- /* Check if kernel tags fit into page flags */
- if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) {
- shutdown_mem_profiling(false); /* allocinfo file does not exist yet */
- pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n",
- kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS);
- return;
- }
-
- alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS);
- alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1);
- pr_debug("Memory allocation profiling compression is using %d page flag bits!\n",
- NR_UNUSED_PAGEFLAG_BITS);
-}
-
-#ifdef CONFIG_MODULES
-
-static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);
-static struct vm_struct *vm_module_tags;
-/* A dummy object used to indicate an unloaded module */
-static struct module unloaded_mod;
-/* A dummy object used to indicate a module prepended area */
-static struct module prepend_mod;
-
-struct alloc_tag_module_section module_tags;
-
-static inline unsigned long alloc_tag_align(unsigned long val)
-{
- if (!static_key_enabled(&mem_profiling_compressed)) {
- /* No alignment requirements when we are not indexing the tags */
- return val;
- }
-
- if (val % sizeof(struct alloc_tag) == 0)
- return val;
- return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag);
-}
-
-static bool ensure_alignment(unsigned long align, unsigned int *prepend)
-{
- if (!static_key_enabled(&mem_profiling_compressed)) {
- /* No alignment requirements when we are not indexing the tags */
- return true;
- }
-
- /*
- * If alloc_tag size is not a multiple of required alignment, tag
- * indexing does not work.
- */
- if (!IS_ALIGNED(sizeof(struct alloc_tag), align))
- return false;
-
- /* Ensure prepend consumes multiple of alloc_tag-sized blocks */
- if (*prepend)
- *prepend = alloc_tag_align(*prepend);
-
- return true;
-}
-
-static inline bool tags_addressable(void)
-{
- unsigned long tag_idx_count;
-
- if (!static_key_enabled(&mem_profiling_compressed))
- return true; /* with page_ext tags are always addressable */
-
- tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count +
- module_tags.size / sizeof(struct alloc_tag);
-
- return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS);
-}
-
-static bool needs_section_mem(struct module *mod, unsigned long size)
-{
- if (!mem_profiling_support)
- return false;
-
- return size >= sizeof(struct alloc_tag);
-}
-
-static bool clean_unused_counters(struct alloc_tag *start_tag,
- struct alloc_tag *end_tag)
-{
- struct alloc_tag *tag;
- bool ret = true;
-
- for (tag = start_tag; tag <= end_tag; tag++) {
- struct alloc_tag_counters counter;
-
- if (!tag->counters)
- continue;
-
- counter = alloc_tag_read(tag);
- if (!counter.bytes) {
- free_percpu(tag->counters);
- tag->counters = NULL;
- } else {
- ret = false;
- }
- }
-
- return ret;
-}
-
-/* Called with mod_area_mt locked */
-static void clean_unused_module_areas_locked(void)
-{
- MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
- struct module *val;
-
- mas_for_each(&mas, val, module_tags.size) {
- struct alloc_tag *start_tag;
- struct alloc_tag *end_tag;
-
- if (val != &unloaded_mod)
- continue;
-
- /* Release area if all tags are unused */
- start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
- end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
- if (clean_unused_counters(start_tag, end_tag))
- mas_erase(&mas);
- }
-}
-
-/* Called with mod_area_mt locked */
-static bool find_aligned_area(struct ma_state *mas, unsigned long section_size,
- unsigned long size, unsigned int prepend, unsigned long align)
-{
- bool cleanup_done = false;
-
-repeat:
- /* Try finding exact size and hope the start is aligned */
- if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) {
- if (IS_ALIGNED(mas->index + prepend, align))
- return true;
-
- /* Try finding larger area to align later */
- mas_reset(mas);
- if (!mas_empty_area(mas, 0, section_size - 1,
- size + prepend + align - 1))
- return true;
- }
-
- /* No free area, try cleanup stale data and repeat the search once */
- if (!cleanup_done) {
- clean_unused_module_areas_locked();
- cleanup_done = true;
- mas_reset(mas);
- goto repeat;
- }
-
- return false;
-}
-
-static int vm_module_tags_populate(void)
-{
- unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) +
- (vm_module_tags->nr_pages << PAGE_SHIFT);
- unsigned long new_end = module_tags.start_addr + module_tags.size;
-
- if (phys_end < new_end) {
- struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages;
- unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN);
- unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN);
- unsigned long more_pages;
- unsigned long nr = 0;
-
- more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT;
- while (nr < more_pages) {
- unsigned long allocated;
-
- allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN,
- NUMA_NO_NODE, more_pages - nr, next_page + nr);
-
- if (!allocated)
- break;
- nr += allocated;
- }
-
- if (nr < more_pages ||
- vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL,
- next_page, PAGE_SHIFT) < 0) {
- release_pages_arg arg = { .pages = next_page };
-
- /* Clean up and error out */
- release_pages(arg, nr);
- return -ENOMEM;
- }
-
- vm_module_tags->nr_pages += nr;
-
- /*
- * Kasan allocates 1 byte of shadow for every 8 bytes of data.
- * When kasan_alloc_module_shadow allocates shadow memory,
- * its unit of allocation is a page.
- * Therefore, here we need to align to MODULE_ALIGN.
- */
- if (old_shadow_end < new_shadow_end)
- kasan_alloc_module_shadow((void *)old_shadow_end,
- new_shadow_end - old_shadow_end,
- GFP_KERNEL);
- }
-
- /*
- * Mark the pages as accessible, now that they are mapped.
- * With hardware tag-based KASAN, marking is skipped for
- * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
- */
- kasan_unpoison_vmalloc((void *)module_tags.start_addr,
- new_end - module_tags.start_addr,
- KASAN_VMALLOC_PROT_NORMAL);
-
- return 0;
-}
-
-static void *reserve_module_tags(struct module *mod, unsigned long size,
- unsigned int prepend, unsigned long align)
-{
- unsigned long section_size = module_tags.end_addr - module_tags.start_addr;
- MA_STATE(mas, &mod_area_mt, 0, section_size - 1);
- unsigned long offset;
- void *ret = NULL;
-
- /* If no tags return error */
- if (size < sizeof(struct alloc_tag))
- return ERR_PTR(-EINVAL);
-
- /*
- * align is always power of 2, so we can use IS_ALIGNED and ALIGN.
- * align 0 or 1 means no alignment, to simplify set to 1.
- */
- if (!align)
- align = 1;
-
- if (!ensure_alignment(align, &prepend)) {
- shutdown_mem_profiling(true);
- pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n",
- mod->name, align);
- return ERR_PTR(-EINVAL);
- }
-
- mas_lock(&mas);
- if (!find_aligned_area(&mas, section_size, size, prepend, align)) {
- ret = ERR_PTR(-ENOMEM);
- goto unlock;
- }
-
- /* Mark found area as reserved */
- offset = mas.index;
- offset += prepend;
- offset = ALIGN(offset, align);
- if (offset != mas.index) {
- unsigned long pad_start = mas.index;
-
- mas.last = offset - 1;
- mas_store(&mas, &prepend_mod);
- if (mas_is_err(&mas)) {
- ret = ERR_PTR(xa_err(mas.node));
- goto unlock;
- }
- mas.index = offset;
- mas.last = offset + size - 1;
- mas_store(&mas, mod);
- if (mas_is_err(&mas)) {
- mas.index = pad_start;
- mas_erase(&mas);
- ret = ERR_PTR(xa_err(mas.node));
- }
- } else {
- mas.last = offset + size - 1;
- mas_store(&mas, mod);
- if (mas_is_err(&mas))
- ret = ERR_PTR(xa_err(mas.node));
- }
-unlock:
- mas_unlock(&mas);
-
- if (IS_ERR(ret))
- return ret;
-
- if (module_tags.size < offset + size) {
- int grow_res;
-
- module_tags.size = offset + size;
- if (mem_alloc_profiling_enabled() && !tags_addressable()) {
- shutdown_mem_profiling(true);
- pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n",
- mod->name, NR_UNUSED_PAGEFLAG_BITS);
- }
-
- grow_res = vm_module_tags_populate();
- if (grow_res) {
- shutdown_mem_profiling(true);
- pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n",
- mod->name);
- return ERR_PTR(grow_res);
- }
- }
-
- return (struct alloc_tag *)(module_tags.start_addr + offset);
-}
-
-static void release_module_tags(struct module *mod, bool used)
-{
- MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size);
- struct alloc_tag *start_tag;
- struct alloc_tag *end_tag;
- struct module *val;
-
- mas_lock(&mas);
- mas_for_each_rev(&mas, val, 0)
- if (val == mod)
- break;
-
- if (!val) /* module not found */
- goto out;
-
- if (!used)
- goto release_area;
-
- start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
- end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
- if (!clean_unused_counters(start_tag, end_tag)) {
- struct alloc_tag *tag;
-
- for (tag = start_tag; tag <= end_tag; tag++) {
- struct alloc_tag_counters counter;
-
- if (!tag->counters)
- continue;
-
- counter = alloc_tag_read(tag);
- pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n",
- tag->ct.filename, tag->ct.lineno, tag->ct.modname,
- tag->ct.function, counter.bytes);
- }
- } else {
- used = false;
- }
-release_area:
- mas_store(&mas, used ? &unloaded_mod : NULL);
- val = mas_prev_range(&mas, 0);
- if (val == &prepend_mod)
- mas_store(&mas, NULL);
-out:
- mas_unlock(&mas);
-}
-
-static int load_module(struct module *mod, struct codetag *start, struct codetag *stop)
-{
- /* Allocate module alloc_tag percpu counters */
- struct alloc_tag *start_tag;
- struct alloc_tag *stop_tag;
- struct alloc_tag *tag;
-
- /* percpu counters for core allocations are already statically allocated */
- if (!mod)
- return 0;
-
- start_tag = ct_to_alloc_tag(start);
- stop_tag = ct_to_alloc_tag(stop);
- for (tag = start_tag; tag < stop_tag; tag++) {
- WARN_ON(tag->counters);
- tag->counters = alloc_percpu(struct alloc_tag_counters);
- if (!tag->counters) {
- while (--tag >= start_tag) {
- free_percpu(tag->counters);
- tag->counters = NULL;
- }
- pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n",
- mod->name);
- return -ENOMEM;
- }
-
- /*
- * Avoid a kmemleak false positive. The pointer to the counters is stored
- * in the alloc_tag section of the module and cannot be directly accessed.
- */
- kmemleak_ignore_percpu(tag->counters);
- }
- return 0;
-}
-
-static void replace_module(struct module *mod, struct module *new_mod)
-{
- MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
- struct module *val;
-
- mas_lock(&mas);
- mas_for_each(&mas, val, module_tags.size) {
- if (val != mod)
- continue;
-
- mas_store_gfp(&mas, new_mod, GFP_KERNEL);
- break;
- }
- mas_unlock(&mas);
-}
-
-static int __init alloc_mod_tags_mem(void)
-{
- /* Map space to copy allocation tags */
- vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE);
- if (!vm_module_tags) {
- pr_err("Failed to map %lu bytes for module allocation tags\n",
- MODULE_ALLOC_TAG_VMAP_SIZE);
- module_tags.start_addr = 0;
- return -ENOMEM;
- }
-
- vm_module_tags->pages = kmalloc_objs(struct page *,
- get_vm_area_size(vm_module_tags) >> PAGE_SHIFT,
- GFP_KERNEL | __GFP_ZERO);
- if (!vm_module_tags->pages) {
- free_vm_area(vm_module_tags);
- return -ENOMEM;
- }
-
- module_tags.start_addr = (unsigned long)vm_module_tags->addr;
- module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE;
- /* Ensure the base is alloc_tag aligned when required for indexing */
- module_tags.start_addr = alloc_tag_align(module_tags.start_addr);
-
- return 0;
-}
-
-static void __init free_mod_tags_mem(void)
-{
- release_pages_arg arg = { .pages = vm_module_tags->pages };
-
- module_tags.start_addr = 0;
- release_pages(arg, vm_module_tags->nr_pages);
- kfree(vm_module_tags->pages);
- free_vm_area(vm_module_tags);
-}
-
-#else /* CONFIG_MODULES */
-
-static inline int alloc_mod_tags_mem(void) { return 0; }
-static inline void free_mod_tags_mem(void) {}
-
-#endif /* CONFIG_MODULES */
-
-/* See: Documentation/mm/allocation-profiling.rst */
-static int __init setup_early_mem_profiling(char *str)
-{
- bool compressed = false;
- bool enable;
-
- if (!str || !str[0])
- return -EINVAL;
-
- if (!strncmp(str, "never", 5)) {
- enable = false;
- mem_profiling_support = false;
- pr_info("Memory allocation profiling is disabled!\n");
- } else {
- char *token = strsep(&str, ",");
-
- if (kstrtobool(token, &enable))
- return -EINVAL;
-
- if (str) {
-
- if (strcmp(str, "compressed"))
- return -EINVAL;
-
- compressed = true;
- }
- mem_profiling_support = true;
- pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n",
- compressed ? "with" : "without", str_on_off(enable));
- }
-
- if (enable != mem_alloc_profiling_enabled()) {
- if (enable)
- static_branch_enable(&mem_alloc_profiling_key);
- else
- static_branch_disable(&mem_alloc_profiling_key);
- }
- if (compressed != static_key_enabled(&mem_profiling_compressed)) {
- if (compressed)
- static_branch_enable(&mem_profiling_compressed);
- else
- static_branch_disable(&mem_profiling_compressed);
- }
-
- return 0;
-}
-early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
-
-static __init bool need_page_alloc_tagging(void)
-{
- if (static_key_enabled(&mem_profiling_compressed))
- return false;
-
- return mem_profiling_support;
-}
-
-#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
-/*
- * Track page allocations before page_ext is initialized.
- * Some pages are allocated before page_ext becomes available, leaving
- * their codetag uninitialized. Track these early PFNs so we can clear
- * their codetag refs later to avoid warnings when they are freed.
- *
- * Each page is cast to a pfn_pool: the first few bytes hold metadata
- * (next pointer and slot count), the remainder stores PFNs.
- */
-struct pfn_pool {
- struct pfn_pool *next;
- atomic_t count;
- unsigned long pfns[];
-};
-
-#define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \
- sizeof(unsigned long))
-
-/*
- * Skip early PFN recording for a page allocation. Reuses the
- * %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid
- * recursion when allocating pages for the early PFN tracking list
- * itself.
- *
- * Codetags of the pages allocated with __GFP_NO_CODETAG should be
- * cleared (via clear_page_tag_ref()) before freeing the pages to prevent
- * alloc_tag_sub_check() from triggering a warning.
- */
-#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT
-
-static struct pfn_pool *current_pfn_pool __initdata;
-
-static void __init __alloc_tag_add_early_pfn(unsigned long pfn)
-{
- struct pfn_pool *pool;
- int idx;
-
- do {
- pool = READ_ONCE(current_pfn_pool);
- if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) {
- struct page *new_page = alloc_page(__GFP_HIGH | __GFP_NO_CODETAG);
- struct pfn_pool *new;
-
- if (!new_page) {
- pr_warn_once("early PFN tracking page allocation failed\n");
- return;
- }
- new = page_address(new_page);
- new->next = pool;
- atomic_set(&new->count, 0);
- if (cmpxchg(¤t_pfn_pool, pool, new) != pool) {
- clear_page_tag_ref(new_page);
- __free_page(new_page);
- continue;
- }
- pool = new;
- }
- idx = atomic_read(&pool->count);
- if (idx >= PFN_POOL_SIZE)
- continue;
- if (atomic_cmpxchg(&pool->count, idx, idx + 1) == idx)
- break;
- } while (1);
-
- pool->pfns[idx] = pfn;
-}
-
-typedef void alloc_tag_add_func(unsigned long pfn);
-static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata =
- RCU_INITIALIZER(__alloc_tag_add_early_pfn);
-
-void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags)
-{
- alloc_tag_add_func *alloc_tag_add;
-
- if (static_key_enabled(&mem_profiling_compressed))
- return;
-
- /* Skip allocations for the tracking list itself to avoid recursion. */
- if (gfp_flags & __GFP_NO_CODETAG)
- return;
-
- rcu_read_lock();
- alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr);
- if (alloc_tag_add)
- alloc_tag_add(pfn);
- rcu_read_unlock();
-}
-
-static void __init clear_early_alloc_pfn_tag_refs(void)
-{
- struct pfn_pool *pool, *next;
- struct page *page;
- int i;
-
- if (static_key_enabled(&mem_profiling_compressed))
- return;
-
- rcu_assign_pointer(alloc_tag_add_early_pfn_ptr, NULL);
- /* Make sure we are not racing with __alloc_tag_add_early_pfn() */
- synchronize_rcu();
-
- for (pool = current_pfn_pool; pool; pool = next) {
- int nr_pfns = atomic_read(&pool->count);
-
- for (i = 0; i < nr_pfns; i++) {
- unsigned long pfn = pool->pfns[i];
-
- if (pfn_valid(pfn)) {
- union pgtag_ref_handle handle;
- union codetag_ref ref;
-
- if (get_page_tag_ref(pfn_to_page(pfn), &ref, &handle)) {
- /*
- * An early-allocated page could be freed and reallocated
- * after its page_ext is initialized but before we clear it.
- * In that case, it already has a valid tag set.
- * We should not overwrite that valid tag
- * with CODETAG_EMPTY.
- *
- * Note: there is still a small race window between checking
- * ref.ct and calling set_codetag_empty(). We accept this
- * race as it's unlikely and the extra complexity of atomic
- * cmpxchg is not worth it for this debug-only code path.
- */
- if (ref.ct) {
- put_page_tag_ref(handle);
- continue;
- }
-
- set_codetag_empty(&ref);
- update_page_tag_ref(handle, &ref);
- put_page_tag_ref(handle);
- }
- }
- }
-
- next = pool->next;
- page = virt_to_page(pool);
- clear_page_tag_ref(page);
- __free_page(page);
- }
-}
-#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */
-static inline void __init clear_early_alloc_pfn_tag_refs(void) {}
-#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
-
-static __init void init_page_alloc_tagging(void)
-{
- clear_early_alloc_pfn_tag_refs();
-}
-
-struct page_ext_operations page_alloc_tagging_ops = {
- .size = sizeof(union codetag_ref),
- .need = need_page_alloc_tagging,
- .init = init_page_alloc_tagging,
-};
-EXPORT_SYMBOL(page_alloc_tagging_ops);
-
-#ifdef CONFIG_SYSCTL
-/*
- * Not using proc_do_static_key() directly to prevent enabling profiling
- * after it was shut down.
- */
-static int proc_mem_profiling_handler(const struct ctl_table *table, int write,
- void *buffer, size_t *lenp, loff_t *ppos)
-{
- if (write) {
- /*
- * Call from do_sysctl_args() which is a no-op since the same
- * value was already set by setup_early_mem_profiling.
- * Return success to avoid warnings from do_sysctl_args().
- */
- if (!current->mm)
- return 0;
-
-#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
- /* User can't toggle profiling while debugging */
- return -EACCES;
-#endif
- if (!mem_profiling_support)
- return -EINVAL;
- }
-
- return proc_do_static_key(table, write, buffer, lenp, ppos);
-}
-
-
-static const struct ctl_table memory_allocation_profiling_sysctls[] = {
- {
- .procname = "mem_profiling",
- .data = &mem_alloc_profiling_key,
- .mode = 0644,
- .proc_handler = proc_mem_profiling_handler,
- },
-};
-
-static void __init sysctl_init(void)
-{
- register_sysctl_init("vm", memory_allocation_profiling_sysctls);
-}
-#else /* CONFIG_SYSCTL */
-static inline void sysctl_init(void) {}
-#endif /* CONFIG_SYSCTL */
-
-static int __init alloc_tag_init(void)
-{
- const struct codetag_type_desc desc = {
- .section = ALLOC_TAG_SECTION_NAME,
- .tag_size = sizeof(struct alloc_tag),
-#ifdef CONFIG_MODULES
- .needs_section_mem = needs_section_mem,
- .alloc_section_mem = reserve_module_tags,
- .free_section_mem = release_module_tags,
- .module_load = load_module,
- .module_replaced = replace_module,
-#endif
- };
- int res;
-
- sysctl_init();
-
- if (!mem_profiling_support) {
- pr_info("Memory allocation profiling is not supported!\n");
- return 0;
- }
-
- if (!proc_create_seq_private(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op,
- sizeof(struct allocinfo_private), NULL)) {
- pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME);
- shutdown_mem_profiling(false);
- return -ENOMEM;
- }
-
- res = alloc_mod_tags_mem();
- if (res) {
- pr_err("Failed to reserve address space for module tags, errno = %d\n", res);
- shutdown_mem_profiling(true);
- return res;
- }
-
- alloc_tag_cttype = codetag_register_type(&desc);
- if (IS_ERR(alloc_tag_cttype)) {
- pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype);
- free_mod_tags_mem();
- shutdown_mem_profiling(true);
- return PTR_ERR(alloc_tag_cttype);
- }
-
- return 0;
-}
-module_init(alloc_tag_init);
diff --git a/mm/filemap.c b/mm/filemap.c
index 6afec63..00fd89c 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -1616,7 +1616,7 @@ static void filemap_end_dropbehind(struct folio *folio)
return;
if (!folio_test_clear_dropbehind(folio))
return;
- if (mapping)
+ if (mapping && !folio_mapped(folio))
folio_unmap_invalidate(mapping, folio, 0);
}
diff --git a/mm/folio.c b/mm/folio.c
index c02dcea..50a6dbe 100644
--- a/mm/folio.c
+++ b/mm/folio.c
@@ -33,6 +33,7 @@
#include <linux/page_idle.h>
#include <linux/local_lock.h>
#include <linux/buffer_head.h>
+#include <linux/kvm_types.h>
#include "internal.h"
#include "page_alloc.h"
@@ -926,6 +927,7 @@ void lru_cache_drain_for_folio(const struct folio *folio,
*drained = LRU_CACHE_DRAINED_ALL;
}
}
+EXPORT_SYMBOL_FOR_KVM(lru_cache_drain_for_folio);
atomic_t lru_disable_count = ATOMIC_INIT(0);
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index afbb597..1e5d68a 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -92,7 +92,7 @@ unsigned long huge_anon_orders_madvise __read_mostly;
unsigned long huge_anon_orders_inherit __read_mostly;
static bool anon_orders_configured __initdata;
-static inline bool file_thp_enabled(struct vm_area_struct *vma)
+static inline bool file_thp_enabled(const struct vm_area_struct *vma)
{
struct inode *inode;
@@ -118,6 +118,67 @@ static bool vma_is_special_huge(const struct vm_area_struct *vma)
return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT);
}
+static bool vma_file_bypass_thp_tuneables(const struct vm_area_struct *vma,
+ enum tva_type type)
+{
+ const bool has_huge_fault = vma->vm_ops->huge_fault;
+
+ /* MADV_COLLAPSE ignores tuneables. */
+ if (type == TVA_FORCED_COLLAPSE)
+ return true;
+ /* Huge PFN mappings are uncompactable so the policy doesn't apply. */
+ if (vma_test(vma, VMA_PFNMAP_BIT) && has_huge_fault)
+ return true;
+ return false;
+}
+
+static bool vma_file_allow_thp_tuneables(vm_flags_t vm_flags)
+{
+ /* THP=always? */
+ if (hugepage_global_always())
+ return true;
+ /* THP=madvise and marked MADV_HUGEPAGE? */
+ if (hugepage_global_enabled() && (vm_flags & VM_HUGEPAGE))
+ return true;
+ return false;
+}
+
+static bool vma_file_check_thp_tuneables(const struct vm_area_struct *vma,
+ vm_flags_t vm_flags, enum tva_type type)
+{
+ return vma_file_bypass_thp_tuneables(vma, type) ||
+ vma_file_allow_thp_tuneables(vm_flags);
+}
+
+static bool vma_can_map_huge_file(const struct vm_area_struct *vma,
+ vm_flags_t vm_flags, enum tva_type type)
+{
+ const bool has_huge_fault = vma->vm_ops->huge_fault;
+
+ /*
+ * Enforce THP collapse requirements as necessary. Anonymous vmas
+ * were already handled in thp_vma_allowable_orders().
+ */
+ if (!vma_file_check_thp_tuneables(vma, vm_flags, type))
+ return false;
+
+ switch (type) {
+ case TVA_PAGEFAULT:
+ /*
+ * Trust that ->huge_fault() handlers know what they are doing
+ * in fault path.
+ */
+ return has_huge_fault;
+ case TVA_SMAPS:
+ if (has_huge_fault)
+ return true;
+ fallthrough;
+ default:
+ /* Only regular file is valid in collapse path. */
+ return file_thp_enabled(vma);
+ }
+}
+
unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
vm_flags_t vm_flags,
enum tva_type type,
@@ -190,27 +251,8 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
vma, vma_start_pgoff(vma), 0,
forced_collapse);
- if (!vma_is_anonymous(vma)) {
- /*
- * Enforce THP collapse requirements as necessary. Anonymous vmas
- * were already handled in thp_vma_allowable_orders().
- */
- if (!forced_collapse &&
- (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) &&
- !hugepage_global_always())))
- return 0;
-
- /*
- * Trust that ->huge_fault() handlers know what they are doing
- * in fault path.
- */
- if (((in_pf || smaps)) && vma->vm_ops->huge_fault)
- return orders;
- /* Only regular file is valid in collapse path */
- if (((!in_pf || smaps)) && file_thp_enabled(vma))
- return orders;
- return 0;
- }
+ if (!vma_is_anonymous(vma))
+ return vma_can_map_huge_file(vma, vm_flags, type) ? orders : 0;
if (vma_is_temporary_stack(vma))
return 0;
diff --git a/mm/memblock.c b/mm/memblock.c
index 9ce8634..021db49 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -2908,14 +2908,18 @@ static int memblock_debug_show(struct seq_file *m, void *private)
else
seq_printf(m, "%4c ", 'x');
if (reg->flags) {
- for (j = 0; j < count; j++) {
- if (reg->flags & (1U << j)) {
- seq_printf(m, "%s\n", flagname[j]);
- break;
- }
+ unsigned int flags = reg->flags;
+ bool first = true;
+
+ for (j = 0; flags; j++, flags >>= 1) {
+ if (!(flags & 1))
+ continue;
+ if (!first)
+ seq_putc(m, '|');
+ seq_puts(m, j < count ? flagname[j] : "UNKNOWN");
+ first = false;
}
- if (j == count)
- seq_printf(m, "%s\n", "UNKNOWN");
+ seq_putc(m, '\n');
} else {
seq_printf(m, "%s\n", "NONE");
}
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 1271d39..856a7d0 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -3158,7 +3158,7 @@ static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp,
memcg = get_mem_cgroup_from_objcg(objcg);
- ret = try_charge_memcg(memcg, gfp, nr_pages);
+ ret = try_charge(memcg, gfp, nr_pages);
if (ret)
goto out;
diff --git a/mm/mlock.c b/mm/mlock.c
index efa6716..39215a3 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -141,7 +141,7 @@ static struct lruvec *__munlock_folio(struct folio *folio, struct lruvec *lruvec
munlock:
if (folio_test_clear_mlocked(folio)) {
- __zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages);
+ zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages);
if (isolated || !folio_test_unevictable(folio))
__count_vm_events(UNEVICTABLE_PGMUNLOCKED, nr_pages);
else
diff --git a/mm/mremap.c b/mm/mremap.c
index 2b4b523..7c36844 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -1355,12 +1355,11 @@ static void dontunmap_complete(struct vma_remap_struct *vrm,
if (vma_is_anonymous(vma) && !vma->vm_file)
vma_set_pgoff(vma, pgoff_unfaulted);
}
-
- /* Because we won't unmap we don't need to touch locked_vm. */
}
static unsigned long move_vma(struct vma_remap_struct *vrm)
{
+ const bool is_dontunmap = vrm->flags & MREMAP_DONTUNMAP;
struct mm_struct *mm = current->mm;
struct vm_area_struct *new_vma;
unsigned long hiwater_vm;
@@ -1401,10 +1400,10 @@ static unsigned long move_vma(struct vma_remap_struct *vrm)
*/
hiwater_vm = mm->hiwater_vm;
- vrm_stat_account(vrm, vrm->new_len);
- if (unlikely(!err && (vrm->flags & MREMAP_DONTUNMAP)))
+ if (unlikely(is_dontunmap && !err))
dontunmap_complete(vrm, new_vma);
- else
+ vrm_stat_account(vrm, vrm->new_len);
+ if (!is_dontunmap || err)
unmap_source_vma(vrm);
mm->hiwater_vm = hiwater_vm;
diff --git a/mm/shrinker.c b/mm/shrinker.c
index a70aab1..7ec2a97 100644
--- a/mm/shrinker.c
+++ b/mm/shrinker.c
@@ -227,6 +227,8 @@ static int shrinker_memcg_alloc(struct shrinker *shrinker)
{
int id;
+ shrinker->id = -1;
+
if (mem_cgroup_disabled())
return -ENOSYS;
if (mem_cgroup_kmem_disabled() && !(shrinker->flags & SHRINKER_NONSLAB))
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 53bf01d..601979b 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -156,7 +156,7 @@ static struct swap_info_struct *swap_entry_to_info(swp_entry_t entry)
* This bit will be set if the device is not on the plist and not
* usable, will be cleared if the device is on the plist.
*/
-#define SWAP_USAGE_OFFLIST_BIT (1UL << (BITS_PER_TYPE(atomic_t) - 2))
+#define SWAP_USAGE_OFFLIST_BIT (1UL << (BITS_PER_TYPE(atomic_long_t) - 2))
#define SWAP_USAGE_COUNTER_MASK (~SWAP_USAGE_OFFLIST_BIT)
static long swap_usage_in_pages(struct swap_info_struct *si)
{
diff --git a/mm/vma.c b/mm/vma.c
index 35e7a64..f29abb3 100644
--- a/mm/vma.c
+++ b/mm/vma.c
@@ -2859,10 +2859,12 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr,
map.check_ksm_early = can_set_ksm_flags_early(&map);
error = __mmap_setup(&map, &desc, uf);
- if (!error && have_mmap_prepare)
- error = call_mmap_prepare(&map, &desc);
if (error)
goto abort_munmap;
+ if (have_mmap_prepare)
+ error = call_mmap_prepare(&map, &desc);
+ if (error)
+ goto unacct_error;
if (map.check_ksm_early)
update_ksm_flags(&map);
diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h
index 76f5e02..2ddb01a 100644
--- a/tools/sched_ext/include/scx/common.bpf.h
+++ b/tools/sched_ext/include/scx/common.bpf.h
@@ -113,6 +113,7 @@ s32 scx_bpf_this_cid(void) __ksym __weak;
struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak;
u32 scx_bpf_nr_cids(void) __ksym __weak;
u32 scx_bpf_nr_online_cids(void) __ksym __weak;
+const void __arena *scx_bpf_online_cmask(void) __ksym __weak;
u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak;
u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak;
s32 scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak;
diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c
index 9f6e61d..67b7c01 100644
--- a/tools/sched_ext/scx_qmap.bpf.c
+++ b/tools/sched_ext/scx_qmap.bpf.c
@@ -24,6 +24,9 @@
* time-share that stays self-local.
* self - The excl cpus the node kept for itself, plus all of held_shared.
* owner - Who holds a cid - a child slot, CID_SELF, or CID_NONE.
+ * avail - Cpus whose caps are in effect, per ops.sub_ecaps_updated().
+ * usable - self AND avail. Placement decisions use this: self is the
+ * delegation split and can run ahead of what the cpus honor.
*
* The scheduler splits its held-excl cpus among self and the children in
* proportion to each node's cpu.weight, handing each the floor of its share as
@@ -208,8 +211,8 @@ static int qmap_spin_lock(struct bpf_res_spin_lock *lock)
}
/*
- * Try prev_cid, then scan cpus_allowed AND idle_cids AND self_cids round-robin
- * from prev_cid + 1. Atomic claim retries on race; bounded by
+ * Try prev_cid, then scan cpus_allowed AND idle_cids AND usable_cids
+ * round-robin from prev_cid + 1. Atomic claim retries on race; bounded by
* IDLE_PICK_RETRIES to keep the verifier's insn budget in check.
*/
#define IDLE_PICK_RETRIES 16
@@ -221,7 +224,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid,
s32 cid;
u32 i;
- if (cmask_test(prev_cid, &qa.self_cids.mask) &&
+ if (cmask_test(prev_cid, &qa.usable_cids.mask) &&
cmask_test_and_clear(prev_cid, &qa.idle_cids.mask))
return prev_cid;
@@ -229,7 +232,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid,
bpf_for(i, 0, IDLE_PICK_RETRIES) {
cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed,
&qa.idle_cids.mask,
- &qa.self_cids.mask, cid + 1);
+ &qa.usable_cids.mask, cid + 1);
barrier_var(cid);
if (cid >= nr_cids)
return -1;
@@ -358,8 +361,8 @@ s32 BPF_STRUCT_OPS(qmap_select_cid, struct task_struct *p,
}
/*
- * A received time-shared cid is held ENQ_IMMED-only, so inserts must set
- * SCX_ENQ_IMMED.
+ * A received time-shared cid is held ENQ_IMMED-only, so inserts meant to run
+ * there must set SCX_ENQ_IMMED.
*/
static u64 needs_immed(s32 cid)
{
@@ -444,9 +447,11 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
* didn't grant them or we delegated them to children - would starve in
* SHARED/FIFO since we only pull from those on self cids.
*
- * Force it onto its first allowed cid's local DSQ. If we hold that cid
- * it runs. Otherwise the insert carries SCX_ENQ_RESCUE and the kernel
- * diverts the task to its rescue path.
+ * Force it onto its first allowed cid's local DSQ with SCX_ENQ_RESCUE.
+ * If we hold ENQ on that cid it runs. Otherwise the kernel diverts the
+ * task to its rescue path. IMMED would turn the insert into a legal
+ * placement on a time-shared cid and the kernel would bounce it back
+ * here instead of rescuing it.
*/
if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) {
s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0);
@@ -455,7 +460,7 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
taskc->force_local = false;
__sync_fetch_and_add(&qa.nr_rescue_dsp, 1);
scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | c, slice_ns,
- enq_flags | needs_immed(c) | SCX_ENQ_RESCUE);
+ enq_flags | SCX_ENQ_RESCUE);
return;
}
}
@@ -540,7 +545,7 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
scx_bpf_dsq_insert(p, SHARED_DSQ, 0, enq_flags);
cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed,
&qa.idle_cids.mask,
- &qa.self_cids.mask, 0);
+ &qa.usable_cids.mask, 0);
if (cid < scx_bpf_nr_cids())
scx_bpf_kick_cid(cid, SCX_KICK_IDLE);
return;
@@ -618,7 +623,7 @@ static bool scan_shared_dsq(bool from_timer)
if (c >= 0 && c < scx_bpf_nr_cids()) {
__sync_fetch_and_add(&qa.nr_rescue_dsp, 1);
scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c,
- needs_immed(c) | SCX_ENQ_RESCUE);
+ SCX_ENQ_RESCUE);
}
continue;
}
@@ -644,22 +649,27 @@ static bool scan_shared_dsq(bool from_timer)
if (!(taskc = lookup_task_ctx(p)))
return false;
- /* only run highpri tasks on cids this node holds, not delegated ones */
+ /* only run highpri tasks on cids this node can use right now */
if (cmask_test(this_cid, &taskc->cpus_allowed) &&
- cmask_test(this_cid, &qa.self_cids.mask))
+ cmask_test(this_cid, &qa.usable_cids.mask))
cid = this_cid;
else
cid = cmask_next_and_set_wrap(&taskc->cpus_allowed,
- &qa.self_cids.mask,
+ &qa.usable_cids.mask,
this_cid + 1);
if (cid >= nr_cids) {
- /* stranded after the cull - rescue it from here */
- s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0);
+ s32 c;
+ /* self cids lack caps in effect yet, leave it queued */
+ if (cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask))
+ continue;
+
+ /* stranded after the cull - rescue it from here */
+ c = cmask_next_set_wrap(&taskc->cpus_allowed, 0);
if (c >= 0 && c < nr_cids) {
__sync_fetch_and_add(&qa.nr_rescue_dsp, 1);
scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c,
- needs_immed(c) | SCX_ENQ_RESCUE);
+ SCX_ENQ_RESCUE);
}
continue;
}
@@ -808,10 +818,10 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev)
batch--;
cpuc->dsp_cnt--;
if (!batch || !scx_bpf_dispatch_nr_slots()) {
- if (scan_shared_dsq(false))
+ if (scan_shared_dsq(false) ||
+ scx_bpf_dsq_move_to_local(SHARED_DSQ, needs_immed(cid)))
return;
- scx_bpf_dsq_move_to_local(SHARED_DSQ, needs_immed(cid));
- return;
+ goto prev;
}
if (!cpuc->dsp_cnt)
break;
@@ -822,10 +832,14 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev)
if (scan_shared_dsq(false))
return;
-
+prev:
/*
* No other tasks. @prev will keep running. Update its core_sched_seq as
* if the task were enqueued and dispatched immediately.
+ *
+ * No @prev to keep running means the CPU goes idle. If its claim was
+ * never used, that is not a transition and ops.update_idle() stays
+ * silent. Restore the claim here.
*/
if (prev) {
taskc = lookup_task_ctx(prev);
@@ -834,6 +848,8 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev)
taskc->core_sched_seq =
qa.core_sched_tail_seqs[weight_to_idx(prev->scx.weight)]++;
+ } else {
+ cmask_set(cid, &qa.idle_cids.mask);
}
}
@@ -1113,7 +1129,7 @@ void BPF_STRUCT_OPS(qmap_update_idle, s32 cid, bool idle)
/*
* The kernel delivers update_idle() for every cid this node holds
* SCX_CAP_BASE on. Track every cid's idle state regardless of
- * delegation: the direct-dispatch pick masks idle_cids with self_cids
+ * delegation: the direct-dispatch pick masks idle_cids with usable_cids
* at selection, so a cid already idle when it returns to self needs no
* reseed here.
*/
@@ -1285,11 +1301,16 @@ struct {
__type(value, struct round_robin_timer);
} round_robin_timer SEC(".maps");
+enum part_pending_flags {
+ PART_REFRESH = BIT_U64(0),
+ PART_REDISTRIBUTE = BIT_U64(1),
+};
+
/*
* Partition update synchronization. qa.part can be written from concurrent
* contexts. This single-runner guard admits one writer at a time without
* holding a lock across the grant/revoke kfuncs. part_pending coalesces
- * repartition requests that arrive while it is held.
+ * refresh and repartition requests that arrive while it is held.
*
* They live in .bss, not the arena: rr_advance() runs from a bpf_timer
* callback, where the verifier rejects atomic ops on arena memory.
@@ -1538,6 +1559,19 @@ static __noinline void account_alloc(void)
}
/*
+ * usable_cids = self_cids & avail_cids. The inputs have separate writers,
+ * apply_partition() and qmap_sub_ecaps_updated(), so the result is rebuilt in
+ * full under the partition guard, in scratch first so that readers never see
+ * self_cids alone.
+ */
+static void refresh_usable(void)
+{
+ cmask_copy(&qa.usable_scratch.mask, &qa.self_cids.mask);
+ cmask_and(&qa.usable_scratch.mask, &qa.avail_cids.mask);
+ cmask_copy(&qa.usable_cids.mask, &qa.usable_scratch.mask);
+}
+
+/*
* apply_partition - execute the plan compute_partition() built
*
* Turn the owner map into the per-child, shared and self cmasks and issue the
@@ -1559,6 +1593,7 @@ __noinline void apply_partition(void)
/* no excl cpu: run own tasks on the held shares, evict children */
if (!qa.part.nr_excl) {
cmask_copy(&qa.self_cids.mask, &qa.held_shared.mask);
+ refresh_usable();
bpf_for(i, 0, MAX_SUB_SCHEDS)
if (qa.sub_sched_ctxs[i].cgroup_id)
scx_bpf_sub_kill(qa.sub_sched_ctxs[i].cgroup_id,
@@ -1596,6 +1631,7 @@ __noinline void apply_partition(void)
else if (o == CID_SELF)
cmask_set(cid, &qa.self_cids.mask);
}
+ refresh_usable();
/*
* Apply each child's exclusive cids as a delta against its previous
@@ -1643,33 +1679,46 @@ __noinline void apply_partition(void)
}
}
-/*
- * Recompute the split off the node's held caps and apply it. The contexts this
- * runs from (the sub-sched and cgroup callbacks, the rr timer) are not
- * serialized by the kernel, so a single runner does the work. A caller that
- * finds the guard held leaves part_pending set; the holder drains it before
- * releasing, with the rr timer as a backstop.
+/**
+ * execute_partition - Run pending partition updates
+ *
+ * The rr timer is the backstop if the loop reaches its iteration limit.
*/
-static void redistribute(void)
+static void execute_partition(void)
{
+ u64 pending;
s32 i;
- __sync_fetch_and_or(&part_pending, 1);
-
- if (!part_try_start())
- return;
-
bpf_for(i, 0, 1024) {
- __sync_fetch_and_and(&part_pending, 0);
- /* charge elapsed time to the current partition before rebuilding it */
- account_alloc();
- compute_partition();
- apply_partition();
+ if (!part_try_start())
+ break;
+
+ pending = __sync_fetch_and_and(&part_pending, 0);
+ if (pending & PART_REDISTRIBUTE) {
+ /* charge elapsed time before repartitioning */
+ account_alloc();
+ compute_partition();
+ apply_partition();
+ } else if (pending & PART_REFRESH) {
+ refresh_usable();
+ }
+
+ /*
+ * Requests are published before trying the guard. Releasing it
+ * before checking pending work ensures a racing request is
+ * either observed here or handled by a caller that acquires the
+ * guard.
+ */
+ part_end();
if (!__sync_fetch_and_or(&part_pending, 0))
break;
}
+}
- part_end();
+static void redistribute(void)
+{
+ __sync_fetch_and_or(&part_pending, PART_REDISTRIBUTE);
+ execute_partition();
}
/*
@@ -1683,6 +1732,7 @@ int flush_alloc(void *ctx)
if (part_try_start()) {
account_alloc();
part_end();
+ execute_partition();
}
return 0;
}
@@ -1740,9 +1790,7 @@ static void rr_advance(void)
part_end();
- /* a resplit queued while we held the guard supersedes this rotation */
- if (__sync_fetch_and_or(&part_pending, 0))
- redistribute();
+ execute_partition();
}
/* advance the time-shared cid pool every round_robin_ns */
@@ -1837,8 +1885,11 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init)
cmask_init(&qa.rr_cids.mask, 0, nr_cids);
cmask_init(&qa.prev_rr_cids.mask, 0, nr_cids);
cmask_init(&qa.self_cids.mask, 0, nr_cids);
+ cmask_init(&qa.avail_cids.mask, 0, nr_cids);
+ cmask_init(&qa.usable_cids.mask, 0, nr_cids);
cmask_init(&qa.to_revoke_cids.mask, 0, nr_cids);
cmask_init(&qa.to_grant_cids.mask, 0, nr_cids);
+ cmask_init(&qa.usable_scratch.mask, 0, nr_cids);
cmask_init(&qa.held_excl.mask, 0, nr_cids);
cmask_init(&qa.held_shared.mask, 0, nr_cids);
@@ -1852,14 +1903,16 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init)
}
/*
- * The root starts holding every cid. qmap_sub_ecaps_updated() maintains
- * per-cid shared state as effective caps settle, and redistribute()
- * rebuilds owner and self from held caps. A non-root node starts with
- * nothing.
+ * The root starts holding every cid and gets no ecaps notifications, so
+ * its avail set is fixed here. qmap_sub_ecaps_updated() maintains the
+ * per-cid state as effective caps settle, and redistribute() rebuilds
+ * owner and self from held caps. A non-root node starts with nothing.
*/
bpf_for(i, 0, nr_cids) {
if (!sub_cgroup_id) {
cmask_set(i, &qa.self_cids.mask);
+ cmask_set(i, &qa.avail_cids.mask);
+ cmask_set(i, &qa.usable_cids.mask);
qa.part.cid_owner[i] = CID_SELF;
} else {
qa.part.cid_owner[i] = CID_NONE;
@@ -2000,12 +2053,19 @@ void BPF_STRUCT_OPS(qmap_sub_ecaps_updated, s32 cid, u64 before, u64 after)
{
/*
* Effective caps updated. Track which cids hold shared caps so a self
- * task placed there enqueues IMMED.
+ * task placed there enqueues IMMED, and which cids have ENQ_IMMED in
+ * effect at all (avail, see the header comment).
*/
- if (after & SCX_CAP_ENQ_IMMED)
+ if (after & SCX_CAP_ENQ_IMMED) {
qa.cid_shared[cid] = (after & SCX_CAP_ENQ) ? 0 : 1;
- else
+ cmask_set(cid, &qa.avail_cids.mask);
+ } else {
qa.cid_shared[cid] = 0;
+ cmask_clear(cid, &qa.avail_cids.mask);
+ }
+
+ __sync_fetch_and_or(&part_pending, PART_REFRESH);
+ execute_partition();
}
SCX_OPS_CID_DEFINE(qmap_ops,
diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h
index c78d618..e95fffc 100644
--- a/tools/sched_ext/scx_qmap.h
+++ b/tools/sched_ext/scx_qmap.h
@@ -165,12 +165,15 @@ struct qmap_arena {
/* bpf-internal cmasks (embedded, see struct qmap_cmask) */
struct qmap_cmask self_cids; /* cids this node runs its own tasks on */
+ struct qmap_cmask avail_cids; /* cids with caps in effect on the cpu */
+ struct qmap_cmask usable_cids; /* self_cids & avail_cids, placeable right now */
struct qmap_cmask idle_cids; /* idle state of all cids regardless of delegation */
struct qmap_cmask rr_cids; /* the shared pool, as a mask for grant/revoke */
/* scratch cmasks */
struct qmap_cmask to_revoke_cids; /* delta cids to revoke */
struct qmap_cmask to_grant_cids; /* delta cids to grant */
+ struct qmap_cmask usable_scratch; /* refresh_usable() build area */
struct qmap_cmask prev_rr_cids; /* previous shared pool, to clear stale grants */
struct qmap_cmask held_excl; /* cids held excl (ENQ): delegatable */
struct qmap_cmask held_shared; /* cids held shared (ENQ_IMMED only): self-local */
