kernel/trace/pid_list.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2021 VMware Inc, Steven Rostedt <rostedt@goodmis.org>
  */
 #include <linux/spinlock.h>
 #include <linux/irq_work.h>
 #include <linux/slab.h>
 #include "trace.h"

 /* See pid_list.h for details */

 static inline union lower_chunk *get_lower_chunk(struct trace_pid_list *pid_list)
 {
 	union lower_chunk *chunk;

 	lockdep_assert_held(&pid_list->lock);

 	if (!pid_list->lower_list)
 		return NULL;

 	chunk = pid_list->lower_list;
 	pid_list->lower_list = chunk->next;
 	pid_list->free_lower_chunks--;
 	WARN_ON_ONCE(pid_list->free_lower_chunks < 0);
 	chunk->next = NULL;
 	/*
 	 * If a refill needs to happen, it can not happen here
 	 * as the scheduler run queue locks are held.
 	 */
 	if (pid_list->free_lower_chunks <= CHUNK_REALLOC)
 		irq_work_queue(&pid_list->refill_irqwork);

 	return chunk;
 }

 static inline union upper_chunk *get_upper_chunk(struct trace_pid_list *pid_list)
 {
 	union upper_chunk *chunk;

 	lockdep_assert_held(&pid_list->lock);

 	if (!pid_list->upper_list)
 		return NULL;

 	chunk = pid_list->upper_list;
 	pid_list->upper_list = chunk->next;
 	pid_list->free_upper_chunks--;
 	WARN_ON_ONCE(pid_list->free_upper_chunks < 0);
 	chunk->next = NULL;
 	/*
 	 * If a refill needs to happen, it can not happen here
 	 * as the scheduler run queue locks are held.
 	 */
 	if (pid_list->free_upper_chunks <= CHUNK_REALLOC)
 		irq_work_queue(&pid_list->refill_irqwork);

 	return chunk;
 }

 static inline void put_lower_chunk(struct trace_pid_list *pid_list,
 				   union lower_chunk *chunk)
 {
 	lockdep_assert_held(&pid_list->lock);

 	chunk->next = pid_list->lower_list;
 	pid_list->lower_list = chunk;
 	pid_list->free_lower_chunks++;
 }

 static inline void put_upper_chunk(struct trace_pid_list *pid_list,
 				   union upper_chunk *chunk)
 {
 	lockdep_assert_held(&pid_list->lock);

 	chunk->next = pid_list->upper_list;
 	pid_list->upper_list = chunk;
 	pid_list->free_upper_chunks++;
 }

 static inline bool upper_empty(union upper_chunk *chunk)
 {
 	/*
 	 * If chunk->data has no lower chunks, it will be the same
 	 * as a zeroed bitmask. Use find_first_bit() to test it
 	 * and if it doesn't find any bits set, then the array
 	 * is empty.
 	 */
 	int bit = find_first_bit((unsigned long *)chunk->data,
 				 sizeof(chunk->data) * 8);
 	return bit >= sizeof(chunk->data) * 8;
 }

 static inline int pid_split(unsigned int pid, unsigned int *upper1,
 			     unsigned int *upper2, unsigned int *lower)
 {
 	/* MAX_PID should cover all pids */
 	BUILD_BUG_ON(MAX_PID < PID_MAX_LIMIT);

 	/* In case a bad pid is passed in, then fail */
 	if (unlikely(pid >= MAX_PID))
 		return -1;

 	*upper1 = (pid >> UPPER1_SHIFT) & UPPER_MASK;
 	*upper2 = (pid >> UPPER2_SHIFT) & UPPER_MASK;
 	*lower = pid & LOWER_MASK;

 	return 0;
 }

 static inline unsigned int pid_join(unsigned int upper1,
 				    unsigned int upper2, unsigned int lower)
 {
 	return ((upper1 & UPPER_MASK) << UPPER1_SHIFT) |
 		((upper2 & UPPER_MASK) << UPPER2_SHIFT) |
 		(lower & LOWER_MASK);
 }

 /**
  * trace_pid_list_is_set - test if the pid is set in the list
  * @pid_list: The pid list to test
  * @pid: The pid to see if set in the list.
  *
  * Tests if @pid is set in the @pid_list. This is usually called
  * from the scheduler when a task is scheduled. Its pid is checked
  * if it should be traced or not.
  *
  * Return true if the pid is in the list, false otherwise.
  */
 bool trace_pid_list_is_set(struct trace_pid_list *pid_list, unsigned int pid)
 {
 	union upper_chunk *upper_chunk;
 	union lower_chunk *lower_chunk;
 	unsigned long flags;
 	unsigned int upper1;
 	unsigned int upper2;
 	unsigned int lower;
 	bool ret = false;

 	if (!pid_list)
 		return false;

 	if (pid_split(pid, &upper1, &upper2, &lower) < 0)
 		return false;

 	raw_spin_lock_irqsave(&pid_list->lock, flags);
 	upper_chunk = pid_list->upper[upper1];
 	if (upper_chunk) {
 		lower_chunk = upper_chunk->data[upper2];
 		if (lower_chunk)
 			ret = test_bit(lower, lower_chunk->data);
 	}
 	raw_spin_unlock_irqrestore(&pid_list->lock, flags);

 	return ret;
 }

 /**
  * trace_pid_list_set - add a pid to the list
  * @pid_list: The pid list to add the @pid to.
  * @pid: The pid to add.
  *
  * Adds @pid to @pid_list. This is usually done explicitly by a user
  * adding a task to be traced, or indirectly by the fork function
  * when children should be traced and a task's pid is in the list.
  *
  * Return 0 on success, negative otherwise.
  */
 int trace_pid_list_set(struct trace_pid_list *pid_list, unsigned int pid)
 {
 	union upper_chunk *upper_chunk;
 	union lower_chunk *lower_chunk;
 	unsigned long flags;
 	unsigned int upper1;
 	unsigned int upper2;
 	unsigned int lower;
 	int ret;

 	if (!pid_list)
 		return -ENODEV;

 	if (pid_split(pid, &upper1, &upper2, &lower) < 0)
 		return -EINVAL;

 	raw_spin_lock_irqsave(&pid_list->lock, flags);
 	upper_chunk = pid_list->upper[upper1];
 	if (!upper_chunk) {
 		upper_chunk = get_upper_chunk(pid_list);
 		if (!upper_chunk) {
 			ret = -ENOMEM;
 			goto out;
 		}
 		pid_list->upper[upper1] = upper_chunk;
 	}
 	lower_chunk = upper_chunk->data[upper2];
 	if (!lower_chunk) {
 		lower_chunk = get_lower_chunk(pid_list);
 		if (!lower_chunk) {
 			ret = -ENOMEM;
 			goto out;
 		}
 		upper_chunk->data[upper2] = lower_chunk;
 	}
 	set_bit(lower, lower_chunk->data);
 	ret = 0;
  out:
 	raw_spin_unlock_irqrestore(&pid_list->lock, flags);
 	return ret;
 }

 /**
  * trace_pid_list_clear - remove a pid from the list
  * @pid_list: The pid list to remove the @pid from.
  * @pid: The pid to remove.
  *
  * Removes @pid from @pid_list. This is usually done explicitly by a user
  * removing tasks from tracing, or indirectly by the exit function
  * when a task that is set to be traced exits.
  *
  * Return 0 on success, negative otherwise.
  */
 int trace_pid_list_clear(struct trace_pid_list *pid_list, unsigned int pid)
 {
 	union upper_chunk *upper_chunk;
 	union lower_chunk *lower_chunk;
 	unsigned long flags;
 	unsigned int upper1;
 	unsigned int upper2;
 	unsigned int lower;

 	if (!pid_list)
 		return -ENODEV;

 	if (pid_split(pid, &upper1, &upper2, &lower) < 0)
 		return -EINVAL;

 	raw_spin_lock_irqsave(&pid_list->lock, flags);
 	upper_chunk = pid_list->upper[upper1];
 	if (!upper_chunk)
 		goto out;

 	lower_chunk = upper_chunk->data[upper2];
 	if (!lower_chunk)
 		goto out;

 	clear_bit(lower, lower_chunk->data);

 	/* if there's no more bits set, add it to the free list */
 	if (find_first_bit(lower_chunk->data, LOWER_MAX) >= LOWER_MAX) {
 		put_lower_chunk(pid_list, lower_chunk);
 		upper_chunk->data[upper2] = NULL;
 		if (upper_empty(upper_chunk)) {
 			put_upper_chunk(pid_list, upper_chunk);
 			pid_list->upper[upper1] = NULL;
 		}
 	}
  out:
 	raw_spin_unlock_irqrestore(&pid_list->lock, flags);
 	return 0;
 }

 /**
  * trace_pid_list_next - return the next pid in the list
  * @pid_list: The pid list to examine.
  * @pid: The pid to start from
  * @next: The pointer to place the pid that is set starting from @pid.
  *
  * Looks for the next consecutive pid that is in @pid_list starting
  * at the pid specified by @pid. If one is set (including @pid), then
  * that pid is placed into @next.
  *
  * Return 0 when a pid is found, -1 if there are no more pids included.
  */
 int trace_pid_list_next(struct trace_pid_list *pid_list, unsigned int pid,
 			unsigned int *next)
 {
 	union upper_chunk *upper_chunk;
 	union lower_chunk *lower_chunk;
 	unsigned long flags;
 	unsigned int upper1;
 	unsigned int upper2;
 	unsigned int lower;

 	if (!pid_list)
 		return -ENODEV;

 	if (pid_split(pid, &upper1, &upper2, &lower) < 0)
 		return -EINVAL;

 	raw_spin_lock_irqsave(&pid_list->lock, flags);
 	for (; upper1 <= UPPER_MASK; upper1++, upper2 = 0) {
 		upper_chunk = pid_list->upper[upper1];

 		if (!upper_chunk)
 			continue;

 		for (; upper2 <= UPPER_MASK; upper2++, lower = 0) {
 			lower_chunk = upper_chunk->data[upper2];
 			if (!lower_chunk)
 				continue;

 			lower = find_next_bit(lower_chunk->data, LOWER_MAX,
 					    lower);
 			if (lower < LOWER_MAX)
 				goto found;
 		}
 	}

  found:
 	raw_spin_unlock_irqrestore(&pid_list->lock, flags);
 	if (upper1 > UPPER_MASK)
 		return -1;

 	*next = pid_join(upper1, upper2, lower);
 	return 0;
 }

 /**
  * trace_pid_list_first - return the first pid in the list
  * @pid_list: The pid list to examine.
  * @pid: The pointer to place the pid first found pid that is set.
  *
  * Looks for the first pid that is set in @pid_list, and places it
  * into @pid if found.
  *
  * Return 0 when a pid is found, -1 if there are no pids set.
  */
 int trace_pid_list_first(struct trace_pid_list *pid_list, unsigned int *pid)
 {
 	return trace_pid_list_next(pid_list, 0, pid);
 }

 static void pid_list_refill_irq(struct irq_work *iwork)
 {
 	struct trace_pid_list *pid_list = container_of(iwork, struct trace_pid_list,
 						       refill_irqwork);
 	union upper_chunk *upper = NULL;
 	union lower_chunk *lower = NULL;
 	union upper_chunk **upper_next = &upper;
 	union lower_chunk **lower_next = &lower;
 	int upper_count;
 	int lower_count;
 	int ucnt = 0;
 	int lcnt = 0;

  again:
 	raw_spin_lock(&pid_list->lock);
 	upper_count = CHUNK_ALLOC - pid_list->free_upper_chunks;
 	lower_count = CHUNK_ALLOC - pid_list->free_lower_chunks;
 	raw_spin_unlock(&pid_list->lock);

 	if (upper_count <= 0 && lower_count <= 0)
 		return;

 	while (upper_count-- > 0) {
 		union upper_chunk *chunk;

 		chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT);
 		if (!chunk)
 			break;
 		*upper_next = chunk;
 		upper_next = &chunk->next;
 		ucnt++;
 	}

 	while (lower_count-- > 0) {
 		union lower_chunk *chunk;

 		chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT);
 		if (!chunk)
 			break;
 		*lower_next = chunk;
 		lower_next = &chunk->next;
 		lcnt++;
 	}

 	raw_spin_lock(&pid_list->lock);
 	if (upper) {
 		*upper_next = pid_list->upper_list;
 		pid_list->upper_list = upper;
 		pid_list->free_upper_chunks += ucnt;
 	}
 	if (lower) {
 		*lower_next = pid_list->lower_list;
 		pid_list->lower_list = lower;
 		pid_list->free_lower_chunks += lcnt;
 	}
 	raw_spin_unlock(&pid_list->lock);

 	/*
 	 * On success of allocating all the chunks, both counters
 	 * will be less than zero. If they are not, then an allocation
 	 * failed, and we should not try again.
 	 */
 	if (upper_count >= 0 || lower_count >= 0)
 		return;
 	/*
 	 * When the locks were released, free chunks could have
 	 * been used and allocation needs to be done again. Might as
 	 * well allocate it now.
 	 */
 	goto again;
 }

 /**
  * trace_pid_list_alloc - create a new pid_list
  *
  * Allocates a new pid_list to store pids into.
  *
  * Returns the pid_list on success, NULL otherwise.
  */
 struct trace_pid_list *trace_pid_list_alloc(void)
 {
 	struct trace_pid_list *pid_list;
 	int i;

 	/* According to linux/thread.h, pids can be no bigger that 30 bits */
 	WARN_ON_ONCE(pid_max > (1 << 30));

 	pid_list = kzalloc(sizeof(*pid_list), GFP_KERNEL);
 	if (!pid_list)
 		return NULL;

 	init_irq_work(&pid_list->refill_irqwork, pid_list_refill_irq);

 	raw_spin_lock_init(&pid_list->lock);

 	for (i = 0; i < CHUNK_ALLOC; i++) {
 		union upper_chunk *chunk;

 		chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
 		if (!chunk)
 			break;
 		chunk->next = pid_list->upper_list;
 		pid_list->upper_list = chunk;
 		pid_list->free_upper_chunks++;
 	}

 	for (i = 0; i < CHUNK_ALLOC; i++) {
 		union lower_chunk *chunk;

 		chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
 		if (!chunk)
 			break;
 		chunk->next = pid_list->lower_list;
 		pid_list->lower_list = chunk;
 		pid_list->free_lower_chunks++;
 	}

 	return pid_list;
 }

 /**
  * trace_pid_list_free - Frees an allocated pid_list.
  * @pid_list: The pid list to free.
  *
  * Frees the memory for a pid_list that was allocated.
  */
 void trace_pid_list_free(struct trace_pid_list *pid_list)
 {
 	union upper_chunk *upper;
 	union lower_chunk *lower;
 	int i, j;

 	if (!pid_list)
 		return;

 	irq_work_sync(&pid_list->refill_irqwork);

 	while (pid_list->lower_list) {
 		union lower_chunk *chunk;

 		chunk = pid_list->lower_list;
 		pid_list->lower_list = pid_list->lower_list->next;
 		kfree(chunk);
 	}

 	while (pid_list->upper_list) {
 		union upper_chunk *chunk;

 		chunk = pid_list->upper_list;
 		pid_list->upper_list = pid_list->upper_list->next;
 		kfree(chunk);
 	}

 	for (i = 0; i < UPPER1_SIZE; i++) {
 		upper = pid_list->upper[i];
 		if (upper) {
 			for (j = 0; j < UPPER2_SIZE; j++) {
 				lower = upper->data[j];
 				kfree(lower);
 			}
 			kfree(upper);
 		}
 	}
 	kfree(pid_list);
 }
	// SPDX-License-Identifier: GPL-2.0
	/*
	* Copyright (C) 2021 VMware Inc, Steven Rostedt <rostedt@goodmis.org>
	*/
	#include <linux/spinlock.h>
	#include <linux/irq_work.h>
	#include <linux/slab.h>
	#include "trace.h"

	/* See pid_list.h for details */

	static inline union lower_chunk get_lower_chunk(struct trace_pid_list pid_list)
	{
	union lower_chunk *chunk;

	lockdep_assert_held(&pid_list->lock);

	if (!pid_list->lower_list)
	return NULL;

	chunk = pid_list->lower_list;
	pid_list->lower_list = chunk->next;
	pid_list->free_lower_chunks--;
	WARN_ON_ONCE(pid_list->free_lower_chunks < 0);
	chunk->next = NULL;
	/*
	* If a refill needs to happen, it can not happen here
	* as the scheduler run queue locks are held.
	*/
	if (pid_list->free_lower_chunks <= CHUNK_REALLOC)
	irq_work_queue(&pid_list->refill_irqwork);

	return chunk;
	}

	static inline union upper_chunk get_upper_chunk(struct trace_pid_list pid_list)
	{
	union upper_chunk *chunk;

	lockdep_assert_held(&pid_list->lock);

	if (!pid_list->upper_list)
	return NULL;

	chunk = pid_list->upper_list;
	pid_list->upper_list = chunk->next;
	pid_list->free_upper_chunks--;
	WARN_ON_ONCE(pid_list->free_upper_chunks < 0);
	chunk->next = NULL;
	/*
	* If a refill needs to happen, it can not happen here
	* as the scheduler run queue locks are held.
	*/
	if (pid_list->free_upper_chunks <= CHUNK_REALLOC)
	irq_work_queue(&pid_list->refill_irqwork);

	return chunk;
	}

	static inline void put_lower_chunk(struct trace_pid_list *pid_list,
	union lower_chunk *chunk)
	{
	lockdep_assert_held(&pid_list->lock);

	chunk->next = pid_list->lower_list;
	pid_list->lower_list = chunk;
	pid_list->free_lower_chunks++;
	}

	static inline void put_upper_chunk(struct trace_pid_list *pid_list,
	union upper_chunk *chunk)
	{
	lockdep_assert_held(&pid_list->lock);

	chunk->next = pid_list->upper_list;
	pid_list->upper_list = chunk;
	pid_list->free_upper_chunks++;
	}

	static inline bool upper_empty(union upper_chunk *chunk)
	{
	/*
	* If chunk->data has no lower chunks, it will be the same
	* as a zeroed bitmask. Use find_first_bit() to test it
	* and if it doesn't find any bits set, then the array
	* is empty.
	*/
	int bit = find_first_bit((unsigned long *)chunk->data,
	sizeof(chunk->data) * 8);
	return bit >= sizeof(chunk->data) * 8;
	}

	static inline int pid_split(unsigned int pid, unsigned int *upper1,
	unsigned int upper2, unsigned int lower)
	{
	/* MAX_PID should cover all pids */
	BUILD_BUG_ON(MAX_PID < PID_MAX_LIMIT);

	/* In case a bad pid is passed in, then fail */
	if (unlikely(pid >= MAX_PID))
	return -1;

	*upper1 = (pid >> UPPER1_SHIFT) & UPPER_MASK;
	*upper2 = (pid >> UPPER2_SHIFT) & UPPER_MASK;
	*lower = pid & LOWER_MASK;

	return 0;
	}

	static inline unsigned int pid_join(unsigned int upper1,
	unsigned int upper2, unsigned int lower)
	{
	return ((upper1 & UPPER_MASK) << UPPER1_SHIFT) \|
	((upper2 & UPPER_MASK) << UPPER2_SHIFT) \|
	(lower & LOWER_MASK);
	}

	/**
	* trace_pid_list_is_set - test if the pid is set in the list
	* @pid_list: The pid list to test
	* @pid: The pid to see if set in the list.
	*
	* Tests if @pid is set in the @pid_list. This is usually called
	* from the scheduler when a task is scheduled. Its pid is checked
	* if it should be traced or not.
	*
	* Return true if the pid is in the list, false otherwise.
	*/
	bool trace_pid_list_is_set(struct trace_pid_list *pid_list, unsigned int pid)
	{
	union upper_chunk *upper_chunk;
	union lower_chunk *lower_chunk;
	unsigned long flags;
	unsigned int upper1;
	unsigned int upper2;
	unsigned int lower;
	bool ret = false;

	if (!pid_list)
	return false;

	if (pid_split(pid, &upper1, &upper2, &lower) < 0)
	return false;

	raw_spin_lock_irqsave(&pid_list->lock, flags);
	upper_chunk = pid_list->upper[upper1];
	if (upper_chunk) {
	lower_chunk = upper_chunk->data[upper2];
	if (lower_chunk)
	ret = test_bit(lower, lower_chunk->data);
	}
	raw_spin_unlock_irqrestore(&pid_list->lock, flags);

	return ret;
	}

	/**
	* trace_pid_list_set - add a pid to the list
	* @pid_list: The pid list to add the @pid to.
	* @pid: The pid to add.
	*
	* Adds @pid to @pid_list. This is usually done explicitly by a user
	* adding a task to be traced, or indirectly by the fork function
	* when children should be traced and a task's pid is in the list.
	*
	* Return 0 on success, negative otherwise.
	*/
	int trace_pid_list_set(struct trace_pid_list *pid_list, unsigned int pid)
	{
	union upper_chunk *upper_chunk;
	union lower_chunk *lower_chunk;
	unsigned long flags;
	unsigned int upper1;
	unsigned int upper2;
	unsigned int lower;
	int ret;

	if (!pid_list)
	return -ENODEV;

	if (pid_split(pid, &upper1, &upper2, &lower) < 0)
	return -EINVAL;

	raw_spin_lock_irqsave(&pid_list->lock, flags);
	upper_chunk = pid_list->upper[upper1];
	if (!upper_chunk) {
	upper_chunk = get_upper_chunk(pid_list);
	if (!upper_chunk) {
	ret = -ENOMEM;
	goto out;
	}
	pid_list->upper[upper1] = upper_chunk;
	}
	lower_chunk = upper_chunk->data[upper2];
	if (!lower_chunk) {
	lower_chunk = get_lower_chunk(pid_list);
	if (!lower_chunk) {
	ret = -ENOMEM;
	goto out;
	}
	upper_chunk->data[upper2] = lower_chunk;
	}
	set_bit(lower, lower_chunk->data);
	ret = 0;
	out:
	raw_spin_unlock_irqrestore(&pid_list->lock, flags);
	return ret;
	}

	/**
	* trace_pid_list_clear - remove a pid from the list
	* @pid_list: The pid list to remove the @pid from.
	* @pid: The pid to remove.
	*
	* Removes @pid from @pid_list. This is usually done explicitly by a user
	* removing tasks from tracing, or indirectly by the exit function
	* when a task that is set to be traced exits.
	*
	* Return 0 on success, negative otherwise.
	*/
	int trace_pid_list_clear(struct trace_pid_list *pid_list, unsigned int pid)
	{
	union upper_chunk *upper_chunk;
	union lower_chunk *lower_chunk;
	unsigned long flags;
	unsigned int upper1;
	unsigned int upper2;
	unsigned int lower;

	if (!pid_list)
	return -ENODEV;

	if (pid_split(pid, &upper1, &upper2, &lower) < 0)
	return -EINVAL;

	raw_spin_lock_irqsave(&pid_list->lock, flags);
	upper_chunk = pid_list->upper[upper1];
	if (!upper_chunk)
	goto out;

	lower_chunk = upper_chunk->data[upper2];
	if (!lower_chunk)
	goto out;

	clear_bit(lower, lower_chunk->data);

	/* if there's no more bits set, add it to the free list */
	if (find_first_bit(lower_chunk->data, LOWER_MAX) >= LOWER_MAX) {
	put_lower_chunk(pid_list, lower_chunk);
	upper_chunk->data[upper2] = NULL;
	if (upper_empty(upper_chunk)) {
	put_upper_chunk(pid_list, upper_chunk);
	pid_list->upper[upper1] = NULL;
	}
	}
	out:
	raw_spin_unlock_irqrestore(&pid_list->lock, flags);
	return 0;
	}

	/**
	* trace_pid_list_next - return the next pid in the list
	* @pid_list: The pid list to examine.
	* @pid: The pid to start from
	* @next: The pointer to place the pid that is set starting from @pid.
	*
	* Looks for the next consecutive pid that is in @pid_list starting
	* at the pid specified by @pid. If one is set (including @pid), then
	* that pid is placed into @next.
	*
	* Return 0 when a pid is found, -1 if there are no more pids included.
	*/
	int trace_pid_list_next(struct trace_pid_list *pid_list, unsigned int pid,
	unsigned int *next)
	{
	union upper_chunk *upper_chunk;
	union lower_chunk *lower_chunk;
	unsigned long flags;
	unsigned int upper1;
	unsigned int upper2;
	unsigned int lower;

	if (!pid_list)
	return -ENODEV;

	if (pid_split(pid, &upper1, &upper2, &lower) < 0)
	return -EINVAL;

	raw_spin_lock_irqsave(&pid_list->lock, flags);
	for (; upper1 <= UPPER_MASK; upper1++, upper2 = 0) {
	upper_chunk = pid_list->upper[upper1];

	if (!upper_chunk)
	continue;

	for (; upper2 <= UPPER_MASK; upper2++, lower = 0) {
	lower_chunk = upper_chunk->data[upper2];
	if (!lower_chunk)
	continue;

	lower = find_next_bit(lower_chunk->data, LOWER_MAX,
	lower);
	if (lower < LOWER_MAX)
	goto found;
	}
	}

	found:
	raw_spin_unlock_irqrestore(&pid_list->lock, flags);
	if (upper1 > UPPER_MASK)
	return -1;

	*next = pid_join(upper1, upper2, lower);
	return 0;
	}

	/**
	* trace_pid_list_first - return the first pid in the list
	* @pid_list: The pid list to examine.
	* @pid: The pointer to place the pid first found pid that is set.
	*
	* Looks for the first pid that is set in @pid_list, and places it
	* into @pid if found.
	*
	* Return 0 when a pid is found, -1 if there are no pids set.
	*/
	int trace_pid_list_first(struct trace_pid_list pid_list, unsigned int pid)
	{
	return trace_pid_list_next(pid_list, 0, pid);
	}

	static void pid_list_refill_irq(struct irq_work *iwork)
	{
	struct trace_pid_list *pid_list = container_of(iwork, struct trace_pid_list,
	refill_irqwork);
	union upper_chunk *upper = NULL;
	union lower_chunk *lower = NULL;
	union upper_chunk **upper_next = &upper;
	union lower_chunk **lower_next = &lower;
	int upper_count;
	int lower_count;
	int ucnt = 0;
	int lcnt = 0;

	again:
	raw_spin_lock(&pid_list->lock);
	upper_count = CHUNK_ALLOC - pid_list->free_upper_chunks;
	lower_count = CHUNK_ALLOC - pid_list->free_lower_chunks;
	raw_spin_unlock(&pid_list->lock);

	if (upper_count <= 0 && lower_count <= 0)
	return;

	while (upper_count-- > 0) {
	union upper_chunk *chunk;

	chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT);
	if (!chunk)
	break;
	*upper_next = chunk;
	upper_next = &chunk->next;
	ucnt++;
	}

	while (lower_count-- > 0) {
	union lower_chunk *chunk;

	chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT);
	if (!chunk)
	break;
	*lower_next = chunk;
	lower_next = &chunk->next;
	lcnt++;
	}

	raw_spin_lock(&pid_list->lock);
	if (upper) {
	*upper_next = pid_list->upper_list;
	pid_list->upper_list = upper;
	pid_list->free_upper_chunks += ucnt;
	}
	if (lower) {
	*lower_next = pid_list->lower_list;
	pid_list->lower_list = lower;
	pid_list->free_lower_chunks += lcnt;
	}
	raw_spin_unlock(&pid_list->lock);

	/*
	* On success of allocating all the chunks, both counters
	* will be less than zero. If they are not, then an allocation
	* failed, and we should not try again.
	*/
	if (upper_count >= 0 \|\| lower_count >= 0)
	return;
	/*
	* When the locks were released, free chunks could have
	* been used and allocation needs to be done again. Might as
	* well allocate it now.
	*/
	goto again;
	}

	/**
	* trace_pid_list_alloc - create a new pid_list
	*
	* Allocates a new pid_list to store pids into.
	*
	* Returns the pid_list on success, NULL otherwise.
	*/
	struct trace_pid_list *trace_pid_list_alloc(void)
	{
	struct trace_pid_list *pid_list;
	int i;

	/* According to linux/thread.h, pids can be no bigger that 30 bits */
	WARN_ON_ONCE(pid_max > (1 << 30));

	pid_list = kzalloc(sizeof(*pid_list), GFP_KERNEL);
	if (!pid_list)
	return NULL;

	init_irq_work(&pid_list->refill_irqwork, pid_list_refill_irq);

	raw_spin_lock_init(&pid_list->lock);

	for (i = 0; i < CHUNK_ALLOC; i++) {
	union upper_chunk *chunk;

	chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
	if (!chunk)
	break;
	chunk->next = pid_list->upper_list;
	pid_list->upper_list = chunk;
	pid_list->free_upper_chunks++;
	}

	for (i = 0; i < CHUNK_ALLOC; i++) {
	union lower_chunk *chunk;

	chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
	if (!chunk)
	break;
	chunk->next = pid_list->lower_list;
	pid_list->lower_list = chunk;
	pid_list->free_lower_chunks++;
	}

	return pid_list;
	}

	/**
	* trace_pid_list_free - Frees an allocated pid_list.
	* @pid_list: The pid list to free.
	*
	* Frees the memory for a pid_list that was allocated.
	*/
	void trace_pid_list_free(struct trace_pid_list *pid_list)
	{
	union upper_chunk *upper;
	union lower_chunk *lower;
	int i, j;

	if (!pid_list)
	return;

	irq_work_sync(&pid_list->refill_irqwork);

	while (pid_list->lower_list) {
	union lower_chunk *chunk;

	chunk = pid_list->lower_list;
	pid_list->lower_list = pid_list->lower_list->next;
	kfree(chunk);
	}

	while (pid_list->upper_list) {
	union upper_chunk *chunk;

	chunk = pid_list->upper_list;
	pid_list->upper_list = pid_list->upper_list->next;
	kfree(chunk);
	}

	for (i = 0; i < UPPER1_SIZE; i++) {
	upper = pid_list->upper[i];
	if (upper) {
	for (j = 0; j < UPPER2_SIZE; j++) {
	lower = upper->data[j];
	kfree(lower);
	}
	kfree(upper);
	}
	}
	kfree(pid_list);
	}