cregit-Linux how code gets into the kernel

Release 4.15 net/sunrpc/sched.c

Directory: net/sunrpc
/*
 * linux/net/sunrpc/sched.c
 *
 * Scheduling for synchronous and asynchronous RPC requests.
 *
 * Copyright (C) 1996 Olaf Kirch, <okir@monad.swb.de>
 *
 * TCP NFS related read + write fixes
 * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie>
 */

#include <linux/module.h>

#include <linux/sched.h>
#include <linux/interrupt.h>
#include <linux/slab.h>
#include <linux/mempool.h>
#include <linux/smp.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/freezer.h>

#include <linux/sunrpc/clnt.h>

#include "sunrpc.h"

#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)

#define RPCDBG_FACILITY		RPCDBG_SCHED
#endif


#define CREATE_TRACE_POINTS
#include <trace/events/sunrpc.h>

/*
 * RPC slabs and memory pools
 */

#define RPC_BUFFER_MAXSIZE	(2048)

#define RPC_BUFFER_POOLSIZE	(8)

#define RPC_TASK_POOLSIZE	(8)

static struct kmem_cache	*rpc_task_slabp __read_mostly;

static struct kmem_cache	*rpc_buffer_slabp __read_mostly;

static mempool_t	*rpc_task_mempool __read_mostly;

static mempool_t	*rpc_buffer_mempool __read_mostly;

static void			rpc_async_schedule(struct work_struct *);
static void			 rpc_release_task(struct rpc_task *task);
static void __rpc_queue_timer_fn(struct timer_list *t);

/*
 * RPC tasks sit here while waiting for conditions to improve.
 */

static struct rpc_wait_queue delay_queue;

/*
 * rpciod-related stuff
 */

struct workqueue_struct *rpciod_workqueue __read_mostly;

struct workqueue_struct *xprtiod_workqueue __read_mostly;

/*
 * Disable the timer for a given RPC task. Should be called with
 * queue->lock and bh_disabled in order to avoid races within
 * rpc_run_timer().
 */

static void __rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task) { if (task->tk_timeout == 0) return; dprintk("RPC: %5u disabling timer\n", task->tk_pid); task->tk_timeout = 0; list_del(&task->u.tk_wait.timer_list); if (list_empty(&queue->timer_list.list)) del_timer(&queue->timer_list.timer); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust4966.22%250.00%
Linus Torvalds (pre-git)2432.43%125.00%
Chuck Lever11.35%125.00%
Total74100.00%4100.00%


static void rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires) { queue->timer_list.expires = expires; mod_timer(&queue->timer_list.timer, expires); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust3497.14%150.00%
Linus Torvalds (pre-git)12.86%150.00%
Total35100.00%2100.00%

/* * Set up a timer for the current task. */
static void __rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task) { if (!task->tk_timeout) return; dprintk("RPC: %5u setting alarm for %u ms\n", task->tk_pid, jiffies_to_msecs(task->tk_timeout)); task->u.tk_wait.expires = jiffies + task->tk_timeout; if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.expires)) rpc_set_queue_timer(queue, task->u.tk_wait.expires); list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust6052.17%333.33%
Linus Torvalds (pre-git)5144.35%444.44%
Nicholas Mc Guire43.48%222.22%
Total115100.00%9100.00%


static void rpc_rotate_queue_owner(struct rpc_wait_queue *queue) { struct list_head *q = &queue->tasks[queue->priority]; struct rpc_task *task; if (!list_empty(q)) { task = list_first_entry(q, struct rpc_task, u.tk_wait.list); if (task->tk_owner == queue->owner) list_move_tail(&task->u.tk_wait.list, q); } }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust81100.00%1100.00%
Total81100.00%1100.00%


static void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority) { if (queue->priority != priority) { /* Fairness: rotate the list when changing priority */ rpc_rotate_queue_owner(queue); queue->priority = priority; } }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust36100.00%2100.00%
Total36100.00%2100.00%


static void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid) { queue->owner = pid; queue->nr = RPC_BATCH_COUNT; }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust26100.00%1100.00%
Total26100.00%1100.00%


static void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue) { rpc_set_waitqueue_priority(queue, queue->maxpriority); rpc_set_waitqueue_owner(queue, 0); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust27100.00%1100.00%
Total27100.00%1100.00%

/* * Add new request to a priority queue. */
static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue, struct rpc_task *task, unsigned char queue_priority) { struct list_head *q; struct rpc_task *t; INIT_LIST_HEAD(&task->u.tk_wait.links); if (unlikely(queue_priority > queue->maxpriority)) queue_priority = queue->maxpriority; if (queue_priority > queue->priority) rpc_set_waitqueue_priority(queue, queue_priority); q = &queue->tasks[queue_priority]; list_for_each_entry(t, q, u.tk_wait.list) { if (t->tk_owner == task->tk_owner) { list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links); return; } } list_add_tail(&task->u.tk_wait.list, q); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust142100.00%5100.00%
Total142100.00%5100.00%

/* * Add new request to wait queue. * * Swapper tasks always get inserted at the head of the queue. * This should avoid many nasty memory deadlocks and hopefully * improve overall performance. * Everyone else gets appended to the queue to ensure proper FIFO behavior. */
static void __rpc_add_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task, unsigned char queue_priority) { WARN_ON_ONCE(RPC_IS_QUEUED(task)); if (RPC_IS_QUEUED(task)) return; if (RPC_IS_PRIORITY(queue)) __rpc_add_wait_queue_priority(queue, task, queue_priority); else if (RPC_IS_SWAPPER(task)) list_add(&task->u.tk_wait.list, &queue->tasks[0]); else list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]); task->tk_waitqueue = queue; queue->qlen++; /* barrier matches the read in rpc_wake_up_task_queue_locked() */ smp_wmb(); rpc_set_queued(task); dprintk("RPC: %5u added to queue %p \"%s\"\n", task->tk_pid, queue, rpc_qname(queue)); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)5842.34%218.18%
Trond Myklebust5237.96%545.45%
Neil Brown128.76%19.09%
Weston Andros Adamson96.57%19.09%
Chuck Lever64.38%218.18%
Total137100.00%11100.00%

/* * Remove request from a priority queue. */
static void __rpc_remove_wait_queue_priority(struct rpc_task *task) { struct rpc_task *t; if (!list_empty(&task->u.tk_wait.links)) { t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list); list_move(&t->u.tk_wait.list, &task->u.tk_wait.list); list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links); } }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust99100.00%2100.00%
Total99100.00%2100.00%

/* * Remove request from queue. * Note: must be called with spin lock held. */
static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task) { __rpc_disable_timer(queue, task); if (RPC_IS_PRIORITY(queue)) __rpc_remove_wait_queue_priority(task); list_del(&task->u.tk_wait.list); queue->qlen--; dprintk("RPC: %5u removed from queue %p \"%s\"\n", task->tk_pid, queue, rpc_qname(queue)); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust6291.18%571.43%
Chuck Lever68.82%228.57%
Total68100.00%7100.00%


static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigned char nr_queues) { int i; spin_lock_init(&queue->lock); for (i = 0; i < ARRAY_SIZE(queue->tasks); i++) INIT_LIST_HEAD(&queue->tasks[i]); queue->maxpriority = nr_queues - 1; rpc_reset_waitqueue_priority(queue); queue->qlen = 0; timer_setup(&queue->timer_list.timer, __rpc_queue_timer_fn, 0); INIT_LIST_HEAD(&queue->timer_list.list); rpc_assign_waitqueue_name(queue, qname); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust8678.18%555.56%
Linus Torvalds (pre-git)2119.09%222.22%
Kees Cook21.82%111.11%
Neil Brown10.91%111.11%
Total110100.00%9100.00%


void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname) { __rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust1562.50%266.67%
Linus Torvalds (pre-git)937.50%133.33%
Total24100.00%3100.00%

EXPORT_SYMBOL_GPL(rpc_init_priority_wait_queue);
void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname) { __rpc_init_priority_wait_queue(queue, qname, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust2187.50%266.67%
Linus Torvalds (pre-git)312.50%133.33%
Total24100.00%3100.00%

EXPORT_SYMBOL_GPL(rpc_init_wait_queue);
void rpc_destroy_wait_queue(struct rpc_wait_queue *queue) { del_timer_sync(&queue->timer_list.timer); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust20100.00%2100.00%
Total20100.00%2100.00%

EXPORT_SYMBOL_GPL(rpc_destroy_wait_queue);
static int rpc_wait_bit_killable(struct wait_bit_key *key, int mode) { freezable_schedule_unsafe(); if (signal_pending_state(mode, current)) return -ERESTARTSYS; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust2060.61%125.00%
Peter Zijlstra927.27%125.00%
Neil Brown39.09%125.00%
Matthew Wilcox13.03%125.00%
Total33100.00%4100.00%

#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) || IS_ENABLED(CONFIG_TRACEPOINTS)
static void rpc_task_set_debuginfo(struct rpc_task *task) { static atomic_t rpc_pid; task->tk_pid = atomic_inc_return(&rpc_pid); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust25100.00%1100.00%
Total25100.00%1100.00%

#else
static inline void rpc_task_set_debuginfo(struct rpc_task *task) { }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust11100.00%1100.00%
Total11100.00%1100.00%

#endif
static void rpc_set_active(struct rpc_task *task) { rpc_task_set_debuginfo(task); set_bit(RPC_TASK_ACTIVE, &task->tk_runstate); trace_rpc_task_begin(task->tk_client, task, NULL); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust2670.27%480.00%
Chuck Lever1129.73%120.00%
Total37100.00%5100.00%

/* * Mark an RPC call as having completed by clearing the 'active' bit * and then waking up all tasks that were sleeping. */
static int rpc_complete_task(struct rpc_task *task) { void *m = &task->tk_runstate; wait_queue_head_t *wq = bit_waitqueue(m, RPC_TASK_ACTIVE); struct wait_bit_key k = __WAIT_BIT_KEY_INITIALIZER(m, RPC_TASK_ACTIVE); unsigned long flags; int ret; trace_rpc_task_complete(task->tk_client, task, NULL); spin_lock_irqsave(&wq->lock, flags); clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate); ret = atomic_dec_and_test(&task->tk_count); if (waitqueue_active(wq)) __wake_up_locked_key(wq, TASK_NORMAL, &k); spin_unlock_irqrestore(&wq->lock, flags); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust120100.00%4100.00%
Total120100.00%4100.00%

/* * Allow callers to wait for completion of an RPC call * * Note the use of out_of_line_wait_on_bit() rather than wait_on_bit() * to enforce taking of the wq->lock and hence avoid races with * rpc_complete_task(). */
int __rpc_wait_for_completion_task(struct rpc_task *task, wait_bit_action_f *action) { if (action == NULL) action = rpc_wait_bit_killable; return out_of_line_wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE, action, TASK_KILLABLE); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust3692.31%250.00%
Matthew Wilcox25.13%125.00%
Neil Brown12.56%125.00%
Total39100.00%4100.00%

EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task); /* * Make an RPC task runnable. * * Note: If the task is ASYNC, and is being made runnable after sitting on an * rpc_wait_queue, this must be called with the queue spinlock held to protect * the wait queue operation. * Note the ordering of rpc_test_and_set_running() and rpc_clear_queued(), * which is needed to ensure that __rpc_execute() doesn't loop (due to the * lockless RPC_IS_QUEUED() test) before we've had a chance to test * the RPC_TASK_RUNNING flag. */
static void rpc_make_runnable(struct workqueue_struct *wq, struct rpc_task *task) { bool need_wakeup = !rpc_test_and_set_running(task); rpc_clear_queued(task); if (!need_wakeup) return; if (RPC_IS_ASYNC(task)) { INIT_WORK(&task->u.tk_work, rpc_async_schedule); queue_work(wq, &task->u.tk_work); } else wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust4050.00%444.44%
Linus Torvalds (pre-git)3543.75%444.44%
Christophe Saout56.25%111.11%
Total80100.00%9100.00%

/* * Prepare for sleeping on a wait queue. * By always appending tasks to the list we ensure FIFO behavior. * NB: An RPC task will only receive interrupt-driven events as long * as it's on a wait queue. */
static void __rpc_sleep_on_priority(struct rpc_wait_queue *q, struct rpc_task *task, rpc_action action, unsigned char queue_priority) { dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n", task->tk_pid, rpc_qname(q), jiffies); trace_rpc_task_sleep(task->tk_client, task, q); __rpc_add_wait_queue(q, task, queue_priority); WARN_ON_ONCE(task->tk_callback != NULL); task->tk_callback = action; __rpc_add_timer(q, task); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)5770.37%650.00%
Trond Myklebust2227.16%433.33%
Weston Andros Adamson11.23%18.33%
Chuck Lever11.23%18.33%
Total81100.00%12100.00%


void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task, rpc_action action) { /* We shouldn't ever put an inactive task to sleep */ WARN_ON_ONCE(!RPC_IS_ACTIVATED(task)); if (!RPC_IS_ACTIVATED(task)) { task->tk_status = -EIO; rpc_put_task_async(task); return; } /* * Protect the queue operations. */ spin_lock_bh(&q->lock); __rpc_sleep_on_priority(q, task, action, task->tk_priority); spin_unlock_bh(&q->lock); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)3644.44%337.50%
Weston Andros Adamson2429.63%112.50%
Trond Myklebust2125.93%450.00%
Total81100.00%8100.00%

EXPORT_SYMBOL_GPL(rpc_sleep_on);
void rpc_sleep_on_priority(struct rpc_wait_queue *q, struct rpc_task *task, rpc_action action, int priority) { /* We shouldn't ever put an inactive task to sleep */ WARN_ON_ONCE(!RPC_IS_ACTIVATED(task)); if (!RPC_IS_ACTIVATED(task)) { task->tk_status = -EIO; rpc_put_task_async(task); return; } /* * Protect the queue operations. */ spin_lock_bh(&q->lock); __rpc_sleep_on_priority(q, task, action, priority - RPC_PRIORITY_LOW); spin_unlock_bh(&q->lock); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust6071.43%150.00%
Weston Andros Adamson2428.57%150.00%
Total84100.00%2100.00%

EXPORT_SYMBOL_GPL(rpc_sleep_on_priority); /** * __rpc_do_wake_up_task_on_wq - wake up a single rpc_task * @wq: workqueue on which to run task * @queue: wait queue * @task: task to be woken up * * Caller must hold queue->lock, and have cleared the task queued flag. */
static void __rpc_do_wake_up_task_on_wq(struct workqueue_struct *wq, struct rpc_wait_queue *queue, struct rpc_task *task) { dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n", task->tk_pid, jiffies); /* Has the task been executed yet? If not, we cannot wake it up! */ if (!RPC_IS_ACTIVATED(task)) { printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task); return; } trace_rpc_task_wakeup(task->tk_client, task, queue); __rpc_remove_wait_queue(queue, task); rpc_make_runnable(wq, task); dprintk("RPC: __rpc_wake_up_task done\n"); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)4352.44%228.57%
Trond Myklebust3745.12%457.14%
Chuck Lever22.44%114.29%
Total82100.00%7100.00%

/* * Wake up a queued task while the queue lock is being held */
static void rpc_wake_up_task_on_wq_queue_locked(struct workqueue_struct *wq, struct rpc_wait_queue *queue, struct rpc_task *task) { if (RPC_IS_QUEUED(task)) { smp_rmb(); if (task->tk_waitqueue == queue) __rpc_do_wake_up_task_on_wq(wq, queue, task); } }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust4284.00%571.43%
Linus Torvalds (pre-git)816.00%228.57%
Total50100.00%7100.00%

/* * Wake up a queued task while the queue lock is being held */
static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task) { rpc_wake_up_task_on_wq_queue_locked(rpciod_workqueue, queue, task); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust25100.00%1100.00%
Total25100.00%1100.00%

/* * Wake up a task on a specific queue */
void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task) { spin_lock_bh(&queue->lock); rpc_wake_up_task_queue_locked(queue, task); spin_unlock_bh(&queue->lock); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust38100.00%5100.00%
Total38100.00%5100.00%

EXPORT_SYMBOL_GPL(rpc_wake_up_queued_task); /* * Wake up the next task on a priority queue. */
static struct rpc_task *__rpc_find_next_queued_priority(struct rpc_wait_queue *queue) { struct list_head *q; struct rpc_task *task; /* * Service a batch of tasks from a single owner. */ q = &queue->tasks[queue->priority]; if (!list_empty(q)) { task = list_entry(q->next, struct rpc_task, u.tk_wait.list); if (queue->owner == task->tk_owner) { if (--queue->nr) goto out; list_move_tail(&task->u.tk_wait.list, q); } /* * Check if we need to switch queues. */ goto new_owner; } /* * Service the next queue. */ do { if (q == &queue->tasks[0]) q = &queue->tasks[queue->maxpriority]; else q = q - 1; if (!list_empty(q)) { task = list_entry(q->next, struct rpc_task, u.tk_wait.list); goto new_queue; } } while (q != &queue->tasks[queue->priority]); rpc_reset_waitqueue_priority(queue); return NULL; new_queue: rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0])); new_owner: rpc_set_waitqueue_owner(queue, task->tk_owner); out: return task; }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust21190.95%457.14%
Linus Torvalds (pre-git)219.05%342.86%
Total232100.00%7100.00%


static struct rpc_task *__rpc_find_next_queued(struct rpc_wait_queue *queue) { if (RPC_IS_PRIORITY(queue)) return __rpc_find_next_queued_priority(queue); if (!list_empty(&queue->tasks[0])) return list_first_entry(&queue->tasks[0], struct rpc_task, u.tk_wait.list); return NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust6195.31%375.00%
Linus Torvalds (pre-git)34.69%125.00%
Total64100.00%4100.00%

/* * Wake up the first task on the wait queue. */
struct rpc_task *rpc_wake_up_first_on_wq(struct workqueue_struct *wq, struct rpc_wait_queue *queue, bool (*func)(struct rpc_task *, void *), void *data) { struct rpc_task *task = NULL; dprintk("RPC: wake_up_first(%p \"%s\")\n", queue, rpc_qname(queue)); spin_lock_bh(&queue->lock); task = __rpc_find_next_queued(queue); if (task != NULL) { if (func(task, data)) rpc_wake_up_task_on_wq_queue_locked(wq, queue, task); else task = NULL; } spin_unlock_bh(&queue->lock); return task; }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust6457.66%666.67%
Linus Torvalds (pre-git)4237.84%222.22%
Neil Brown54.50%111.11%
Total111100.00%9100.00%

/* * Wake up the first task on the wait queue. */
struct rpc_task *rpc_wake_up_first(struct rpc_wait_queue *queue, bool (*func)(struct rpc_task *, void *), void *data) { return rpc_wake_up_first_on_wq(rpciod_workqueue, queue, func, data); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust42100.00%1100.00%
Total42100.00%1100.00%

EXPORT_SYMBOL_GPL(rpc_wake_up_first);
static bool rpc_wake_up_next_func(struct rpc_task *task, void *data) { return true; }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust18100.00%1100.00%
Total18100.00%1100.00%

/* * Wake up the next task on the wait queue. */
struct rpc_task *rpc_wake_up_next(struct rpc_wait_queue *queue) { return rpc_wake_up_first(queue, rpc_wake_up_next_func, NULL); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust22100.00%1100.00%
Total22100.00%1100.00%

EXPORT_SYMBOL_GPL(rpc_wake_up_next); /** * rpc_wake_up - wake up all rpc_tasks * @queue: rpc_wait_queue on which the tasks are sleeping * * Grabs queue->lock */
void rpc_wake_up(struct rpc_wait_queue *queue) { struct list_head *head; spin_lock_bh(&queue->lock); head = &queue->tasks[queue->maxpriority]; for (;;) { while (!list_empty(head)) { struct rpc_task *task; task = list_first_entry(head, struct rpc_task, u.tk_wait.list); rpc_wake_up_task_queue_locked(queue, task); } if (head == &queue->tasks[0]) break; head--; } spin_unlock_bh(&queue->lock); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust8076.92%666.67%
Linus Torvalds (pre-git)2120.19%222.22%
Neil Brown32.88%111.11%
Total104100.00%9100.00%

EXPORT_SYMBOL_GPL(rpc_wake_up); /** * rpc_wake_up_status - wake up all rpc_tasks and set their status value. * @queue: rpc_wait_queue on which the tasks are sleeping * @status: status value to set * * Grabs queue->lock */
void rpc_wake_up_status(struct rpc_wait_queue *queue, int status) { struct list_head *head; spin_lock_bh(&queue->lock); head = &queue->tasks[queue->maxpriority]; for (;;) { while (!list_empty(head)) { struct rpc_task *task; task = list_first_entry(head, struct rpc_task, u.tk_wait.list); task->tk_status = status; rpc_wake_up_task_queue_locked(queue, task); } if (head == &queue->tasks[0]) break; head--; } spin_unlock_bh(&queue->lock); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust7566.37%666.67%
Linus Torvalds (pre-git)3430.09%222.22%
Neil Brown43.54%111.11%
Total113100.00%9100.00%

EXPORT_SYMBOL_GPL(rpc_wake_up_status);
static void __rpc_queue_timer_fn(struct timer_list *t) { struct rpc_wait_queue *queue = from_timer(queue, t, timer_list.timer); struct rpc_task *task, *n; unsigned long expires, now, timeo; spin_lock(&queue->lock); expires = now = jiffies; list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) { timeo = task->u.tk_wait.expires; if (time_after_eq(now, timeo)) { dprintk("RPC: %5u timeout\n", task->tk_pid); task->tk_status = -ETIMEDOUT; rpc_wake_up_task_queue_locked(queue, task); continue; } if (expires == now || time_after(expires, timeo)) expires = timeo; } if (!list_empty(&queue->timer_list.list)) rpc_set_queue_timer(queue, expires); spin_unlock(&queue->lock); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust14591.19%150.00%
Kees Cook148.81%150.00%
Total159100.00%2100.00%


static void __rpc_atrun(struct rpc_task *task) { if (task->tk_status == -ETIMEDOUT) task->tk_status = 0; }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust26100.00%3100.00%
Total26100.00%3100.00%

/* * Run a task at a later time */
void rpc_delay(struct rpc_task *task, unsigned long delay) { task->tk_timeout = delay; rpc_sleep_on(&delay_queue, task, __rpc_atrun); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)2996.67%150.00%
Trond Myklebust13.33%150.00%
Total30100.00%2100.00%

EXPORT_SYMBOL_GPL(rpc_delay); /* * Helper to call task->tk_ops->rpc_call_prepare */
void rpc_prepare_task(struct rpc_task *task) { task->tk_ops->rpc_call_prepare(task, task->tk_calldata); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust23100.00%1100.00%
Total23100.00%1100.00%


static void rpc_init_task_statistics(struct rpc_task *task) { /* Initialize retry counters */ task->tk_garb_retry = 2; task->tk_cred_retry = 2; task->tk_rebind_retry = 2; /* starting timestamp */ task->tk_start = ktime_get(); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust38100.00%1100.00%
Total38100.00%1100.00%


static void rpc_reset_task_statistics(struct rpc_task *task) { task->tk_timeouts = 0; task->tk_flags &= ~(RPC_CALL_MAJORSEEN|RPC_TASK_KILLED|RPC_TASK_SENT); rpc_init_task_statistics(task); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust35100.00%1100.00%
Total35100.00%1100.00%

/* * Helper that calls task->tk_ops->rpc_call_done if it exists */
void rpc_exit_task(struct rpc_task *task) { task->tk_action = NULL; if (task->tk_ops->rpc_call_done != NULL) { task->tk_ops->rpc_call_done(task, task->tk_calldata); if (task->tk_action != NULL) { WARN_ON(RPC_ASSASSINATED(task)); /* Always release the RPC slot and buffer memory */ xprt_release(task); rpc_reset_task_statistics(task); } } }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust70100.00%4100.00%
Total70100.00%4100.00%


void rpc_exit(struct rpc_task *task, int status) { task->tk_status = status; task->tk_action = rpc_exit_task; if (RPC_IS_QUEUED(task)) rpc_wake_up_queued_task(task->tk_waitqueue, task); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust41100.00%1100.00%
Total41100.00%1100.00%

EXPORT_SYMBOL_GPL(rpc_exit);
void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata) { if (ops->rpc_release != NULL) ops->rpc_release(calldata); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust30100.00%1100.00%
Total30100.00%1100.00%

/* * This is the RPC `scheduler' (or rather, the finite state machine). */
static void __rpc_execute(struct rpc_task *task) { struct rpc_wait_queue *queue; int task_is_async = RPC_IS_ASYNC(task); int status = 0; dprintk("RPC: %5u __rpc_execute flags=0x%x\n", task->tk_pid, task->tk_flags); WARN_ON_ONCE(RPC_IS_QUEUED(task)); if (RPC_IS_QUEUED(task)) return; for (;;) { void (*do_action)(struct rpc_task *); /* * Execute any pending callback first. */ do_action = task->tk_callback; task->tk_callback = NULL; if (do_action == NULL) { /* * Perform the next FSM step. * tk_action may be NULL if the task has been killed. * In particular, note that rpc_killall_tasks may * do this at any time, so beware when dereferencing. */ do_action = task->tk_action; if (do_action == NULL) break; } trace_rpc_task_run_action(task->tk_client, task, task->tk_action); do_action(task); /* * Lockless check for whether task is sleeping or not. */ if (!RPC_IS_QUEUED(task)) continue; /* * The queue->lock protects against races with * rpc_make_runnable(). * * Note that once we clear RPC_TASK_RUNNING on an asynchronous * rpc_task, rpc_make_runnable() can assign it to a * different workqueue. We therefore cannot assume that the * rpc_task pointer may still be dereferenced. */ queue = task->tk_waitqueue; spin_lock_bh(&queue->lock); if (!RPC_IS_QUEUED(task)) { spin_unlock_bh(&queue->lock); continue; } rpc_clear_running(task); spin_unlock_bh(&queue->lock); if (task_is_async) return; /* sync task: sleep here */ dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid); status = out_of_line_wait_on_bit(&task->tk_runstate, RPC_TASK_QUEUED, rpc_wait_bit_killable, TASK_KILLABLE); if (status == -ERESTARTSYS) { /* * When a sync task receives a signal, it exits with * -ERESTARTSYS. In order to catch any callbacks that * clean up after sleeping on some queue, we don't * break the loop here, but go around once more. */ dprintk("RPC: %5u got signal\n", task->tk_pid); task->tk_flags |= RPC_TASK_KILLED; rpc_exit(task, -ERESTARTSYS); } dprintk("RPC: %5u sync task resuming\n", task->tk_pid); } dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status, task->tk_status); /* Release all resources associated with the task */ rpc_release_task(task); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)14050.00%527.78%
Trond Myklebust12444.29%1055.56%
Weston Andros Adamson93.21%15.56%
Chuck Lever51.79%15.56%
Matthew Wilcox20.71%15.56%
Total280100.00%18100.00%

/* * User-visible entry point to the scheduler. * * This may be called recursively if e.g. an async NFS task updates * the attributes and finds that dirty pages must be flushed. * NOTE: Upon exit of this function the task is guaranteed to be * released. In particular note that tk_release() will have * been called, so your task memory may have been freed. */
void rpc_execute(struct rpc_task *task) { bool is_async = RPC_IS_ASYNC(task); rpc_set_active(task); rpc_make_runnable(rpciod_workqueue, task); if (!is_async) __rpc_execute(task); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)2050.00%545.45%
Trond Myklebust2050.00%654.55%
Total40100.00%11100.00%


static void rpc_async_schedule(struct work_struct *work) { __rpc_execute(container_of(work, struct rpc_task, u.tk_work)); }

Contributors

PersonTokensPropCommitsCommitProp
David Howells1246.15%125.00%
Trond Myklebust1038.46%125.00%
Linus Torvalds (pre-git)415.38%250.00%
Total26100.00%4100.00%

/** * rpc_malloc - allocate RPC buffer resources * @task: RPC task * * A single memory region is allocated, which is split between the * RPC call and RPC reply that this task is being used for. When * this RPC is retired, the memory is released by calling rpc_free. * * To prevent rpciod from hanging, this allocator never sleeps, * returning -ENOMEM and suppressing warning if the request cannot * be serviced immediately. The caller can arrange to sleep in a * way that is safe for rpciod. * * Most requests are 'small' (under 2KiB) and can be serviced from a * mempool, ensuring that NFS reads and writes can always proceed, * and that there is good locality of reference for these buffers. * * In order to avoid memory starvation triggering more writebacks of * NFS requests, we avoid using GFP_KERNEL. */
int rpc_malloc(struct rpc_task *task) { struct rpc_rqst *rqst = task->tk_rqstp; size_t size = rqst->rq_callsize + rqst->rq_rcvsize; struct rpc_buffer *buf; gfp_t gfp = GFP_NOIO | __GFP_NOWARN; if (RPC_IS_SWAPPER(task)) gfp = __GFP_MEMALLOC | GFP_NOWAIT | __GFP_NOWARN; size += sizeof(struct rpc_buffer); if (size <= RPC_BUFFER_MAXSIZE) buf = mempool_alloc(rpc_buffer_mempool, gfp); else buf = kmalloc(size, gfp); if (!buf) return -ENOMEM; buf->len = size; dprintk("RPC: %5u allocated buffer of size %zu at %p\n", task->tk_pid, size, buf); rqst->rq_buffer = buf->data; rqst->rq_rbuffer = (char *)rqst->rq_buffer + rqst->rq_callsize; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever8760.00%533.33%
Trond Myklebust2416.55%320.00%
Linus Torvalds (pre-git)1611.03%213.33%
Peter Zijlstra74.83%16.67%
Mel Gorman74.83%16.67%
David Rientjes21.38%16.67%
Geert Uytterhoeven10.69%16.67%
Al Viro10.69%16.67%
Total145100.00%15100.00%

EXPORT_SYMBOL_GPL(rpc_malloc); /** * rpc_free - free RPC buffer resources allocated via rpc_malloc * @task: RPC task * */
void rpc_free(struct rpc_task *task) { void *buffer = task->tk_rqstp->rq_buffer; size_t size; struct rpc_buffer *buf; buf = container_of(buffer, struct rpc_buffer, data); size = buf->len; dprintk("RPC: freeing buffer of size %zu at %p\n", size, buf); if (size <= RPC_BUFFER_MAXSIZE) mempool_free(buf, rpc_buffer_mempool); else kfree(buf); }

Contributors

PersonTokensPropCommitsCommitProp
Chuck Lever5270.27%457.14%
Trond Myklebust1216.22%114.29%
Linus Torvalds (pre-git)912.16%114.29%
Geert Uytterhoeven11.35%114.29%
Total74100.00%7100.00%

EXPORT_SYMBOL_GPL(rpc_free); /* * Creation and deletion of RPC task structures */
static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data) { memset(task, 0, sizeof(*task)); atomic_set(&task->tk_count, 1); task->tk_flags = task_setup_data->flags; task->tk_ops = task_setup_data->callback_ops; task->tk_calldata = task_setup_data->callback_data; INIT_LIST_HEAD(&task->tk_task); task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW; task->tk_owner = current->tgid; /* Initialize workqueue for async tasks */ task->tk_workqueue = task_setup_data->workqueue; task->tk_xprt = xprt_get(task_setup_data->rpc_xprt); if (task->tk_ops->rpc_call_prepare != NULL) task->tk_action = rpc_prepare_task; rpc_init_task_statistics(task); dprintk("RPC: new task initialized, procpid %u\n", task_pid_nr(current)); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust9668.09%1372.22%
Linus Torvalds (pre-git)3927.66%211.11%
Pavel Emelyanov32.13%15.56%
Chuck Lever32.13%211.11%
Total141100.00%18100.00%


static struct rpc_task * rpc_alloc_task(void) { return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOIO); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust2191.30%133.33%
Linus Torvalds (pre-git)14.35%133.33%
Mel Gorman14.35%133.33%
Total23100.00%3100.00%

/* * Create a new task for the specified client. */
struct rpc_task *rpc_new_task(const struct rpc_task_setup *setup_data) { struct rpc_task *task = setup_data->task; unsigned short flags = 0; if (task == NULL) { task = rpc_alloc_task(); flags = RPC_TASK_DYNAMIC; } rpc_init_task(task, setup_data); task->tk_flags |= flags; dprintk("RPC: allocated task %p\n", task); return task; }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust3652.94%457.14%
Linus Torvalds (pre-git)3145.59%228.57%
Chuck Lever11.47%114.29%
Total68100.00%7100.00%

/* * rpc_free_task - release rpc task and perform cleanups * * Note that we free up the rpc_task _after_ rpc_release_calldata() * in order to work around a workqueue dependency issue. * * Tejun Heo states: * "Workqueue currently considers two work items to be the same if they're * on the same address and won't execute them concurrently - ie. it * makes a work item which is queued again while being executed wait * for the previous execution to complete. * * If a work function frees the work item, and then waits for an event * which should be performed by another work item and *that* work item * recycles the freed work item, it can create a false dependency loop. * There really is no reliable way to detect this short of verifying * every memory free." * */
static void rpc_free_task(struct rpc_task *task) { unsigned short tk_flags = task->tk_flags; rpc_release_calldata(task->tk_ops, task->tk_calldata); if (tk_flags & RPC_TASK_DYNAMIC) { dprintk("RPC: %5u freeing task\n", task->tk_pid); mempool_free(task, rpc_task_mempool); } }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust4787.04%685.71%
Linus Torvalds (pre-git)712.96%114.29%
Total54100.00%7100.00%


static void rpc_async_release(struct work_struct *work) { rpc_free_task(container_of(work, struct rpc_task, u.tk_work)); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust26100.00%1100.00%
Total26100.00%1100.00%


static void rpc_release_resources_task(struct rpc_task *task) { xprt_release(task); if (task->tk_msg.rpc_cred) { put_rpccred(task->tk_msg.rpc_cred); task->tk_msg.rpc_cred = NULL; } rpc_task_release_client(task); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)2143.75%228.57%
Trond Myklebust1735.42%457.14%
Hirofumi Ogawa1020.83%114.29%
Total48100.00%7100.00%


static void rpc_final_put_task(struct rpc_task *task, struct workqueue_struct *q) { if (q != NULL) { INIT_WORK(&task->u.tk_work, rpc_async_release); queue_work(q, &task->u.tk_work); } else rpc_free_task(task); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust4685.19%583.33%
Linus Torvalds (pre-git)814.81%116.67%
Total54100.00%6100.00%


static void rpc_do_put_task(struct rpc_task *task, struct workqueue_struct *q) { if (atomic_dec_and_test(&task->tk_count)) { rpc_release_resources_task(task); rpc_final_put_task(task, q); } }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust40100.00%1100.00%
Total40100.00%1100.00%


void rpc_put_task(struct rpc_task *task) { rpc_do_put_task(task, NULL); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust17100.00%1100.00%
Total17100.00%1100.00%

EXPORT_SYMBOL_GPL(rpc_put_task);
void rpc_put_task_async(struct rpc_task *task) { rpc_do_put_task(task, task->tk_workqueue); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust19100.00%1100.00%
Total19100.00%1100.00%

EXPORT_SYMBOL_GPL(rpc_put_task_async);
static void rpc_release_task(struct rpc_task *task) { dprintk("RPC: %5u release task\n", task->tk_pid); WARN_ON_ONCE(RPC_IS_QUEUED(task)); rpc_release_resources_task(task); /* * Note: at this point we have been removed from rpc_clnt->cl_tasks, * so it should be safe to use task->tk_count as a test for whether * or not any other processes still hold references to our rpc_task. */ if (atomic_read(&task->tk_count) != 1 + !RPC_IS_ASYNC(task)) { /* Wake up anyone who may be waiting for task completion */ if (!rpc_complete_task(task)) return; } else { if (!atomic_dec_and_test(&task->tk_count)) return; } rpc_final_put_task(task, task->tk_workqueue); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust8697.73%360.00%
Chuck Lever11.14%120.00%
Weston Andros Adamson11.14%120.00%
Total88100.00%5100.00%


int rpciod_up(void) { return try_module_get(THIS_MODULE) ? 0 : -EINVAL; }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust1161.11%250.00%
Linus Torvalds (pre-git)738.89%250.00%
Total18100.00%4100.00%


void rpciod_down(void) { module_put(THIS_MODULE); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust975.00%150.00%
Linus Torvalds (pre-git)325.00%150.00%
Total12100.00%2100.00%

/* * Start up the rpciod workqueue. */
static int rpciod_start(void) { struct workqueue_struct *wq; /* * Create the rpciod thread and wait for it to start. */ dprintk("RPC: creating workqueue rpciod\n"); wq = alloc_workqueue("rpciod", WQ_MEM_RECLAIM, 0); if (!wq) goto out_failed; rpciod_workqueue = wq; /* Note: highpri because network receive is latency sensitive */ wq = alloc_workqueue("xprtiod", WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); if (!wq) goto free_rpciod; xprtiod_workqueue = wq; return 1; free_rpciod: wq = rpciod_workqueue; rpciod_workqueue = NULL; destroy_workqueue(wq); out_failed: return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust8087.91%666.67%
Linus Torvalds (pre-git)1010.99%222.22%
Tejun Heo11.10%111.11%
Total91100.00%9100.00%


static void rpciod_stop(void) { struct workqueue_struct *wq = NULL; if (rpciod_workqueue == NULL) return; dprintk("RPC: destroying workqueue rpciod\n"); wq = rpciod_workqueue; rpciod_workqueue = NULL; destroy_workqueue(wq); wq = xprtiod_workqueue; xprtiod_workqueue = NULL; destroy_workqueue(wq); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust3973.58%466.67%
Linus Torvalds (pre-git)1426.42%233.33%
Total53100.00%6100.00%


void rpc_destroy_mempool(void) { rpciod_stop(); mempool_destroy(rpc_buffer_mempool); mempool_destroy(rpc_task_mempool); kmem_cache_destroy(rpc_task_slabp); kmem_cache_destroy(rpc_buffer_slabp); rpc_destroy_wait_queue(&delay_queue); }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust36100.00%3100.00%
Total36100.00%3100.00%


int rpc_init_mempool(void) { /* * The following is not strictly a mempool initialisation, * but there is no harm in doing it here */ rpc_init_wait_queue(&delay_queue, "delayq"); if (!rpciod_start()) goto err_nomem; rpc_task_slabp = kmem_cache_create("rpc_tasks", sizeof(struct rpc_task), 0, SLAB_HWCACHE_ALIGN, NULL); if (!rpc_task_slabp) goto err_nomem; rpc_buffer_slabp = kmem_cache_create("rpc_buffers", RPC_BUFFER_MAXSIZE, 0, SLAB_HWCACHE_ALIGN, NULL); if (!rpc_buffer_slabp) goto err_nomem; rpc_task_mempool = mempool_create_slab_pool(RPC_TASK_POOLSIZE, rpc_task_slabp); if (!rpc_task_mempool) goto err_nomem; rpc_buffer_mempool = mempool_create_slab_pool(RPC_BUFFER_POOLSIZE, rpc_buffer_slabp); if (!rpc_buffer_mempool) goto err_nomem; return 0; err_nomem: rpc_destroy_mempool(); return -ENOMEM; }

Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust11998.35%266.67%
Matthew Dobson21.65%133.33%
Total121100.00%3100.00%


Overall Contributors

PersonTokensPropCommitsCommitProp
Trond Myklebust342472.76%7255.81%
Linus Torvalds (pre-git)86918.47%1410.85%
Chuck Lever1783.78%107.75%
Weston Andros Adamson681.44%43.10%
Neil Brown300.64%21.55%
Kees Cook200.42%10.78%
Peter Zijlstra160.34%21.55%
David Howells140.30%10.78%
Jeff Layton130.28%32.33%
Talpey, Thomas100.21%10.78%
Hirofumi Ogawa100.21%10.78%
Mel Gorman80.17%10.78%
Christophe Saout50.11%10.78%
Alexandros Batsakis50.11%10.78%
Matthew Wilcox50.11%10.78%
Christoph Lameter40.08%10.78%
Nicholas Mc Guire40.08%21.55%
Eric Dumazet40.08%10.78%
Benny Halevy30.06%10.78%
Arjan van de Ven30.06%10.78%
Pavel Emelyanov30.06%10.78%
Geert Uytterhoeven20.04%10.78%
Matthew Dobson20.04%10.78%
David Rientjes20.04%10.78%
Linus Torvalds10.02%10.78%
Hideaki Yoshifuji / 吉藤英明10.02%10.78%
Al Viro10.02%10.78%
Tejun Heo10.02%10.78%
Total4706100.00%129100.00%
Directory: net/sunrpc
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.