| /* |
| * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR |
| * policies) |
| */ |
| |
| /* |
| * Update the current task's runtime statistics. Skip current tasks that |
| * are not in our scheduling class. |
| */ |
| static inline void update_curr_rt(struct rq *rq) |
| { |
| struct task_struct *curr = rq->curr; |
| u64 delta_exec; |
| |
| if (!task_has_rt_policy(curr)) |
| return; |
| |
| delta_exec = rq->clock - curr->se.exec_start; |
| if (unlikely((s64)delta_exec < 0)) |
| delta_exec = 0; |
| |
| schedstat_set(curr->se.exec_max, max(curr->se.exec_max, delta_exec)); |
| |
| curr->se.sum_exec_runtime += delta_exec; |
| curr->se.exec_start = rq->clock; |
| } |
| |
| static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup) |
| { |
| struct rt_prio_array *array = &rq->rt.active; |
| |
| list_add_tail(&p->run_list, array->queue + p->prio); |
| __set_bit(p->prio, array->bitmap); |
| } |
| |
| /* |
| * Adding/removing a task to/from a priority array: |
| */ |
| static void |
| dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep, u64 now) |
| { |
| struct rt_prio_array *array = &rq->rt.active; |
| |
| update_curr_rt(rq); |
| |
| list_del(&p->run_list); |
| if (list_empty(array->queue + p->prio)) |
| __clear_bit(p->prio, array->bitmap); |
| } |
| |
| /* |
| * Put task to the end of the run list without the overhead of dequeue |
| * followed by enqueue. |
| */ |
| static void requeue_task_rt(struct rq *rq, struct task_struct *p) |
| { |
| struct rt_prio_array *array = &rq->rt.active; |
| |
| list_move_tail(&p->run_list, array->queue + p->prio); |
| } |
| |
| static void |
| yield_task_rt(struct rq *rq, struct task_struct *p) |
| { |
| requeue_task_rt(rq, p); |
| } |
| |
| /* |
| * Preempt the current task with a newly woken task if needed: |
| */ |
| static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p) |
| { |
| if (p->prio < rq->curr->prio) |
| resched_task(rq->curr); |
| } |
| |
| static struct task_struct *pick_next_task_rt(struct rq *rq, u64 now) |
| { |
| struct rt_prio_array *array = &rq->rt.active; |
| struct task_struct *next; |
| struct list_head *queue; |
| int idx; |
| |
| idx = sched_find_first_bit(array->bitmap); |
| if (idx >= MAX_RT_PRIO) |
| return NULL; |
| |
| queue = array->queue + idx; |
| next = list_entry(queue->next, struct task_struct, run_list); |
| |
| next->se.exec_start = rq->clock; |
| |
| return next; |
| } |
| |
| static void put_prev_task_rt(struct rq *rq, struct task_struct *p, u64 now) |
| { |
| update_curr_rt(rq); |
| p->se.exec_start = 0; |
| } |
| |
| /* |
| * Load-balancing iterator. Note: while the runqueue stays locked |
| * during the whole iteration, the current task might be |
| * dequeued so the iterator has to be dequeue-safe. Here we |
| * achieve that by always pre-iterating before returning |
| * the current task: |
| */ |
| static struct task_struct *load_balance_start_rt(void *arg) |
| { |
| struct rq *rq = arg; |
| struct rt_prio_array *array = &rq->rt.active; |
| struct list_head *head, *curr; |
| struct task_struct *p; |
| int idx; |
| |
| idx = sched_find_first_bit(array->bitmap); |
| if (idx >= MAX_RT_PRIO) |
| return NULL; |
| |
| head = array->queue + idx; |
| curr = head->prev; |
| |
| p = list_entry(curr, struct task_struct, run_list); |
| |
| curr = curr->prev; |
| |
| rq->rt.rt_load_balance_idx = idx; |
| rq->rt.rt_load_balance_head = head; |
| rq->rt.rt_load_balance_curr = curr; |
| |
| return p; |
| } |
| |
| static struct task_struct *load_balance_next_rt(void *arg) |
| { |
| struct rq *rq = arg; |
| struct rt_prio_array *array = &rq->rt.active; |
| struct list_head *head, *curr; |
| struct task_struct *p; |
| int idx; |
| |
| idx = rq->rt.rt_load_balance_idx; |
| head = rq->rt.rt_load_balance_head; |
| curr = rq->rt.rt_load_balance_curr; |
| |
| /* |
| * If we arrived back to the head again then |
| * iterate to the next queue (if any): |
| */ |
| if (unlikely(head == curr)) { |
| int next_idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx+1); |
| |
| if (next_idx >= MAX_RT_PRIO) |
| return NULL; |
| |
| idx = next_idx; |
| head = array->queue + idx; |
| curr = head->prev; |
| |
| rq->rt.rt_load_balance_idx = idx; |
| rq->rt.rt_load_balance_head = head; |
| } |
| |
| p = list_entry(curr, struct task_struct, run_list); |
| |
| curr = curr->prev; |
| |
| rq->rt.rt_load_balance_curr = curr; |
| |
| return p; |
| } |
| |
| static unsigned long |
| load_balance_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, |
| unsigned long max_nr_move, unsigned long max_load_move, |
| struct sched_domain *sd, enum cpu_idle_type idle, |
| int *all_pinned, int *this_best_prio) |
| { |
| int nr_moved; |
| struct rq_iterator rt_rq_iterator; |
| unsigned long load_moved; |
| |
| rt_rq_iterator.start = load_balance_start_rt; |
| rt_rq_iterator.next = load_balance_next_rt; |
| /* pass 'busiest' rq argument into |
| * load_balance_[start|next]_rt iterators |
| */ |
| rt_rq_iterator.arg = busiest; |
| |
| nr_moved = balance_tasks(this_rq, this_cpu, busiest, max_nr_move, |
| max_load_move, sd, idle, all_pinned, &load_moved, |
| this_best_prio, &rt_rq_iterator); |
| |
| return load_moved; |
| } |
| |
| static void task_tick_rt(struct rq *rq, struct task_struct *p) |
| { |
| /* |
| * RR tasks need a special form of timeslice management. |
| * FIFO tasks have no timeslices. |
| */ |
| if (p->policy != SCHED_RR) |
| return; |
| |
| if (--p->time_slice) |
| return; |
| |
| p->time_slice = static_prio_timeslice(p->static_prio); |
| set_tsk_need_resched(p); |
| |
| /* put it at the end of the queue: */ |
| requeue_task_rt(rq, p); |
| } |
| |
| static struct sched_class rt_sched_class __read_mostly = { |
| .enqueue_task = enqueue_task_rt, |
| .dequeue_task = dequeue_task_rt, |
| .yield_task = yield_task_rt, |
| |
| .check_preempt_curr = check_preempt_curr_rt, |
| |
| .pick_next_task = pick_next_task_rt, |
| .put_prev_task = put_prev_task_rt, |
| |
| .load_balance = load_balance_rt, |
| |
| .task_tick = task_tick_rt, |
| }; |