| /* | |
| * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR | |
| * policies) | |
| */ | |
| /* | |
| * Update the current task's runtime statistics. Skip current tasks that | |
| * are not in our scheduling class. | |
| */ | |
| static inline void update_curr_rt(struct rq *rq, u64 now) | |
| { | |
| struct task_struct *curr = rq->curr; | |
| u64 delta_exec; | |
| if (!task_has_rt_policy(curr)) | |
| return; | |
| delta_exec = now - curr->se.exec_start; | |
| if (unlikely((s64)delta_exec < 0)) | |
| delta_exec = 0; | |
| if (unlikely(delta_exec > curr->se.exec_max)) | |
| curr->se.exec_max = delta_exec; | |
| curr->se.sum_exec_runtime += delta_exec; | |
| curr->se.exec_start = now; | |
| } | |
| static void | |
| enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup, u64 now) | |
| { | |
| struct rt_prio_array *array = &rq->rt.active; | |
| list_add_tail(&p->run_list, array->queue + p->prio); | |
| __set_bit(p->prio, array->bitmap); | |
| } | |
| /* | |
| * Adding/removing a task to/from a priority array: | |
| */ | |
| static void | |
| dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep, u64 now) | |
| { | |
| struct rt_prio_array *array = &rq->rt.active; | |
| update_curr_rt(rq, now); | |
| list_del(&p->run_list); | |
| if (list_empty(array->queue + p->prio)) | |
| __clear_bit(p->prio, array->bitmap); | |
| } | |
| /* | |
| * Put task to the end of the run list without the overhead of dequeue | |
| * followed by enqueue. | |
| */ | |
| static void requeue_task_rt(struct rq *rq, struct task_struct *p) | |
| { | |
| struct rt_prio_array *array = &rq->rt.active; | |
| list_move_tail(&p->run_list, array->queue + p->prio); | |
| } | |
| static void | |
| yield_task_rt(struct rq *rq, struct task_struct *p) | |
| { | |
| requeue_task_rt(rq, p); | |
| } | |
| /* | |
| * Preempt the current task with a newly woken task if needed: | |
| */ | |
| static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p) | |
| { | |
| if (p->prio < rq->curr->prio) | |
| resched_task(rq->curr); | |
| } | |
| static struct task_struct *pick_next_task_rt(struct rq *rq, u64 now) | |
| { | |
| struct rt_prio_array *array = &rq->rt.active; | |
| struct task_struct *next; | |
| struct list_head *queue; | |
| int idx; | |
| idx = sched_find_first_bit(array->bitmap); | |
| if (idx >= MAX_RT_PRIO) | |
| return NULL; | |
| queue = array->queue + idx; | |
| next = list_entry(queue->next, struct task_struct, run_list); | |
| next->se.exec_start = now; | |
| return next; | |
| } | |
| static void put_prev_task_rt(struct rq *rq, struct task_struct *p, u64 now) | |
| { | |
| update_curr_rt(rq, now); | |
| p->se.exec_start = 0; | |
| } | |
| /* | |
| * Load-balancing iterator. Note: while the runqueue stays locked | |
| * during the whole iteration, the current task might be | |
| * dequeued so the iterator has to be dequeue-safe. Here we | |
| * achieve that by always pre-iterating before returning | |
| * the current task: | |
| */ | |
| static struct task_struct *load_balance_start_rt(void *arg) | |
| { | |
| struct rq *rq = arg; | |
| struct rt_prio_array *array = &rq->rt.active; | |
| struct list_head *head, *curr; | |
| struct task_struct *p; | |
| int idx; | |
| idx = sched_find_first_bit(array->bitmap); | |
| if (idx >= MAX_RT_PRIO) | |
| return NULL; | |
| head = array->queue + idx; | |
| curr = head->prev; | |
| p = list_entry(curr, struct task_struct, run_list); | |
| curr = curr->prev; | |
| rq->rt.rt_load_balance_idx = idx; | |
| rq->rt.rt_load_balance_head = head; | |
| rq->rt.rt_load_balance_curr = curr; | |
| return p; | |
| } | |
| static struct task_struct *load_balance_next_rt(void *arg) | |
| { | |
| struct rq *rq = arg; | |
| struct rt_prio_array *array = &rq->rt.active; | |
| struct list_head *head, *curr; | |
| struct task_struct *p; | |
| int idx; | |
| idx = rq->rt.rt_load_balance_idx; | |
| head = rq->rt.rt_load_balance_head; | |
| curr = rq->rt.rt_load_balance_curr; | |
| /* | |
| * If we arrived back to the head again then | |
| * iterate to the next queue (if any): | |
| */ | |
| if (unlikely(head == curr)) { | |
| int next_idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx+1); | |
| if (next_idx >= MAX_RT_PRIO) | |
| return NULL; | |
| idx = next_idx; | |
| head = array->queue + idx; | |
| curr = head->prev; | |
| rq->rt.rt_load_balance_idx = idx; | |
| rq->rt.rt_load_balance_head = head; | |
| } | |
| p = list_entry(curr, struct task_struct, run_list); | |
| curr = curr->prev; | |
| rq->rt.rt_load_balance_curr = curr; | |
| return p; | |
| } | |
| static int | |
| load_balance_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, | |
| unsigned long max_nr_move, unsigned long max_load_move, | |
| struct sched_domain *sd, enum cpu_idle_type idle, | |
| int *all_pinned, unsigned long *load_moved) | |
| { | |
| int this_best_prio, best_prio, best_prio_seen = 0; | |
| int nr_moved; | |
| struct rq_iterator rt_rq_iterator; | |
| best_prio = sched_find_first_bit(busiest->rt.active.bitmap); | |
| this_best_prio = sched_find_first_bit(this_rq->rt.active.bitmap); | |
| /* | |
| * Enable handling of the case where there is more than one task | |
| * with the best priority. If the current running task is one | |
| * of those with prio==best_prio we know it won't be moved | |
| * and therefore it's safe to override the skip (based on load) | |
| * of any task we find with that prio. | |
| */ | |
| if (busiest->curr->prio == best_prio) | |
| best_prio_seen = 1; | |
| rt_rq_iterator.start = load_balance_start_rt; | |
| rt_rq_iterator.next = load_balance_next_rt; | |
| /* pass 'busiest' rq argument into | |
| * load_balance_[start|next]_rt iterators | |
| */ | |
| rt_rq_iterator.arg = busiest; | |
| nr_moved = balance_tasks(this_rq, this_cpu, busiest, max_nr_move, | |
| max_load_move, sd, idle, all_pinned, load_moved, | |
| this_best_prio, best_prio, best_prio_seen, | |
| &rt_rq_iterator); | |
| return nr_moved; | |
| } | |
| static void task_tick_rt(struct rq *rq, struct task_struct *p) | |
| { | |
| /* | |
| * RR tasks need a special form of timeslice management. | |
| * FIFO tasks have no timeslices. | |
| */ | |
| if (p->policy != SCHED_RR) | |
| return; | |
| if (--p->time_slice) | |
| return; | |
| p->time_slice = static_prio_timeslice(p->static_prio); | |
| set_tsk_need_resched(p); | |
| /* put it at the end of the queue: */ | |
| requeue_task_rt(rq, p); | |
| } | |
| /* | |
| * No parent/child timeslice management necessary for RT tasks, | |
| * just activate them: | |
| */ | |
| static void task_new_rt(struct rq *rq, struct task_struct *p) | |
| { | |
| activate_task(rq, p, 1); | |
| } | |
| static struct sched_class rt_sched_class __read_mostly = { | |
| .enqueue_task = enqueue_task_rt, | |
| .dequeue_task = dequeue_task_rt, | |
| .yield_task = yield_task_rt, | |
| .check_preempt_curr = check_preempt_curr_rt, | |
| .pick_next_task = pick_next_task_rt, | |
| .put_prev_task = put_prev_task_rt, | |
| .load_balance = load_balance_rt, | |
| .task_tick = task_tick_rt, | |
| .task_new = task_new_rt, | |
| }; |