put_prev_task_balance(rq, prev, rf);
smt_mask = cpu_smt_mask(cpu);
+ need_sync = !!rq->core->core_cookie;
+
+ /* reset state */
+ rq->core->core_cookie = 0UL;
+ if (rq->core->core_forceidle) {
+ need_sync = true;
+ fi_before = true;
+ rq->core->core_forceidle = false;
+ }
/*
* core->core_task_seq, core->core_pick_seq, rq->core_sched_seq
* 'Fix' this by also increasing @task_seq for every pick.
*/
rq->core->core_task_seq++;
- need_sync = !!rq->core->core_cookie;
- /* reset state */
- rq->core->core_cookie = 0UL;
- if (rq->core->core_forceidle) {
+ /*
+ * Optimize for common case where this CPU has no cookies
+ * and there are no cookied tasks running on siblings.
+ */
+ if (!need_sync) {
+ for_each_class(class) {
+ next = class->pick_task(rq);
+ if (next)
+ break;
+ }
+
+ if (!next->core_cookie) {
+ rq->core_pick = NULL;
+ goto done;
+ }
need_sync = true;
- rq->core->core_forceidle = false;
}
+
for_each_cpu(i, smt_mask) {
struct rq *rq_i = cpu_rq(i);
* core.
*/
p = pick_task(rq_i, class, max);
- if (!p) {
- /*
- * If there weren't no cookies; we don't need to
- * bother with the other siblings.
- * If the rest of the core is not running a tagged
- * task, i.e. need_sync == 0, and the current CPU
- * which called into the schedule() loop does not
- * have any tasks for this class, skip selecting for
- * other siblings since there's no point. We don't skip
- * for RT/DL because that could make CFS force-idle RT.
- */
- if (i == cpu && !need_sync && class == &fair_sched_class)
- goto next_class;
-
+ if (!p)
continue;
- }
-
- /*
- * Optimize the 'normal' case where there aren't any
- * cookies and we don't need to sync up.
- */
- if (i == cpu && !need_sync && !p->core_cookie) {
- next = p;
- goto done;
- }
rq_i->core_pick = p;
cpu_rq(j)->core_pick = NULL;
}
goto again;
- } else {
- /*
- * Once we select a task for a cpu, we
- * should not be doing an unconstrained
- * pick because it might starve a task
- * on a forced idle cpu.
- */
- need_sync = true;
}
-
}
}
-next_class:;
}
rq->core->core_pick_seq = rq->core->core_task_seq;