e0bbe2d80c
Work claiming wants to be SMP-safe.
And by the time we try to claim a work, if it is already executing
concurrently on another CPU, we want to succeed the claiming and queue
the work again because the other CPU may have missed the data we wanted
to handle in our work if it's about to complete there.
This scenario is summarized below:
CPU 1 CPU 2
----- -----
(flags = 0)
cmpxchg(flags, 0, IRQ_WORK_FLAGS)
(flags = 3)
[...]
xchg(flags, IRQ_WORK_BUSY)
(flags = 2)
func()
if (flags & IRQ_WORK_PENDING)
(not true)
cmpxchg(flags, flags, IRQ_WORK_FLAGS)
(flags = 3)
[...]
cmpxchg(flags, IRQ_WORK_BUSY, 0);
(fail, pending on CPU 2)
This state machine is synchronized using [cmp]xchg() on the flags.
As such, the early IRQ_WORK_PENDING check in CPU 2 above is racy.
By the time we check it, we may be dealing with a stale value because
we aren't using an atomic accessor. As a result, CPU 2 may "see"
that the work is still pending on another CPU while it may be
actually completing the work function exection already, leaving
our data unprocessed.
To fix this, we start by speculating about the value we wish to be
in the work->flags but we only make any conclusion after the value
returned by the cmpxchg() call that either claims the work or let
the current owner handle the pending work for us.
Changelog-heavily-inspired-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Acked-by: Steven Rostedt <rostedt@goodmis.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: Anish Kumar <anish198519851985@gmail.com>
155 lines
3.5 KiB
C
155 lines
3.5 KiB
C
/*
|
|
* Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
|
|
*
|
|
* Provides a framework for enqueueing and running callbacks from hardirq
|
|
* context. The enqueueing is NMI-safe.
|
|
*/
|
|
|
|
#include <linux/bug.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/export.h>
|
|
#include <linux/irq_work.h>
|
|
#include <linux/percpu.h>
|
|
#include <linux/hardirq.h>
|
|
#include <linux/irqflags.h>
|
|
#include <asm/processor.h>
|
|
|
|
/*
|
|
* An entry can be in one of four states:
|
|
*
|
|
* free NULL, 0 -> {claimed} : free to be used
|
|
* claimed NULL, 3 -> {pending} : claimed to be enqueued
|
|
* pending next, 3 -> {busy} : queued, pending callback
|
|
* busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
|
|
*/
|
|
|
|
#define IRQ_WORK_PENDING 1UL
|
|
#define IRQ_WORK_BUSY 2UL
|
|
#define IRQ_WORK_FLAGS 3UL
|
|
|
|
static DEFINE_PER_CPU(struct llist_head, irq_work_list);
|
|
|
|
/*
|
|
* Claim the entry so that no one else will poke at it.
|
|
*/
|
|
static bool irq_work_claim(struct irq_work *work)
|
|
{
|
|
unsigned long flags, oflags, nflags;
|
|
|
|
/*
|
|
* Start with our best wish as a premise but only trust any
|
|
* flag value after cmpxchg() result.
|
|
*/
|
|
flags = work->flags & ~IRQ_WORK_PENDING;
|
|
for (;;) {
|
|
nflags = flags | IRQ_WORK_FLAGS;
|
|
oflags = cmpxchg(&work->flags, flags, nflags);
|
|
if (oflags == flags)
|
|
break;
|
|
if (oflags & IRQ_WORK_PENDING)
|
|
return false;
|
|
flags = oflags;
|
|
cpu_relax();
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void __weak arch_irq_work_raise(void)
|
|
{
|
|
/*
|
|
* Lame architectures will get the timer tick callback
|
|
*/
|
|
}
|
|
|
|
/*
|
|
* Queue the entry and raise the IPI if needed.
|
|
*/
|
|
static void __irq_work_queue(struct irq_work *work)
|
|
{
|
|
bool empty;
|
|
|
|
preempt_disable();
|
|
|
|
empty = llist_add(&work->llnode, &__get_cpu_var(irq_work_list));
|
|
/* The list was empty, raise self-interrupt to start processing. */
|
|
if (empty)
|
|
arch_irq_work_raise();
|
|
|
|
preempt_enable();
|
|
}
|
|
|
|
/*
|
|
* Enqueue the irq_work @entry, returns true on success, failure when the
|
|
* @entry was already enqueued by someone else.
|
|
*
|
|
* Can be re-enqueued while the callback is still in progress.
|
|
*/
|
|
bool irq_work_queue(struct irq_work *work)
|
|
{
|
|
if (!irq_work_claim(work)) {
|
|
/*
|
|
* Already enqueued, can't do!
|
|
*/
|
|
return false;
|
|
}
|
|
|
|
__irq_work_queue(work);
|
|
return true;
|
|
}
|
|
EXPORT_SYMBOL_GPL(irq_work_queue);
|
|
|
|
/*
|
|
* Run the irq_work entries on this cpu. Requires to be ran from hardirq
|
|
* context with local IRQs disabled.
|
|
*/
|
|
void irq_work_run(void)
|
|
{
|
|
struct irq_work *work;
|
|
struct llist_head *this_list;
|
|
struct llist_node *llnode;
|
|
|
|
this_list = &__get_cpu_var(irq_work_list);
|
|
if (llist_empty(this_list))
|
|
return;
|
|
|
|
BUG_ON(!in_irq());
|
|
BUG_ON(!irqs_disabled());
|
|
|
|
llnode = llist_del_all(this_list);
|
|
while (llnode != NULL) {
|
|
work = llist_entry(llnode, struct irq_work, llnode);
|
|
|
|
llnode = llist_next(llnode);
|
|
|
|
/*
|
|
* Clear the PENDING bit, after this point the @work
|
|
* can be re-used.
|
|
* Make it immediately visible so that other CPUs trying
|
|
* to claim that work don't rely on us to handle their data
|
|
* while we are in the middle of the func.
|
|
*/
|
|
xchg(&work->flags, IRQ_WORK_BUSY);
|
|
work->func(work);
|
|
/*
|
|
* Clear the BUSY bit and return to the free state if
|
|
* no-one else claimed it meanwhile.
|
|
*/
|
|
(void)cmpxchg(&work->flags, IRQ_WORK_BUSY, 0);
|
|
}
|
|
}
|
|
EXPORT_SYMBOL_GPL(irq_work_run);
|
|
|
|
/*
|
|
* Synchronize against the irq_work @entry, ensures the entry is not
|
|
* currently in use.
|
|
*/
|
|
void irq_work_sync(struct irq_work *work)
|
|
{
|
|
WARN_ON_ONCE(irqs_disabled());
|
|
|
|
while (work->flags & IRQ_WORK_BUSY)
|
|
cpu_relax();
|
|
}
|
|
EXPORT_SYMBOL_GPL(irq_work_sync);
|