mirror of
git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git
synced 2026-10-28 21:02:43 +01:00
Karl Mehltretter
28fa9af353
hrtimer_rearm_deferred_user_irq() removes the rearm bit from the local
copy of the TIF work with:
*tif_work &= ~TIF_HRTIMER_REARM;
TIF_HRTIMER_REARM is the bit number (12), not the mask. That clears
TIF_NOTIFY_SIGNAL (bit 2) and TIF_MEMDIE (bit 3) from the copy and
leaves bit 12 set.
So the function never returns true, and the copy handed to
exit_to_user_mode_loop() lacks TIF_NOTIFY_SIGNAL. If that was the only
work for the loop, the task returns to user space with the task work
still pending. hrtimer_interrupt() sets TIF_HRTIMER_REARM every time,
so the next tick repeats that. Task work queued with TWA_SIGNAL from a
hrtimer callback stays pending until the task does a syscall, takes an
interrupt without deferred rearm or has to reschedule.
A task which polls the io_uring completion ring in user space sees a
timeout completion after 30ms (median) instead of 70us. 2 CPU QEMU
guest, HZ=1000.
Use the mask.
Fixes: 15dd3a9488 ("hrtimer: Push reprogramming timers into the interrupt return path")
Signed-off-by: Karl Mehltretter <[email protected]>
Signed-off-by: Thomas Gleixner <[email protected]>
Assisted-by: LLM
Cc: [email protected]
Link: https://patch.msgid.link/[email protected]
89 lines
2.8 KiB
C
89 lines
2.8 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
#ifndef _LINUX_HRTIMER_REARM_H
|
|
#define _LINUX_HRTIMER_REARM_H
|
|
|
|
#include <linux/types.h>
|
|
|
|
#ifdef CONFIG_HRTIMER_REARM_DEFERRED
|
|
#include <linux/irqflags.h>
|
|
#include <linux/lockdep.h>
|
|
#include <linux/preempt.h>
|
|
#include <linux/thread_info.h>
|
|
|
|
void __hrtimer_rearm_deferred(void);
|
|
|
|
/*
|
|
* This is purely CPU local, so check the TIF bit first to avoid the overhead of
|
|
* the atomic test_and_clear_bit() operation for the common case where the bit
|
|
* is not set.
|
|
*/
|
|
static __always_inline bool hrtimer_test_and_clear_rearm_deferred_tif(unsigned long tif_work)
|
|
{
|
|
lockdep_assert_irqs_disabled();
|
|
|
|
if (unlikely(tif_work & _TIF_HRTIMER_REARM)) {
|
|
clear_thread_flag(TIF_HRTIMER_REARM);
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
#define TIF_REARM_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY | _TIF_HRTIMER_REARM)
|
|
|
|
/* Invoked from the exit to user before invoking exit_to_user_mode_loop() */
|
|
static __always_inline bool
|
|
hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif_mask)
|
|
{
|
|
/* Help the compiler to optimize the function out for syscall returns */
|
|
if (!(tif_mask & _TIF_HRTIMER_REARM))
|
|
return false;
|
|
/*
|
|
* Rearm the timer if none of the resched flags is set before going into
|
|
* the loop which re-enables interrupts.
|
|
*/
|
|
if (unlikely((*tif_work & TIF_REARM_MASK) == _TIF_HRTIMER_REARM)) {
|
|
clear_thread_flag(TIF_HRTIMER_REARM);
|
|
__hrtimer_rearm_deferred();
|
|
/* Don't go into the loop if HRTIMER_REARM was the only flag */
|
|
*tif_work &= ~_TIF_HRTIMER_REARM;
|
|
return !*tif_work;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/* Invoked from the time slice extension decision function */
|
|
static __always_inline void hrtimer_rearm_deferred_tif(unsigned long tif_work)
|
|
{
|
|
if (hrtimer_test_and_clear_rearm_deferred_tif(tif_work))
|
|
__hrtimer_rearm_deferred();
|
|
}
|
|
|
|
/*
|
|
* This is to be called on all irqentry_exit() paths that will enable
|
|
* interrupts.
|
|
*/
|
|
static __always_inline void hrtimer_rearm_deferred(void)
|
|
{
|
|
hrtimer_rearm_deferred_tif(read_thread_flags());
|
|
}
|
|
|
|
/*
|
|
* Invoked from the scheduler on entry to __schedule() so it can defer
|
|
* rearming after the load balancing callbacks which might change hrtick.
|
|
*/
|
|
static __always_inline bool hrtimer_test_and_clear_rearm_deferred(void)
|
|
{
|
|
return hrtimer_test_and_clear_rearm_deferred_tif(read_thread_flags());
|
|
}
|
|
|
|
#else /* CONFIG_HRTIMER_REARM_DEFERRED */
|
|
static __always_inline void __hrtimer_rearm_deferred(void) { }
|
|
static __always_inline void hrtimer_rearm_deferred(void) { }
|
|
static __always_inline void hrtimer_rearm_deferred_tif(unsigned long tif_work) { }
|
|
static __always_inline bool
|
|
hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif_mask) { return false; }
|
|
static __always_inline bool hrtimer_test_and_clear_rearm_deferred(void) { return false; }
|
|
#endif /* !CONFIG_HRTIMER_REARM_DEFERRED */
|
|
|
|
#endif
|