mirror of
git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git
synced 2026-10-30 13:52:50 +01:00
Usama Arif
4febfe7d98
blk_cgroup_congested() walks the current task's blkcg ancestor chain on every readahead decision and, once swap is in use, on every anonymous and shmem folio allocation. The answer is almost always "no", but finding that out costs two loads per level on two cold cache lines, plus an out-of-line kthread_blkcg() and an RCU read-side pair. On a fleet profile of hosts running containers with 5-10 level hierarchies it costs about as much as all of mutex_lock(), 99.4% of it under __folio_throttle_swaprate(). Gate the walk on a global count of blkcgs with a non-zero congestion_count. The counter only moves on the 0 <-> 1 transitions of each blkcg's congestion_count, so the extra atomic stays in the throttle arm/disarm paths and never appears in steady state. When something is throttled the counter is non-zero and the walk runs as before. Signed-off-by: Usama Arif <[email protected]> Acked-by: Tejun Heo <[email protected]> Link: https://patch.msgid.link/[email protected] Signed-off-by: Jens Axboe <[email protected]>
73 lines
2.2 KiB
C
73 lines
2.2 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _BLK_CGROUP_H
|
|
#define _BLK_CGROUP_H
|
|
/*
|
|
* Common Block IO controller cgroup interface
|
|
*
|
|
* Based on ideas and code from CFQ, CFS and BFQ:
|
|
* Copyright (C) 2003 Jens Axboe <[email protected]>
|
|
*
|
|
* Copyright (C) 2008 Fabio Checconi <[email protected]>
|
|
* Paolo Valente <[email protected]>
|
|
*
|
|
* Copyright (C) 2009 Vivek Goyal <[email protected]>
|
|
* Nauman Rafique <[email protected]>
|
|
*/
|
|
|
|
#include <linux/atomic.h>
|
|
#include <linux/compiler.h>
|
|
#include <linux/types.h>
|
|
|
|
struct bio;
|
|
struct cgroup_subsys_state;
|
|
struct gendisk;
|
|
|
|
#define FC_APPID_LEN 129
|
|
|
|
#ifdef CONFIG_BLK_CGROUP
|
|
extern struct cgroup_subsys_state * const blkcg_root_css;
|
|
extern atomic_t blkcg_nr_congested;
|
|
|
|
void blkcg_schedule_throttle(struct gendisk *disk, bool use_memdelay);
|
|
void blkcg_maybe_throttle_current(void);
|
|
bool __blk_cgroup_congested(void);
|
|
|
|
/**
|
|
* blk_cgroup_congested - is the current task in a throttled blkcg?
|
|
*
|
|
* Called from mm hot paths where the answer is almost always false, so keep
|
|
* that case to a load and a branch and only walk the hierarchy out of line
|
|
* when something in the system really is throttled.
|
|
*
|
|
* Return: %true if the current task's blkcg or any of its ancestors is
|
|
* throttled, %false otherwise.
|
|
*/
|
|
static inline bool blk_cgroup_congested(void)
|
|
{
|
|
if (likely(!atomic_read(&blkcg_nr_congested)))
|
|
return false;
|
|
return __blk_cgroup_congested();
|
|
}
|
|
|
|
void blkcg_pin_online(struct cgroup_subsys_state *blkcg_css);
|
|
void blkcg_unpin_online(struct cgroup_subsys_state *blkcg_css);
|
|
struct list_head *blkcg_get_cgwb_list(struct cgroup_subsys_state *css);
|
|
struct cgroup_subsys_state *bio_blkcg_css(struct bio *bio);
|
|
|
|
#else /* CONFIG_BLK_CGROUP */
|
|
|
|
#define blkcg_root_css ((struct cgroup_subsys_state *)ERR_PTR(-EINVAL))
|
|
|
|
static inline void blkcg_maybe_throttle_current(void) { }
|
|
static inline bool blk_cgroup_congested(void) { return false; }
|
|
static inline struct cgroup_subsys_state *bio_blkcg_css(struct bio *bio)
|
|
{
|
|
return NULL;
|
|
}
|
|
#endif /* CONFIG_BLK_CGROUP */
|
|
|
|
int blkcg_set_fc_appid(char *app_id, u64 cgrp_id, size_t app_id_len);
|
|
char *blkcg_get_fc_appid(struct bio *bio);
|
|
|
|
#endif /* _BLK_CGROUP_H */
|