mirror of
git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git
synced 2026-11-03 16:13:17 +01:00
mm: shrinker: make memcg slab shrink lockless
Like global slab shrink, this commit also uses refcount+RCU method to make
memcg slab shrink lockless.
Use the following script to do slab shrink stress test:
```
DIR="/root/shrinker/memcg/mnt"
do_create()
{
mkdir -p /sys/fs/cgroup/memory/test
echo 4G > /sys/fs/cgroup/memory/test/memory.limit_in_bytes
for i in `seq 0 $1`;
do
mkdir -p /sys/fs/cgroup/memory/test/$i;
echo $$ > /sys/fs/cgroup/memory/test/$i/cgroup.procs;
mkdir -p $DIR/$i;
done
}
do_mount()
{
for i in `seq $1 $2`;
do
mount -t tmpfs $i $DIR/$i;
done
}
do_touch()
{
for i in `seq $1 $2`;
do
echo $$ > /sys/fs/cgroup/memory/test/$i/cgroup.procs;
dd if=/dev/zero of=$DIR/$i/file$i bs=1M count=1 &
done
}
case "$1" in
touch)
do_touch $2 $3
;;
test)
do_create 4000
do_mount 0 4000
do_touch 0 3000
;;
*)
exit 1
;;
esac
```
Save the above script, then run test and touch commands. Then we can use
the following perf command to view hotspots:
perf top -U -F 999
1) Before applying this patchset:
33.15% [kernel] [k] down_read_trylock
25.38% [kernel] [k] shrink_slab
21.75% [kernel] [k] up_read
4.45% [kernel] [k] _find_next_bit
2.27% [kernel] [k] do_shrink_slab
1.80% [kernel] [k] intel_idle_irq
1.79% [kernel] [k] shrink_lruvec
0.67% [kernel] [k] xas_descend
0.41% [kernel] [k] mem_cgroup_iter
0.40% [kernel] [k] shrink_node
0.38% [kernel] [k] list_lru_count_one
2) After applying this patchset:
64.56% [kernel] [k] shrink_slab
12.18% [kernel] [k] do_shrink_slab
3.30% [kernel] [k] __rcu_read_unlock
2.61% [kernel] [k] shrink_lruvec
2.49% [kernel] [k] __rcu_read_lock
1.93% [kernel] [k] intel_idle_irq
0.89% [kernel] [k] shrink_node
0.81% [kernel] [k] mem_cgroup_iter
0.77% [kernel] [k] mem_cgroup_calculate_protection
0.66% [kernel] [k] list_lru_count_one
We can see that the first perf hotspot becomes shrink_slab, which is what
we expect.
Link: https://lkml.kernel.org/r/[email protected]
Signed-off-by: Qi Zheng <[email protected]>
Cc: Abhinav Kumar <[email protected]>
Cc: Alasdair Kergon <[email protected]>
Cc: Alexander Viro <[email protected]>
Cc: Alyssa Rosenzweig <[email protected]>
Cc: Andreas Dilger <[email protected]>
Cc: Andreas Gruenbacher <[email protected]>
Cc: Anna Schumaker <[email protected]>
Cc: Arnd Bergmann <[email protected]>
Cc: Bob Peterson <[email protected]>
Cc: Borislav Petkov <[email protected]>
Cc: Carlos Llamas <[email protected]>
Cc: Chandan Babu R <[email protected]>
Cc: Chao Yu <[email protected]>
Cc: Chris Mason <[email protected]>
Cc: Christian Brauner <[email protected]>
Cc: Christian Koenig <[email protected]>
Cc: Chuck Lever <[email protected]>
Cc: Coly Li <[email protected]>
Cc: Dai Ngo <[email protected]>
Cc: Daniel Vetter <[email protected]>
Cc: Daniel Vetter <[email protected]>
Cc: "Darrick J. Wong" <[email protected]>
Cc: Dave Chinner <[email protected]>
Cc: Dave Hansen <[email protected]>
Cc: David Airlie <[email protected]>
Cc: David Hildenbrand <[email protected]>
Cc: David Sterba <[email protected]>
Cc: Dmitry Baryshkov <[email protected]>
Cc: Gao Xiang <[email protected]>
Cc: Greg Kroah-Hartman <[email protected]>
Cc: Huang Rui <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Jaegeuk Kim <[email protected]>
Cc: Jani Nikula <[email protected]>
Cc: Jan Kara <[email protected]>
Cc: Jason Wang <[email protected]>
Cc: Jeff Layton <[email protected]>
Cc: Jeffle Xu <[email protected]>
Cc: Joel Fernandes (Google) <[email protected]>
Cc: Joonas Lahtinen <[email protected]>
Cc: Josef Bacik <[email protected]>
Cc: Juergen Gross <[email protected]>
Cc: Kent Overstreet <[email protected]>
Cc: Kirill Tkhai <[email protected]>
Cc: Marijn Suijten <[email protected]>
Cc: "Michael S. Tsirkin" <[email protected]>
Cc: Mike Snitzer <[email protected]>
Cc: Minchan Kim <[email protected]>
Cc: Muchun Song <[email protected]>
Cc: Muchun Song <[email protected]>
Cc: Nadav Amit <[email protected]>
Cc: Neil Brown <[email protected]>
Cc: Oleksandr Tyshchenko <[email protected]>
Cc: Olga Kornievskaia <[email protected]>
Cc: Paul E. McKenney <[email protected]>
Cc: Richard Weinberger <[email protected]>
Cc: Rob Clark <[email protected]>
Cc: Rob Herring <[email protected]>
Cc: Rodrigo Vivi <[email protected]>
Cc: Roman Gushchin <[email protected]>
Cc: Sean Paul <[email protected]>
Cc: Sergey Senozhatsky <[email protected]>
Cc: Song Liu <[email protected]>
Cc: Stefano Stabellini <[email protected]>
Cc: Steven Price <[email protected]>
Cc: "Theodore Ts'o" <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: Tomeu Vizoso <[email protected]>
Cc: Tom Talpey <[email protected]>
Cc: Trond Myklebust <[email protected]>
Cc: Tvrtko Ursulin <[email protected]>
Cc: Vlastimil Babka <[email protected]>
Cc: Xuan Zhuo <[email protected]>
Cc: Yue Hu <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
This commit is contained in:
Qi Zheng authored and Andrew Morton committed
1 parent
ca1d36b823
commit
50d09da8e1
1 file changed
+66
-19
+66
-19
@@ -219,7 +219,6 @@ static int shrinker_memcg_alloc(struct shrinker *shrinker)
|
||||
return -ENOSYS;
|
||||
|
||||
down_write(&shrinker_rwsem);
|
||||
/* This may call shrinker, so it must use down_read_trylock() */
|
||||
id = idr_alloc(&shrinker_idr, shrinker, 0, 0, GFP_KERNEL);
|
||||
if (id < 0)
|
||||
goto unlock;
|
||||
@@ -253,10 +252,15 @@ static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
|
||||
{
|
||||
struct shrinker_info *info;
|
||||
struct shrinker_info_unit *unit;
|
||||
long nr_deferred;
|
||||
|
||||
info = shrinker_info_protected(memcg, nid);
|
||||
rcu_read_lock();
|
||||
info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
|
||||
unit = info->unit[shrinker_id_to_index(shrinker->id)];
|
||||
return atomic_long_xchg(&unit->nr_deferred[shrinker_id_to_offset(shrinker->id)], 0);
|
||||
nr_deferred = atomic_long_xchg(&unit->nr_deferred[shrinker_id_to_offset(shrinker->id)], 0);
|
||||
rcu_read_unlock();
|
||||
|
||||
return nr_deferred;
|
||||
}
|
||||
|
||||
static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
|
||||
@@ -264,10 +268,16 @@ static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
|
||||
{
|
||||
struct shrinker_info *info;
|
||||
struct shrinker_info_unit *unit;
|
||||
long nr_deferred;
|
||||
|
||||
info = shrinker_info_protected(memcg, nid);
|
||||
rcu_read_lock();
|
||||
info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
|
||||
unit = info->unit[shrinker_id_to_index(shrinker->id)];
|
||||
return atomic_long_add_return(nr, &unit->nr_deferred[shrinker_id_to_offset(shrinker->id)]);
|
||||
nr_deferred =
|
||||
atomic_long_add_return(nr, &unit->nr_deferred[shrinker_id_to_offset(shrinker->id)]);
|
||||
rcu_read_unlock();
|
||||
|
||||
return nr_deferred;
|
||||
}
|
||||
|
||||
void reparent_shrinker_deferred(struct mem_cgroup *memcg)
|
||||
@@ -464,18 +474,54 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
|
||||
if (!mem_cgroup_online(memcg))
|
||||
return 0;
|
||||
|
||||
if (!down_read_trylock(&shrinker_rwsem))
|
||||
return 0;
|
||||
|
||||
info = shrinker_info_protected(memcg, nid);
|
||||
/*
|
||||
* lockless algorithm of memcg shrink.
|
||||
*
|
||||
* The shrinker_info may be freed asynchronously via RCU in the
|
||||
* expand_one_shrinker_info(), so the rcu_read_lock() needs to be used
|
||||
* to ensure the existence of the shrinker_info.
|
||||
*
|
||||
* The shrinker_info_unit is never freed unless its corresponding memcg
|
||||
* is destroyed. Here we already hold the refcount of memcg, so the
|
||||
* memcg will not be destroyed, and of course shrinker_info_unit will
|
||||
* not be freed.
|
||||
*
|
||||
* So in the memcg shrink:
|
||||
* step 1: use rcu_read_lock() to guarantee existence of the
|
||||
* shrinker_info.
|
||||
* step 2: after getting shrinker_info_unit we can safely release the
|
||||
* RCU lock.
|
||||
* step 3: traverse the bitmap and calculate shrinker_id
|
||||
* step 4: use rcu_read_lock() to guarantee existence of the shrinker.
|
||||
* step 5: use shrinker_id to find the shrinker, then use
|
||||
* shrinker_try_get() to guarantee existence of the shrinker,
|
||||
* then we can release the RCU lock to do do_shrink_slab() that
|
||||
* may sleep.
|
||||
* step 6: do shrinker_put() paired with step 5 to put the refcount,
|
||||
* if the refcount reaches 0, then wake up the waiter in
|
||||
* shrinker_free() by calling complete().
|
||||
* Note: here is different from the global shrink, we don't
|
||||
* need to acquire the RCU lock to guarantee existence of
|
||||
* the shrinker, because we don't need to use this
|
||||
* shrinker to traverse the next shrinker in the bitmap.
|
||||
* step 7: we have already exited the read-side of rcu critical section
|
||||
* before calling do_shrink_slab(), the shrinker_info may be
|
||||
* released in expand_one_shrinker_info(), so go back to step 1
|
||||
* to reacquire the shrinker_info.
|
||||
*/
|
||||
again:
|
||||
rcu_read_lock();
|
||||
info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
|
||||
if (unlikely(!info))
|
||||
goto unlock;
|
||||
|
||||
for (; index < shrinker_id_to_index(info->map_nr_max); index++) {
|
||||
if (index < shrinker_id_to_index(info->map_nr_max)) {
|
||||
struct shrinker_info_unit *unit;
|
||||
|
||||
unit = info->unit[index];
|
||||
|
||||
rcu_read_unlock();
|
||||
|
||||
for_each_set_bit(offset, unit->map, SHRINKER_UNIT_BITS) {
|
||||
struct shrink_control sc = {
|
||||
.gfp_mask = gfp_mask,
|
||||
@@ -485,12 +531,14 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
|
||||
struct shrinker *shrinker;
|
||||
int shrinker_id = calc_shrinker_id(index, offset);
|
||||
|
||||
rcu_read_lock();
|
||||
shrinker = idr_find(&shrinker_idr, shrinker_id);
|
||||
if (unlikely(!shrinker || !(shrinker->flags & SHRINKER_REGISTERED))) {
|
||||
if (!shrinker)
|
||||
clear_bit(offset, unit->map);
|
||||
if (unlikely(!shrinker || !shrinker_try_get(shrinker))) {
|
||||
clear_bit(offset, unit->map);
|
||||
rcu_read_unlock();
|
||||
continue;
|
||||
}
|
||||
rcu_read_unlock();
|
||||
|
||||
/* Call non-slab shrinkers even though kmem is disabled */
|
||||
if (!memcg_kmem_online() &&
|
||||
@@ -523,15 +571,14 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
|
||||
set_shrinker_bit(memcg, nid, shrinker_id);
|
||||
}
|
||||
freed += ret;
|
||||
|
||||
if (rwsem_is_contended(&shrinker_rwsem)) {
|
||||
freed = freed ? : 1;
|
||||
goto unlock;
|
||||
}
|
||||
shrinker_put(shrinker);
|
||||
}
|
||||
|
||||
index++;
|
||||
goto again;
|
||||
}
|
||||
unlock:
|
||||
up_read(&shrinker_rwsem);
|
||||
rcu_read_unlock();
|
||||
return freed;
|
||||
}
|
||||
#else /* !CONFIG_MEMCG */
|
||||
|
||||
Reference in new issue
Block a user