Skip to content

Test remove ls percpu locks v2 #9850

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 12 commits into
base: bpf-next_base
Choose a base branch
from
Open

Test remove ls percpu locks v2 #9850

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
c038bae
bpf: Select bpf_local_storage_map_bucket based on bpf_local_storage
ameryhung Aug 25, 2025
39657e9
bpf: Convert bpf_selem_unlink_map to failable
ameryhung Jul 25, 2025
1666dbd
bpf: Convert bpf_selem_link_map to failable
ameryhung Jul 25, 2025
31cdf4b
bpf: Open code bpf_selem_unlink_storage in bpf_selem_unlink
ameryhung Jul 25, 2025
f27c039
bpf: Convert bpf_selem_unlink to failable
ameryhung Jul 26, 2025
1fd7741
bpf: Change local_storage->lock and b->lock to rqspinlock
ameryhung Jul 26, 2025
594ad99
bpf: Remove task local storage percpu counter
ameryhung Jul 28, 2025
ec3cb0d
bpf: Remove cgroup local storage percpu counter
ameryhung Jul 28, 2025
19ed0c0
bpf: Remove unused percpu counter from bpf_local_storage_map_free
ameryhung Jul 28, 2025
0893053
selftests/bpf: Update task_local_storage/recursion test
ameryhung Jul 28, 2025
4055d76
selftests/bpf: Remove test_task_storage_map_stress_lookup
ameryhung Jul 29, 2025
c334195
selftests/bpf: Choose another percpu variable in bpf for btf_dump test
ameryhung Jul 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions include/linux/bpf_local_storage.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include <linux/types.h>
#include <linux/bpf_mem_alloc.h>
#include <uapi/linux/btf.h>
#include <asm/rqspinlock.h>

#define BPF_LOCAL_STORAGE_CACHE_SIZE 16

Expand All @@ -23,7 +24,7 @@
rcu_read_lock_bh_held())
struct bpf_local_storage_map_bucket {
struct hlist_head list;
raw_spinlock_t lock;
rqspinlock_t lock;
};

/* Thp map is not the primary owner of a bpf_local_storage_elem.
Expand Down Expand Up @@ -99,7 +100,7 @@ struct bpf_local_storage {
* bpf_local_storage_elem.
*/
struct rcu_head rcu;
raw_spinlock_t lock; /* Protect adding/removing from the "list" */
rqspinlock_t lock; /* Protect adding/removing from the "list" */
};

/* U16_MAX is much more than enough for sk local storage
Expand Down Expand Up @@ -169,8 +170,7 @@ bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
void bpf_local_storage_destroy(struct bpf_local_storage *local_storage);

void bpf_local_storage_map_free(struct bpf_map *map,
struct bpf_local_storage_cache *cache,
int __percpu *busy_counter);
struct bpf_local_storage_cache *cache);

int bpf_local_storage_map_check_btf(const struct bpf_map *map,
const struct btf *btf,
Expand All @@ -180,10 +180,10 @@ int bpf_local_storage_map_check_btf(const struct bpf_map *map,
void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
struct bpf_local_storage_elem *selem);

void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool reuse_now);
int bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool reuse_now);

void bpf_selem_link_map(struct bpf_local_storage_map *smap,
struct bpf_local_storage_elem *selem);
int bpf_selem_link_map(struct bpf_local_storage_map *smap,
struct bpf_local_storage_elem *selem);

struct bpf_local_storage_elem *
bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
Expand Down
60 changes: 7 additions & 53 deletions kernel/bpf/bpf_cgrp_storage.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,29 +11,6 @@

DEFINE_BPF_STORAGE_CACHE(cgroup_cache);

static DEFINE_PER_CPU(int, bpf_cgrp_storage_busy);

static void bpf_cgrp_storage_lock(void)
{
cant_migrate();
this_cpu_inc(bpf_cgrp_storage_busy);
}

static void bpf_cgrp_storage_unlock(void)
{
this_cpu_dec(bpf_cgrp_storage_busy);
}

static bool bpf_cgrp_storage_trylock(void)
{
cant_migrate();
if (unlikely(this_cpu_inc_return(bpf_cgrp_storage_busy) != 1)) {
this_cpu_dec(bpf_cgrp_storage_busy);
return false;
}
return true;
}

static struct bpf_local_storage __rcu **cgroup_storage_ptr(void *owner)
{
struct cgroup *cg = owner;
Expand All @@ -45,18 +22,14 @@ void bpf_cgrp_storage_free(struct cgroup *cgroup)
{
struct bpf_local_storage *local_storage;

migrate_disable();
rcu_read_lock();
local_storage = rcu_dereference(cgroup->bpf_cgrp_storage);
if (!local_storage)
goto out;

bpf_cgrp_storage_lock();
bpf_local_storage_destroy(local_storage);
bpf_cgrp_storage_unlock();
out:
rcu_read_unlock();
migrate_enable();
}

static struct bpf_local_storage_data *
Expand Down Expand Up @@ -85,9 +58,7 @@ static void *bpf_cgrp_storage_lookup_elem(struct bpf_map *map, void *key)
if (IS_ERR(cgroup))
return ERR_CAST(cgroup);

bpf_cgrp_storage_lock();
sdata = cgroup_storage_lookup(cgroup, map, true);
bpf_cgrp_storage_unlock();
cgroup_put(cgroup);
return sdata ? sdata->data : NULL;
}
Expand All @@ -104,10 +75,8 @@ static long bpf_cgrp_storage_update_elem(struct bpf_map *map, void *key,
if (IS_ERR(cgroup))
return PTR_ERR(cgroup);

bpf_cgrp_storage_lock();
sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
value, map_flags, false, GFP_ATOMIC);
bpf_cgrp_storage_unlock();
cgroup_put(cgroup);
return PTR_ERR_OR_ZERO(sdata);
}
Expand All @@ -120,8 +89,7 @@ static int cgroup_storage_delete(struct cgroup *cgroup, struct bpf_map *map)
if (!sdata)
return -ENOENT;

bpf_selem_unlink(SELEM(sdata), false);
return 0;
return bpf_selem_unlink(SELEM(sdata), false);
}

static long bpf_cgrp_storage_delete_elem(struct bpf_map *map, void *key)
Expand All @@ -134,9 +102,7 @@ static long bpf_cgrp_storage_delete_elem(struct bpf_map *map, void *key)
if (IS_ERR(cgroup))
return PTR_ERR(cgroup);

bpf_cgrp_storage_lock();
err = cgroup_storage_delete(cgroup, map);
bpf_cgrp_storage_unlock();
cgroup_put(cgroup);
return err;
}
Expand All @@ -153,15 +119,14 @@ static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr)

static void cgroup_storage_map_free(struct bpf_map *map)
{
bpf_local_storage_map_free(map, &cgroup_cache, &bpf_cgrp_storage_busy);
bpf_local_storage_map_free(map, &cgroup_cache);
}

/* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,
void *, value, u64, flags, gfp_t, gfp_flags)
{
struct bpf_local_storage_data *sdata;
bool nobusy;

WARN_ON_ONCE(!bpf_rcu_lock_held());
if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
Expand All @@ -170,38 +135,27 @@ BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,
if (!cgroup)
return (unsigned long)NULL;

nobusy = bpf_cgrp_storage_trylock();

sdata = cgroup_storage_lookup(cgroup, map, nobusy);
sdata = cgroup_storage_lookup(cgroup, map, NULL);
if (sdata)
goto unlock;
goto out;

/* only allocate new storage, when the cgroup is refcounted */
if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&
(flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy)
(flags & BPF_LOCAL_STORAGE_GET_F_CREATE))
sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
value, BPF_NOEXIST, false, gfp_flags);

unlock:
if (nobusy)
bpf_cgrp_storage_unlock();
out:
return IS_ERR_OR_NULL(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data;
}

BPF_CALL_2(bpf_cgrp_storage_delete, struct bpf_map *, map, struct cgroup *, cgroup)
{
int ret;

WARN_ON_ONCE(!bpf_rcu_lock_held());
if (!cgroup)
return -EINVAL;

if (!bpf_cgrp_storage_trylock())
return -EBUSY;

ret = cgroup_storage_delete(cgroup, map);
bpf_cgrp_storage_unlock();
return ret;
return cgroup_storage_delete(cgroup, map);
}

const struct bpf_map_ops cgrp_storage_map_ops = {
Expand Down
6 changes: 2 additions & 4 deletions kernel/bpf/bpf_inode_storage.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,7 @@ static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
if (!sdata)
return -ENOENT;

bpf_selem_unlink(SELEM(sdata), false);

return 0;
return bpf_selem_unlink(SELEM(sdata), false);
}

static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
Expand Down Expand Up @@ -188,7 +186,7 @@ static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)

static void inode_storage_map_free(struct bpf_map *map)
{
bpf_local_storage_map_free(map, &inode_cache, NULL);
bpf_local_storage_map_free(map, &inode_cache);
}

const struct bpf_map_ops inode_storage_map_ops = {
Expand Down
Loading