Skip to content

Commit e26d876

Browse files
Kent Overstreetakpm00
Kent Overstreet
authored and
akpm00
committed
mempool: hook up to memory allocation profiling
This adds hooks to mempools for correctly annotating mempool-backed allocations at the correct source line, so they show up correctly in /sys/kernel/debug/allocations. Various inline functions are converted to wrappers so that we can invoke alloc_hooks() in fewer places. [[email protected]: undo _noprof additions in the documentation] Link: https://lkml.kernel.org/r/[email protected] [[email protected]: add missing mempool_create_node documentation] Link: https://lkml.kernel.org/r/[email protected] Link: https://lkml.kernel.org/r/[email protected] Signed-off-by: Kent Overstreet <[email protected]> Signed-off-by: Suren Baghdasaryan <[email protected]> Tested-by: Kees Cook <[email protected]> Cc: Alexander Viro <[email protected]> Cc: Alex Gaynor <[email protected]> Cc: Alice Ryhl <[email protected]> Cc: Andreas Hindborg <[email protected]> Cc: Benno Lossin <[email protected]> Cc: "Björn Roy Baron" <[email protected]> Cc: Boqun Feng <[email protected]> Cc: Christoph Lameter <[email protected]> Cc: Dennis Zhou <[email protected]> Cc: Gary Guo <[email protected]> Cc: Miguel Ojeda <[email protected]> Cc: Pasha Tatashin <[email protected]> Cc: Peter Zijlstra <[email protected]> Cc: Tejun Heo <[email protected]> Cc: Vlastimil Babka <[email protected]> Cc: Wedson Almeida Filho <[email protected]> Signed-off-by: Andrew Morton <[email protected]>
1 parent 7bd230a commit e26d876

File tree

2 files changed

+50
-59
lines changed

2 files changed

+50
-59
lines changed

‎include/linux/mempool.h

Lines changed: 35 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
#ifndef _LINUX_MEMPOOL_H
66
#define _LINUX_MEMPOOL_H
77

8+
#include <linux/sched.h>
9+
#include <linux/alloc_tag.h>
810
#include <linux/wait.h>
911
#include <linux/compiler.h>
1012

@@ -39,18 +41,32 @@ void mempool_exit(mempool_t *pool);
3941
int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
4042
mempool_free_t *free_fn, void *pool_data,
4143
gfp_t gfp_mask, int node_id);
42-
int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
44+
45+
int mempool_init_noprof(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
4346
mempool_free_t *free_fn, void *pool_data);
47+
#define mempool_init(...) \
48+
alloc_hooks(mempool_init_noprof(__VA_ARGS__))
4449

4550
extern mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
4651
mempool_free_t *free_fn, void *pool_data);
47-
extern mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
52+
53+
extern mempool_t *mempool_create_node_noprof(int min_nr, mempool_alloc_t *alloc_fn,
4854
mempool_free_t *free_fn, void *pool_data,
4955
gfp_t gfp_mask, int nid);
56+
#define mempool_create_node(...) \
57+
alloc_hooks(mempool_create_node_noprof(__VA_ARGS__))
58+
59+
#define mempool_create(_min_nr, _alloc_fn, _free_fn, _pool_data) \
60+
mempool_create_node(_min_nr, _alloc_fn, _free_fn, _pool_data, \
61+
GFP_KERNEL, NUMA_NO_NODE)
5062

5163
extern int mempool_resize(mempool_t *pool, int new_min_nr);
5264
extern void mempool_destroy(mempool_t *pool);
53-
extern void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask) __malloc;
65+
66+
extern void *mempool_alloc_noprof(mempool_t *pool, gfp_t gfp_mask) __malloc;
67+
#define mempool_alloc(...) \
68+
alloc_hooks(mempool_alloc_noprof(__VA_ARGS__))
69+
5470
extern void *mempool_alloc_preallocated(mempool_t *pool) __malloc;
5571
extern void mempool_free(void *element, mempool_t *pool);
5672

@@ -62,19 +78,10 @@ extern void mempool_free(void *element, mempool_t *pool);
6278
void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data);
6379
void mempool_free_slab(void *element, void *pool_data);
6480

65-
static inline int
66-
mempool_init_slab_pool(mempool_t *pool, int min_nr, struct kmem_cache *kc)
67-
{
68-
return mempool_init(pool, min_nr, mempool_alloc_slab,
69-
mempool_free_slab, (void *) kc);
70-
}
71-
72-
static inline mempool_t *
73-
mempool_create_slab_pool(int min_nr, struct kmem_cache *kc)
74-
{
75-
return mempool_create(min_nr, mempool_alloc_slab, mempool_free_slab,
76-
(void *) kc);
77-
}
81+
#define mempool_init_slab_pool(_pool, _min_nr, _kc) \
82+
mempool_init(_pool, (_min_nr), mempool_alloc_slab, mempool_free_slab, (void *)(_kc))
83+
#define mempool_create_slab_pool(_min_nr, _kc) \
84+
mempool_create((_min_nr), mempool_alloc_slab, mempool_free_slab, (void *)(_kc))
7885

7986
/*
8087
* a mempool_alloc_t and a mempool_free_t to kmalloc and kfree the
@@ -83,17 +90,12 @@ mempool_create_slab_pool(int min_nr, struct kmem_cache *kc)
8390
void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data);
8491
void mempool_kfree(void *element, void *pool_data);
8592

86-
static inline int mempool_init_kmalloc_pool(mempool_t *pool, int min_nr, size_t size)
87-
{
88-
return mempool_init(pool, min_nr, mempool_kmalloc,
89-
mempool_kfree, (void *) size);
90-
}
91-
92-
static inline mempool_t *mempool_create_kmalloc_pool(int min_nr, size_t size)
93-
{
94-
return mempool_create(min_nr, mempool_kmalloc, mempool_kfree,
95-
(void *) size);
96-
}
93+
#define mempool_init_kmalloc_pool(_pool, _min_nr, _size) \
94+
mempool_init(_pool, (_min_nr), mempool_kmalloc, mempool_kfree, \
95+
(void *)(unsigned long)(_size))
96+
#define mempool_create_kmalloc_pool(_min_nr, _size) \
97+
mempool_create((_min_nr), mempool_kmalloc, mempool_kfree, \
98+
(void *)(unsigned long)(_size))
9799

98100
void *mempool_kvmalloc(gfp_t gfp_mask, void *pool_data);
99101
void mempool_kvfree(void *element, void *pool_data);
@@ -115,16 +117,11 @@ static inline mempool_t *mempool_create_kvmalloc_pool(int min_nr, size_t size)
115117
void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data);
116118
void mempool_free_pages(void *element, void *pool_data);
117119

118-
static inline int mempool_init_page_pool(mempool_t *pool, int min_nr, int order)
119-
{
120-
return mempool_init(pool, min_nr, mempool_alloc_pages,
121-
mempool_free_pages, (void *)(long)order);
122-
}
123-
124-
static inline mempool_t *mempool_create_page_pool(int min_nr, int order)
125-
{
126-
return mempool_create(min_nr, mempool_alloc_pages, mempool_free_pages,
127-
(void *)(long)order);
128-
}
120+
#define mempool_init_page_pool(_pool, _min_nr, _order) \
121+
mempool_init(_pool, (_min_nr), mempool_alloc_pages, \
122+
mempool_free_pages, (void *)(long)(_order))
123+
#define mempool_create_page_pool(_min_nr, _order) \
124+
mempool_create((_min_nr), mempool_alloc_pages, \
125+
mempool_free_pages, (void *)(long)(_order))
129126

130127
#endif /* _LINUX_MEMPOOL_H */

‎mm/mempool.c

Lines changed: 15 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -240,22 +240,24 @@ EXPORT_SYMBOL(mempool_init_node);
240240
*
241241
* Return: %0 on success, negative error code otherwise.
242242
*/
243-
int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
244-
mempool_free_t *free_fn, void *pool_data)
243+
int mempool_init_noprof(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
244+
mempool_free_t *free_fn, void *pool_data)
245245
{
246246
return mempool_init_node(pool, min_nr, alloc_fn, free_fn,
247247
pool_data, GFP_KERNEL, NUMA_NO_NODE);
248248

249249
}
250-
EXPORT_SYMBOL(mempool_init);
250+
EXPORT_SYMBOL(mempool_init_noprof);
251251

252252
/**
253-
* mempool_create - create a memory pool
253+
* mempool_create_node - create a memory pool
254254
* @min_nr: the minimum number of elements guaranteed to be
255255
* allocated for this pool.
256256
* @alloc_fn: user-defined element-allocation function.
257257
* @free_fn: user-defined element-freeing function.
258258
* @pool_data: optional private data available to the user-defined functions.
259+
* @gfp_mask: memory allocation flags
260+
* @node_id: numa node to allocate on
259261
*
260262
* this function creates and allocates a guaranteed size, preallocated
261263
* memory pool. The pool can be used from the mempool_alloc() and mempool_free()
@@ -265,17 +267,9 @@ EXPORT_SYMBOL(mempool_init);
265267
*
266268
* Return: pointer to the created memory pool object or %NULL on error.
267269
*/
268-
mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
269-
mempool_free_t *free_fn, void *pool_data)
270-
{
271-
return mempool_create_node(min_nr, alloc_fn, free_fn, pool_data,
272-
GFP_KERNEL, NUMA_NO_NODE);
273-
}
274-
EXPORT_SYMBOL(mempool_create);
275-
276-
mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
277-
mempool_free_t *free_fn, void *pool_data,
278-
gfp_t gfp_mask, int node_id)
270+
mempool_t *mempool_create_node_noprof(int min_nr, mempool_alloc_t *alloc_fn,
271+
mempool_free_t *free_fn, void *pool_data,
272+
gfp_t gfp_mask, int node_id)
279273
{
280274
mempool_t *pool;
281275

@@ -291,7 +285,7 @@ mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
291285

292286
return pool;
293287
}
294-
EXPORT_SYMBOL(mempool_create_node);
288+
EXPORT_SYMBOL(mempool_create_node_noprof);
295289

296290
/**
297291
* mempool_resize - resize an existing memory pool
@@ -387,7 +381,7 @@ EXPORT_SYMBOL(mempool_resize);
387381
*
388382
* Return: pointer to the allocated element or %NULL on error.
389383
*/
390-
void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
384+
void *mempool_alloc_noprof(mempool_t *pool, gfp_t gfp_mask)
391385
{
392386
void *element;
393387
unsigned long flags;
@@ -454,7 +448,7 @@ void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
454448
finish_wait(&pool->wait, &wait);
455449
goto repeat_alloc;
456450
}
457-
EXPORT_SYMBOL(mempool_alloc);
451+
EXPORT_SYMBOL(mempool_alloc_noprof);
458452

459453
/**
460454
* mempool_alloc_preallocated - allocate an element from preallocated elements
@@ -562,7 +556,7 @@ void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data)
562556
{
563557
struct kmem_cache *mem = pool_data;
564558
VM_BUG_ON(mem->ctor);
565-
return kmem_cache_alloc(mem, gfp_mask);
559+
return kmem_cache_alloc_noprof(mem, gfp_mask);
566560
}
567561
EXPORT_SYMBOL(mempool_alloc_slab);
568562

@@ -580,7 +574,7 @@ EXPORT_SYMBOL(mempool_free_slab);
580574
void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data)
581575
{
582576
size_t size = (size_t)pool_data;
583-
return kmalloc(size, gfp_mask);
577+
return kmalloc_noprof(size, gfp_mask);
584578
}
585579
EXPORT_SYMBOL(mempool_kmalloc);
586580

@@ -610,7 +604,7 @@ EXPORT_SYMBOL(mempool_kvfree);
610604
void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data)
611605
{
612606
int order = (int)(long)pool_data;
613-
return alloc_pages(gfp_mask, order);
607+
return alloc_pages_noprof(gfp_mask, order);
614608
}
615609
EXPORT_SYMBOL(mempool_alloc_pages);
616610

0 commit comments

Comments
 (0)