← 返回首页
bpo-40609: _tracemalloc allocates traces by vstinner · Pull Request #20064 · python/cpython · GitHub
Skip to content

Navigation Menu

Toggle navigation
Sign in
Appearance settings
Search or jump to...

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Include my email address so I can be contacted
Cancel Submit feedback

Saved searches

Use saved searches to filter your results more quickly

Cancel Create saved search
Appearance settings
Resetting focus

bpo-40609: _tracemalloc allocates traces #20064

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vstinner merged 1 commit into from
May 13, 2020
Conversation 0 Commits 1 Checks 0 Files changed
Merged

bpo-40609: _tracemalloc allocates traces #20064

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension .c  (2) .h  (1) All 2 file types selected Viewed files
Conversations
Failed to load comments. Retry
Loading
Jump to
Jump to file
Failed to load files. Retry
Loading
Diff view
Unified
Split
Hide whitespace
Apply and reload
Show whitespace
Diff view
Unified
Split
Hide whitespace
Apply and reload
10 changes: 0 additions & 10 deletions Include/internal/pycore_hashtable.h
Show comments
View file Edit file Delete file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,6 @@ typedef struct {
sizeof(DATA)); \
} while (0)

#define _Py_HASHTABLE_ENTRY_WRITE_DATA(TABLE, ENTRY, DATA) \
do { \
assert(sizeof(DATA) == (TABLE)->data_size); \
memcpy((void *)_Py_HASHTABLE_ENTRY_PDATA(ENTRY), \
&(DATA), sizeof(DATA)); \
} while (0)


/* _Py_hashtable: prototypes */

Expand Down Expand Up @@ -118,9 +111,6 @@ PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full(

PyAPI_FUNC(void) _Py_hashtable_destroy(_Py_hashtable_t *ht);

/* Return a copy of the hash table */
PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_copy(_Py_hashtable_t *src);

PyAPI_FUNC(void) _Py_hashtable_clear(_Py_hashtable_t *ht);

typedef int (*_Py_hashtable_foreach_func) (_Py_hashtable_t *ht,
Expand Down
160 changes: 117 additions & 43 deletions Modules/_tracemalloc.c
Show comments
View file Edit file Delete file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ static traceback_t *tracemalloc_traceback = NULL;
Protected by the GIL */
static _Py_hashtable_t *tracemalloc_tracebacks = NULL;

/* pointer (void*) => trace (trace_t).
/* pointer (void*) => trace (trace_t*).
Protected by TABLES_LOCK(). */
static _Py_hashtable_t *tracemalloc_traces = NULL;

Expand Down Expand Up @@ -467,13 +467,23 @@ traceback_new(void)
}


static void
tracemalloc_destroy_trace_cb(_Py_hashtable_t *traces,
_Py_hashtable_entry_t *entry)
{
trace_t *trace;
_Py_HASHTABLE_ENTRY_READ_DATA(traces, entry, trace);
raw_free(trace);
}


static _Py_hashtable_t*
tracemalloc_create_traces_table(void)
{
return hashtable_new(sizeof(trace_t),
return hashtable_new(sizeof(trace_t*),
_Py_hashtable_hash_ptr,
_Py_hashtable_compare_direct,
NULL);
tracemalloc_destroy_trace_cb);
}


Expand Down Expand Up @@ -528,12 +538,13 @@ tracemalloc_remove_trace(unsigned int domain, uintptr_t ptr)
return;
}

trace_t trace;
trace_t *trace;
if (!_Py_HASHTABLE_POP(traces, TO_PTR(ptr), trace)) {
return;
}
assert(tracemalloc_traced_memory >= trace.size);
tracemalloc_traced_memory -= trace.size;
assert(tracemalloc_traced_memory >= trace->size);
tracemalloc_traced_memory -= trace->size;
raw_free(trace);
}

#define REMOVE_TRACE(ptr) \
Expand Down Expand Up @@ -565,23 +576,27 @@ tracemalloc_add_trace(unsigned int domain, uintptr_t ptr,
}

_Py_hashtable_entry_t* entry = _Py_HASHTABLE_GET_ENTRY(traces, ptr);
trace_t trace;
if (entry != NULL) {
/* the memory block is already tracked */
trace_t *trace;
_Py_HASHTABLE_ENTRY_READ_DATA(traces, entry, trace);
assert(tracemalloc_traced_memory >= trace.size);
tracemalloc_traced_memory -= trace.size;
assert(tracemalloc_traced_memory >= trace->size);
tracemalloc_traced_memory -= trace->size;

trace.size = size;
trace.traceback = traceback;
_Py_HASHTABLE_ENTRY_WRITE_DATA(traces, entry, trace);
trace->size = size;
trace->traceback = traceback;
}
else {
trace.size = size;
trace.traceback = traceback;
trace_t *trace = raw_malloc(sizeof(trace_t));
if (trace == NULL) {
return -1;
}
trace->size = size;
trace->traceback = traceback;

int res = _Py_HASHTABLE_SET(traces, TO_PTR(ptr), trace);
if (res != 0) {
raw_free(trace);
return res;
}
}
Expand Down Expand Up @@ -1225,38 +1240,100 @@ typedef struct {
unsigned int domain;
} get_traces_t;


static int
tracemalloc_get_traces_copy_domain(_Py_hashtable_t *domains,
_Py_hashtable_entry_t *entry,
void *user_data)
tracemalloc_copy_trace(_Py_hashtable_t *traces,
_Py_hashtable_entry_t *entry,
void *traces2_raw)
{
get_traces_t *get_traces = user_data;
_Py_hashtable_t *traces2 = (_Py_hashtable_t *)traces2_raw;

trace_t *trace;
_Py_HASHTABLE_ENTRY_READ_DATA(traces, entry, trace);

trace_t *trace2 = raw_malloc(sizeof(trace_t));
if (traces2 == NULL) {
return -1;
}
*trace2 = *trace;
if (_Py_HASHTABLE_SET(traces2, entry->key, trace2) < 0) {
raw_free(trace2);
return -1;
}
return 0;
}


static _Py_hashtable_t*
tracemalloc_copy_traces(_Py_hashtable_t *traces)
{
_Py_hashtable_t *traces2 = tracemalloc_create_traces_table();
if (traces2 == NULL) {
return NULL;
}

int err = _Py_hashtable_foreach(traces,
tracemalloc_copy_trace,
traces2);
if (err) {
_Py_hashtable_destroy(traces2);
return NULL;
}
return traces2;
}


static int
tracemalloc_copy_domain(_Py_hashtable_t *domains,
_Py_hashtable_entry_t *entry,
void *domains2_raw)
{
_Py_hashtable_t *domains2 = (_Py_hashtable_t *)domains2_raw;

unsigned int domain = (unsigned int)FROM_PTR(entry->key);
_Py_hashtable_t *traces;
_Py_HASHTABLE_ENTRY_READ_DATA(domains, entry, traces);

_Py_hashtable_t *traces2 = _Py_hashtable_copy(traces);
if (_Py_HASHTABLE_SET(get_traces->domains, TO_PTR(domain), traces2) < 0) {
_Py_hashtable_t *traces2 = tracemalloc_copy_traces(traces);
if (_Py_HASHTABLE_SET(domains2, TO_PTR(domain), traces2) < 0) {
_Py_hashtable_destroy(traces2);
return -1;
}
return 0;
}


static _Py_hashtable_t*
tracemalloc_copy_domains(_Py_hashtable_t *domains)
{
_Py_hashtable_t *domains2 = tracemalloc_create_domains_table();
if (domains2 == NULL) {
return NULL;
}

int err = _Py_hashtable_foreach(domains,
tracemalloc_copy_domain,
domains2);
if (err) {
_Py_hashtable_destroy(domains2);
return NULL;
}
return domains2;
}


static int
tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entry,
void *user_data)
{
get_traces_t *get_traces = user_data;
trace_t trace;
trace_t *trace;
PyObject *tracemalloc_obj;
int res;

_Py_HASHTABLE_ENTRY_READ_DATA(traces, entry, trace);

tracemalloc_obj = trace_to_pyobject(get_traces->domain, &trace, get_traces->tracebacks);
tracemalloc_obj = trace_to_pyobject(get_traces->domain, trace, get_traces->tracebacks);
if (tracemalloc_obj == NULL)
return 1;

Expand Down Expand Up @@ -1335,37 +1412,34 @@ _tracemalloc__get_traces_impl(PyObject *module)
goto no_memory;
}

get_traces.domains = tracemalloc_create_domains_table();
if (get_traces.domains == NULL) {
goto no_memory;
}

int err;

// Copy all traces so tracemalloc_get_traces_fill() doesn't have to disable
// temporarily tracemalloc which would impact other threads and so would
// miss allocations while get_traces() is called.
TABLES_LOCK();
get_traces.traces = _Py_hashtable_copy(tracemalloc_traces);
err = _Py_hashtable_foreach(tracemalloc_domains,
tracemalloc_get_traces_copy_domain,
&get_traces);
get_traces.traces = tracemalloc_copy_traces(tracemalloc_traces);
TABLES_UNLOCK();

if (get_traces.traces == NULL) {
goto no_memory;
}
if (err) {

TABLES_LOCK();
get_traces.domains = tracemalloc_copy_domains(tracemalloc_domains);
TABLES_UNLOCK();

if (get_traces.domains == NULL) {
goto no_memory;
}

// Convert traces to a list of tuples
set_reentrant(1);
err = _Py_hashtable_foreach(get_traces.traces,
tracemalloc_get_traces_fill, &get_traces);
int err = _Py_hashtable_foreach(get_traces.traces,
tracemalloc_get_traces_fill,
&get_traces);
if (!err) {
err = _Py_hashtable_foreach(get_traces.domains,
tracemalloc_get_traces_domain, &get_traces);
tracemalloc_get_traces_domain,
&get_traces);
}
set_reentrant(0);
if (err) {
Expand Down Expand Up @@ -1398,7 +1472,7 @@ _tracemalloc__get_traces_impl(PyObject *module)
static traceback_t*
tracemalloc_get_traceback(unsigned int domain, uintptr_t ptr)
{
trace_t trace;
trace_t *trace;
int found;

if (!_Py_tracemalloc_config.tracing)
Expand All @@ -1414,10 +1488,11 @@ tracemalloc_get_traceback(unsigned int domain, uintptr_t ptr)
}
TABLES_UNLOCK();

if (!found)
if (!found) {
return NULL;
}

return trace.traceback;
return trace->traceback;
}


Expand Down Expand Up @@ -1758,10 +1833,9 @@ _PyTraceMalloc_NewReference(PyObject *op)
/* update the traceback of the memory block */
traceback_t *traceback = traceback_new();
if (traceback != NULL) {
trace_t trace;
trace_t *trace;
_Py_HASHTABLE_ENTRY_READ_DATA(tracemalloc_traces, entry, trace);
trace.traceback = traceback;
_Py_HASHTABLE_ENTRY_WRITE_DATA(tracemalloc_traces, entry, trace);
trace->traceback = traceback;
res = 0;
}
}
Expand Down
49 changes: 0 additions & 49 deletions Python/hashtable.c
Show comments
View file Edit file Delete file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -350,21 +350,6 @@ _Py_hashtable_pop(_Py_hashtable_t *ht, const void *key,
}


/* Code commented since the function is not needed in Python */
#if 0
void
_Py_hashtable_delete(_Py_hashtable_t *ht, size_t const void *key)
{
#ifndef NDEBUG
int found = _Py_hashtable_pop_entry(ht, key, NULL, 0);
assert(found);
#else
(void)_Py_hashtable_pop_entry(ht, key, NULL, 0);
#endif
}
#endif


int
_Py_hashtable_foreach(_Py_hashtable_t *ht,
_Py_hashtable_foreach_func func,
Expand Down Expand Up @@ -538,37 +523,3 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht)
ht->alloc.free(ht->buckets);
ht->alloc.free(ht);
}


_Py_hashtable_t *
_Py_hashtable_copy(_Py_hashtable_t *src)
{
const size_t data_size = src->data_size;
_Py_hashtable_t *dst;
_Py_hashtable_entry_t *entry;
size_t bucket;
int err;

dst = _Py_hashtable_new_full(data_size, src->num_buckets,
src->hash_func,
src->compare_func,
src->key_destroy_func,
src->value_destroy_func,
&src->alloc);
if (dst == NULL)
return NULL;

for (bucket=0; bucket < src->num_buckets; bucket++) {
entry = TABLE_HEAD(src, bucket);
for (; entry; entry = ENTRY_NEXT(entry)) {
const void *key = entry->key;
const void *pdata = _Py_HASHTABLE_ENTRY_PDATA(entry);
err = _Py_hashtable_set(dst, key, data_size, pdata);
if (err) {
_Py_hashtable_destroy(dst);
return NULL;
}
}
}
return dst;
}
Toggle all file notes Toggle all file annotations

Footer

© 2026 GitHub, Inc.