diff --git a/src/gc-common.c b/src/gc-common.c index ee461b576ea9e7..a299b8aec21502 100644 --- a/src/gc-common.c +++ b/src/gc-common.c @@ -6,9 +6,6 @@ #include "julia_gcext.h" #include "julia_assert.h" #include "threading.h" -#ifdef __GLIBC__ -#include // for malloc_trim -#endif #ifdef __cplusplus extern "C" { @@ -115,6 +112,21 @@ JL_DLLEXPORT void jl_gc_set_cb_notify_gc_pressure(jl_gc_cb_notify_gc_pressure_t jl_gc_deregister_callback(&gc_cblist_notify_gc_pressure, (jl_gc_cb_func_t)cb); } +// =========================================================================== // +// malloc wrappers, aligned allocation +// =========================================================================== // + +size_t memory_block_usable_size(void *p) JL_NOTSAFEPOINT +{ +#if defined(_OS_WINDOWS_) + return _aligned_msize(p, JL_CACHE_BYTE_ALIGNMENT, 0); +#elif defined(_OS_DARWIN_) + return malloc_size(p); +#else + return malloc_usable_size(p); +#endif +} + // =========================================================================== // // Finalization // =========================================================================== // diff --git a/src/gc-common.h b/src/gc-common.h index 4d53830442a7dc..d721cb75ffde12 100644 --- a/src/gc-common.h +++ b/src/gc-common.h @@ -12,6 +12,12 @@ #endif #endif +#if defined(_OS_DARWIN_) +#include +#else +#include // for malloc_trim +#endif + #ifdef __cplusplus extern "C" { #endif diff --git a/src/gc-stock.c b/src/gc-stock.c index 1fc2087da6503f..8a58474052e845 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -9,9 +9,6 @@ #include "julia_atomics.h" #include "julia_gcext.h" #include "julia_assert.h" -#ifdef __GLIBC__ -#include // for malloc_trim -#endif #ifdef __cplusplus extern "C" { @@ -579,11 +576,6 @@ void jl_gc_count_allocd(size_t sz) JL_NOTSAFEPOINT jl_batch_accum_heap_size(ptls, sz); } -void jl_gc_count_freed(size_t sz) JL_NOTSAFEPOINT -{ - jl_batch_accum_free_size(jl_current_task->ptls, sz); -} - // Only safe to update the heap inside the GC static void combine_thread_gc_counts(jl_gc_num_t *dest, int update_heap) JL_NOTSAFEPOINT { @@ -664,13 +656,15 @@ static void jl_gc_free_memory(jl_value_t *v, int isaligned) JL_NOTSAFEPOINT jl_genericmemory_t *m = (jl_genericmemory_t*)v; assert(jl_genericmemory_how(m) == 1 || jl_genericmemory_how(m) == 2); char *d = (char*)m->ptr; + size_t freed_bytes = memory_block_usable_size(d); + assert(freed_bytes != 0); if (isaligned) jl_free_aligned(d); else free(d); jl_atomic_store_relaxed(&gc_heap_stats.heap_size, - jl_atomic_load_relaxed(&gc_heap_stats.heap_size) - jl_genericmemory_nbytes(m)); - gc_num.freed += jl_genericmemory_nbytes(m); + jl_atomic_load_relaxed(&gc_heap_stats.heap_size) - freed_bytes); + gc_num.freed += freed_bytes; gc_num.freecall++; } @@ -3739,11 +3733,13 @@ JL_DLLEXPORT void *jl_gc_managed_malloc(size_t sz) if (b == NULL) jl_throw(jl_memory_exception); + size_t allocated_bytes = memory_block_usable_size(b); + assert(allocated_bytes >= allocsz); jl_atomic_store_relaxed(&ptls->gc_tls.gc_num.allocd, - jl_atomic_load_relaxed(&ptls->gc_tls.gc_num.allocd) + allocsz); + jl_atomic_load_relaxed(&ptls->gc_tls.gc_num.allocd) + allocated_bytes); jl_atomic_store_relaxed(&ptls->gc_tls.gc_num.malloc, jl_atomic_load_relaxed(&ptls->gc_tls.gc_num.malloc) + 1); - jl_batch_accum_heap_size(ptls, allocsz); + jl_batch_accum_heap_size(ptls, allocated_bytes); #ifdef _OS_WINDOWS_ SetLastError(last_error); #endif diff --git a/src/genericmemory.c b/src/genericmemory.c index ea52fca66ba48b..3ad0776d272e95 100644 --- a/src/genericmemory.c +++ b/src/genericmemory.c @@ -165,7 +165,8 @@ JL_DLLEXPORT jl_genericmemory_t *jl_ptr_to_genericmemory(jl_value_t *mtype, void if (own_buffer) { int isaligned = 0; // TODO: allow passing memalign'd buffers jl_gc_track_malloced_genericmemory(ct->ptls, m, isaligned); - jl_gc_count_allocd(nel*elsz); + size_t allocated_bytes = memory_block_usable_size(data); + jl_gc_count_allocd(allocated_bytes); } return m; } @@ -208,8 +209,6 @@ JL_DLLEXPORT jl_value_t *jl_genericmemory_to_string(jl_genericmemory_t *m, size_ JL_GC_PUSH1(&o); jl_value_t *str = jl_pchar_to_string((const char*)m->ptr, len); JL_GC_POP(); - if (how == 1) // TODO: we might like to early-call jl_gc_free_memory here instead actually, but hopefully `m` will die soon - jl_gc_count_freed(mlength); return str; } // n.b. how == 0 is always pool-allocated, so the freed bytes are computed from the pool not the object diff --git a/src/julia_internal.h b/src/julia_internal.h index d4d1a3239785ca..a867c3fa7854c4 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -601,8 +601,8 @@ jl_svec_t *jl_perm_symsvec(size_t n, ...); void jl_gc_track_malloced_genericmemory(jl_ptls_t ptls, jl_genericmemory_t *m, int isaligned) JL_NOTSAFEPOINT; size_t jl_genericmemory_nbytes(jl_genericmemory_t *a) JL_NOTSAFEPOINT; +size_t memory_block_usable_size(void *mem) JL_NOTSAFEPOINT; void jl_gc_count_allocd(size_t sz) JL_NOTSAFEPOINT; -void jl_gc_count_freed(size_t sz) JL_NOTSAFEPOINT; void jl_gc_run_all_finalizers(jl_task_t *ct); void jl_release_task_stack(jl_ptls_t ptls, jl_task_t *task); void jl_gc_add_finalizer_(jl_ptls_t ptls, void *v, void *f) JL_NOTSAFEPOINT;