From 2892dfd42edd1becc93b80e9d97f6be944ca92e8 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Wed, 20 Nov 2024 15:05:16 +0100 Subject: [PATCH 01/12] no need for custom INI parsing --- profiling/build.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/profiling/build.rs b/profiling/build.rs index a704705732..36f7dcaa96 100644 --- a/profiling/build.rs +++ b/profiling/build.rs @@ -366,6 +366,7 @@ fn cfg_php_feature_flags(vernum: u64) { fn cfg_zts() { let output = Command::new("php") + .arg("-n") .arg("-r") .arg("echo PHP_ZTS, PHP_EOL;") .output() From e03757672648d7116b5d87368cf592e53745fb7d Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Wed, 20 Nov 2024 13:22:42 +0100 Subject: [PATCH 02/12] move allocation profiling into own mod --- profiling/build.rs | 1 + .../allocation_le83.rs} | 96 +------------- profiling/src/allocation/mod.rs | 120 ++++++++++++++++++ profiling/src/config.rs | 2 +- 4 files changed, 124 insertions(+), 95 deletions(-) rename profiling/src/{allocation.rs => allocation/allocation_le83.rs} (85%) create mode 100644 profiling/src/allocation/mod.rs diff --git a/profiling/build.rs b/profiling/build.rs index 36f7dcaa96..01142cd8c9 100644 --- a/profiling/build.rs +++ b/profiling/build.rs @@ -361,6 +361,7 @@ fn cfg_php_feature_flags(vernum: u64) { if vernum >= 80400 { println!("cargo:rustc-cfg=php_frameless"); println!("cargo:rustc-cfg=php_opcache_restart_hook"); + println!("cargo:rustc-cfg=php_new_zendmm_hooks"); } } diff --git a/profiling/src/allocation.rs b/profiling/src/allocation/allocation_le83.rs similarity index 85% rename from profiling/src/allocation.rs rename to profiling/src/allocation/allocation_le83.rs index a61b22ce2a..917aa4ffb0 100644 --- a/profiling/src/allocation.rs +++ b/profiling/src/allocation/allocation_le83.rs @@ -1,41 +1,18 @@ +use crate::allocation::{ALLOCATION_PROFILING_COUNT, ALLOCATION_PROFILING_SIZE, ALLOCATION_PROFILING_STATS}; use crate::bindings::{ self as zend, datadog_php_install_handler, datadog_php_zif_handler, ddog_php_prof_copy_long_into_zval, }; -use crate::profiling::Profiler; use crate::{PROFILER_NAME, REQUEST_LOCALS}; use lazy_static::lazy_static; use libc::{c_char, c_int, c_void, size_t}; use log::{debug, error, trace, warn}; -use rand::rngs::ThreadRng; -use rand_distr::{Distribution, Poisson}; -use std::cell::{RefCell, UnsafeCell}; +use std::cell::UnsafeCell; use std::ptr; -use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering::{Relaxed, SeqCst}; static mut GC_MEM_CACHES_HANDLER: zend::InternalFunctionHandler = None; -/// take a sample every 4096 KiB -pub const ALLOCATION_PROFILING_INTERVAL: f64 = 1024.0 * 4096.0; - -/// This will store the count of allocations (including reallocations) during -/// a profiling period. This will overflow when doing more than u64::MAX -/// allocations, which seems big enough to ignore. -pub static ALLOCATION_PROFILING_COUNT: AtomicU64 = AtomicU64::new(0); - -/// This will store the accumulated size of all allocations in bytes during the -/// profiling period. This will overflow when allocating more than 18 exabyte -/// of memory (u64::MAX) which might not happen, so we can ignore this. -pub static ALLOCATION_PROFILING_SIZE: AtomicU64 = AtomicU64::new(0); - -pub struct AllocationProfilingStats { - /// number of bytes until next sample collection - next_sample: i64, - poisson: Poisson, - rng: ThreadRng, -} - type ZendHeapPrepareFn = unsafe fn(heap: *mut zend::_zend_mm_heap) -> c_int; type ZendHeapRestoreFn = unsafe fn(heap: *mut zend::_zend_mm_heap, custom_heap: c_int); @@ -69,49 +46,7 @@ struct ZendMMState { free: unsafe fn(*mut c_void), } -impl AllocationProfilingStats { - fn new() -> AllocationProfilingStats { - // Safety: this will only error if lambda <= 0 - let poisson = Poisson::new(ALLOCATION_PROFILING_INTERVAL).unwrap(); - let mut stats = AllocationProfilingStats { - next_sample: 0, - poisson, - rng: rand::thread_rng(), - }; - stats.next_sampling_interval(); - stats - } - - fn next_sampling_interval(&mut self) { - self.next_sample = self.poisson.sample(&mut self.rng) as i64; - } - - fn track_allocation(&mut self, len: size_t) { - self.next_sample -= len as i64; - - if self.next_sample > 0 { - return; - } - - self.next_sampling_interval(); - - if let Some(profiler) = Profiler::get() { - // Safety: execute_data was provided by the engine, and the profiler doesn't mutate it. - unsafe { - profiler.collect_allocations( - zend::ddog_php_prof_get_current_execute_data(), - 1_i64, - len as i64, - ) - }; - } - } -} - thread_local! { - static ALLOCATION_PROFILING_STATS: RefCell = - RefCell::new(AllocationProfilingStats::new()); - /// Using an `UnsafeCell` here should be okay. There might not be any /// synchronisation issues, as it is used in as thread local and only /// mutated in RINIT and RSHUTDOWN. @@ -167,23 +102,6 @@ pub fn first_rinit_should_disable_due_to_jit() -> bool { } pub fn alloc_prof_rinit() { - let allocation_profiling: bool = REQUEST_LOCALS.with(|cell| { - match cell.try_borrow() { - Ok(locals) => { - let system_settings = locals.system_settings(); - system_settings.profiling_allocation_enabled - }, - Err(_err) => { - error!("Memory allocation was not initialized correctly due to a borrow error. Please report this to Datadog."); - false - } - } - }); - - if !allocation_profiling { - return; - } - ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); @@ -247,16 +165,6 @@ pub fn alloc_prof_rinit() { } pub fn alloc_prof_rshutdown() { - let allocation_profiling = REQUEST_LOCALS.with(|cell| { - cell.try_borrow() - .map(|locals| locals.system_settings().profiling_allocation_enabled) - .unwrap_or(false) - }); - - if !allocation_profiling { - return; - } - // If `is_zend_mm()` is true, the custom handlers have been reset to `None` // already. This is unexpected, therefore we will not touch the ZendMM // handlers anymore as resetting to prev handlers might result in segfaults diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs new file mode 100644 index 0000000000..0b2c3a12e2 --- /dev/null +++ b/profiling/src/allocation/mod.rs @@ -0,0 +1,120 @@ +use crate::bindings::{self as zend}; +use crate::profiling::Profiler; +use crate::REQUEST_LOCALS; +use libc::size_t; +use log::{error, trace}; +use rand::rngs::ThreadRng; +use rand_distr::{Distribution, Poisson}; +use std::cell::RefCell; +use std::sync::atomic::AtomicU64; + +pub mod allocation_le83; + +/// take a sample every 4096 KiB +pub const ALLOCATION_PROFILING_INTERVAL: f64 = 1024.0 * 4096.0; + +/// This will store the count of allocations (including reallocations) during +/// a profiling period. This will overflow when doing more than u64::MAX +/// allocations, which seems big enough to ignore. +pub static ALLOCATION_PROFILING_COUNT: AtomicU64 = AtomicU64::new(0); + +/// This will store the accumulated size of all allocations in bytes during the +/// profiling period. This will overflow when allocating more than 18 exabyte +/// of memory (u64::MAX) which might not happen, so we can ignore this. +pub static ALLOCATION_PROFILING_SIZE: AtomicU64 = AtomicU64::new(0); + +pub struct AllocationProfilingStats { + /// number of bytes until next sample collection + next_sample: i64, + poisson: Poisson, + rng: ThreadRng, +} + +impl AllocationProfilingStats { + fn new() -> AllocationProfilingStats { + // Safety: this will only error if lambda <= 0 + let poisson = Poisson::new(ALLOCATION_PROFILING_INTERVAL).unwrap(); + let mut stats = AllocationProfilingStats { + next_sample: 0, + poisson, + rng: rand::thread_rng(), + }; + stats.next_sampling_interval(); + stats + } + + fn next_sampling_interval(&mut self) { + self.next_sample = self.poisson.sample(&mut self.rng) as i64; + } + + fn track_allocation(&mut self, len: size_t) { + self.next_sample -= len as i64; + + if self.next_sample > 0 { + return; + } + + self.next_sampling_interval(); + + if let Some(profiler) = Profiler::get() { + // Safety: execute_data was provided by the engine, and the profiler doesn't mutate it. + unsafe { + profiler.collect_allocations( + zend::ddog_php_prof_get_current_execute_data(), + 1_i64, + len as i64, + ) + }; + } + } +} + +thread_local! { + static ALLOCATION_PROFILING_STATS: RefCell = + RefCell::new(AllocationProfilingStats::new()); +} + +pub fn alloc_prof_minit() { + allocation_le83::alloc_prof_minit(); +} + +pub fn alloc_prof_startup() { + allocation_le83::alloc_prof_startup(); +} + +pub fn alloc_prof_rinit() { + let allocation_profiling: bool = REQUEST_LOCALS.with(|cell| { + match cell.try_borrow() { + Ok(locals) => { + let system_settings = locals.system_settings(); + system_settings.profiling_allocation_enabled + }, + Err(_err) => { + error!("Memory allocation was not initialized correctly due to a borrow error. Please report this to Datadog."); + false + } + } + }); + + if !allocation_profiling { + return; + } + + allocation_le83::alloc_prof_rinit(); + + trace!("Memory allocation profiling enabled.") +} + +pub fn alloc_prof_rshutdown() { + let allocation_profiling = REQUEST_LOCALS.with(|cell| { + cell.try_borrow() + .map(|locals| locals.system_settings().profiling_allocation_enabled) + .unwrap_or(false) + }); + + if !allocation_profiling { + return; + } + + allocation_le83::alloc_prof_rshutdown(); +} diff --git a/profiling/src/config.rs b/profiling/src/config.rs index d8a9c4fb18..a64ca0d94d 100644 --- a/profiling/src/config.rs +++ b/profiling/src/config.rs @@ -101,7 +101,7 @@ impl SystemSettings { } // Work around version-specific issues. - if allocation::first_rinit_should_disable_due_to_jit() { + if allocation::allocation_le83::first_rinit_should_disable_due_to_jit() { system_settings.profiling_allocation_enabled = false; } swap(&mut system_settings, SYSTEM_SETTINGS.assume_init_mut()); From bd10234bb0820b050383fcec104009513ca706fd Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Wed, 20 Nov 2024 14:47:23 +0100 Subject: [PATCH 03/12] Add new ZendMM hooks --- profiling/src/allocation/allocation_ge84.rs | 375 ++++++++++++++++++++ profiling/src/allocation/allocation_le83.rs | 4 +- profiling/src/allocation/mod.rs | 11 + profiling/src/bindings/mod.rs | 4 + profiling/src/config.rs | 2 + 5 files changed, 395 insertions(+), 1 deletion(-) create mode 100644 profiling/src/allocation/allocation_ge84.rs diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs new file mode 100644 index 0000000000..0cddf0736e --- /dev/null +++ b/profiling/src/allocation/allocation_ge84.rs @@ -0,0 +1,375 @@ +use crate::allocation::ALLOCATION_PROFILING_COUNT; +use crate::allocation::ALLOCATION_PROFILING_SIZE; +use crate::allocation::ALLOCATION_PROFILING_STATS; +use crate::bindings::{self as zend}; +use crate::PROFILER_NAME; +use libc::{c_char, c_void, size_t}; +use log::{debug, trace, warn}; +use std::cell::UnsafeCell; +use std::ptr; +use std::sync::atomic::Ordering::SeqCst; + +struct ZendMMState { + /// The heap we create and set as the current heap in ZendMM + heap: Option<*mut zend::zend_mm_heap>, + /// The heap installed in ZendMM at the time we install our custom handlers + prev_heap: Option<*mut zend::zend_mm_heap>, + /// The engine's previous custom allocation function, if there is one. + prev_custom_mm_alloc: Option, + /// The engine's previous custom reallocation function, if there is one. + prev_custom_mm_realloc: Option, + /// The engine's previous custom free function, if there is one. + prev_custom_mm_free: Option, + /// The engine's previous custom gc function, if there is one. + prev_custom_mm_gc: Option, + /// The engine's previous custom shutdown function, if there is one. + prev_custom_mm_shutdown: Option, + /// Safety: this function pointer is only allowed to point to + /// `alloc_prof_prev_alloc()` when at the same time the + /// `ZEND_MM_STATE.prev_custom_mm_alloc` is initialised to a valid function + /// pointer, otherwise there will be dragons. + alloc: unsafe fn(size_t) -> *mut c_void, + /// Safety: this function pointer is only allowed to point to + /// `alloc_prof_prev_realloc()` when at the same time the + /// `ZEND_MM_STATE.prev_custom_mm_realloc` is initialised to a valid + /// function pointer, otherwise there will be dragons. + realloc: unsafe fn(*mut c_void, size_t) -> *mut c_void, + /// Safety: this function pointer is only allowed to point to + /// `alloc_prof_prev_free()` when at the same time the + /// `ZEND_MM_STATE.prev_custom_mm_free` is initialised to a valid function + /// pointer, otherwise there will be dragons. + free: unsafe fn(*mut c_void), + /// Safety: this function pointer is only allowed to point to + /// `alloc_prof_prev_gc()` when at the same time the + /// `ZEND_MM_STATE.prev_custom_mm_gc` is initialised to a valid function + /// pointer, otherwise there will be dragons. + gc: unsafe fn() -> size_t, + /// Safety: this function pointer is only allowed to point to + /// `alloc_prof_prev_shutdown()` when at the same time the + /// `ZEND_MM_STATE.prev_custom_mm_shutdown` is initialised to a valid function + /// pointer, otherwise there will be dragons. + shutdown: unsafe fn(bool, bool), +} + +thread_local! { + /// Using an `UnsafeCell` here should be okay. There might not be any + /// synchronisation issues, as it is used in as thread local and only + /// mutated in RINIT and RSHUTDOWN. + static ZEND_MM_STATE: UnsafeCell = const { + UnsafeCell::new(ZendMMState { + heap: None, + prev_heap: None, + prev_custom_mm_alloc: None, + prev_custom_mm_realloc: None, + prev_custom_mm_free: None, + prev_custom_mm_gc: None, + prev_custom_mm_shutdown: None, + alloc: alloc_prof_orig_alloc, + realloc: alloc_prof_orig_realloc, + free: alloc_prof_orig_free, + gc: alloc_prof_orig_gc, + shutdown: alloc_prof_orig_shutdown, + }) + }; +} + +macro_rules! tls_zend_mm_state { + ($x:ident) => { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + (*zend_mm_state).$x + }) + }; +} + +pub fn alloc_prof_rinit() { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + + // Only need to create an observed heap once per thread. When we have it, we can just + // install the observed hook via `zend::zend_mm_set_heap()` + if unsafe { (*zend_mm_state).heap.is_none() } { + // Safety: `zend_mm_get_heap()` always returns a non-null pointer to a valid heap structure + let prev_heap = unsafe { zend::zend_mm_get_heap() }; + unsafe { ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(Some(prev_heap)) }; + + if !is_zend_mm() { + // Neighboring custom memory handlers found in the currently used ZendMM heap + debug!("Found another extension using the ZendMM custom handler hook"); + unsafe { + zend::zend_mm_get_custom_handlers_ex( + prev_heap, + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_alloc), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_free), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_realloc), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_gc), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_shutdown), + ); + ptr::addr_of_mut!((*zend_mm_state).alloc).write(alloc_prof_prev_alloc); + ptr::addr_of_mut!((*zend_mm_state).free).write(alloc_prof_prev_free); + ptr::addr_of_mut!((*zend_mm_state).realloc).write(alloc_prof_prev_realloc); + // `gc` handler can be NULL + if (*zend_mm_state).prev_custom_mm_gc.is_none() { + ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_orig_gc); + } else { + ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_prev_gc); + } + // `shutdown` handler can be NULL + if (*zend_mm_state).prev_custom_mm_shutdown.is_none() { + ptr::addr_of_mut!((*zend_mm_state).shutdown) + .write(alloc_prof_orig_shutdown); + } else { + ptr::addr_of_mut!((*zend_mm_state).shutdown) + .write(alloc_prof_prev_shutdown); + } + } + } + + // Create our observed heap and prepare custom handlers + let heap = unsafe { zend::zend_mm_startup() }; + unsafe { ptr::addr_of_mut!((*zend_mm_state).heap).write(Some(heap)) }; + + // install our custom handler to ZendMM + unsafe { + zend::zend_mm_set_custom_handlers_ex( + (*zend_mm_state).heap.unwrap(), + Some(alloc_prof_malloc), + Some(alloc_prof_free), + Some(alloc_prof_realloc), + Some(alloc_prof_gc), + Some(alloc_prof_shutdown), + ); + } + } + + // install the observed heap into ZendMM + unsafe { + zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + } + }); + + // `is_zend_mm()` should be false now, as we installed our custom handlers + if is_zend_mm() { + // Can't proceed with it being disabled, because that's a system-wide + // setting, not per-request. + panic!("Memory allocation profiling could not be enabled. Please feel free to fill an issue stating the PHP version and installed modules. Most likely the reason is your PHP binary was compiled with `ZEND_MM_CUSTOM` being disabled."); + } + trace!("Memory allocation profiling enabled.") +} + +pub fn alloc_prof_rshutdown() { + // If `is_zend_mm()` is true, the custom handlers have been reset to `None` or our observed + // heap has been uninstalled. This is unexpected, therefore we will not touch the ZendMM + // handlers anymore as resetting to prev handlers might result in segfaults and other undefined + // behaviour. + if is_zend_mm() { + return; + } + + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + + // Do a sanity check and see if something played with our heap + let mut custom_mm_malloc: Option = None; + let mut custom_mm_free: Option = None; + let mut custom_mm_realloc: Option = None; + let mut custom_mm_gc: Option = None; + let mut custom_mm_shutdown: Option = None; + + // Safety: `unwrap()` is safe here, as `heap` is initialized in `RINIT` + let heap = unsafe { (*zend_mm_state).heap.unwrap() }; + unsafe { + zend::zend_mm_get_custom_handlers_ex( + heap, + &mut custom_mm_malloc, + &mut custom_mm_free, + &mut custom_mm_realloc, + &mut custom_mm_gc, + &mut custom_mm_shutdown, + ); + } + if custom_mm_free != Some(alloc_prof_free) + || custom_mm_malloc != Some(alloc_prof_malloc) + || custom_mm_realloc != Some(alloc_prof_realloc) + || custom_mm_gc != Some(alloc_prof_gc) + || custom_mm_shutdown != Some(alloc_prof_shutdown) + { + // Custom handlers are installed, but it's not us. Someone, somewhere might have + // function pointers to our custom handlers. Best bet to avoid segfaults is to not + // touch custom handlers in ZendMM and make sure our extension will not be + // `dlclose()`-ed so the pointers stay valid + let zend_extension = + unsafe { zend::zend_get_extension(PROFILER_NAME.as_ptr() as *const c_char) }; + if !zend_extension.is_null() { + // Safety: Checked for null pointer above. + unsafe { ptr::addr_of_mut!((*zend_extension).handle).write(ptr::null_mut()) }; + } + warn!("Found another extension using the custom heap which is unexpected at this point, so the extension handle was `null`'ed to avoid being `dlclose()`'ed."); + } else { + // This is the happy path. Restore previous heap. + unsafe { + zend::zend_mm_set_heap( + (*zend_mm_state).prev_heap.unwrap() + ); + } + trace!("Memory allocation profiling shutdown gracefully."); + } + }); +} + +unsafe extern "C" fn alloc_prof_malloc(len: size_t) -> *mut c_void { + ALLOCATION_PROFILING_COUNT.fetch_add(1, SeqCst); + ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, SeqCst); + + let ptr = tls_zend_mm_state!(alloc)(len); + + // during startup, minit, rinit, ... current_execute_data is null + // we are only interested in allocations during userland operations + if zend::ddog_php_prof_get_current_execute_data().is_null() { + return ptr; + } + + ALLOCATION_PROFILING_STATS.with(|cell| { + let mut allocations = cell.borrow_mut(); + allocations.track_allocation(len) + }); + + ptr +} + +unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + // Safety: `ZEND_MM_STATE.alloc` will be initialised in + // `alloc_prof_rinit()` and only point to this function when + // `prev_custom_mm_alloc` is also initialised + let ptr = ((*zend_mm_state).prev_custom_mm_alloc.unwrap())(len); + // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + ptr + }) +} + +unsafe fn alloc_prof_orig_alloc(len: size_t) -> *mut c_void { + let ptr: *mut c_void = zend::_zend_mm_alloc(tls_zend_mm_state!(prev_heap).unwrap(), len); + ptr +} + +/// This function exists because when calling `zend_mm_set_custom_handlers()`, +/// you need to pass a pointer to a `free()` function as well, otherwise your +/// custom handlers won't be installed. We can not just point to the original +/// `zend::_zend_mm_free()` as the function definitions differ. +unsafe extern "C" fn alloc_prof_free(ptr: *mut c_void) { + tls_zend_mm_state!(free)(ptr); +} + +unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + // Safety: `ZEND_MM_STATE.free` will be initialised in + // `alloc_prof_rinit()` and only point to this function when + // `prev_custom_mm_free` is also initialised + ((*zend_mm_state).prev_custom_mm_free.unwrap())(ptr); + // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + }) +} + +unsafe fn alloc_prof_orig_free(ptr: *mut c_void) { + zend::_zend_mm_free(tls_zend_mm_state!(prev_heap).unwrap(), ptr); +} + +unsafe extern "C" fn alloc_prof_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { + ALLOCATION_PROFILING_COUNT.fetch_add(1, SeqCst); + ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, SeqCst); + + let ptr = tls_zend_mm_state!(realloc)(prev_ptr, len); + + // during startup, minit, rinit, ... current_execute_data is null + // we are only interested in allocations during userland operations + if zend::ddog_php_prof_get_current_execute_data().is_null() || ptr == prev_ptr { + return ptr; + } + + ALLOCATION_PROFILING_STATS.with(|cell| { + let mut allocations = cell.borrow_mut(); + allocations.track_allocation(len) + }); + + ptr +} + +unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + // Safety: `ZEND_MM_STATE.realloc` will be initialised in + // `alloc_prof_rinit()` and only point to this function when + // `prev_custom_mm_realloc` is also initialised + let ptr = ((*zend_mm_state).prev_custom_mm_realloc.unwrap())(prev_ptr, len); + // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + ptr + }) +} + +unsafe fn alloc_prof_orig_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { + zend::_zend_mm_realloc(tls_zend_mm_state!(prev_heap).unwrap(), prev_ptr, len) +} + +unsafe extern "C" fn alloc_prof_gc() -> size_t { + tls_zend_mm_state!(gc)() +} + +unsafe fn alloc_prof_prev_gc() -> size_t { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + // Safety: `ZEND_MM_STATE.gc` will be initialised in + // `alloc_prof_rinit()` and only point to this function when + // `prev_custom_mm_gc` is also initialised + let freed = ((*zend_mm_state).prev_custom_mm_gc.unwrap())(); + // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + freed + }) +} + +unsafe fn alloc_prof_orig_gc() -> size_t { + zend::zend_mm_gc(tls_zend_mm_state!(prev_heap).unwrap()) +} + +unsafe extern "C" fn alloc_prof_shutdown(full: bool, silent: bool) { + tls_zend_mm_state!(shutdown)(full, silent); +} + +unsafe fn alloc_prof_prev_shutdown(full: bool, silent: bool) { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + // Safety: `ZEND_MM_STATE.shutdown` will be initialised in + // `alloc_prof_rinit()` and only point to this function when + // `prev_custom_mm_shutdown` is also initialised + ((*zend_mm_state).prev_custom_mm_shutdown.unwrap())(full, silent); + // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + }) +} + +unsafe fn alloc_prof_orig_shutdown(full: bool, silent: bool) { + zend::zend_mm_shutdown(tls_zend_mm_state!(prev_heap).unwrap(), full, silent) +} + +/// safe wrapper for `zend::is_zend_mm()`. +/// `true` means the internal ZendMM is being used, `false` means that a custom memory manager is +/// installed +fn is_zend_mm() -> bool { + unsafe { zend::is_zend_mm() } +} diff --git a/profiling/src/allocation/allocation_le83.rs b/profiling/src/allocation/allocation_le83.rs index 917aa4ffb0..f77ecb5a9e 100644 --- a/profiling/src/allocation/allocation_le83.rs +++ b/profiling/src/allocation/allocation_le83.rs @@ -1,4 +1,6 @@ -use crate::allocation::{ALLOCATION_PROFILING_COUNT, ALLOCATION_PROFILING_SIZE, ALLOCATION_PROFILING_STATS}; +use crate::allocation::{ + ALLOCATION_PROFILING_COUNT, ALLOCATION_PROFILING_SIZE, ALLOCATION_PROFILING_STATS, +}; use crate::bindings::{ self as zend, datadog_php_install_handler, datadog_php_zif_handler, ddog_php_prof_copy_long_into_zval, diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs index 0b2c3a12e2..f7303d112d 100644 --- a/profiling/src/allocation/mod.rs +++ b/profiling/src/allocation/mod.rs @@ -8,6 +8,9 @@ use rand_distr::{Distribution, Poisson}; use std::cell::RefCell; use std::sync::atomic::AtomicU64; +#[cfg(php_new_zendmm_hooks)] +mod allocation_ge84; +#[cfg(not(php_new_zendmm_hooks))] pub mod allocation_le83; /// take a sample every 4096 KiB @@ -75,10 +78,12 @@ thread_local! { } pub fn alloc_prof_minit() { + #[cfg(not(php_new_zendmm_hooks))] allocation_le83::alloc_prof_minit(); } pub fn alloc_prof_startup() { + #[cfg(not(php_new_zendmm_hooks))] allocation_le83::alloc_prof_startup(); } @@ -100,7 +105,10 @@ pub fn alloc_prof_rinit() { return; } + #[cfg(not(php_new_zendmm_hooks))] allocation_le83::alloc_prof_rinit(); + #[cfg(php_new_zendmm_hooks)] + allocation_ge84::alloc_prof_rinit(); trace!("Memory allocation profiling enabled.") } @@ -116,5 +124,8 @@ pub fn alloc_prof_rshutdown() { return; } + #[cfg(not(php_new_zendmm_hooks))] allocation_le83::alloc_prof_rshutdown(); + #[cfg(php_new_zendmm_hooks)] + allocation_ge84::alloc_prof_rshutdown(); } diff --git a/profiling/src/bindings/mod.rs b/profiling/src/bindings/mod.rs index c2e01d2f8c..70ddafd038 100644 --- a/profiling/src/bindings/mod.rs +++ b/profiling/src/bindings/mod.rs @@ -46,6 +46,10 @@ pub type VmMmCustomAllocFn = unsafe extern "C" fn(size_t) -> *mut c_void; pub type VmMmCustomReallocFn = unsafe extern "C" fn(*mut c_void, size_t) -> *mut c_void; #[cfg(feature = "allocation_profiling")] pub type VmMmCustomFreeFn = unsafe extern "C" fn(*mut c_void); +#[cfg(all(feature = "allocation_profiling", php_new_zendmm_hooks))] +pub type VmMmCustomGcFn = unsafe extern "C" fn() -> size_t; +#[cfg(all(feature = "allocation_profiling", php_new_zendmm_hooks))] +pub type VmMmCustomShutdownFn = unsafe extern "C" fn(bool, bool); // todo: this a lie on some PHP versions; is it a problem even though zend_bool // was always supposed to be 0 or 1 anyway? diff --git a/profiling/src/config.rs b/profiling/src/config.rs index a64ca0d94d..b84650fbb2 100644 --- a/profiling/src/config.rs +++ b/profiling/src/config.rs @@ -1,3 +1,4 @@ +#[cfg(not(php_new_zendmm_hooks))] use crate::allocation; use crate::bindings::zai_config_type::*; use crate::bindings::{ @@ -101,6 +102,7 @@ impl SystemSettings { } // Work around version-specific issues. + #[cfg(not(php_new_zendmm_hooks))] if allocation::allocation_le83::first_rinit_should_disable_due_to_jit() { system_settings.profiling_allocation_enabled = false; } From bd07e725ee4f08b4761df4dba0e3af81213f7d16 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Thu, 21 Nov 2024 10:59:30 +0100 Subject: [PATCH 04/12] fix comments --- profiling/src/allocation/allocation_le83.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/profiling/src/allocation/allocation_le83.rs b/profiling/src/allocation/allocation_le83.rs index f77ecb5a9e..e58b1953b2 100644 --- a/profiling/src/allocation/allocation_le83.rs +++ b/profiling/src/allocation/allocation_le83.rs @@ -311,7 +311,7 @@ unsafe extern "C" fn alloc_prof_malloc(len: size_t) -> *mut c_void { } unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { - // Safety: `ALLOCATION_PROFILING_ALLOC` will be initialised in + // Safety: `ZEND_MM_STATE.prev_custom_mm_alloc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_alloc` is also initialised let alloc = tls_zend_mm_state!(prev_custom_mm_alloc).unwrap(); @@ -336,8 +336,8 @@ unsafe extern "C" fn alloc_prof_free(ptr: *mut c_void) { } unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { - // Safety: `ALLOCATION_PROFILING_FREE` will be initialised in - // `alloc_prof_free()` and only point to this function when + // Safety: `ZEND_MM_STATE.prev_custom_mm_free` will be initialised in + // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_free` is also initialised let free = tls_zend_mm_state!(prev_custom_mm_free).unwrap(); free(ptr) @@ -369,8 +369,8 @@ unsafe extern "C" fn alloc_prof_realloc(prev_ptr: *mut c_void, len: size_t) -> * } unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - // Safety: `ALLOCATION_PROFILING_REALLOC` will be initialised in - // `alloc_prof_realloc()` and only point to this function when + // Safety: `ZEND_MM_STATE.prev_custom_mm_realloc` will be initialised in + // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_realloc` is also initialised let realloc = tls_zend_mm_state!(prev_custom_mm_realloc).unwrap(); realloc(prev_ptr, len) From 8645119e660da3578a0fc75ea306560ef55c3bb0 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Mon, 25 Nov 2024 14:16:55 +0100 Subject: [PATCH 05/12] Remove some `unwrap()` calls --- profiling/src/allocation/allocation_ge84.rs | 57 +++++++++++---------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index 0cddf0736e..10266b8156 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -11,9 +11,9 @@ use std::sync::atomic::Ordering::SeqCst; struct ZendMMState { /// The heap we create and set as the current heap in ZendMM - heap: Option<*mut zend::zend_mm_heap>, + heap: *mut zend::zend_mm_heap, /// The heap installed in ZendMM at the time we install our custom handlers - prev_heap: Option<*mut zend::zend_mm_heap>, + prev_heap: *mut zend::zend_mm_heap, /// The engine's previous custom allocation function, if there is one. prev_custom_mm_alloc: Option, /// The engine's previous custom reallocation function, if there is one. @@ -57,8 +57,13 @@ thread_local! { /// mutated in RINIT and RSHUTDOWN. static ZEND_MM_STATE: UnsafeCell = const { UnsafeCell::new(ZendMMState { - heap: None, - prev_heap: None, + // Safety: Using `ptr::null_mut()` might seem dangerous but actually it is okay in this + // case. The `heap` and `prev_heap` fields will be initialized in the first call to + // RINIT and only used after that. By using this "trick" we can get rid of all + // `unwrap()` calls when using the `heap` or `prev_heap` field. Alternatively we could + // use `unwrap_unchecked()` for the same performance characteristics. + heap: ptr::null_mut(), + prev_heap: ptr::null_mut(), prev_custom_mm_alloc: None, prev_custom_mm_realloc: None, prev_custom_mm_free: None, @@ -88,10 +93,10 @@ pub fn alloc_prof_rinit() { // Only need to create an observed heap once per thread. When we have it, we can just // install the observed hook via `zend::zend_mm_set_heap()` - if unsafe { (*zend_mm_state).heap.is_none() } { + if unsafe { (*zend_mm_state).heap.is_null() } { // Safety: `zend_mm_get_heap()` always returns a non-null pointer to a valid heap structure let prev_heap = unsafe { zend::zend_mm_get_heap() }; - unsafe { ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(Some(prev_heap)) }; + unsafe { ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(prev_heap) }; if !is_zend_mm() { // Neighboring custom memory handlers found in the currently used ZendMM heap @@ -127,12 +132,12 @@ pub fn alloc_prof_rinit() { // Create our observed heap and prepare custom handlers let heap = unsafe { zend::zend_mm_startup() }; - unsafe { ptr::addr_of_mut!((*zend_mm_state).heap).write(Some(heap)) }; + unsafe { ptr::addr_of_mut!((*zend_mm_state).heap).write(heap) }; // install our custom handler to ZendMM unsafe { zend::zend_mm_set_custom_handlers_ex( - (*zend_mm_state).heap.unwrap(), + (*zend_mm_state).heap, Some(alloc_prof_malloc), Some(alloc_prof_free), Some(alloc_prof_realloc), @@ -144,7 +149,7 @@ pub fn alloc_prof_rinit() { // install the observed heap into ZendMM unsafe { - zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).heap); } }); @@ -177,7 +182,7 @@ pub fn alloc_prof_rshutdown() { let mut custom_mm_shutdown: Option = None; // Safety: `unwrap()` is safe here, as `heap` is initialized in `RINIT` - let heap = unsafe { (*zend_mm_state).heap.unwrap() }; + let heap = unsafe { (*zend_mm_state).heap }; unsafe { zend::zend_mm_get_custom_handlers_ex( heap, @@ -209,7 +214,7 @@ pub fn alloc_prof_rshutdown() { // This is the happy path. Restore previous heap. unsafe { zend::zend_mm_set_heap( - (*zend_mm_state).prev_heap.unwrap() + (*zend_mm_state).prev_heap ); } trace!("Memory allocation profiling shutdown gracefully."); @@ -241,19 +246,19 @@ unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).prev_heap); // Safety: `ZEND_MM_STATE.alloc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_alloc` is also initialised let ptr = ((*zend_mm_state).prev_custom_mm_alloc.unwrap())(len); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).heap); ptr }) } unsafe fn alloc_prof_orig_alloc(len: size_t) -> *mut c_void { - let ptr: *mut c_void = zend::_zend_mm_alloc(tls_zend_mm_state!(prev_heap).unwrap(), len); + let ptr: *mut c_void = zend::_zend_mm_alloc(tls_zend_mm_state!(prev_heap), len); ptr } @@ -269,18 +274,18 @@ unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).prev_heap); // Safety: `ZEND_MM_STATE.free` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_free` is also initialised ((*zend_mm_state).prev_custom_mm_free.unwrap())(ptr); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).heap); }) } unsafe fn alloc_prof_orig_free(ptr: *mut c_void) { - zend::_zend_mm_free(tls_zend_mm_state!(prev_heap).unwrap(), ptr); + zend::_zend_mm_free(tls_zend_mm_state!(prev_heap), ptr); } unsafe extern "C" fn alloc_prof_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { @@ -307,19 +312,19 @@ unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_ ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).prev_heap); // Safety: `ZEND_MM_STATE.realloc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_realloc` is also initialised let ptr = ((*zend_mm_state).prev_custom_mm_realloc.unwrap())(prev_ptr, len); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).heap); ptr }) } unsafe fn alloc_prof_orig_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - zend::_zend_mm_realloc(tls_zend_mm_state!(prev_heap).unwrap(), prev_ptr, len) + zend::_zend_mm_realloc(tls_zend_mm_state!(prev_heap), prev_ptr, len) } unsafe extern "C" fn alloc_prof_gc() -> size_t { @@ -330,19 +335,19 @@ unsafe fn alloc_prof_prev_gc() -> size_t { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).prev_heap); // Safety: `ZEND_MM_STATE.gc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_gc` is also initialised let freed = ((*zend_mm_state).prev_custom_mm_gc.unwrap())(); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).heap); freed }) } unsafe fn alloc_prof_orig_gc() -> size_t { - zend::zend_mm_gc(tls_zend_mm_state!(prev_heap).unwrap()) + zend::zend_mm_gc(tls_zend_mm_state!(prev_heap)) } unsafe extern "C" fn alloc_prof_shutdown(full: bool, silent: bool) { @@ -353,18 +358,18 @@ unsafe fn alloc_prof_prev_shutdown(full: bool, silent: bool) { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).prev_heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).prev_heap); // Safety: `ZEND_MM_STATE.shutdown` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_shutdown` is also initialised ((*zend_mm_state).prev_custom_mm_shutdown.unwrap())(full, silent); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` - zend::zend_mm_set_heap((*zend_mm_state).heap.unwrap()); + zend::zend_mm_set_heap((*zend_mm_state).heap); }) } unsafe fn alloc_prof_orig_shutdown(full: bool, silent: bool) { - zend::zend_mm_shutdown(tls_zend_mm_state!(prev_heap).unwrap(), full, silent) + zend::zend_mm_shutdown(tls_zend_mm_state!(prev_heap), full, silent) } /// safe wrapper for `zend::is_zend_mm()`. From 5b9d8c8c2346ca168b081d1c6f1d9a77c40dd697 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Mon, 25 Nov 2024 14:35:22 +0100 Subject: [PATCH 06/12] fix comments --- profiling/src/allocation/allocation_ge84.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index 10266b8156..32eac825ae 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -247,7 +247,7 @@ unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).prev_heap); - // Safety: `ZEND_MM_STATE.alloc` will be initialised in + // Safety: `ZEND_MM_STATE.prev_custom_mm_alloc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_alloc` is also initialised let ptr = ((*zend_mm_state).prev_custom_mm_alloc.unwrap())(len); @@ -275,7 +275,7 @@ unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).prev_heap); - // Safety: `ZEND_MM_STATE.free` will be initialised in + // Safety: `ZEND_MM_STATE.prev_custom_mm_free` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_free` is also initialised ((*zend_mm_state).prev_custom_mm_free.unwrap())(ptr); @@ -313,7 +313,7 @@ unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_ let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).prev_heap); - // Safety: `ZEND_MM_STATE.realloc` will be initialised in + // Safety: `ZEND_MM_STATE.prev_custom_mm_realloc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_realloc` is also initialised let ptr = ((*zend_mm_state).prev_custom_mm_realloc.unwrap())(prev_ptr, len); @@ -336,7 +336,7 @@ unsafe fn alloc_prof_prev_gc() -> size_t { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).prev_heap); - // Safety: `ZEND_MM_STATE.gc` will be initialised in + // Safety: `ZEND_MM_STATE.prev_custom_mm_gc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_gc` is also initialised let freed = ((*zend_mm_state).prev_custom_mm_gc.unwrap())(); @@ -359,7 +359,7 @@ unsafe fn alloc_prof_prev_shutdown(full: bool, silent: bool) { let zend_mm_state = cell.get(); // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).prev_heap); - // Safety: `ZEND_MM_STATE.shutdown` will be initialised in + // Safety: `ZEND_MM_STATE.prev_custom_mm_shutdown` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_shutdown` is also initialised ((*zend_mm_state).prev_custom_mm_shutdown.unwrap())(full, silent); From 0e78168491fd8631ce9c5600fcb50e38a352c08f Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Mon, 25 Nov 2024 17:23:29 +0100 Subject: [PATCH 07/12] These `unwrap()`'s can go unchecked :tada: --- profiling/src/allocation/allocation_ge84.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index 32eac825ae..6a8dafda87 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -250,7 +250,7 @@ unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { // Safety: `ZEND_MM_STATE.prev_custom_mm_alloc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_alloc` is also initialised - let ptr = ((*zend_mm_state).prev_custom_mm_alloc.unwrap())(len); + let ptr = ((*zend_mm_state).prev_custom_mm_alloc.unwrap_unchecked())(len); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).heap); ptr @@ -278,7 +278,7 @@ unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { // Safety: `ZEND_MM_STATE.prev_custom_mm_free` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_free` is also initialised - ((*zend_mm_state).prev_custom_mm_free.unwrap())(ptr); + ((*zend_mm_state).prev_custom_mm_free.unwrap_unchecked())(ptr); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).heap); }) @@ -316,7 +316,7 @@ unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_ // Safety: `ZEND_MM_STATE.prev_custom_mm_realloc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_realloc` is also initialised - let ptr = ((*zend_mm_state).prev_custom_mm_realloc.unwrap())(prev_ptr, len); + let ptr = ((*zend_mm_state).prev_custom_mm_realloc.unwrap_unchecked())(prev_ptr, len); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).heap); ptr @@ -339,7 +339,7 @@ unsafe fn alloc_prof_prev_gc() -> size_t { // Safety: `ZEND_MM_STATE.prev_custom_mm_gc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_gc` is also initialised - let freed = ((*zend_mm_state).prev_custom_mm_gc.unwrap())(); + let freed = ((*zend_mm_state).prev_custom_mm_gc.unwrap_unchecked())(); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).heap); freed @@ -362,7 +362,7 @@ unsafe fn alloc_prof_prev_shutdown(full: bool, silent: bool) { // Safety: `ZEND_MM_STATE.prev_custom_mm_shutdown` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_shutdown` is also initialised - ((*zend_mm_state).prev_custom_mm_shutdown.unwrap())(full, silent); + ((*zend_mm_state).prev_custom_mm_shutdown.unwrap_unchecked())(full, silent); // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` zend::zend_mm_set_heap((*zend_mm_state).heap); }) From 58b6e044240bfc05d7715275f411652d5e3d800f Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Mon, 2 Dec 2024 16:11:46 +0100 Subject: [PATCH 08/12] cleanup --- profiling/build.rs | 2 +- profiling/src/allocation/allocation_ge84.rs | 67 +++++++++++++++++++-- profiling/src/allocation/mod.rs | 39 +++++++++--- profiling/src/bindings/mod.rs | 4 +- profiling/src/config.rs | 4 +- profiling/src/lib.rs | 15 ++++- 6 files changed, 110 insertions(+), 21 deletions(-) diff --git a/profiling/build.rs b/profiling/build.rs index 01142cd8c9..37db945a45 100644 --- a/profiling/build.rs +++ b/profiling/build.rs @@ -361,7 +361,7 @@ fn cfg_php_feature_flags(vernum: u64) { if vernum >= 80400 { println!("cargo:rustc-cfg=php_frameless"); println!("cargo:rustc-cfg=php_opcache_restart_hook"); - println!("cargo:rustc-cfg=php_new_zendmm_hooks"); + println!("cargo:rustc-cfg=php_zend_mm_set_custom_handlers_ex"); } } diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index 6a8dafda87..c3fb11ae52 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -87,12 +87,24 @@ macro_rules! tls_zend_mm_state { }; } -pub fn alloc_prof_rinit() { +#[allow(dead_code)] +pub fn alloc_prof_minit() { + #[cfg(not(php_zts))] + alloc_prof_ginit(); +} + +#[allow(dead_code)] +pub fn alloc_prof_mshutdown() { + #[cfg(not(php_zts))] + alloc_prof_gshutdown(); +} + +pub fn alloc_prof_ginit() { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); // Only need to create an observed heap once per thread. When we have it, we can just - // install the observed hook via `zend::zend_mm_set_heap()` + // install the observed heap via `zend::zend_mm_set_heap()` if unsafe { (*zend_mm_state).heap.is_null() } { // Safety: `zend_mm_get_heap()` always returns a non-null pointer to a valid heap structure let prev_heap = unsafe { zend::zend_mm_get_heap() }; @@ -130,7 +142,7 @@ pub fn alloc_prof_rinit() { } } - // Create our observed heap and prepare custom handlers + // Create a new (to be observed) heap and prepare custom handlers let heap = unsafe { zend::zend_mm_startup() }; unsafe { ptr::addr_of_mut!((*zend_mm_state).heap).write(heap) }; @@ -145,10 +157,55 @@ pub fn alloc_prof_rinit() { Some(alloc_prof_shutdown), ); } + debug!("New observed heap created"); } + }); +} - // install the observed heap into ZendMM +pub fn alloc_prof_gshutdown() { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + unsafe { + // remove custom handlers to allow for ZendMM internal shutdown + zend::zend_mm_set_custom_handlers_ex( + (*zend_mm_state).heap, + None, + None, + None, + None, + None, + ); + + // reset to defaults + ptr::addr_of_mut!((*zend_mm_state).alloc).write(alloc_prof_orig_alloc); + ptr::addr_of_mut!((*zend_mm_state).free).write(alloc_prof_orig_free); + ptr::addr_of_mut!((*zend_mm_state).realloc).write(alloc_prof_orig_realloc); + ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_orig_gc); + ptr::addr_of_mut!((*zend_mm_state).shutdown).write(alloc_prof_orig_shutdown); + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_alloc).write(None); + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_free).write(None); + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_realloc).write(None); + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_gc).write(None); + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_shutdown).write(None); + + // This shutdown will free the observed heap we created in minit + zend::zend_mm_shutdown((*zend_mm_state).heap, true, true); + + ptr::addr_of_mut!((*zend_mm_state).heap).write(ptr::null_mut()); + ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(ptr::null_mut()); + + debug!("Observed heap was freed and `zend_mm_state` reset"); + } + }); +} + +pub fn alloc_prof_rinit() { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `zend_mm_state.heap` got initialized in `MINIT` and is guaranteed to + // be a non null pointer to a valid `zend::zend_mm_heap` struct unsafe { + // Install our observed heap into ZendMM zend::zend_mm_set_heap((*zend_mm_state).heap); } }); @@ -181,7 +238,7 @@ pub fn alloc_prof_rshutdown() { let mut custom_mm_gc: Option = None; let mut custom_mm_shutdown: Option = None; - // Safety: `unwrap()` is safe here, as `heap` is initialized in `RINIT` + // Safety: `unwrap()` is safe here, as `heap` is initialized in `MINIT` let heap = unsafe { (*zend_mm_state).heap }; unsafe { zend::zend_mm_get_custom_handlers_ex( diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs index f7303d112d..9980db0adf 100644 --- a/profiling/src/allocation/mod.rs +++ b/profiling/src/allocation/mod.rs @@ -8,9 +8,9 @@ use rand_distr::{Distribution, Poisson}; use std::cell::RefCell; use std::sync::atomic::AtomicU64; -#[cfg(php_new_zendmm_hooks)] +#[cfg(php_zend_mm_set_custom_handlers_ex)] mod allocation_ge84; -#[cfg(not(php_new_zendmm_hooks))] +#[cfg(not(php_zend_mm_set_custom_handlers_ex))] pub mod allocation_le83; /// take a sample every 4096 KiB @@ -78,12 +78,35 @@ thread_local! { } pub fn alloc_prof_minit() { - #[cfg(not(php_new_zendmm_hooks))] + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] allocation_le83::alloc_prof_minit(); + #[cfg(php_zend_mm_set_custom_handlers_ex)] + allocation_ge84::alloc_prof_minit(); } +#[allow(dead_code)] +pub fn alloc_prof_mshutdown() { + #[cfg(php_zend_mm_set_custom_handlers_ex)] + allocation_ge84::alloc_prof_mshutdown(); +} + +#[allow(dead_code)] +#[cfg(php_zts)] +pub fn alloc_prof_ginit() { + #[cfg(php_zend_mm_set_custom_handlers_ex)] + allocation_ge84::alloc_prof_ginit(); +} + +#[allow(dead_code)] +#[cfg(php_zts)] +pub fn alloc_prof_gshutdown() { + #[cfg(php_zend_mm_set_custom_handlers_ex)] + allocation_ge84::alloc_prof_gshutdown(); +} + +#[allow(dead_code)] pub fn alloc_prof_startup() { - #[cfg(not(php_new_zendmm_hooks))] + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] allocation_le83::alloc_prof_startup(); } @@ -105,9 +128,9 @@ pub fn alloc_prof_rinit() { return; } - #[cfg(not(php_new_zendmm_hooks))] + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] allocation_le83::alloc_prof_rinit(); - #[cfg(php_new_zendmm_hooks)] + #[cfg(php_zend_mm_set_custom_handlers_ex)] allocation_ge84::alloc_prof_rinit(); trace!("Memory allocation profiling enabled.") @@ -124,8 +147,8 @@ pub fn alloc_prof_rshutdown() { return; } - #[cfg(not(php_new_zendmm_hooks))] + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] allocation_le83::alloc_prof_rshutdown(); - #[cfg(php_new_zendmm_hooks)] + #[cfg(php_zend_mm_set_custom_handlers_ex)] allocation_ge84::alloc_prof_rshutdown(); } diff --git a/profiling/src/bindings/mod.rs b/profiling/src/bindings/mod.rs index 70ddafd038..93f8ad5a91 100644 --- a/profiling/src/bindings/mod.rs +++ b/profiling/src/bindings/mod.rs @@ -46,9 +46,9 @@ pub type VmMmCustomAllocFn = unsafe extern "C" fn(size_t) -> *mut c_void; pub type VmMmCustomReallocFn = unsafe extern "C" fn(*mut c_void, size_t) -> *mut c_void; #[cfg(feature = "allocation_profiling")] pub type VmMmCustomFreeFn = unsafe extern "C" fn(*mut c_void); -#[cfg(all(feature = "allocation_profiling", php_new_zendmm_hooks))] +#[cfg(all(feature = "allocation_profiling", php_zend_mm_set_custom_handlers_ex))] pub type VmMmCustomGcFn = unsafe extern "C" fn() -> size_t; -#[cfg(all(feature = "allocation_profiling", php_new_zendmm_hooks))] +#[cfg(all(feature = "allocation_profiling", php_zend_mm_set_custom_handlers_ex))] pub type VmMmCustomShutdownFn = unsafe extern "C" fn(bool, bool); // todo: this a lie on some PHP versions; is it a problem even though zend_bool diff --git a/profiling/src/config.rs b/profiling/src/config.rs index b84650fbb2..c21185e838 100644 --- a/profiling/src/config.rs +++ b/profiling/src/config.rs @@ -1,4 +1,4 @@ -#[cfg(not(php_new_zendmm_hooks))] +#[cfg(not(php_zend_mm_set_custom_handlers_ex))] use crate::allocation; use crate::bindings::zai_config_type::*; use crate::bindings::{ @@ -102,7 +102,7 @@ impl SystemSettings { } // Work around version-specific issues. - #[cfg(not(php_new_zendmm_hooks))] + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] if allocation::allocation_le83::first_rinit_should_disable_due_to_jit() { system_settings.profiling_allocation_enabled = false; } diff --git a/profiling/src/lib.rs b/profiling/src/lib.rs index 3b3db288a0..e15efb26b1 100644 --- a/profiling/src/lib.rs +++ b/profiling/src/lib.rs @@ -203,12 +203,18 @@ pub extern "C" fn get_module() -> &'static mut zend::ModuleEntry { unsafe extern "C" fn ginit(_globals_ptr: *mut c_void) { #[cfg(all(feature = "timeline", php_zts))] timeline::timeline_ginit(); + + #[cfg(feature = "allocation_profiling")] + allocation::alloc_prof_ginit(); } #[cfg(php_zts)] unsafe extern "C" fn gshutdown(_globals_ptr: *mut c_void) { #[cfg(all(feature = "timeline", php_zts))] timeline::timeline_gshutdown(); + + #[cfg(feature = "allocation_profiling")] + allocation::alloc_prof_gshutdown(); } /* Important note on the PHP lifecycle: @@ -337,15 +343,15 @@ extern "C" fn minit(_type: c_int, module_number: c_int) -> ZendResult { */ unsafe { zend::zend_register_extension(&extension, handle) }; - #[cfg(feature = "allocation_profiling")] - allocation::alloc_prof_minit(); - #[cfg(feature = "timeline")] timeline::timeline_minit(); #[cfg(feature = "exception_profiling")] exception::exception_profiling_minit(); + #[cfg(feature = "allocation_profiling")] + allocation::alloc_prof_minit(); + // There are a few things which need to do something on the first rinit of // each minit/mshutdown cycle. In Apache, when doing `apachectl graceful`, // there can be more than one of these cycles per process. @@ -851,6 +857,9 @@ extern "C" fn mshutdown(_type: c_int, _module_number: c_int) -> ZendResult { #[cfg(feature = "exception_profiling")] exception::exception_profiling_mshutdown(); + #[cfg(feature = "allocation_profiling")] + allocation::alloc_prof_mshutdown(); + Profiler::stop(Duration::from_secs(1)); ZendResult::Success From 0cff885d3eeb3f514e4b553a77d3acf8cf868742 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Tue, 3 Dec 2024 16:56:16 +0100 Subject: [PATCH 09/12] cleanup --- profiling/src/allocation/allocation_ge84.rs | 40 +++++++++++++++------ 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index c3fb11ae52..b9dd34d2e6 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -51,12 +51,9 @@ struct ZendMMState { shutdown: unsafe fn(bool, bool), } -thread_local! { - /// Using an `UnsafeCell` here should be okay. There might not be any - /// synchronisation issues, as it is used in as thread local and only - /// mutated in RINIT and RSHUTDOWN. - static ZEND_MM_STATE: UnsafeCell = const { - UnsafeCell::new(ZendMMState { +impl ZendMMState { + const fn new() -> ZendMMState { + ZendMMState { // Safety: Using `ptr::null_mut()` might seem dangerous but actually it is okay in this // case. The `heap` and `prev_heap` fields will be initialized in the first call to // RINIT and only used after that. By using this "trick" we can get rid of all @@ -74,7 +71,18 @@ thread_local! { free: alloc_prof_orig_free, gc: alloc_prof_orig_gc, shutdown: alloc_prof_orig_shutdown, - }) + } + } +} + +impl ZendMMState {} + +thread_local! { + /// Using an `UnsafeCell` here should be okay. There might not be any + /// synchronisation issues, as it is used in as thread local and only + /// mutated in RINIT and RSHUTDOWN. + static ZEND_MM_STATE: UnsafeCell = const { + UnsafeCell::new(ZendMMState::new()) }; } @@ -90,16 +98,28 @@ macro_rules! tls_zend_mm_state { #[allow(dead_code)] pub fn alloc_prof_minit() { #[cfg(not(php_zts))] - alloc_prof_ginit(); + alloc_prof_hook(); } #[allow(dead_code)] pub fn alloc_prof_mshutdown() { #[cfg(not(php_zts))] - alloc_prof_gshutdown(); + alloc_prof_unhook(); } +#[allow(dead_code)] pub fn alloc_prof_ginit() { + #[cfg(php_zts)] + alloc_prof_hook(); +} + +#[allow(dead_code)] +pub fn alloc_prof_gshutdown() { + #[cfg(php_zts)] + alloc_prof_unhook(); +} + +fn alloc_prof_hook() { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); @@ -162,7 +182,7 @@ pub fn alloc_prof_ginit() { }); } -pub fn alloc_prof_gshutdown() { +fn alloc_prof_unhook() { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); unsafe { From 82c7e52e4a1927974e46ffb3ec2d8f90027ae163 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Wed, 4 Dec 2024 12:48:58 +0100 Subject: [PATCH 10/12] add moar comments --- profiling/src/allocation/allocation_ge84.rs | 129 +++++++++++--------- 1 file changed, 70 insertions(+), 59 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index b9dd34d2e6..1d19bc6181 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -98,95 +98,105 @@ macro_rules! tls_zend_mm_state { #[allow(dead_code)] pub fn alloc_prof_minit() { #[cfg(not(php_zts))] - alloc_prof_hook(); + alloc_prof_custom_heap_init(); } #[allow(dead_code)] pub fn alloc_prof_mshutdown() { #[cfg(not(php_zts))] - alloc_prof_unhook(); + alloc_prof_custom_heap_reset(); } #[allow(dead_code)] pub fn alloc_prof_ginit() { #[cfg(php_zts)] - alloc_prof_hook(); + alloc_prof_custom_heap_init(); } #[allow(dead_code)] pub fn alloc_prof_gshutdown() { #[cfg(php_zts)] - alloc_prof_unhook(); + alloc_prof_custom_heap_reset(); } -fn alloc_prof_hook() { +/// This initializes the thread locale variable `ZEND_MM_STATE` with respect to the currently +/// installed `zend_mm_heap` in ZendMM. It guarantees compliance with the safety guarantees +/// described in the `ZendMMState` structure, specifically for `ZendMMState::alloc`, +/// `ZendMMState::realloc`, `ZendMMState::free`, `ZendMMState::gc` and `ZendMMState::shutdown`. +/// This function may panic if called out of order! +fn alloc_prof_custom_heap_init() { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); // Only need to create an observed heap once per thread. When we have it, we can just // install the observed heap via `zend::zend_mm_set_heap()` - if unsafe { (*zend_mm_state).heap.is_null() } { - // Safety: `zend_mm_get_heap()` always returns a non-null pointer to a valid heap structure - let prev_heap = unsafe { zend::zend_mm_get_heap() }; - unsafe { ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(prev_heap) }; - - if !is_zend_mm() { - // Neighboring custom memory handlers found in the currently used ZendMM heap - debug!("Found another extension using the ZendMM custom handler hook"); - unsafe { - zend::zend_mm_get_custom_handlers_ex( - prev_heap, - ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_alloc), - ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_free), - ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_realloc), - ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_gc), - ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_shutdown), - ); - ptr::addr_of_mut!((*zend_mm_state).alloc).write(alloc_prof_prev_alloc); - ptr::addr_of_mut!((*zend_mm_state).free).write(alloc_prof_prev_free); - ptr::addr_of_mut!((*zend_mm_state).realloc).write(alloc_prof_prev_realloc); - // `gc` handler can be NULL - if (*zend_mm_state).prev_custom_mm_gc.is_none() { - ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_orig_gc); - } else { - ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_prev_gc); - } - // `shutdown` handler can be NULL - if (*zend_mm_state).prev_custom_mm_shutdown.is_none() { - ptr::addr_of_mut!((*zend_mm_state).shutdown) - .write(alloc_prof_orig_shutdown); - } else { - ptr::addr_of_mut!((*zend_mm_state).shutdown) - .write(alloc_prof_prev_shutdown); - } - } - } + if unsafe { !(*zend_mm_state).heap.is_null() } { + // This can only happen if either MINIT or GINIT is being called out of order. + panic!("MINIT/GINIT was called with an already initialized allocation profiler. Most likely the SAPI did this without going through MSHUTDOWN/GSHUTDOWN before."); + } - // Create a new (to be observed) heap and prepare custom handlers - let heap = unsafe { zend::zend_mm_startup() }; - unsafe { ptr::addr_of_mut!((*zend_mm_state).heap).write(heap) }; + // Safety: `zend_mm_get_heap()` always returns a non-null pointer to a valid heap structure + let prev_heap = unsafe { zend::zend_mm_get_heap() }; + unsafe { ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(prev_heap) }; - // install our custom handler to ZendMM + if !is_zend_mm() { + // Neighboring custom memory handlers found in the currently used ZendMM heap + debug!("Found another extension using the ZendMM custom handler hook"); unsafe { - zend::zend_mm_set_custom_handlers_ex( - (*zend_mm_state).heap, - Some(alloc_prof_malloc), - Some(alloc_prof_free), - Some(alloc_prof_realloc), - Some(alloc_prof_gc), - Some(alloc_prof_shutdown), + zend::zend_mm_get_custom_handlers_ex( + prev_heap, + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_alloc), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_free), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_realloc), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_gc), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_shutdown), ); + ptr::addr_of_mut!((*zend_mm_state).alloc).write(alloc_prof_prev_alloc); + ptr::addr_of_mut!((*zend_mm_state).free).write(alloc_prof_prev_free); + ptr::addr_of_mut!((*zend_mm_state).realloc).write(alloc_prof_prev_realloc); + // `gc` handler can be NULL + if (*zend_mm_state).prev_custom_mm_gc.is_none() { + ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_orig_gc); + } else { + ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_prev_gc); + } + // `shutdown` handler can be NULL + if (*zend_mm_state).prev_custom_mm_shutdown.is_none() { + ptr::addr_of_mut!((*zend_mm_state).shutdown).write(alloc_prof_orig_shutdown); + } else { + ptr::addr_of_mut!((*zend_mm_state).shutdown).write(alloc_prof_prev_shutdown); + } } - debug!("New observed heap created"); } + + // Create a new (to be observed) heap and prepare custom handlers + let heap = unsafe { zend::zend_mm_startup() }; + unsafe { ptr::addr_of_mut!((*zend_mm_state).heap).write(heap) }; + + // install our custom handler to ZendMM + unsafe { + zend::zend_mm_set_custom_handlers_ex( + (*zend_mm_state).heap, + Some(alloc_prof_malloc), + Some(alloc_prof_free), + Some(alloc_prof_realloc), + Some(alloc_prof_gc), + Some(alloc_prof_shutdown), + ); + } + debug!("New observed heap created"); }); } -fn alloc_prof_unhook() { +/// This resets the thread locale variable `ZEND_MM_STATE` and frees allocated memory. It +/// guarantees compliance with the safety guarantees described in the `ZendMMState` structure, +/// specifically for `ZendMMState::alloc`, `ZendMMState::realloc`, `ZendMMState::free`, +/// `ZendMMState::gc` and `ZendMMState::shutdown`. +fn alloc_prof_custom_heap_reset() { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); unsafe { - // remove custom handlers to allow for ZendMM internal shutdown + // Remove custom handlers to allow for ZendMM internal shutdown zend::zend_mm_set_custom_handlers_ex( (*zend_mm_state).heap, None, @@ -196,7 +206,8 @@ fn alloc_prof_unhook() { None, ); - // reset to defaults + // Reset ZEND_MM_STATE to defaults, now that the pointer are not know to the observed + // heap anymore. ptr::addr_of_mut!((*zend_mm_state).alloc).write(alloc_prof_orig_alloc); ptr::addr_of_mut!((*zend_mm_state).free).write(alloc_prof_orig_free); ptr::addr_of_mut!((*zend_mm_state).realloc).write(alloc_prof_orig_realloc); @@ -208,14 +219,14 @@ fn alloc_prof_unhook() { ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_gc).write(None); ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_shutdown).write(None); - // This shutdown will free the observed heap we created in minit + // This shutdown call will free the observed heap we created in `alloc_prof_custom_heap_init` zend::zend_mm_shutdown((*zend_mm_state).heap, true, true); + // Now that the heap is gone, we need to NULL the pointer ptr::addr_of_mut!((*zend_mm_state).heap).write(ptr::null_mut()); ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(ptr::null_mut()); - - debug!("Observed heap was freed and `zend_mm_state` reset"); } + trace!("Observed heap was freed and `zend_mm_state` reset"); }); } From 0941a905ebde4f05868e2d9f65ac316c906caca8 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Thu, 5 Dec 2024 11:19:54 +0100 Subject: [PATCH 11/12] cleanup --- profiling/src/allocation/allocation_ge84.rs | 28 ++------------------- profiling/src/allocation/allocation_le83.rs | 2 +- profiling/src/allocation/mod.rs | 22 +++------------- profiling/src/bindings/mod.rs | 4 +-- profiling/src/lib.rs | 16 +++--------- 5 files changed, 11 insertions(+), 61 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index 1d19bc6181..6be8ba5fcc 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -95,36 +95,12 @@ macro_rules! tls_zend_mm_state { }; } -#[allow(dead_code)] -pub fn alloc_prof_minit() { - #[cfg(not(php_zts))] - alloc_prof_custom_heap_init(); -} - -#[allow(dead_code)] -pub fn alloc_prof_mshutdown() { - #[cfg(not(php_zts))] - alloc_prof_custom_heap_reset(); -} - -#[allow(dead_code)] -pub fn alloc_prof_ginit() { - #[cfg(php_zts)] - alloc_prof_custom_heap_init(); -} - -#[allow(dead_code)] -pub fn alloc_prof_gshutdown() { - #[cfg(php_zts)] - alloc_prof_custom_heap_reset(); -} - /// This initializes the thread locale variable `ZEND_MM_STATE` with respect to the currently /// installed `zend_mm_heap` in ZendMM. It guarantees compliance with the safety guarantees /// described in the `ZendMMState` structure, specifically for `ZendMMState::alloc`, /// `ZendMMState::realloc`, `ZendMMState::free`, `ZendMMState::gc` and `ZendMMState::shutdown`. /// This function may panic if called out of order! -fn alloc_prof_custom_heap_init() { +pub fn alloc_prof_ginit() { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); @@ -192,7 +168,7 @@ fn alloc_prof_custom_heap_init() { /// guarantees compliance with the safety guarantees described in the `ZendMMState` structure, /// specifically for `ZendMMState::alloc`, `ZendMMState::realloc`, `ZendMMState::free`, /// `ZendMMState::gc` and `ZendMMState::shutdown`. -fn alloc_prof_custom_heap_reset() { +pub fn alloc_prof_gshutdown() { ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); unsafe { diff --git a/profiling/src/allocation/allocation_le83.rs b/profiling/src/allocation/allocation_le83.rs index e58b1953b2..8484fb5304 100644 --- a/profiling/src/allocation/allocation_le83.rs +++ b/profiling/src/allocation/allocation_le83.rs @@ -87,7 +87,7 @@ lazy_static! { static ref JIT_ENABLED: bool = unsafe { zend::ddog_php_jit_enabled() }; } -pub fn alloc_prof_minit() { +pub fn alloc_prof_ginit() { unsafe { zend::ddog_php_opcache_init_handle() }; } diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs index 9980db0adf..f20e2e31be 100644 --- a/profiling/src/allocation/mod.rs +++ b/profiling/src/allocation/mod.rs @@ -77,36 +77,20 @@ thread_local! { RefCell::new(AllocationProfilingStats::new()); } -pub fn alloc_prof_minit() { - #[cfg(not(php_zend_mm_set_custom_handlers_ex))] - allocation_le83::alloc_prof_minit(); - #[cfg(php_zend_mm_set_custom_handlers_ex)] - allocation_ge84::alloc_prof_minit(); -} - -#[allow(dead_code)] -pub fn alloc_prof_mshutdown() { - #[cfg(php_zend_mm_set_custom_handlers_ex)] - allocation_ge84::alloc_prof_mshutdown(); -} - -#[allow(dead_code)] -#[cfg(php_zts)] pub fn alloc_prof_ginit() { + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] + allocation_le83::alloc_prof_ginit(); #[cfg(php_zend_mm_set_custom_handlers_ex)] allocation_ge84::alloc_prof_ginit(); } -#[allow(dead_code)] -#[cfg(php_zts)] pub fn alloc_prof_gshutdown() { #[cfg(php_zend_mm_set_custom_handlers_ex)] allocation_ge84::alloc_prof_gshutdown(); } -#[allow(dead_code)] +#[cfg(not(php_zend_mm_set_custom_handlers_ex))] pub fn alloc_prof_startup() { - #[cfg(not(php_zend_mm_set_custom_handlers_ex))] allocation_le83::alloc_prof_startup(); } diff --git a/profiling/src/bindings/mod.rs b/profiling/src/bindings/mod.rs index 93f8ad5a91..2fee76570c 100644 --- a/profiling/src/bindings/mod.rs +++ b/profiling/src/bindings/mod.rs @@ -179,13 +179,13 @@ pub struct ModuleEntry { /// thread for module globals. The function pointers in [`ModuleEntry::globals_ctor`] and /// [`ModuleEntry::globals_dtor`] will only be called if this is a non-zero. pub globals_size: size_t, - #[cfg(php_zts)] /// Pointer to a `ts_rsrc_id` (which is a [`i32`]). For C-Extension this is created using the /// `ZEND_DECLARE_MODULE_GLOBALS(module_name)` macro. /// See + #[cfg(php_zts)] pub globals_id_ptr: *mut ts_rsrc_id, - #[cfg(not(php_zts))] /// Pointer to the module globals struct in NTS mode + #[cfg(not(php_zts))] pub globals_ptr: *mut c_void, /// Constructor for module globals. /// Be aware this will only be called in case [`ModuleEntry::globals_size`] is non-zero and for diff --git a/profiling/src/lib.rs b/profiling/src/lib.rs index e15efb26b1..f8cf7c44be 100644 --- a/profiling/src/lib.rs +++ b/profiling/src/lib.rs @@ -31,7 +31,6 @@ use core::ptr; use ddcommon::{cstr, tag, tag::Tag}; use lazy_static::lazy_static; use libc::c_char; -#[cfg(php_zts)] use libc::c_void; use log::{debug, error, info, trace, warn}; use once_cell::sync::{Lazy, OnceCell}; @@ -184,14 +183,13 @@ pub extern "C" fn get_module() -> &'static mut zend::ModuleEntry { version: PROFILER_VERSION.as_ptr(), post_deactivate_func: Some(prshutdown), deps: DEPS.as_ptr(), - #[cfg(php_zts)] globals_ctor: Some(ginit), - #[cfg(php_zts)] globals_dtor: Some(gshutdown), - #[cfg(php_zts)] globals_size: 1, #[cfg(php_zts)] globals_id_ptr: unsafe { &mut GLOBALS_ID_PTR }, + #[cfg(not(php_zts))] + globals_ptr: ptr::null_mut(), ..Default::default() }); @@ -199,7 +197,6 @@ pub extern "C" fn get_module() -> &'static mut zend::ModuleEntry { unsafe { &mut *ptr::addr_of_mut!(MODULE) } } -#[cfg(php_zts)] unsafe extern "C" fn ginit(_globals_ptr: *mut c_void) { #[cfg(all(feature = "timeline", php_zts))] timeline::timeline_ginit(); @@ -208,7 +205,6 @@ unsafe extern "C" fn ginit(_globals_ptr: *mut c_void) { allocation::alloc_prof_ginit(); } -#[cfg(php_zts)] unsafe extern "C" fn gshutdown(_globals_ptr: *mut c_void) { #[cfg(all(feature = "timeline", php_zts))] timeline::timeline_gshutdown(); @@ -349,9 +345,6 @@ extern "C" fn minit(_type: c_int, module_number: c_int) -> ZendResult { #[cfg(feature = "exception_profiling")] exception::exception_profiling_minit(); - #[cfg(feature = "allocation_profiling")] - allocation::alloc_prof_minit(); - // There are a few things which need to do something on the first rinit of // each minit/mshutdown cycle. In Apache, when doing `apachectl graceful`, // there can be more than one of these cycles per process. @@ -857,9 +850,6 @@ extern "C" fn mshutdown(_type: c_int, _module_number: c_int) -> ZendResult { #[cfg(feature = "exception_profiling")] exception::exception_profiling_mshutdown(); - #[cfg(feature = "allocation_profiling")] - allocation::alloc_prof_mshutdown(); - Profiler::stop(Duration::from_secs(1)); ZendResult::Success @@ -884,7 +874,7 @@ extern "C" fn startup(extension: *mut ZendExtension) -> ZendResult { timeline::timeline_startup(); } - #[cfg(feature = "allocation_profiling")] + #[cfg(all(feature = "allocation_profiling", not(php_zend_mm_set_custom_handlers_ex)))] allocation::alloc_prof_startup(); ZendResult::Success From af5825e40cc5cbb48d39cb38aa32588232bf9f07 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Tue, 14 Jan 2025 09:02:47 +0100 Subject: [PATCH 12/12] fix clippy warning --- profiling/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiling/src/lib.rs b/profiling/src/lib.rs index f8cf7c44be..37ecddedb0 100644 --- a/profiling/src/lib.rs +++ b/profiling/src/lib.rs @@ -187,7 +187,7 @@ pub extern "C" fn get_module() -> &'static mut zend::ModuleEntry { globals_dtor: Some(gshutdown), globals_size: 1, #[cfg(php_zts)] - globals_id_ptr: unsafe { &mut GLOBALS_ID_PTR }, + globals_id_ptr: unsafe { ptr::addr_of_mut!(GLOBALS_ID_PTR) }, #[cfg(not(php_zts))] globals_ptr: ptr::null_mut(), ..Default::default()