From 46f0760e7ed2b80c46acd91bacc130049620bee6 Mon Sep 17 00:00:00 2001 From: "Niall Douglas (s [underscore] sourceforge {at} nedprod [dot] com)" Date: Sat, 21 Aug 2021 01:53:24 +0100 Subject: [PATCH] Reimplemented current_process_memory_usage() for Linux so it's a lot faster for very large programs. Debugged the map handle cache somewhat, but it's still failing. Tomorrow! --- cmake/tests.cmake | 1 + include/llfio/revision.hpp | 6 +- include/llfio/v2.0/detail/impl/map_handle.ipp | 17 +- .../v2.0/detail/impl/posix/map_handle.ipp | 22 +- .../llfio/v2.0/detail/impl/posix/utils.ipp | 246 ++++++++++++------ .../v2.0/detail/impl/windows/map_handle.ipp | 19 +- .../llfio/v2.0/detail/impl/windows/utils.ipp | 4 +- include/llfio/v2.0/llfio.hpp | 2 +- include/llfio/v2.0/map_handle.hpp | 9 +- include/llfio/v2.0/utils.hpp | 40 ++- test/tests/map_handle_cache.cpp | 108 ++++++++ test/tests/utils.cpp | 7 +- 12 files changed, 372 insertions(+), 109 deletions(-) create mode 100644 test/tests/map_handle_cache.cpp diff --git a/cmake/tests.cmake b/cmake/tests.cmake index de7d54d49..c8eabb3a1 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -18,6 +18,7 @@ set(llfio_TESTS "test/tests/issue0028.cpp" "test/tests/issue0073.cpp" "test/tests/large_pages.cpp" + "test/tests/map_handle_cache.cpp" "test/tests/map_handle_create_close/kernel_map_handle.cpp.hpp" "test/tests/map_handle_create_close/runner.cpp" "test/tests/mapped.cpp" diff --git a/include/llfio/revision.hpp b/include/llfio/revision.hpp index 809c1844d..8227a794d 100644 --- a/include/llfio/revision.hpp +++ b/include/llfio/revision.hpp @@ -1,4 +1,4 @@ // Note the second line of this file must ALWAYS be the git SHA, third line ALWAYS the git SHA update time -#define LLFIO_PREVIOUS_COMMIT_REF e43ef4f6953230803a30453bbab20061b009fe05 -#define LLFIO_PREVIOUS_COMMIT_DATE "2021-08-20 09:36:50 +00:00" -#define LLFIO_PREVIOUS_COMMIT_UNIQUE e43ef4f6 +#define LLFIO_PREVIOUS_COMMIT_REF d3ff87fd8b91f06d4f7bd240860ef68f15a03621 +#define LLFIO_PREVIOUS_COMMIT_DATE "2021-08-20 20:17:59 +00:00" +#define LLFIO_PREVIOUS_COMMIT_UNIQUE d3ff87fd diff --git a/include/llfio/v2.0/detail/impl/map_handle.ipp b/include/llfio/v2.0/detail/impl/map_handle.ipp index b4df17245..f15de0806 100644 --- a/include/llfio/v2.0/detail/impl/map_handle.ipp +++ b/include/llfio/v2.0/detail/impl/map_handle.ipp @@ -73,7 +73,7 @@ namespace detail { const auto _bytes = bytes >> page_size_shift; _lock_guard g(lock); - auto it = _base::find_equal_or_larger(_bytes, 7 /* 99% close to the key after page size shift */); + auto it = _base::find(_bytes); for(; it != _base::end() && page_size != it->page_size && _bytes == it->trie_key; ++it) { } @@ -106,11 +106,22 @@ namespace detail { for(auto it = _base::begin(); it != _base::end();) { - if(it->when_added < older_than) + if(it->when_added <= older_than) { auto *p = *it; it = _base::erase(it); const auto _bytes = p->trie_key << page_size_shift; +#ifdef _WIN32 + if(!win32_release_nonfile_allocations((byte *) p->addr, _bytes, MEM_RELEASE)) +#else + if(-1 == ::munmap(p->addr, _bytes)) +#endif + { + LLFIO_LOG_FATAL(nullptr, "map_handle cache failed to trim a map! If on Linux, you may have exceeded the " + "64k VMA process limit, set the LLFIO_DEBUG_LINUX_MUNMAP macro at the top of posix/map_handle.ipp to cause dumping of VMAs to " + "/tmp/llfio_unmap_debug_smaps.txt, and combine with strace to figure it out."); + abort(); + } _base::bytes_in_cache -= _bytes; ret.bytes_just_trimmed += _bytes; ret.items_just_trimmed++; @@ -148,7 +159,7 @@ result map_handle::_recycled_map(size_type bytes, section_handle::fl void *addr = detail::map_handle_cache().get(bytes, pagesize); if(addr == nullptr) { - return _new_map(bytes, _flag); + return _new_map(bytes, false, _flag); } #ifdef _WIN32 { diff --git a/include/llfio/v2.0/detail/impl/posix/map_handle.ipp b/include/llfio/v2.0/detail/impl/posix/map_handle.ipp index f77954caf..bc6bf3450 100644 --- a/include/llfio/v2.0/detail/impl/posix/map_handle.ipp +++ b/include/llfio/v2.0/detail/impl/posix/map_handle.ipp @@ -298,7 +298,7 @@ static inline result do_mmap(native_handle_type &nativeh, void *ataddr, bool have_backing = (section != nullptr); int prot = 0, flags = have_backing ? MAP_SHARED : (MAP_PRIVATE | MAP_ANONYMOUS); void *addr = nullptr; - if(_flag == section_handle::flag::none) + if(!(_flag & section_handle::flag::read) && !(_flag & section_handle::flag::write) && !(_flag & section_handle::flag::execute)) { prot |= PROT_NONE; #ifdef MAP_GUARD @@ -424,7 +424,7 @@ static inline result do_mmap(native_handle_type &nativeh, void *ataddr, return addr; } -result map_handle::_new_map(size_type bytes, section_handle::flag _flag) noexcept +result map_handle::_new_map(size_type bytes, bool fallback, section_handle::flag _flag) noexcept { if(bytes == 0u) { @@ -434,8 +434,22 @@ result map_handle::_new_map(size_type bytes, section_handle::flag _f result ret(map_handle(nullptr, _flag)); native_handle_type &nativeh = ret.value()._v; OUTCOME_TRY(auto &&pagesize, detail::pagesize_from_flags(ret.value()._flag)); - OUTCOME_TRY(auto &&addr, do_mmap(nativeh, nullptr, 0, nullptr, pagesize, bytes, 0, ret.value()._flag)); - ret.value()._addr = static_cast(addr); + auto addr = do_mmap(nativeh, nullptr, 0, nullptr, pagesize, bytes, 0, ret.value()._flag); + if(!addr) + { + if(fallback && addr.error() == errc::not_enough_memory) + { + // Try the cache + auto r = _recycled_map(bytes, _flag); + if(r) + { + memset(r.assume_value().address(), 0, r.assume_value().length()); + return r; + } + } + return std::move(addr).error(); + } + ret.value()._addr = static_cast(addr.assume_value()); ret.value()._reservation = bytes; ret.value()._length = bytes; ret.value()._pagesize = pagesize; diff --git a/include/llfio/v2.0/detail/impl/posix/utils.ipp b/include/llfio/v2.0/detail/impl/posix/utils.ipp index 7b734a45f..2332ae91f 100644 --- a/include/llfio/v2.0/detail/impl/posix/utils.ipp +++ b/include/llfio/v2.0/detail/impl/posix/utils.ipp @@ -214,11 +214,96 @@ namespace utils return false; } - result current_process_memory_usage() noexcept + result current_process_memory_usage(process_memory_usage::want want) noexcept { #ifdef __linux__ try { + auto fill_buffer = [](std::vector &buffer, const char *path) -> result { + for(;;) + { + int ih = ::open(path, O_RDONLY); + if(ih == -1) + { + return posix_error(); + } + size_t totalbytesread = 0; + for(;;) + { + auto bytesread = ::read(ih, buffer.data() + totalbytesread, buffer.size() - totalbytesread); + if(bytesread < 0) + { + ::close(ih); + return posix_error(); + } + if(bytesread == 0) + { + break; + } + totalbytesread += bytesread; + } + ::close(ih); + if(totalbytesread < buffer.size()) + { + buffer.resize(totalbytesread); + break; + } + buffer.resize(buffer.size() * 2); + } + return success(); + }; + auto parse = [](string_view item, string_view what) -> result { + auto idx = item.find(what); + if(string_view::npos == idx) + { + return (uint64_t) -1; + } + idx += what.size(); + for(; item[idx] == ' '; idx++) + ; + auto eidx = idx; + for(; item[eidx] != '\n'; eidx++) + ; + string_view unit(item.substr(eidx - 2, 2)); + uint64_t value = atoll(item.data() + idx); + if(unit == "kB") + { + value *= 1024ULL; + } + else if(unit == "mB") + { + value *= 1024ULL * 1024; + } + else if(unit == "gB") + { + value *= 1024ULL * 1024 * 1024; + } + else if(unit == "tB") + { + value *= 1024ULL * 1024 * 1024 * 1024; + } + else if(unit == "pB") + { + value *= 1024ULL * 1024 * 1024 * 1024 * 1024; + } + else if(unit == "eB") + { + value *= 1024ULL * 1024 * 1024 * 1024 * 1024 * 1024; + } + else if(unit == "zB") + { + value *= 1024ULL * 1024 * 1024 * 1024 * 1024 * 1024 * 1024; + } + else if(unit == "yB") + { + value *= 1024ULL * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024; + } + else + { + return errc::illegal_byte_sequence; + } + return value; + }; /* /proc/[pid]/status: total_address_space_in_use = VmSize @@ -230,40 +315,90 @@ namespace utils total_address_space_in_use = Sum of Size total_address_space_paged_in = Sum of Rss - private_committed = Sum of Size for all entries with VmFlags containing ac, and inode = 0? - private_paged_in = (Sum of Anonymous - Sum of LazyFree) for all entries with VmFlags containing ac, and inode = 0? + private_committed = (Sum of Anonymous - Sum of LazyFree) for all entries with VmFlags containing ac, and inode = 0? + private_paged_in = Sum of Rss for all entries with VmFlags containing ac, and inode = 0? + + /proc/[pid]/maps: + + hexstart-hexend rw-p offset devid:devid inode pathname + + total_address_space_in_use = Sum of regions + total_address_space_paged_in = ??? MISSING + private_committed = Sum of Size for all entries with rw-p, and inode = 0? + private_paged_in = ??? MISSING + + /proc/[pid]/statm: + + %zu %zu %zu ... total_address_space_in_use total_address_space_paged_in file_shared_pages_paged_in + + (values are in pages) + + /proc/[pid]/smaps_rollup: + + total_address_space_in_use = ??? MISSING + total_address_space_paged_in = ??? MISSING + private_committed = Anonymous - LazyFree (but, can't distinguish reservations!) + private_paged_in = ??? MISSING + */ - std::vector buffer(65536); - for(;;) + if(want & process_memory_usage::want::private_committed_inaccurate) { - int ih = ::open("/proc/self/smaps", O_RDONLY); - if(ih == -1) + process_memory_usage ret; + if((want & process_memory_usage::want::total_address_space_in_use) || (want & process_memory_usage::want::total_address_space_paged_in) || + (want & process_memory_usage::want::private_paged_in)) { - return posix_error(); + std::vector buffer(256); + OUTCOME_TRY(fill_buffer(buffer, "/proc/self/statm")); + if(buffer.size() > 1) + { + size_t file_and_shared_pages_paged_in = 0; + sscanf(buffer.data(), "%zu %zu %zu", &ret.total_address_space_in_use, &ret.total_address_space_paged_in, &file_and_shared_pages_paged_in); + ret.private_paged_in = ret.total_address_space_paged_in - file_and_shared_pages_paged_in; + ret.total_address_space_in_use *= page_size(); + ret.total_address_space_paged_in *= page_size(); + ret.private_paged_in *= page_size(); + //std::cout << string_view(buffer.data(), buffer.size()) << std::endl; + } } - size_t totalbytesread = 0; - for(;;) + if(want & process_memory_usage::want::private_committed) { - auto bytesread = ::read(ih, buffer.data() + totalbytesread, buffer.size() - totalbytesread); - if(bytesread < 0) + std::vector smaps_rollup(256), maps(65536); + OUTCOME_TRY(fill_buffer(smaps_rollup, "/proc/self/smaps_rollup")); + OUTCOME_TRY(fill_buffer(maps, "/proc/self/maps")); + uint64_t lazyfree = 0; { - ::close(ih); - return posix_error(); + string_view i(smaps_rollup.data(), smaps_rollup.size()); + OUTCOME_TRY(lazyfree, parse(i, "\nLazyFree:")); } - if(bytesread == 0) + string_view i(maps.data(), maps.size()); + size_t anonymous = 0; + for(size_t idx = 0;;) { - break; + idx = i.find("\n", idx); + if(idx == i.npos) + { + break; + } + idx++; + size_t start = 0, end = 0, inode = 1; + char read = 0, write = 0, executable = 0, private_ = 0; + sscanf(i.data() + idx, "%zx-%zx %c%c%c%c %*u %*u:%*u %zd", &start, &end, &read, &write, &executable, &private_, &inode); + if(inode == 0 && read == 'r' && write == 'w' && executable == '-' && private_ == 'p') + { + anonymous += end - start; + // std::cout << (end - start) << " " << i.substr(idx, 40) << std::endl; + } } - totalbytesread += bytesread; - } - ::close(ih); - if(totalbytesread < buffer.size()) - { - buffer.resize(totalbytesread); - break; + if(lazyfree != (uint64_t) -1) + { + anonymous -= (size_t) lazyfree; + } + ret.private_committed = anonymous; } - buffer.resize(buffer.size() * 2); + return ret; } + std::vector buffer(1024 * 1024); + OUTCOME_TRY(fill_buffer(buffer, "/proc/self/smaps")); const string_view totalview(buffer.data(), buffer.size()); // std::cerr << totalview << std::endl; std::vector anon_entries, non_anon_entries; @@ -288,7 +423,7 @@ namespace utils }; for(string_view item = find_item(totalview.size()); item != string_view(); item = find_item(item.data() - totalview.data())) { - //std::cout << "***" << item << "***"; + // std::cout << "***" << item << "***"; // hexaddr-hexaddr flags offset dev:id inode [path] size_t inode = 1; sscanf(item.data(), "%*x-%*x %*c%*c%*c%*c %*x %*c%*c:%*c%*c %zu", &inode); @@ -309,58 +444,6 @@ namespace utils non_anon_entries.push_back(item); } } - auto parse = [](string_view item, string_view what) -> result { - auto idx = item.find(what); - if(string_view::npos == idx) - { - return (uint64_t) -1; - } - idx += what.size(); - for(; item[idx] == ' '; idx++) - ; - auto eidx = idx; - for(; item[eidx] != '\n'; eidx++) - ; - string_view unit(item.substr(eidx - 2, 2)); - uint64_t value = atoll(item.data() + idx); - if(unit == "kB") - { - value *= 1024ULL; - } - else if(unit == "mB") - { - value *= 1024ULL * 1024; - } - else if(unit == "gB") - { - value *= 1024ULL * 1024 * 1024; - } - else if(unit == "tB") - { - value *= 1024ULL * 1024 * 1024 * 1024; - } - else if(unit == "pB") - { - value *= 1024ULL * 1024 * 1024 * 1024 * 1024; - } - else if(unit == "eB") - { - value *= 1024ULL * 1024 * 1024 * 1024 * 1024 * 1024; - } - else if(unit == "zB") - { - value *= 1024ULL * 1024 * 1024 * 1024 * 1024 * 1024 * 1024; - } - else if(unit == "yB") - { - value *= 1024ULL * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024; - } - else - { - return errc::illegal_byte_sequence; - } - return value; - }; process_memory_usage ret; // std::cerr << "Anon entries:"; for(auto &i : anon_entries) @@ -373,13 +456,13 @@ namespace utils { ret.total_address_space_in_use += size; ret.total_address_space_paged_in += rss; - ret.private_committed += size; - ret.private_paged_in += anonymous; + ret.private_committed += anonymous; if(lazyfree != (uint64_t) -1) { ret.total_address_space_paged_in -= lazyfree; - ret.private_paged_in -= lazyfree; + ret.private_committed -= lazyfree; } + ret.private_paged_in += rss; } // std::cerr << i << "\nSize = " << size << " Rss = " << rss << std::endl; } @@ -395,7 +478,7 @@ namespace utils ret.total_address_space_paged_in += rss; if(lazyfree != (uint64_t) -1) { - ret.total_address_space_paged_in -= lazyfree; + ret.total_address_space_in_use -= lazyfree; } } // std::cerr << i << "\nSize = " << size << " Rss = " << rss << std::endl; @@ -407,6 +490,7 @@ namespace utils return error_from_exception(); } #elif defined(__APPLE__) + (void) want; kern_return_t error; mach_msg_type_number_t outCount; task_vm_info_data_t vmInfo; diff --git a/include/llfio/v2.0/detail/impl/windows/map_handle.ipp b/include/llfio/v2.0/detail/impl/windows/map_handle.ipp index 4a175f892..2ee3e5e39 100644 --- a/include/llfio/v2.0/detail/impl/windows/map_handle.ipp +++ b/include/llfio/v2.0/detail/impl/windows/map_handle.ipp @@ -622,7 +622,7 @@ map_handle::io_result map_handle::_do_barrier(ma } -result map_handle::_new_map(size_type bytes, section_handle::flag _flag) noexcept +result map_handle::_new_map(size_type bytes, bool fallback, section_handle::flag _flag) noexcept { if(bytes == 0u) { @@ -642,7 +642,18 @@ result map_handle::_new_map(size_type bytes, section_handle::flag _f addr = VirtualAlloc(nullptr, bytes, allocation, prot); if(addr == nullptr) { - return win32_error(); + auto err = win32_error(); + if(fallback && err == errc::not_enough_memory) + { + // Try the cache + auto r = _recycled_map(bytes, _flag); + if(r) + { + memset(r.assume_value().address(), 0, r.assume_value().length()); + return r; + } + } + return err; } ret.value()._addr = static_cast(addr); ret.value()._reservation = bytes; @@ -690,7 +701,7 @@ result map_handle::map(section_handle §ion, size_type bytes, ext ret.value()._addr = static_cast(addr); ret.value()._offset = offset; ret.value()._reservation = _bytes; - ret.value()._length = (size_type) (section.length().value() - offset); + ret.value()._length = (size_type)(section.length().value() - offset); ret.value()._pagesize = pagesize; // Make my handle borrow the native handle of my backing storage ret.value()._v.h = section.backing_native_handle().h; @@ -790,7 +801,7 @@ result map_handle::commit(buffer_type region, section_h return errc::invalid_argument; } DWORD prot = 0; - if(flag == section_handle::flag::none) + if(flag == section_handle::flag::none || flag == section_handle::flag::nocommit) { OUTCOME_TRYV(win32_maps_apply(region.data(), region.size(), win32_map_sought::committed, [](byte *addr, size_t bytes) -> result { DWORD _ = 0; diff --git a/include/llfio/v2.0/detail/impl/windows/utils.ipp b/include/llfio/v2.0/detail/impl/windows/utils.ipp index d4516000e..e12db462c 100644 --- a/include/llfio/v2.0/detail/impl/windows/utils.ipp +++ b/include/llfio/v2.0/detail/impl/windows/utils.ipp @@ -1,5 +1,5 @@ /* Misc utilities -(C) 2015-2017 Niall Douglas (7 commits) +(C) 2015-2021 Niall Douglas (7 commits) File Created: Dec 2015 @@ -207,7 +207,7 @@ namespace utils return success(); } - result current_process_memory_usage() noexcept + result current_process_memory_usage(process_memory_usage::want /*unused*/) noexcept { // Amazingly Win32 doesn't expose private working set, so to avoid having // to iterate all the pages in the process and calculate, use a hidden diff --git a/include/llfio/v2.0/llfio.hpp b/include/llfio/v2.0/llfio.hpp index 0cb651e6e..9ecf38451 100644 --- a/include/llfio/v2.0/llfio.hpp +++ b/include/llfio/v2.0/llfio.hpp @@ -34,7 +34,7 @@ // If C++ Modules are on and we are not compiling the library, // we are either generating the interface or importing -#if !defined(__cpp_modules) || defined(GENERATING_LLFIO_MODULE_INTERFACE) || LLFIO_DISABLE_CXX_MODULES +#if !LLFIO_ENABLE_CXX_MODULES || !defined(__cpp_modules) || defined(GENERATING_LLFIO_MODULE_INTERFACE) || LLFIO_DISABLE_CXX_MODULES // C++ Modules not on, therefore include as usual #define LLFIO_INCLUDE_ALL #else diff --git a/include/llfio/v2.0/map_handle.hpp b/include/llfio/v2.0/map_handle.hpp index ba6b83ef9..8f3b39d75 100644 --- a/include/llfio/v2.0/map_handle.hpp +++ b/include/llfio/v2.0/map_handle.hpp @@ -543,7 +543,7 @@ class LLFIO_DECL map_handle : public lockable_io_handle LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result _do_read(io_request reqs, deadline d = deadline()) noexcept override; LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result _do_write(io_request reqs, deadline d = deadline()) noexcept override; - static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result _new_map(size_type bytes, section_handle::flag _flag) noexcept; + static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result _new_map(size_type bytes, bool fallback, section_handle::flag _flag) noexcept; static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result _recycled_map(size_type bytes, section_handle::flag _flag) noexcept; @@ -595,7 +595,7 @@ class LLFIO_DECL map_handle : public lockable_io_handle LLFIO_MAKE_FREE_FUNCTION static inline result map(size_type bytes, bool zeroed = false, section_handle::flag _flag = section_handle::flag::readwrite) noexcept { - return (zeroed || (_flag & section_handle::flag::nocommit)) ? _new_map(bytes, _flag) : _recycled_map(bytes, _flag); + return (zeroed || (_flag & section_handle::flag::nocommit)) ? _new_map(bytes, true, _flag) : _recycled_map(bytes, _flag); } /*! Reserve address space within which individual pages can later be committed. Reserved address @@ -612,7 +612,10 @@ class LLFIO_DECL map_handle : public lockable_io_handle \errors Any of the values POSIX `mmap()` or `VirtualAlloc()` can return. */ LLFIO_MAKE_FREE_FUNCTION - static inline result reserve(size_type bytes) noexcept { return _new_map(bytes, section_handle::flag::none | section_handle::flag::nocommit); } + static inline result reserve(size_type bytes) noexcept + { + return _new_map(bytes, false, section_handle::flag::none | section_handle::flag::nocommit); + } /*! Create a memory mapped view of a backing storage, optionally reserving additional address space for later growth. diff --git a/include/llfio/v2.0/utils.hpp b/include/llfio/v2.0/utils.hpp index fba01cd4d..1877d43a8 100644 --- a/include/llfio/v2.0/utils.hpp +++ b/include/llfio/v2.0/utils.hpp @@ -103,7 +103,8 @@ namespace utils { assert(pagesize > 0); const auto base = LLFIO_V2_NAMESPACE::detail::unsigned_integer_cast(i.first); - i = {static_cast((base + pagesize - 1) & ~(pagesize - 1)), static_cast(((base + i.second) & ~(pagesize - 1)) - ((base + pagesize - 1) & ~(pagesize - 1)))}; + i = {static_cast((base + pagesize - 1) & ~(pagesize - 1)), + static_cast(((base + i.second) & ~(pagesize - 1)) - ((base + pagesize - 1) & ~(pagesize - 1)))}; return i; } @@ -198,9 +199,21 @@ namespace utils */ struct process_memory_usage { + //! Fields wanted + QUICKCPPLIB_BITFIELD_BEGIN(want){ + total_address_space_in_use = 1U << 0U, + total_address_space_paged_in = 1U << 1U, + private_committed = 1U << 2U, + private_paged_in = 1U << 3U, + private_committed_inaccurate = 1U << 8U, + + all = (unsigned) -1 // + } QUICKCPPLIB_BITFIELD_END(want) + //! The total virtual address space in use. size_t total_address_space_in_use{0}; - //! The total memory currently paged into the process. Always `<= total_address_space_in_use`. Also known as "working set", or "resident set size including shared". + //! The total memory currently paged into the process. Always `<= total_address_space_in_use`. Also known as "working set", or "resident set size including + //! shared". size_t total_address_space_paged_in{0}; //! The total anonymous memory committed. Also known as "commit charge". @@ -212,12 +225,25 @@ namespace utils /*! \brief Retrieve the current memory usage statistics for this process. - \note Mac OS provides no way of reading how much memory a process has committed. We therefore supply as `private_committed` the same value as `private_paged_in`. + Be aware that because Linux provides no summary counter for `private_committed`, we + have to manually parse through `/proc/pid/smaps` to calculate it. This can start to + take seconds for a process with a complex virtual memory space. If you are sure that + you never use `section_handle::flag::nocommit` without `section_handle::flag::none` + (i.e. you don't nocommit accessible memory), then specifying the flag + `process_memory_usage::want::private_committed_inaccurate` can yield significant + performance gains. If you set `process_memory_usage::want::private_committed_inaccurate`, + we use `/proc/pid/smaps_rollup` and `/proc/pid/maps` to calculate the results. This + cannot distinguish between regions with the accounted + flag enabled or disabled. By default, this fast path is enabled. + + \note Mac OS provides no way of reading how much memory a process has committed. + We therefore supply as `private_committed` the same value as `private_paged_in`. */ - LLFIO_HEADERS_ONLY_FUNC_SPEC result current_process_memory_usage() noexcept; + LLFIO_HEADERS_ONLY_FUNC_SPEC result + current_process_memory_usage(process_memory_usage::want want = process_memory_usage::want::all) noexcept; /*! \brief CPU usage statistics for a process. - */ + */ struct process_cpu_usage { //! The amount of nanoseconds all processes ever have spent in user mode. @@ -257,7 +283,7 @@ namespace utils The simplest way to use this API is to call it whilst also taking the current monotonic clock/CPU TSC and then calculating the delta change over that period of time. - + \note The returned values may not be a snapshot accurate against one another as they may get derived from multiple sources. Also, granularity is probably either a lot more than one nanosecond on most platforms, but may be CPU TSC based on others (you can test @@ -372,7 +398,7 @@ namespace utils detail::deallocate_large_pages(p, n * sizeof(T)); } - template void construct(U *p, Args &&... args) { ::new(reinterpret_cast(p)) U(std::forward(args)...); } + template void construct(U *p, Args &&...args) { ::new(reinterpret_cast(p)) U(std::forward(args)...); } template void destroy(U *p) { p->~U(); } }; diff --git a/test/tests/map_handle_cache.cpp b/test/tests/map_handle_cache.cpp new file mode 100644 index 000000000..1756273b5 --- /dev/null +++ b/test/tests/map_handle_cache.cpp @@ -0,0 +1,108 @@ +/* Integration test kernel for whether the map handle cache works +(C) 2021 Niall Douglas (2 commits) +File Created: Aug 2021 + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License in the accompanying file +Licence.txt or at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Distributed under the Boost Software License, Version 1.0. + (See accompanying file Licence.txt or copy at + http://www.boost.org/LICENSE_1_0.txt) +*/ + +#include "../test_kernel_decl.hpp" + +#include +#include + +static inline void TestMapHandleCache() +{ + static constexpr size_t ITEMS_COUNT = 10000; + namespace llfio = LLFIO_V2_NAMESPACE; + bool free_cache_immediately = false; + auto test = [&] { + auto fault = [](llfio::map_handle &mh) { + for(auto *p = (volatile char *) mh.address(); p < (volatile char *) mh.address() + mh.length(); p += mh.page_size()) + { + *p = 1; + } + }; + QUICKCPPLIB_NAMESPACE::algorithm::small_prng::small_prng rand; + std::vector maps; + for(size_t n = 0; n < ITEMS_COUNT; n++) + { + auto v = rand(); + auto toallocate = (v >> 2) & (128 * 1024 - 1); + if(toallocate == 0) + { + toallocate = 1; + } + maps.push_back(llfio::map_handle::map(toallocate, free_cache_immediately).value()); + fault(maps.back()); + } + if(free_cache_immediately) + { + auto stats = llfio::map_handle::trim_cache(std::chrono::steady_clock::now()); + BOOST_REQUIRE(stats.bytes_in_cache == 0); + BOOST_REQUIRE(stats.items_in_cache == 0); + } + auto begin = std::chrono::steady_clock::now(); + for(size_t n = 0; n < ITEMS_COUNT * 10; n++) + { + auto v = rand(); + auto toallocate = (v >> 2) & (128 * 1024 - 1); + if(toallocate == 0) + { + toallocate = 1; + } + if(v & 1) + { + maps[n % ITEMS_COUNT].close().value(); + } + else + { + fault((maps[n % ITEMS_COUNT] = llfio::map_handle::map(toallocate, false).value())); + } + if(free_cache_immediately) + { + auto stats = llfio::map_handle::trim_cache(std::chrono::steady_clock::now()); + BOOST_CHECK(stats.bytes_in_cache == 0); + BOOST_CHECK(stats.items_in_cache == 0); + } + } + auto end = std::chrono::steady_clock::now(); + { + auto stats = llfio::map_handle::trim_cache(); + std::cout << "\nIn the map_handle cache after churn there are " << stats.bytes_in_cache << " bytes in the cache in " << stats.items_in_cache << " items." + << std::endl; + } + for(auto &i : maps) + { + i.close().value(); + } + { + auto stats = llfio::map_handle::trim_cache(); + std::cout << "\nIn the map_handle cache after releasing everything there are " << stats.bytes_in_cache << " bytes in the cache in " + << stats.items_in_cache << " items." << std::endl; + } + std::cout << "With free_cache_immediately = " << free_cache_immediately << " it took " + << (std::chrono::duration_cast(end - begin).count() / 1000.0 / ITEMS_COUNT) << " us per allocation-free." << std::endl; + }; + test(); + free_cache_immediately = true; + test(); +} + +KERNELTEST_TEST_KERNEL(integration, llfio, map_handle, cache, "Tests that the map_handle cache works as expected", TestMapHandleCache()) diff --git a/test/tests/utils.cpp b/test/tests/utils.cpp index 22f39073f..077122298 100644 --- a/test/tests/utils.cpp +++ b/test/tests/utils.cpp @@ -100,6 +100,11 @@ static inline void TestCurrentProcessMemoryUsage() auto maph = llfio::map_handle::map(1024 * 1024 * 1024).value(); auto mapfileh = llfio::mapped_file_handle::mapped_temp_inode(1024 * 1024 * 1024).value(); } // namespace llfio=LLFIO_V2_NAMESPACE; + { + auto stats = llfio::map_handle::trim_cache(std::chrono::steady_clock::now()); + BOOST_REQUIRE(stats.bytes_in_cache == 0); + BOOST_REQUIRE(stats.items_in_cache == 0); + } std::cout << "For page allocation:\n"; { llfio::utils::process_memory_usage before_anything, after_reserve, after_commit, after_fault, after_decommit, after_zero, after_do_not_store; @@ -107,7 +112,7 @@ static inline void TestCurrentProcessMemoryUsage() std::cout << " Before anything:\n" << print(before_anything) << std::endl; { // Should raise total_address_space_in_use by 1Gb - auto maph = llfio::map_handle::map(1024 * 1024 * 1024, false, llfio::section_handle::flag::nocommit).value(); + auto maph = llfio::map_handle::reserve(1024 * 1024 * 1024).value(); std::cout << " After reserving 1Gb:\n" << print(after_reserve) << std::endl; } {