Reimplemented current_process_memory_usage() for Linux so it's a lot …

…faster for very large programs.

Debugged the map handle cache somewhat, but it's still failing. Tomorrow!

cmake/tests.cmake

@@ -18,6 +18,7 @@ set(llfio_TESTS
   "test/tests/issue0028.cpp"
   "test/tests/issue0073.cpp"
   "test/tests/large_pages.cpp"
+  "test/tests/map_handle_cache.cpp"
   "test/tests/map_handle_create_close/kernel_map_handle.cpp.hpp"
   "test/tests/map_handle_create_close/runner.cpp"
   "test/tests/mapped.cpp"

include/llfio/revision.hpp

@@ -1,4 +1,4 @@
 // Note the second line of this file must ALWAYS be the git SHA, third line ALWAYS the git SHA update time
-#define LLFIO_PREVIOUS_COMMIT_REF    e43ef4f6953230803a30453bbab20061b009fe05
-#define LLFIO_PREVIOUS_COMMIT_DATE   "2021-08-20 09:36:50 +00:00"
-#define LLFIO_PREVIOUS_COMMIT_UNIQUE e43ef4f6
+#define LLFIO_PREVIOUS_COMMIT_REF    d3ff87fd8b91f06d4f7bd240860ef68f15a03621
+#define LLFIO_PREVIOUS_COMMIT_DATE   "2021-08-20 20:17:59 +00:00"
+#define LLFIO_PREVIOUS_COMMIT_UNIQUE d3ff87fd

include/llfio/v2.0/detail/impl/map_handle.ipp

@@ -73,7 +73,7 @@ namespace detail
     {
       const auto _bytes = bytes >> page_size_shift;
       _lock_guard g(lock);
-      auto it = _base::find_equal_or_larger(_bytes, 7 /* 99% close to the key after page size shift */);
+      auto it = _base::find(_bytes);
       for(; it != _base::end() && page_size != it->page_size && _bytes == it->trie_key; ++it)
       {
       }
@@ -106,11 +106,22 @@ namespace detail
       {
         for(auto it = _base::begin(); it != _base::end();)
         {
-          if(it->when_added < older_than)
+          if(it->when_added <= older_than)
           {
             auto *p = *it;
             it = _base::erase(it);
             const auto _bytes = p->trie_key << page_size_shift;
+#ifdef _WIN32
+            if(!win32_release_nonfile_allocations((byte *) p->addr, _bytes, MEM_RELEASE))
+#else
+            if(-1 == ::munmap(p->addr, _bytes))
+#endif
+            {
+              LLFIO_LOG_FATAL(nullptr, "map_handle cache failed to trim a map! If on Linux, you may have exceeded the "
+                      "64k VMA process limit, set the LLFIO_DEBUG_LINUX_MUNMAP macro at the top of posix/map_handle.ipp to cause dumping of VMAs to "
+                      "/tmp/llfio_unmap_debug_smaps.txt, and combine with strace to figure it out.");
+              abort();
+            }
             _base::bytes_in_cache -= _bytes;
             ret.bytes_just_trimmed += _bytes;
             ret.items_just_trimmed++;
@@ -148,7 +159,7 @@ result<map_handle> map_handle::_recycled_map(size_type bytes, section_handle::fl
   void *addr = detail::map_handle_cache().get(bytes, pagesize);
   if(addr == nullptr)
   {
-    return _new_map(bytes, _flag);
+    return _new_map(bytes, false, _flag);
   }
 #ifdef _WIN32
   {

include/llfio/v2.0/detail/impl/posix/map_handle.ipp

@@ -298,7 +298,7 @@ static inline result<void *> do_mmap(native_handle_type &nativeh, void *ataddr,
   bool have_backing = (section != nullptr);
   int prot = 0, flags = have_backing ? MAP_SHARED : (MAP_PRIVATE | MAP_ANONYMOUS);
   void *addr = nullptr;
-  if(_flag == section_handle::flag::none)
+  if(!(_flag & section_handle::flag::read) && !(_flag & section_handle::flag::write) && !(_flag & section_handle::flag::execute))
   {
     prot |= PROT_NONE;
 #ifdef MAP_GUARD
@@ -424,7 +424,7 @@ static inline result<void *> do_mmap(native_handle_type &nativeh, void *ataddr,
   return addr;
 }
 
-result<map_handle> map_handle::_new_map(size_type bytes, section_handle::flag _flag) noexcept
+result<map_handle> map_handle::_new_map(size_type bytes, bool fallback, section_handle::flag _flag) noexcept
 {
   if(bytes == 0u)
   {
@@ -434,8 +434,22 @@ result<map_handle> map_handle::_new_map(size_type bytes, section_handle::flag _f
   result<map_handle> ret(map_handle(nullptr, _flag));
   native_handle_type &nativeh = ret.value()._v;
   OUTCOME_TRY(auto &&pagesize, detail::pagesize_from_flags(ret.value()._flag));
-  OUTCOME_TRY(auto &&addr, do_mmap(nativeh, nullptr, 0, nullptr, pagesize, bytes, 0, ret.value()._flag));
-  ret.value()._addr = static_cast<byte *>(addr);
+  auto addr = do_mmap(nativeh, nullptr, 0, nullptr, pagesize, bytes, 0, ret.value()._flag);
+  if(!addr)
+  {
+    if(fallback && addr.error() == errc::not_enough_memory)
+    {
+      // Try the cache
+      auto r = _recycled_map(bytes, _flag);
+      if(r)
+      {
+        memset(r.assume_value().address(), 0, r.assume_value().length());
+        return r;
+      }
+    }
+    return std::move(addr).error();
+  }
+  ret.value()._addr = static_cast<byte *>(addr.assume_value());
   ret.value()._reservation = bytes;
   ret.value()._length = bytes;
   ret.value()._pagesize = pagesize;