diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h
index 60a9a57923..0f81bb6ba3 100644
--- a/include/pybind11/cast.h
+++ b/include/pybind11/cast.h
@@ -586,25 +586,11 @@ class type_caster_generic {
         return inst.release();
     }
 
-    // Base methods for generic caster; there are overridden in copyable_holder_caster
+    // Base methods for generic caster; they are overridden in copyable_holder_caster
     void load_value(value_and_holder &&v_h) {
-        auto *&vptr = v_h.value_ptr();
-        // Lazy allocation for unallocated values:
-        if (vptr == nullptr) {
-            auto *type = v_h.type ? v_h.type : typeinfo;
-            if (type->operator_new) {
-                vptr = type->operator_new(type->type_size);
-            } else {
-                #if defined(__cpp_aligned_new) && (!defined(_MSC_VER) || _MSC_VER >= 1912)
-                    if (type->type_align > __STDCPP_DEFAULT_NEW_ALIGNMENT__)
-                        vptr = ::operator new(type->type_size,
-                                              std::align_val_t(type->type_align));
-                    else
-                #endif
-                vptr = ::operator new(type->type_size);
-            }
-        }
-        value = vptr;
+        value = v_h.value_ptr();
+        if (value == nullptr)
+            throw cast_error("Invalid object instance");
     }
     bool try_implicit_casts(handle src, bool convert) {
         for (auto &cast : typeinfo->implicit_casts) {
@@ -1581,7 +1567,7 @@ struct move_only_holder_caster : public type_caster_base<type> {
     bool load(handle& src, bool convert) {
         bool success = base::template load_impl<move_only_holder_caster<type, holder_type>>(src, convert);
         if (success) // On success, remember src pointer to withdraw later
-            src_handle = std::addressof(src);
+            this->v_h = reinterpret_cast<instance *>(src.ptr())->get_value_and_holder();
         return success;
     }
 
@@ -1593,6 +1579,10 @@ struct move_only_holder_caster : public type_caster_base<type> {
     explicit
     #endif
     operator holder_type&&() {
+        // In load_value() value_ptr was still valid. If it's invalid now, another argument consumed the same value before.
+        if (!v_h.value_ptr())
+            throw cast_error("Multiple access to moved argument");
+        v_h.value_ptr() = nullptr;
         // TODO: release object instance in python
         // clear_instance(src_handle->ptr()); ???
 
@@ -1615,6 +1605,8 @@ struct move_only_holder_caster : public type_caster_base<type> {
         if (v_h.holder_constructed()) {
             holder_ptr = std::addressof(v_h.template holder<holder_type>());
             value = const_cast<void*>(reinterpret_cast<const void*>(holder_helper<holder_type>::get(*holder_ptr)));
+            if (!value)
+                throw cast_error("Invalid object instance");
             return true;
         } else {
             throw cast_error("Unable to cast from non-held to held instance (T& to Holder<T>) "
@@ -1636,7 +1628,7 @@ struct move_only_holder_caster : public type_caster_base<type> {
 
 
     holder_type* holder_ptr = nullptr;
-    handle* src_handle = nullptr;
+    value_and_holder v_h;
 };
 
 template <typename type, typename deleter>