[libc++] Add tombstone traits to use in optional, variant

libcxx/include/CMakeLists.txt

@@ -541,6 +541,7 @@ set(files
   __memory/swap_allocator.h
   __memory/temp_value.h
   __memory/temporary_buffer.h
+  __memory/tombstone_traits.h
   __memory/uninitialized_algorithms.h
   __memory/unique_ptr.h
   __memory/uses_allocator.h

libcxx/include/__expected/expected.h

@@ -17,6 +17,7 @@
 #include <__functional/invoke.h>
 #include <__memory/addressof.h>
 #include <__memory/construct_at.h>
+#include <__memory/tombstone_traits.h>
 #include <__type_traits/conjunction.h>
 #include <__type_traits/disjunction.h>
 #include <__type_traits/integral_constant.h>

libcxx/include/__functional/reference_wrapper.h

@@ -10,12 +10,15 @@
 #ifndef _LIBCPP___FUNCTIONAL_REFERENCE_WRAPPER_H
 #define _LIBCPP___FUNCTIONAL_REFERENCE_WRAPPER_H
 
+#include <__bit/bit_cast.h>
 #include <__compare/synth_three_way.h>
 #include <__concepts/boolean_testable.h>
 #include <__config>
 #include <__functional/invoke.h>
 #include <__functional/weak_result_type.h>
 #include <__memory/addressof.h>
+#include <__memory/tombstone_traits.h>
+#include <__type_traits/datasizeof.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_const.h>
 #include <__type_traits/remove_cvref.h>
@@ -122,6 +125,12 @@ template <class _Tp>
 reference_wrapper(_Tp&) -> reference_wrapper<_Tp>;
 #endif
 
+template <class _Tp>
+struct __tombstone_memory_layout<reference_wrapper<_Tp>> {
+  static constexpr uintptr_t __disengaged_value_  = 0;
+  static constexpr size_t __is_disengaged_offset_ = 0;
+};
+
 template <class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<_Tp> ref(_Tp& __t) _NOEXCEPT {
   return reference_wrapper<_Tp>(__t);

libcxx/include/__locale

@@ -13,6 +13,7 @@
 #include <__config>
 #include <__locale_dir/locale_base_api.h>
 #include <__memory/shared_ptr.h> // __shared_count
+#include <__memory/tombstone_traits.h>
 #include <__mutex/once_flag.h>
 #include <__type_traits/make_unsigned.h>
 #include <__utility/no_destroy.h>
@@ -114,6 +115,9 @@ private:
   friend const _Facet& use_facet(const locale&);
 };
 
+template <>
+struct __tombstone_memory_layout<locale> : __tombstone_pointer_layout {};
+
 class _LIBCPP_EXPORTED_FROM_ABI locale::facet : public __shared_count {
 protected:
   _LIBCPP_HIDE_FROM_ABI explicit facet(size_t __refs = 0) : __shared_count(static_cast<long>(__refs) - 1) {}

libcxx/include/__memory/shared_ptr.h

@@ -832,6 +832,12 @@ template <class _Tp, class _Dp>
 shared_ptr(unique_ptr<_Tp, _Dp>) -> shared_ptr<_Tp>;
 #endif
 
+template <class _Tp>
+struct __tombstone_memory_layout<shared_ptr<_Tp>> {
+  static constexpr auto __disengaged_value_       = __tombstone_pointer_layout::__disengaged_value_;
+  static constexpr size_t __is_disengaged_offset_ = sizeof(void*) + __tombstone_pointer_layout::__is_disengaged_offset_;
+};
+
 //
 // std::allocate_shared and std::make_shared
 //
@@ -1382,6 +1388,12 @@ template <class _Tp>
 weak_ptr(shared_ptr<_Tp>) -> weak_ptr<_Tp>;
 #endif
 
+template <class _Tp>
+struct __tombstone_memory_layout<weak_ptr<_Tp>> {
+  static constexpr auto __disengaged_value_       = __tombstone_pointer_layout::__disengaged_value_;
+  static constexpr size_t __is_disengaged_offset_ = sizeof(void*) + __tombstone_pointer_layout::__is_disengaged_offset_;
+};
+
 template <class _Tp>
 inline _LIBCPP_CONSTEXPR weak_ptr<_Tp>::weak_ptr() _NOEXCEPT : __ptr_(nullptr), __cntrl_(nullptr) {}
 

libcxx/include/__memory/tombstone_traits.h

@@ -0,0 +1,171 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_DISENGAGED_TRAITS_H
+#define _LIBCPP___TYPE_TRAITS_DISENGAGED_TRAITS_H
+
+#include <__config>
+#include <__memory/construct_at.h>
+#include <__type_traits/datasizeof.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_fundamental.h>
+#include <__type_traits/is_trivially_destructible.h>
+#include <__type_traits/void_t.h>
+#include <__utility/forward_like.h>
+#include <__utility/in_place.h>
+#include <__utility/piecewise_construct.h>
+#include <__utility/pointer_int_pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class>
+struct __tombstone_memory_layout;
+
+// bools have always exactly one bit set. If there is more than one set it's disengaged.
+template <>
+struct __tombstone_memory_layout<bool> {
+  static constexpr uint8_t __disengaged_value_    = 3;
+  static constexpr size_t __is_disengaged_offset_ = 0;
+};
+
+struct __tombstone_pointer_layout {
+  static constexpr uint8_t __disengaged_value_ = 1;
+#ifdef _LIBCPP_LITTLE_ENDIAN
+  static constexpr size_t __is_disengaged_offset_ = 0;
+#else
+  static constexpr size_t __is_disengaged_offset_ = sizeof(void*) - 1;
+#endif
+};
+
+// TODO: Look into
+// - filesystem::directory_iterator
+// - vector<T> with alignof(T) == 1
+
+template <class _Tp>
+struct __tombstone_memory_layout<__enable_specialization_if<is_fundamental_v<_Tp> && alignof(_Tp) >= 2, _Tp*>>
+    : __tombstone_pointer_layout {};
+
+template <class _Tp>
+struct __tombstone_memory_layout<_Tp**> : __tombstone_pointer_layout {};
+
+inline constexpr struct __init_engaged_t {
+} __init_engaged;
+inline constexpr struct __init_disengaged_t {
+} __init_disengaged;
+
+template <class _Tp, class _Payload>
+struct __tombstone_data {
+  using _TombstoneLayout = __tombstone_memory_layout<_Tp>;
+  using _IsDisengagedT   = remove_cv_t<decltype(_TombstoneLayout::__disengaged_value_)>;
+
+  _LIBCPP_NO_UNIQUE_ADDRESS _Payload __payload_;
+  char __padding_[_TombstoneLayout::__is_disengaged_offset_ - __datasizeof_v<_Payload>];
+  _IsDisengagedT __is_disengaged_;
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr __tombstone_data(_Args&&... __args)
+      : __payload_(std::forward<_Args>(__args)...), __is_disengaged_(_TombstoneLayout::__disengaged_value_) {}
+};
+
+template <class _Tp, class _Payload>
+  requires(__tombstone_memory_layout<_Tp>::__is_disengaged_offset_ == 0)
+struct __tombstone_data<_Tp, _Payload> {
+  using _TombstoneLayout = __tombstone_memory_layout<_Tp>;
+  using _IsDisengagedT   = remove_cv_t<decltype(_TombstoneLayout::__disengaged_value_)>;
+
+  _IsDisengagedT __is_disengaged_;
+  _LIBCPP_NO_UNIQUE_ADDRESS _Payload __payload_;
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr __tombstone_data(_Args&&... __args)
+      : __is_disengaged_(_TombstoneLayout::__disengaged_value_), __payload_(std::forward<_Args>(__args)...) {}
+};
+
+template <class _Tp, class _Payload>
+struct __tombstone_traits {
+  using _TombstoneLayout = __tombstone_memory_layout<_Tp>;
+  using _TombstoneData   = __tombstone_data<_Tp, _Payload>;
+
+  union {
+    _Tp __value_;
+    _TombstoneData __tombstone_;
+  };
+
+  static_assert(sizeof(__tombstone_data<_Tp, _Payload>) <= sizeof(_Tp));
+  static_assert(is_integral_v<decltype(_TombstoneLayout::__disengaged_value_)>);
+  static_assert(offsetof(_TombstoneData, __is_disengaged_) == _TombstoneLayout::__is_disengaged_offset_);
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr __tombstone_traits(__init_disengaged_t, _Args&&... __args)
+      : __tombstone_(std::forward<_Args>(__args)...) {}
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr __tombstone_traits(__init_engaged_t, _Args&&... __args)
+      : __value_(std::forward<_Args>(__args)...) {}
+
+  template <class _Class>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_value(this _Class&& __self) noexcept {
+    return std::forward<_Class>(__self).__value_;
+  }
+
+  template <class _Class>
+  _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_payload(this _Class&& __self) noexcept {
+    return std::forward<_Class>(__self).__tombstone_.__payload_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __is_engaged() const noexcept {
+    if (__libcpp_is_constant_evaluated())
+      return !__builtin_constant_p(__tombstone_.__is_disengaged_ == _TombstoneLayout::__disengaged_value_);
+    return __tombstone_.__is_disengaged_ != _TombstoneLayout::__disengaged_value_;
+  }
+
+  template <class _Class, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr void __engage(this _Class& __self, piecewise_construct_t, _Args&&... __args) {
+    std::destroy_at(&__self.__tombstone_);
+    std::construct_at(&__self.__value_, std::forward<_Args>(__args)...);
+  }
+
+  template <class _Class, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr void __disengage(this _Class& __self, piecewise_construct_t, _Args&&... __args) {
+    std::destroy_at(&__self.__data_.__value_);
+    std::construct_at(&__self.__data_.__payload_, std::forward<_Args>(__args)...);
+    __self.__data_.__is_disengaged_ = _TombstoneLayout::__disengaged_value_;
+  }
+
+  __tombstone_traits(const __tombstone_traits&)            = default;
+  __tombstone_traits(__tombstone_traits&&)                 = default;
+  __tombstone_traits& operator=(const __tombstone_traits&) = default;
+  __tombstone_traits& operator=(__tombstone_traits&&)      = default;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr ~__tombstone_traits() {
+    if (__is_engaged()) {
+      std::destroy_at(&__value_);
+    } else {
+      std::destroy_at(&__tombstone_);
+    }
+  }
+
+  ~__tombstone_traits()
+    requires is_trivially_destructible_v<_Tp> && is_trivially_destructible_v<_Payload>
+  = default;
+};
+
+template <class _Tp, class = void>
+inline constexpr bool __has_tombstone_v = false;
+
+template <class _Tp>
+inline constexpr bool __has_tombstone_v<_Tp, void_t<decltype(sizeof(__tombstone_memory_layout<_Tp>))>> = true;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_DISENGAGED_TRAITS_H

libcxx/include/__memory/unique_ptr.h

@@ -19,6 +19,7 @@
 #include <__memory/allocator_traits.h> // __pointer
 #include <__memory/auto_ptr.h>
 #include <__memory/compressed_pair.h>
+#include <__memory/tombstone_traits.h>
 #include <__type_traits/add_lvalue_reference.h>
 #include <__type_traits/common_type.h>
 #include <__type_traits/conditional.h>
@@ -281,6 +282,14 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr {
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void swap(unique_ptr& __u) _NOEXCEPT { __ptr_.swap(__u.__ptr_); }
 };
 
+template <class _Tp>
+struct __tombstone_memory_layout<__enable_specialization_if<__has_tombstone_v<_Tp*>, unique_ptr<_Tp>>>
+    : __tombstone_memory_layout<_Tp*> {};
+
+template <class _Tp>
+struct __tombstone_memory_layout<__enable_specialization_if<__has_tombstone_v<_Tp*>, unique_ptr<_Tp[]>>>
+    : __tombstone_memory_layout<_Tp*> {};
+
 template <class _Tp, class _Dp>
 class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp> {
 public:

libcxx/include/__type_traits/datasizeof.h

@@ -26,7 +26,7 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_keyword(__datasizeof) || __has_extension(datasizeof)
+#if __has_extension(datasizeof) && 0
 template <class _Tp>
 inline const size_t __datasizeof_v = __datasizeof(_Tp);
 #else

libcxx/include/__type_traits/enable_if.h

@@ -32,6 +32,9 @@ template <bool _Bp, class _Tp = void>
 using enable_if_t = typename enable_if<_Bp, _Tp>::type;
 #endif
 
+template <bool _Bp, class _Tp, class = enable_if_t<_Bp>>
+using __enable_specialization_if = _Tp;
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___TYPE_TRAITS_ENABLE_IF_H

libcxx/include/__utility/pair.h

@@ -16,6 +16,7 @@
 #include <__fwd/array.h>
 #include <__fwd/pair.h>
 #include <__fwd/tuple.h>
+#include <__memory/tombstone_traits.h>
 #include <__tuple/sfinae_helpers.h>
 #include <__tuple/tuple_element.h>
 #include <__tuple/tuple_indices.h>
@@ -448,6 +449,18 @@ template <class _T1, class _T2>
 pair(_T1, _T2) -> pair<_T1, _T2>;
 #endif
 
+template <class _Tp, class _Up>
+  requires __has_tombstone_v<_Up>
+struct __tombstone_memory_layout<pair<_Tp, _Up>> {
+  static constexpr auto __disengaged_value_ = __tombstone_memory_layout<_Up>::__disengaged_value_;
+  static constexpr size_t __is_disengaged_offset_ =
+      sizeof(_Tp) + __tombstone_memory_layout<_Up>::__is_disengaged_offset_;
+};
+
+template <class _Tp, class _Up>
+  requires(!__has_tombstone_v<_Up> && __has_tombstone_v<_Tp>)
+struct __tombstone_memory_layout<pair<_Tp, _Up>> : __tombstone_memory_layout<_Tp> {};
+
 // [pairs.spec], specialized algorithms
 
 template <class _T1, class _T2, class _U1, class _U2>