[libc++] Optimize std::min_element

libcxx/include/__algorithm/min_element.h

@@ -11,6 +11,7 @@
 
 #include <__algorithm/comp.h>
 #include <__algorithm/comp_ref_type.h>
+#include <__algorithm/iterator_operations.h>
 #include <__config>
 #include <__functional/identity.h>
 #include <__iterator/iterator_traits.h>
@@ -27,20 +28,115 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+template <class _Iter, class _Sent, class = void>
+struct __ConstTimeDistance : false_type {};
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Iter, class _Sent>
+struct __ConstTimeDistance< _Iter, _Sent, __enable_if_t< sized_sentinel_for<_Sent, _Iter> >> : true_type {};
+
+#else
+
+template <class _Iter>
+struct __ConstTimeDistance<
+    _Iter,
+    _Iter,
+    __enable_if_t< is_same<typename iterator_traits<_Iter>::iterator_category, random_access_iterator_tag>::value> >
+    : true_type {};
+
+#endif // _LIBCPP_STD_VER >= 20
+
 template <class _Comp, class _Iter, class _Sent, class _Proj>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter
-__min_element(_Iter __first, _Sent __last, _Comp __comp, _Proj& __proj) {
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter __min_element(
+    _Iter __first,
+    _Sent __last,
+    _Comp __comp,
+    _Proj& __proj,
+    /*_ConstTimeDistance*/ false_type) {
   if (__first == __last)
     return __first;
 
   _Iter __i = __first;
   while (++__i != __last)
     if (std::__invoke(__comp, std::__invoke(__proj, *__i), std::__invoke(__proj, *__first)))
       __first = __i;
-
   return __first;
 }
 
+template <class _Comp, class _Iter, class _Sent, class _Proj>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter __min_element(
+    _Iter __first,
+    _Sent __last,
+    _Comp __comp,
+    _Proj& __proj,
+    /*ConstTimeDistance*/ true_type) {
+  if (__first == __last)
+    return __first;
+
+  typedef typename std::iterator_traits<_Iter>::difference_type diff_type;
+  diff_type __n = std::distance(__first, __last);
+
+  if (__n <= 64) {
+    _Iter __i = __first;
+    while (++__i != __last)
+      if (std::__invoke(__comp, std::__invoke(__proj, *__i), std::__invoke(__proj, *__first)))
+        __first = __i;
+    return __first;
+  }
+
+  diff_type __block_size = 256;
+
+  diff_type __n_blocked = __n - (__n % __block_size);
+  _Iter __block_start = __first, __block_end = __first;
+
+  typedef typename std::iterator_traits<_Iter>::value_type value_type;
+  value_type __min_val = std::__invoke(__proj, *__first);
+
+  _Iter __curr = __first;
+  for (diff_type __i = 0; __i < __n_blocked; __i += __block_size) {
+    _Iter __start          = __curr;
+    value_type __block_min = __min_val;
+    for (diff_type __j = 0; __j < __block_size; __j++) {
+      if (std::__invoke(__comp, std::__invoke(__proj, *__curr), __block_min)) {
+        __block_min = *__curr;
+      }
+      __curr++;
+    }
+    if (std::__invoke(__comp, __block_min, __min_val)) {
+      __min_val     = __block_min;
+      __block_start = __start;
+      __block_end   = __curr;
+    }
+  }
+
+  value_type __epilogue_min = __min_val;
+  _Iter __epilogue_start    = __curr;
+  while (__curr != __last) {
+    if (std::__invoke(__comp, std::__invoke(__proj, *__curr), __epilogue_min)) {
+      __epilogue_min = *__curr;
+    }
+    __curr++;
+  }
+  if (std::__invoke(__comp, __epilogue_min, __min_val)) {
+    __min_val     = __epilogue_min;
+    __block_start = __epilogue_start;
+    __block_end   = __last;
+  }
+
+  for (; __block_start != __block_end; ++__block_start)
+    if (std::__invoke(__proj, *__block_start) == __min_val)
+      break;
+  return __block_start;
+}
+
+template <class _Comp, class _Iter, class _Sent, class _Proj>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter
+__min_element(_Iter __first, _Sent __last, _Comp __comp, _Proj& __proj) {
+  return std::__min_element<_Comp>(
+      std::move(__first), std::move(__last), __comp, __proj, __ConstTimeDistance<_Iter, _Sent>());
+}
+
 template <class _Comp, class _Iter, class _Sent>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter __min_element(_Iter __first, _Sent __last, _Comp __comp) {
   auto __proj = __identity();

libcxx/test/benchmarks/CMakeLists.txt

@@ -120,6 +120,7 @@ set(BENCHMARK_TESTS
     algorithms/make_heap_then_sort_heap.bench.cpp
     algorithms/min.bench.cpp
     algorithms/minmax.bench.cpp
+    algorithms/min_element.bench.cpp
     algorithms/min_max_element.bench.cpp
     algorithms/mismatch.bench.cpp
     algorithms/pop_heap.bench.cpp

libcxx/test/benchmarks/algorithms/min_element.bench.cpp

@@ -0,0 +1,80 @@
+#include <algorithm>
+#include <limits>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+#include <random>
+
+template <typename T>
+static void BM_stdmin_element_decreasing(benchmark::State& state) {
+  std::vector<T> v(state.range(0));
+  T start = std::numeric_limits<T>::max();
+  T end   = std::numeric_limits<T>::min();
+
+  for (size_t i = 0; i < v.size(); i++)
+    v[i] = ((start != end) ? start-- : end);
+
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(v);
+    benchmark::DoNotOptimize(std::min_element(v.begin(), v.end()));
+  }
+}
+
+BENCHMARK(BM_stdmin_element_decreasing<char>)
+    ->DenseRange(1, 8)
+    ->Range(32, 128)
+    ->Range(256, 4096)
+    ->DenseRange(5000, 10000, 1000)
+    ->Range(1 << 14, 1 << 16)
+    ->Arg(70000);
+BENCHMARK(BM_stdmin_element_decreasing<short>)
+    ->DenseRange(1, 8)
+    ->Range(32, 128)
+    ->Range(256, 4096)
+    ->DenseRange(5000, 10000, 1000)
+    ->Range(1 << 14, 1 << 16)
+    ->Arg(70000);
+BENCHMARK(BM_stdmin_element_decreasing<int>)
+    ->DenseRange(1, 8)
+    ->Range(32, 128)
+    ->Range(256, 4096)
+    ->DenseRange(5000, 10000, 1000)
+    ->Range(1 << 14, 1 << 16)
+    ->Arg(70000);
+BENCHMARK(BM_stdmin_element_decreasing<long long>)
+    ->DenseRange(1, 8)
+    ->Range(32, 128)
+    ->Range(256, 4096)
+    ->DenseRange(5000, 10000, 1000)
+    ->Range(1 << 14, 1 << 16)
+    ->Arg(70000);
+BENCHMARK(BM_stdmin_element_decreasing<unsigned char>)
+    ->DenseRange(1, 8)
+    ->Range(32, 128)
+    ->Range(256, 4096)
+    ->DenseRange(5000, 10000, 1000)
+    ->Range(1 << 14, 1 << 16)
+    ->Arg(70000);
+BENCHMARK(BM_stdmin_element_decreasing<unsigned short>)
+    ->DenseRange(1, 8)
+    ->Range(32, 128)
+    ->Range(256, 4096)
+    ->DenseRange(5000, 10000, 1000)
+    ->Range(1 << 14, 1 << 16)
+    ->Arg(70000);
+BENCHMARK(BM_stdmin_element_decreasing<unsigned int>)
+    ->DenseRange(1, 8)
+    ->Range(32, 128)
+    ->Range(256, 4096)
+    ->DenseRange(5000, 10000, 1000)
+    ->Range(1 << 14, 1 << 16)
+    ->Arg(70000);
+BENCHMARK(BM_stdmin_element_decreasing<unsigned long long>)
+    ->DenseRange(1, 8)
+    ->Range(32, 128)
+    ->Range(256, 4096)
+    ->DenseRange(5000, 10000, 1000)
+    ->Range(1 << 14, 1 << 16)
+    ->Arg(70000);
+
+BENCHMARK_MAIN();