Skip to content

Commit 979b71b

Browse files
hkaiserhkaiser
authored
Merge pull request #6539 from Pansysk75/fix-thread-local-caching-allocator
Use thread-safe cache in thread_local_caching_allocator
2 parents 3e6a12f + 108efca commit 979b71b

File tree

19 files changed

+132
-89
lines changed

19 files changed

+132
-89
lines changed

‎libs/core/allocator_support/include/hpx/allocator_support/thread_local_caching_allocator.hpp

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) 2023 Hartmut Kaiser
1+
// Copyright (c) 2023-2024 Hartmut Kaiser
22
//
33
// SPDX-License-Identifier: BSL-1.0
44
// Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -12,7 +12,6 @@
1212
#include <cstddef>
1313
#include <memory>
1414
#include <new>
15-
#include <stack>
1615
#include <type_traits>
1716
#include <utility>
1817

@@ -21,8 +20,10 @@ namespace hpx::util {
2120
#if defined(HPX_ALLOCATOR_SUPPORT_HAVE_CACHING) && \
2221
!((defined(HPX_HAVE_CUDA) && defined(__CUDACC__)) || \
2322
defined(HPX_HAVE_HIP))
23+
2424
///////////////////////////////////////////////////////////////////////////
25-
template <typename T = char, typename Allocator = std::allocator<T>>
25+
template <template <typename, typename> class Stack, typename T = char,
26+
typename Allocator = std::allocator<T>>
2627
struct thread_local_caching_allocator
2728
{
2829
HPX_NO_UNIQUE_ADDRESS Allocator alloc;
@@ -38,7 +39,7 @@ namespace hpx::util {
3839
template <typename U>
3940
struct rebind
4041
{
41-
using other = thread_local_caching_allocator<U,
42+
using other = thread_local_caching_allocator<Stack, U,
4243
typename traits::template rebind_alloc<U>>;
4344
};
4445

@@ -56,6 +57,7 @@ namespace hpx::util {
5657
explicit allocated_cache(Allocator const& a) noexcept(
5758
noexcept(std::is_nothrow_copy_constructible_v<Allocator>))
5859
: alloc(a)
60+
, data(0)
5961
{
6062
}
6163

@@ -72,19 +74,19 @@ namespace hpx::util {
7274
pointer allocate(size_type n)
7375
{
7476
pointer p;
75-
if (data.empty())
77+
std::pair<T*, size_type> pair;
78+
if (data.pop(pair))
79+
{
80+
p = pair.first;
81+
}
82+
else
7683
{
7784
p = traits::allocate(alloc, n);
7885
if (p == nullptr)
7986
{
8087
throw std::bad_alloc();
8188
}
8289
}
83-
else
84-
{
85-
p = data.top().first;
86-
data.pop();
87-
}
8890

8991
++allocated;
9092
return p;
@@ -104,16 +106,15 @@ namespace hpx::util {
104106
private:
105107
void clear_cache() noexcept
106108
{
107-
while (!data.empty())
109+
std::pair<T*, size_type> p;
110+
while (data.pop(p))
108111
{
109-
traits::deallocate(
110-
alloc, data.top().first, data.top().second);
111-
data.pop();
112+
traits::deallocate(alloc, p.first, p.second);
112113
}
113114
}
114115

115116
HPX_NO_UNIQUE_ADDRESS Allocator alloc;
116-
std::stack<std::pair<T*, size_type>> data;
117+
Stack<std::pair<T*, size_type>, Allocator> data;
117118
std::size_t allocated = 0;
118119
std::size_t deallocated = 0;
119120
};
@@ -134,7 +135,7 @@ namespace hpx::util {
134135

135136
template <typename U, typename Alloc>
136137
explicit thread_local_caching_allocator(
137-
thread_local_caching_allocator<U, Alloc> const&
138+
thread_local_caching_allocator<Stack, U, Alloc> const&
138139
rhs) noexcept(noexcept(std::
139140
is_nothrow_copy_constructible_v<Alloc>))
140141
: alloc(rhs.alloc)
@@ -198,7 +199,8 @@ namespace hpx::util {
198199
}
199200
};
200201
#else
201-
template <typename T = char, typename Allocator = std::allocator<T>>
202+
template <template <typename, typename> class Stack, typename T = char,
203+
typename Allocator = std::allocator<T>>
202204
using thread_local_caching_allocator = Allocator;
203205
#endif
204206
} // namespace hpx::util

‎libs/core/async_base/CMakeLists.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2020-2022 The STE||AR-Group
1+
# Copyright (c) 2020-2024 The STE||AR-Group
22
#
33
# SPDX-License-Identifier: BSL-1.0
44
# Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -34,6 +34,6 @@ add_hpx_module(
3434
COMPAT_HEADERS ${async_base_compat_headers}
3535
SOURCES ${async_base_sources}
3636
MODULE_DEPENDENCIES hpx_allocator_support hpx_concepts hpx_config
37-
hpx_coroutines hpx_tag_invoke
37+
hpx_concurrency hpx_coroutines hpx_tag_invoke
3838
CMAKE_SUBDIRS examples tests
3939
)

‎libs/core/async_base/include/hpx/async_base/dataflow.hpp

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) 2007-2022 Hartmut Kaiser
1+
// Copyright (c) 2007-2024 Hartmut Kaiser
22
//
33
// SPDX-License-Identifier: BSL-1.0
44
// Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -29,6 +29,7 @@ namespace hpx {
2929
#else
3030

3131
#include <hpx/config.hpp>
32+
#include <hpx/concurrency/stack.hpp>
3233
#include <hpx/modules/allocator_support.hpp>
3334
#include <hpx/modules/concepts.hpp>
3435
#include <hpx/modules/tag_invoke.hpp>
@@ -60,12 +61,14 @@ namespace hpx {
6061
// clang-format on
6162
friend constexpr HPX_FORCEINLINE auto tag_fallback_invoke(
6263
dataflow_t tag, F&& f, Ts&&... ts)
63-
-> decltype(tag(hpx::util::thread_local_caching_allocator<char,
64+
-> decltype(tag(hpx::util::thread_local_caching_allocator<
65+
hpx::lockfree::variable_size_stack, char,
6466
hpx::util::internal_allocator<>>{},
6567
HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...))
6668
{
6769
using allocator_type =
68-
hpx::util::thread_local_caching_allocator<char,
70+
hpx::util::thread_local_caching_allocator<
71+
hpx::lockfree::variable_size_stack, char,
6972
hpx::util::internal_allocator<>>;
7073
return hpx::functional::tag_invoke(tag, allocator_type{},
7174
HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...);

‎libs/core/async_combinators/CMakeLists.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2019-2023 The STE||AR-Group
1+
# Copyright (c) 2019-2024 The STE||AR-Group
22
#
33
# SPDX-License-Identifier: BSL-1.0
44
# Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -43,6 +43,7 @@ add_hpx_module(
4343
EXCLUDE_FROM_GLOBAL_HEADER "hpx/async_combinators/future_wait.hpp"
4444
MODULE_DEPENDENCIES
4545
hpx_async_base
46+
hpx_concurrency
4647
hpx_config
4748
hpx_errors
4849
hpx_futures

‎libs/core/async_combinators/include/hpx/async_combinators/when_all.hpp

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) 2007-2022 Hartmut Kaiser
1+
// Copyright (c) 2007-2024 Hartmut Kaiser
22
// Copyright (c) 2013 Agustin Berge
33
// Copyright (c) 2017 Denis Blank
44
//
@@ -131,6 +131,7 @@ namespace hpx {
131131
#include <hpx/config.hpp>
132132
#include <hpx/allocator_support/internal_allocator.hpp>
133133
#include <hpx/allocator_support/thread_local_caching_allocator.hpp>
134+
#include <hpx/concurrency/stack.hpp>
134135
#include <hpx/datastructures/tuple.hpp>
135136
#include <hpx/functional/tag_invoke.hpp>
136137
#include <hpx/futures/detail/future_data.hpp>
@@ -201,14 +202,16 @@ namespace hpx::lcos::detail {
201202
return async_visit_future(HPX_FORWARD(T, current));
202203
}
203204

205+
// clang-format off
204206
template <typename T, typename N>
205207
auto operator()(hpx::util::async_traverse_detach_tag, T&& current,
206208
N&& next) -> decltype(async_detach_future(HPX_FORWARD(T, current),
207-
HPX_FORWARD(N, next)))
209+
HPX_FORWARD(N, next)))
208210
{
209211
return async_detach_future(
210212
HPX_FORWARD(T, current), HPX_FORWARD(N, next));
211213
}
214+
// clang-format on
212215

213216
template <typename T>
214217
void operator()(hpx::util::async_traverse_complete_tag, T&& pack)
@@ -226,7 +229,8 @@ namespace hpx::lcos::detail {
226229
using frame_type = async_when_all_frame<result_type>;
227230
using no_addref = typename frame_type::base_type::init_no_addref;
228231

229-
using allocator_type = hpx::util::thread_local_caching_allocator<char,
232+
using allocator_type = hpx::util::thread_local_caching_allocator<
233+
hpx::lockfree::variable_size_stack, char,
230234
hpx::util::internal_allocator<>>;
231235
auto frame = hpx::util::traverse_pack_async_allocator(allocator_type{},
232236
hpx::util::async_traverse_in_place_tag<frame_type>{}, no_addref{},

‎libs/core/concurrency/include/hpx/concurrency/stack.hpp

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
// Copyright (C) 2008-2013 Tim Blechmann
2-
// Copyright (c) 2022-2023 Hartmut Kaiser
2+
// Copyright (c) 2022-2024 Hartmut Kaiser
33
//
44
// SPDX-License-Identifier: BSL-1.0
55
// Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -770,4 +770,10 @@ namespace hpx::lockfree {
770770

771771
pool_t pool;
772772
};
773+
774+
template <typename T, typename Allocator = std::allocator<T>>
775+
class variable_size_stack : public stack<T, Allocator>
776+
{
777+
using stack<T, Allocator>::stack;
778+
};
773779
} // namespace hpx::lockfree

‎libs/core/concurrency/tests/unit/stack.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ void ranged_unsynchronized_push_test()
7373
HPX_TEST(!stk.unsynchronized_pop(out));
7474
}
7575

76-
void fixed_size_stack_test()
76+
void variable_size_stack_test()
7777
{
7878
hpx::lockfree::stack<long, std::allocator<long>, 128> stk;
7979

@@ -89,7 +89,7 @@ void fixed_size_stack_test()
8989
HPX_TEST(stk.empty());
9090
}
9191

92-
void fixed_size_stack_test_exhausted()
92+
void variable_size_stack_test_exhausted()
9393
{
9494
hpx::lockfree::stack<long, std::allocator<long>, 2> stk;
9595

@@ -206,8 +206,8 @@ int main()
206206
unsafe_stack_test();
207207
ranged_push_test();
208208
ranged_unsynchronized_push_test();
209-
fixed_size_stack_test();
210-
fixed_size_stack_test_exhausted();
209+
variable_size_stack_test();
210+
variable_size_stack_test_exhausted();
211211
bounded_stack_test_exhausted();
212212
stack_consume_one_test();
213213
stack_consume_all_test();

‎libs/core/execution/CMakeLists.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2019-2022 The STE||AR-Group
1+
# Copyright (c) 2019-2024 The STE||AR-Group
22
#
33
# SPDX-License-Identifier: BSL-1.0
44
# Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -160,6 +160,7 @@ add_hpx_module(
160160
MODULE_DEPENDENCIES
161161
hpx_async_base
162162
hpx_async_combinators
163+
hpx_concurrency
163164
hpx_config
164165
hpx_threading
165166
hpx_pack_traversal

‎libs/core/execution/include/hpx/execution/detail/future_exec.hpp

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) 2007-2022 Hartmut Kaiser
1+
// Copyright (c) 2007-2024 Hartmut Kaiser
22
// Copyright (c) 2013 Agustin Berge
33
//
44
// SPDX-License-Identifier: BSL-1.0
@@ -13,6 +13,7 @@
1313
#include <hpx/assert.hpp>
1414
#include <hpx/async_base/launch_policy.hpp>
1515
#include <hpx/async_base/traits/is_launch_policy.hpp>
16+
#include <hpx/concurrency/stack.hpp>
1617
#include <hpx/coroutines/thread_enums.hpp>
1718
#include <hpx/execution/traits/executor_traits.hpp>
1819
#include <hpx/execution/traits/future_then_result_exec.hpp>
@@ -64,9 +65,9 @@ namespace hpx::lcos::detail {
6465
using continuation_result_type =
6566
hpx::util::invoke_result_t<F, Future>;
6667

67-
using allocator_type =
68-
hpx::util::thread_local_caching_allocator<char,
69-
hpx::util::internal_allocator<>>;
68+
using allocator_type = hpx::util::thread_local_caching_allocator<
69+
hpx::lockfree::variable_size_stack, char,
70+
hpx::util::internal_allocator<>>;
7071

7172
hpx::traits::detail::shared_state_ptr_t<result_type> p =
7273
detail::make_continuation_alloc<continuation_result_type>(

‎libs/core/executors/CMakeLists.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2020-2022 The STE||AR-Group
1+
# Copyright (c) 2020-2024 The STE||AR-Group
22
#
33
# SPDX-License-Identifier: BSL-1.0
44
# Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -97,6 +97,7 @@ add_hpx_module(
9797
hpx_allocator_support
9898
hpx_async_base
9999
hpx_concepts
100+
hpx_concurrency
100101
hpx_config
101102
hpx_errors
102103
hpx_execution

‎libs/core/executors/include/hpx/executors/parallel_executor.hpp

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
// Copyright (c) 2019-2020 ETH Zurich
2-
// Copyright (c) 2007-2023 Hartmut Kaiser
2+
// Copyright (c) 2007-2024 Hartmut Kaiser
33
// Copyright (c) 2019 Agustin Berge
44
//
55
// SPDX-License-Identifier: BSL-1.0
@@ -12,6 +12,7 @@
1212
#include <hpx/allocator_support/internal_allocator.hpp>
1313
#include <hpx/allocator_support/thread_local_caching_allocator.hpp>
1414
#include <hpx/async_base/launch_policy.hpp>
15+
#include <hpx/concurrency/stack.hpp>
1516
#include <hpx/execution/detail/async_launch_policy_dispatch.hpp>
1617
#include <hpx/execution/detail/future_exec.hpp>
1718
#include <hpx/execution/detail/post_policy_dispatch.hpp>
@@ -389,9 +390,9 @@ namespace hpx::execution {
389390
hpx::bind_back(HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...));
390391
#endif
391392

392-
using allocator_type =
393-
hpx::util::thread_local_caching_allocator<char,
394-
hpx::util::internal_allocator<>>;
393+
using allocator_type = hpx::util::thread_local_caching_allocator<
394+
hpx::lockfree::variable_size_stack, char,
395+
hpx::util::internal_allocator<>>;
395396
hpx::traits::detail::shared_state_ptr_t<result_type> p =
396397
lcos::detail::make_continuation_alloc_nounwrap<result_type>(
397398
allocator_type{}, HPX_FORWARD(Future, predecessor),

‎libs/core/futures/CMakeLists.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2019-2022 The STE||AR-Group
1+
# Copyright (c) 2019-2024 The STE||AR-Group
22
#
33
# SPDX-License-Identifier: BSL-1.0
44
# Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -68,6 +68,7 @@ add_hpx_module(
6868
hpx_assertion
6969
hpx_async_base
7070
hpx_concepts
71+
hpx_concurrency
7172
hpx_config
7273
hpx_errors
7374
hpx_functional

0 commit comments

Comments
 (0)