From 4b23f5764407e47c0d66f0e35dbd01c2f994e9bc Mon Sep 17 00:00:00 2001 From: Oliver Anderson Date: Mon, 4 May 2026 15:47:08 +0200 Subject: [PATCH 1/2] memtouch: SSE2 version of write_page for x86_64 Introduce a `write_pages` method that writes a batch of pages and uses 128-bit non-temporal stores on X86_64 CPUs. Replacing the write_pages loops with calls to this function increases write throughput by roughly a factor between 1.7 and 4x depending on the parameters and CPU. We will refactor the main function to use the functionality introduced here in the next commit, but we make non-temporal stores opt-in in case existing use-cases are relying on the current behavior. Signed-off-by: Oliver Anderson On-behalf-of: SAP oliver.anderson@sap.com --- memtouch.cpp | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/memtouch.cpp b/memtouch.cpp index 3c32d57..4b4341d 100644 --- a/memtouch.cpp +++ b/memtouch.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -10,6 +11,10 @@ #include #include +#if defined(__x86_64__) +#include +#endif + #include #include #include @@ -210,6 +215,44 @@ class WorkerThread { memset(static_cast(mem_base) + page * PAGE_SIZE, PATTERN, PAGE_SIZE); } + void write_pages(uint64_t page_start, uint64_t num_pages) { +#if defined(__x86_64__) + return write_pages_nt(page_start, num_pages); +#endif + + for (uint64_t n{0}; n < num_pages; ++n) { + auto page = page_start + n; + write_page(page); + } + } + +#if defined(__x86_64__) + void write_pages_nt(uint64_t page_start, uint64_t num_pages) { + // This requires SSE2 which is always available on x86_64 + auto pattern = _mm_set1_epi8(static_cast(PATTERN)); + + static_assert((PAGE_SIZE % (16 * 4)) == 0, "PAGE SIZE is not a multiple of 16"); + auto num_iterations = (PAGE_SIZE / (16 * 4)) * num_pages; + + char* mem_addr = ((char*)mem_base) + (page_start * PAGE_SIZE); + + for (uint64_t n{0}; n < num_iterations; ++n) { + // This requires SSE2 which is always available on x86_64. + // The `pre_run` method ensures that mem_base is a multiple of 16 + _mm_stream_si128(reinterpret_cast<__m128i*>(mem_addr), pattern); + _mm_stream_si128(reinterpret_cast<__m128i*>(mem_addr + (16)), pattern); + _mm_stream_si128(reinterpret_cast<__m128i*>(mem_addr + (2 * 16)), pattern); + _mm_stream_si128(reinterpret_cast<__m128i*>(mem_addr + (3 * 16)), pattern); + + mem_addr += 4 * 16; + } + + // Non-temporal stores are weakly ordered hence we apply a fence to ensure that + // our stores are ordered before any subsequent (in program order) loads and stores. + _mm_sfence(); + } +#endif + void read_page(uint64_t page, void* buffer) { memcpy(buffer, static_cast(mem_base) + page * PAGE_SIZE, PAGE_SIZE); } From 77869067d916c1e2c840f0c428d82176971d84b7 Mon Sep 17 00:00:00 2001 From: Oliver Anderson Date: Mon, 4 May 2026 16:15:04 +0200 Subject: [PATCH 2/2] memtouch: Enable opt-in streaming stores on X86_64 Replace the in-line loops with calls to the new write_pages method, but only apply the non-temporal store optimization if the user has opted in through the new "--nt_stores" argument. Signed-off-by: Oliver Anderson On-behalf-of: SAP oliver.anderson@sap.com --- memtouch.cpp | 63 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/memtouch.cpp b/memtouch.cpp index 4b4341d..44a563e 100644 --- a/memtouch.cpp +++ b/memtouch.cpp @@ -57,12 +57,14 @@ class WorkerThread { bool run_once_, unsigned mem_size_mib_, unsigned rw_ratio_, - uint64_t page_log_ival_) + uint64_t page_log_ival_, + bool streaming_writes_) : id(id_), run_once(run_once_), mem_size_mib(mem_size_mib_), rw_ratio(rw_ratio_), page_log_ival(page_log_ival_), + streaming_writes(streaming_writes_), stats() {} ~WorkerThread() { cleanup_memory(); } @@ -76,6 +78,7 @@ class WorkerThread { mem_size_mib(o.mem_size_mib), rw_ratio(o.rw_ratio), page_log_ival(o.page_log_ival), + streaming_writes(o.streaming_writes), mem_base(o.mem_base), read_buffer(), stats(std::move(o.stats)) { @@ -89,13 +92,14 @@ class WorkerThread { cleanup_memory(); - id = o.id; - run_once = o.run_once; - mem_size_mib = o.mem_size_mib; - rw_ratio = o.rw_ratio; - page_log_ival = o.page_log_ival; - mem_base = o.mem_base; - stats = std::move(o.stats); + id = o.id; + run_once = o.run_once; + mem_size_mib = o.mem_size_mib; + rw_ratio = o.rw_ratio; + page_log_ival = o.page_log_ival; + streaming_writes = o.streaming_writes; + mem_base = o.mem_base; + stats = std::move(o.stats); o.mem_base = nullptr; @@ -108,6 +112,15 @@ class WorkerThread { return false; } +#if defined(__x86_64__) + if (streaming_writes) { + // The following assertion should pass because the allocated memory is backed by a + // memory map which are always aligned to the page size. + auto addr_base = reinterpret_cast(mem_base); + assert((addr_base % 16) == 0); + } +#endif + return true; } @@ -121,7 +134,7 @@ class WorkerThread { if (global_terminate.load(std::memory_order_relaxed)) { break; } - write_page(page); + write_pages(page, 1); } if (run_once) { @@ -184,12 +197,8 @@ class WorkerThread { pages_read += pages_to_read_eff; - auto time_write_ns = measure_time_ns([&]() { - for (uint64_t n{0}; n < pages_to_write_eff; ++n) { - auto page = n + pages_read + pages_written; - write_page(page); - } - }); + auto time_write_ns = measure_time_ns( + [&]() { write_pages(pages_read + pages_written, pages_to_write_eff); }); pages_written += pages_to_write_eff; @@ -217,7 +226,9 @@ class WorkerThread { void write_pages(uint64_t page_start, uint64_t num_pages) { #if defined(__x86_64__) - return write_pages_nt(page_start, num_pages); + if (streaming_writes) { + return write_pages_nt(page_start, num_pages); + } #endif for (uint64_t n{0}; n < num_pages; ++n) { @@ -286,6 +297,7 @@ class WorkerThread { unsigned mem_size_mib; unsigned rw_ratio; uint64_t page_log_ival; + bool streaming_writes; void* mem_base{nullptr}; @@ -418,6 +430,13 @@ void setup_argparse(argparse::ArgumentParser& program, int argc, char** argv) { .default_value(false) .implicit_value(true); + program.add_argument("--nt_stores") + .help( + "Provide a non-temporal hint when writing memory pages. This can increase write " + "throughput") + .default_value(false) + .implicit_value(true); + try { program.parse_args(argc, argv); } catch (const std::exception& err) { @@ -433,10 +452,11 @@ int main(int argc, char** argv) { setup_signals(); setup_argparse(program, argc, argv); - auto thread_mem = program.get("--thread_mem"); - auto num_threads = program.get("--num_threads"); - auto rw_ratio = program.get("--rw_ratio"); - auto once = program.get("--once"); + auto thread_mem = program.get("--thread_mem"); + auto num_threads = program.get("--num_threads"); + auto rw_ratio = program.get("--rw_ratio"); + auto once = program.get("--once"); + auto streaming_writes = program.get("--nt_stores"); std::string stats_file; unsigned stats_ival; @@ -489,7 +509,8 @@ int main(int argc, char** argv) { thread_storage.reserve(num_threads + 1 /* statistics thread */); for (unsigned num_thread = 0; num_thread < num_threads; num_thread++) { - worker_storage.emplace_back(num_thread, once, thread_mem, rw_ratio, page_log_ival); + worker_storage.emplace_back(num_thread, once, thread_mem, rw_ratio, page_log_ival, + streaming_writes); if (not worker_storage.back().pre_run()) { worker_storage.clear(); return 1;