From 9569ab7155721be4c181cd09556886f6f274b095 Mon Sep 17 00:00:00 2001 From: Sean Luchen Date: Tue, 13 Feb 2024 18:14:11 +0000 Subject: [PATCH 1/2] [SingleSource/Atomic] Add preliminary tests for atomic builtins. There exist atomic IR unit tests and libatomic unit tests, but neither can test the atomicity and interoperability of atomic builtins and compiler-rt's atomic library. These tests aim to approximate behaviour encountered in user code. These tests have caught issues in Clang. See llvm/llvm-project#74349 and llvm/llvm-project#73176 for LLVM changes inspired by these tests. --- MultiSource/UnitTests/CMakeLists.txt | 2 +- SingleSource/UnitTests/Atomic/CMakeLists.txt | 9 + SingleSource/UnitTests/Atomic/README.txt | 30 ++ SingleSource/UnitTests/Atomic/big_test.cpp | 78 ++++ .../Atomic/big_test.reference_output | 3 + SingleSource/UnitTests/Atomic/float_test.cpp | 109 ++++++ .../Atomic/float_test.reference_output | 5 + .../UnitTests/Atomic/int_aligned_test.cpp | 349 +++++++++++++++++ .../Atomic/int_aligned_test.reference_output | 8 + .../UnitTests/Atomic/int_misaligned_test.cpp | 362 ++++++++++++++++++ .../int_misaligned_test.reference_output | 8 + .../UnitTests/Atomic/misshapen_test.cpp | 155 ++++++++ .../Atomic/misshapen_test.reference_output | 13 + SingleSource/UnitTests/Atomic/util.h | 61 +++ SingleSource/UnitTests/CMakeLists.txt | 1 + 15 files changed, 1192 insertions(+), 1 deletion(-) create mode 100644 SingleSource/UnitTests/Atomic/CMakeLists.txt create mode 100644 SingleSource/UnitTests/Atomic/README.txt create mode 100644 SingleSource/UnitTests/Atomic/big_test.cpp create mode 100644 SingleSource/UnitTests/Atomic/big_test.reference_output create mode 100644 SingleSource/UnitTests/Atomic/float_test.cpp create mode 100644 SingleSource/UnitTests/Atomic/float_test.reference_output create mode 100644 SingleSource/UnitTests/Atomic/int_aligned_test.cpp create mode 100644 SingleSource/UnitTests/Atomic/int_aligned_test.reference_output create mode 100644 SingleSource/UnitTests/Atomic/int_misaligned_test.cpp create mode 100644 SingleSource/UnitTests/Atomic/int_misaligned_test.reference_output create mode 100644 SingleSource/UnitTests/Atomic/misshapen_test.cpp create mode 100644 SingleSource/UnitTests/Atomic/misshapen_test.reference_output create mode 100644 SingleSource/UnitTests/Atomic/util.h diff --git a/MultiSource/UnitTests/CMakeLists.txt b/MultiSource/UnitTests/CMakeLists.txt index 273e48f337..7ce34dd3c4 100644 --- a/MultiSource/UnitTests/CMakeLists.txt +++ b/MultiSource/UnitTests/CMakeLists.txt @@ -1,5 +1,5 @@ add_subdirectory(C++11) +add_subdirectory(Float) if(ARCH STREQUAL "Mips") add_subdirectory(Mips) endif() -add_subdirectory(Float) diff --git a/SingleSource/UnitTests/Atomic/CMakeLists.txt b/SingleSource/UnitTests/Atomic/CMakeLists.txt new file mode 100644 index 0000000000..9a341d187a --- /dev/null +++ b/SingleSource/UnitTests/Atomic/CMakeLists.txt @@ -0,0 +1,9 @@ +# Link the Clang built libatomic. +execute_process(COMMAND ${CMAKE_C_COMPILER} --print-file-name=libclang_rt.atomic.so + OUTPUT_VARIABLE _path_to_libatomic + OUTPUT_STRIP_TRAILING_WHITESPACE) +get_filename_component(_libatomic_dir ${_path_to_libatomic} DIRECTORY) +add_link_options("LINKER:${_path_to_libatomic},-rpath=${_libatomic_dir}") + +llvm_singlesource() + diff --git a/SingleSource/UnitTests/Atomic/README.txt b/SingleSource/UnitTests/Atomic/README.txt new file mode 100644 index 0000000000..df519f1dbc --- /dev/null +++ b/SingleSource/UnitTests/Atomic/README.txt @@ -0,0 +1,30 @@ +Atomic runtime library tests + +======== + +These tests aim to capture real-world multithreaded use cases of atomic +builtins. Each test focuses on a single atomic operation. Those using multiple +operations can be compared with other tests using the same operations to isolate +bugs to a single atomic operation. + +Each test consists of a "looper" body and a test script. The test script +instantiates 10 threads, each running the looper. The loopers contend the same +memory address, performing atomic operations on it. Each looper executes +10^6 times for a total of 10^7 operations. The resultant value in the contended +pointer is compared against a closed-form solution. It's expected that the two +values equate. + +For example, a looper that increments the shared pointer is expected to end up +with a value of 10^7. If its final value is not that, the test fails. + +Each test is performed on all relevant types. + +======== + +Future test writers should be aware that the set of all tests that appear to +test atomicity is not the set of all tests that test atomicity. In fact, tests +that may test atomicity on one processor may not test atomicity on a different +processor. + +As such, test writers are encouraged to write nonatomic variants of their tests, +and verify that they pass in a variety of scenarios. diff --git a/SingleSource/UnitTests/Atomic/big_test.cpp b/SingleSource/UnitTests/Atomic/big_test.cpp new file mode 100644 index 0000000000..2f460e264e --- /dev/null +++ b/SingleSource/UnitTests/Atomic/big_test.cpp @@ -0,0 +1,78 @@ +//===--- big_test.cc -- Testing big (17+ byte) objects ------------ C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file tests atomic operations on big objects with aligned memory +// addresses. +// +// The types tested are: bigs. +// The ops tested are: cmpxchg. +// TODO: Test load/store, xchg. +// +// Please read the README before contributing. +// +//===----------------------------------------------------------------------===// + +#include +#include +#include + +#include "util.h" + +// V >> 56 = 66, so to prevent 32-bit overflow, kExpected must be less than +// 2^31 / 66 = 32 x 10^6. +#ifdef SMALL_PROBLEM_SIZE +static constexpr int kIterations = 1'000'000; +#else +static constexpr int kIterations = 3'000'000; +#endif +static constexpr int kExpected = kThreads * kIterations; +static constexpr int kBigSize = 10; +struct big_t { + int v[kBigSize]; +}; + +// The big struct cmpxchg test is identical to the numeric cmpxchg test, except +// each element of the underlying array is incremented. +void looper_big_cmpxchg(big_t *abig, int success_model, int fail_model) { + for (int n = 0; n < kIterations; ++n) { + big_t desired, expected = {}; + do { + desired = expected; + for (int k = 0; k < kBigSize; ++k) + desired.v[k]++; + } while (!__atomic_compare_exchange(abig, &expected, &desired, true, + success_model, fail_model)); + } +} + +void test_big_cmpxchg() { + std::vector pool; + for (int success_model : atomic_compare_exchange_models) { + for (int fail_model : atomic_compare_exchange_models) { + big_t abig = {}; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_big_cmpxchg, &abig, success_model, fail_model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + for (int n = 0; n < kBigSize; ++n) + if (abig.v[n] != kExpected) + fail(); + } + } +} + +void test_big() { + std::cout << "Testing big\n"; + test_big_cmpxchg(); +} + +int main() { + test_big(); + std::cout << "PASSED\n"; +} diff --git a/SingleSource/UnitTests/Atomic/big_test.reference_output b/SingleSource/UnitTests/Atomic/big_test.reference_output new file mode 100644 index 0000000000..c441a1a30f --- /dev/null +++ b/SingleSource/UnitTests/Atomic/big_test.reference_output @@ -0,0 +1,3 @@ +Testing big +PASSED +exit 0 diff --git a/SingleSource/UnitTests/Atomic/float_test.cpp b/SingleSource/UnitTests/Atomic/float_test.cpp new file mode 100644 index 0000000000..bcbfe11eff --- /dev/null +++ b/SingleSource/UnitTests/Atomic/float_test.cpp @@ -0,0 +1,109 @@ +//===--- float_test.cc -- Testing aligned floating point numbers -- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file tests atomic operations on floating point types with aligned +// memory addresses. +// +// The types tested are: float, double, float128. +// The ops tested are: xchg, cmpxchg. +// +// Please read the README before contributing. +// +//===----------------------------------------------------------------------===// + +#include + +#include +#include +#include + +#include "util.h" + +// There are 23-bits in the mantissa of a single-precision float. +// Therefore, kExpected cannot exceed 2^24. +static constexpr int kIterations = 1'500'000; +static constexpr int kExpected = kThreads * kIterations; + +// See int_aligned_test.cc for an explanation of xchg tests. +template +void looper_float_scalar_xchg(T *afloat, int model) { + __int128_t error = 0; + T next = *afloat + 1; + T result; + for (int n = 0; n < kIterations; ++n) { + __atomic_exchange(afloat, &next, &result, model); + error += + static_cast<__int128_t>(next) - static_cast<__int128_t>(result + 1); + next = result + 1; + } + __atomic_fetch_sub(afloat, static_cast(error), model); +} + +template +void test_float_scalar_xchg() { + std::vector pool; + for (int model : atomic_exchange_models) { + T afloat = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_float_scalar_xchg, &afloat, model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (afloat != kExpected) + fail(); + } +} + +// See int_aligned_test.cc for an explanation of cmpxchg tests. +template +void looper_float_scalar_cmpxchg(T *afloat, int success_model, int fail_model) { + for (int n = 0; n < kIterations; ++n) { + T desired, expected = 0; + do { + desired = expected + 1; + } while (!__atomic_compare_exchange(afloat, &expected, &desired, true, + success_model, fail_model)); + } +} + +template +void test_float_scalar_cmpxchg() { + std::vector pool; + for (int success_model : atomic_compare_exchange_models) { + for (int fail_model : atomic_compare_exchange_models) { + T afloat = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_float_scalar_cmpxchg, &afloat, + success_model, fail_model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (afloat != kExpected) + fail(); + } + } +} + +void test_floating_point() { + std::cout << "Testing float\n"; + test_float_scalar_xchg(); + test_float_scalar_cmpxchg(); + + std::cout << "Testing double\n"; + test_float_scalar_xchg(); + test_float_scalar_cmpxchg(); + + std::cout << "Testing float128\n"; + test_float_scalar_xchg<__float128>(); + test_float_scalar_cmpxchg<__float128>(); +} + +int main() { + test_floating_point(); + std::cout << "PASSED\n"; +} diff --git a/SingleSource/UnitTests/Atomic/float_test.reference_output b/SingleSource/UnitTests/Atomic/float_test.reference_output new file mode 100644 index 0000000000..30d253e2c3 --- /dev/null +++ b/SingleSource/UnitTests/Atomic/float_test.reference_output @@ -0,0 +1,5 @@ +Testing float +Testing double +Testing float128 +PASSED +exit 0 diff --git a/SingleSource/UnitTests/Atomic/int_aligned_test.cpp b/SingleSource/UnitTests/Atomic/int_aligned_test.cpp new file mode 100644 index 0000000000..aa81b6f31e --- /dev/null +++ b/SingleSource/UnitTests/Atomic/int_aligned_test.cpp @@ -0,0 +1,349 @@ +//===--- int_aligned_test.cc -- Testing aligned integers ---------- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file tests atomic operations on signed and unsigned integer types with +// aligned memory addresses. +// +// The types tested are: uint32, uint64, int32, int64, uint128, int128. +// The ops tested are: add, sub, and, or, xor, xchg, xchg_n, cmpxchg, cmpxchg_n. +// The ALU operations are not tested on 128-bit integers. +// +// Please read the README before contributing. +// +//===----------------------------------------------------------------------===// + +#include + +#include +#include +#include +#include + +#include "util.h" + +// V >> 56 = 66, so to prevent 32-bit overflow, kExpected must be less than +// 2^31 / 66 = 32 x 10^6. +#ifdef SMALL_PROBLEM_SIZE +static constexpr int kIterations = 1'000'000; +#else +static constexpr int kIterations = 3'000'000; +#endif +static constexpr int kExpected = kThreads * kIterations; + +template +void looper_int_fetch_add(T *aint, int model) { + static constexpr T val = V >> right_shift(); + for (int n = 0; n < kIterations; ++n) + __atomic_fetch_add(aint, val, model); +} + +template +void test_int_fetch_add() { + static constexpr T val = V >> right_shift(); + std::vector pool; + for (int model : atomic_fetch_models) { + T aint = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_fetch_add, &aint, model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (aint != val * kExpected) + fail(); + } +} + +template +void looper_int_fetch_sub(T *aint, int model) { + static constexpr T val = V >> right_shift(); + for (int n = 0; n < kIterations; ++n) + __atomic_fetch_sub(aint, val, model); +} + +template +void test_int_fetch_sub() { + static constexpr T val = V >> right_shift(); + std::vector pool; + for (int model : atomic_fetch_models) { + T aint = val * kExpected; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_fetch_sub, &aint, model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (aint != 0) + fail(); + } +} + +// The AND + OR tests work as follows: +// +// Each of the 10 threads "owns" one bit of the shared member. Each thread +// attempts to flip the bit it owns, from 1 -> 0 when testing AND, and 0 -> 1 +// when testing OR. If successful, the thread increments a counter, and a +// cmpxchg loop flips the bit back. Once complete, that counter should equal +// 10^7. +template +void __attribute__((optnone)) looper_int_fetch_and(const int id, T *aint, + T *acnt, int model) { + T desired, expected = 0; + for (int n = 0; n < kIterations; ++n) { + const T mask = 1 << id; + __atomic_fetch_and(aint, ~mask, model); + if (~*aint & mask) { + __atomic_fetch_add(acnt, 1, model); + do { + desired = expected | mask; + } while (!__atomic_compare_exchange(aint, &expected, &desired, true, + model, model)); + } + } +} + +template +void test_int_fetch_and() { + std::vector pool; + for (int model : atomic_fetch_models) { + T acnt = 0; + T aint = ~0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_fetch_and, n, &aint, &acnt, model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (acnt != kExpected) + fail(); + } +} + +template +void __attribute__((optnone)) looper_int_fetch_or(const int id, T *aint, + T *acnt, int model) { + T desired, expected = 0; + for (int n = 0; n < kIterations; ++n) { + const T mask = 1 << id; + __atomic_fetch_or(aint, mask, model); + if (*aint & mask) { + __atomic_fetch_add(acnt, 1, model); + do { + desired = expected & ~mask; + } while (!__atomic_compare_exchange(aint, &expected, &desired, true, + model, model)); + } + } +} + +template +void test_int_fetch_or() { + std::vector pool; + for (int model : atomic_fetch_models) { + T acnt = 0; + T aint = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_fetch_or, n, &aint, &acnt, model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (acnt != kExpected) + fail(); + } +} + +template +void looper_int_fetch_xor(T *aint, int model) { + for (int n = 0; n < kIterations; ++n) + __atomic_fetch_xor(aint, n, model); +} + +template +void test_int_fetch_xor() { + std::vector pool; + for (int model : atomic_fetch_models) { + T aint = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_fetch_xor, &aint, model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (aint != 0) + fail(); + } +} + +// The xchg tests work as follows: +// +// Each thread increments a local copy of the shared variable, and exchanges it +// with the shared value. Most of the time, the value moved from shared -> local +// is one less than the value moved from local -> shared. Other times, the +// difference is much bigger (or smaller). When this occurs, the thread +// accumulates the difference in a local error variable. Upon completion, the +// thread subtracts the error from the shared value, all at once. +// +// Like many tests, this test increments by more than 1 -- specifically, a +// number that scales with the width of the type is picked. +// +template +void looper_int_xchg(T *aint, int model) { + static constexpr T val = V >> right_shift(); + __int128_t error = 0; + T next = *aint + val; + T result; + for (int n = 0; n < kIterations; ++n) { + __atomic_exchange(aint, &next, &result, model); + error += + static_cast<__int128_t>(next) - static_cast<__int128_t>(result + val); + next = result + val; + } + __atomic_fetch_sub(aint, static_cast(error), model); +} + +template +void test_int_xchg() { + static constexpr T val = V >> right_shift(); + std::vector pool; + for (int model : atomic_exchange_models) { + T aint = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_xchg, &aint, model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (aint != val * kExpected) + fail(); + } +} + +template +void looper_int_xchg_n(T *aint, int model) { + static constexpr T val = V >> right_shift(); + __int128_t error = 0; + T next = *aint + val; + for (int n = 0; n < kIterations; ++n) { + T result = __atomic_exchange_n(aint, next, model); + error += + static_cast<__int128_t>(next) - static_cast<__int128_t>(result + val); + next = result + val; + } + __atomic_fetch_sub(aint, static_cast(error), model); +} + +template +void test_int_xchg_n() { + static constexpr T val = V >> right_shift(); + std::vector pool; + for (int model : atomic_exchange_models) { + T aint = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_xchg_n, &aint, model); + for (int n = 0; n < kThreads; ++n) pool[n].join(); + pool.clear(); + if (aint != val * kExpected) + fail(); + } +} + +// The cmpxchg tests act similar to fetch_add tests. +template +void looper_int_cmpxchg(T *aint, int success_model, int fail_model) { + static constexpr T val = V >> right_shift(); + for (int n = 0; n < kIterations; ++n) { + T desired, expected = 0; + do { + desired = expected + val; + } while (!__atomic_compare_exchange(aint, &expected, &desired, true, + success_model, fail_model)); + } +} + +template +void test_int_cmpxchg() { + static constexpr T val = V >> right_shift(); + std::vector pool; + for (int success_model : atomic_compare_exchange_models) { + for (int fail_model : atomic_compare_exchange_models) { + T aint = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_cmpxchg, &aint, success_model, + fail_model); + for (int n = 0; n < kThreads; ++n) pool[n].join(); + pool.clear(); + if (aint != static_cast(val) * kExpected) + fail(); + } + } +} + +template +void looper_int_cmpxchg_n(T *aint, int success_model, int fail_model) { + static constexpr T val = V >> right_shift(); + for (int n = 0; n < kIterations; ++n) { + T desired, expected = 0; + do { + desired = expected + val; + } while (!__atomic_compare_exchange_n(aint, &expected, desired, true, + success_model, fail_model)); + } +} + +template +void test_int_cmpxchg_n() { + static constexpr T val = V >> right_shift(); + std::vector pool; + for (int success_model : atomic_compare_exchange_models) { + for (int fail_model : atomic_compare_exchange_models) { + T aint = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_cmpxchg_n, &aint, success_model, + fail_model); + for (int n = 0; n < kThreads; ++n) pool[n].join(); + pool.clear(); + if (aint != static_cast(val) * kExpected) + fail(); + } + } +} + +void test_aligned_int() { +#define INT_SUITE(type) \ + { \ + std::cout << "Testing aligned " #type "\n"; \ + test_int_fetch_add(); \ + test_int_fetch_sub(); \ + test_int_fetch_and(); \ + test_int_fetch_or(); \ + test_int_fetch_xor(); \ + test_int_xchg(); \ + test_int_xchg_n(); \ + test_int_cmpxchg(); \ + test_int_cmpxchg_n(); \ + } + INT_SUITE(uint32_t); + INT_SUITE(uint64_t); + INT_SUITE(int32_t); + INT_SUITE(int64_t); +#undef INT_SUITE + +#if TEST16 +#define INT_SUITE(type) \ + { \ + std::cout << "Testing aligned " #type "\n"; \ + test_int_xchg(); \ + test_int_xchg_n(); \ + test_int_cmpxchg(); \ + test_int_cmpxchg_n(); \ + } + INT_SUITE(__uint128_t); + INT_SUITE(__int128_t); +#undef INT_SUITE +#endif +} + +int main() { + test_aligned_int(); + std::cout << "PASSED\n"; +} diff --git a/SingleSource/UnitTests/Atomic/int_aligned_test.reference_output b/SingleSource/UnitTests/Atomic/int_aligned_test.reference_output new file mode 100644 index 0000000000..560bc21eb9 --- /dev/null +++ b/SingleSource/UnitTests/Atomic/int_aligned_test.reference_output @@ -0,0 +1,8 @@ +Testing aligned uint32_t +Testing aligned uint64_t +Testing aligned int32_t +Testing aligned int64_t +Testing aligned __uint128_t +Testing aligned __int128_t +PASSED +exit 0 diff --git a/SingleSource/UnitTests/Atomic/int_misaligned_test.cpp b/SingleSource/UnitTests/Atomic/int_misaligned_test.cpp new file mode 100644 index 0000000000..8b3f654dae --- /dev/null +++ b/SingleSource/UnitTests/Atomic/int_misaligned_test.cpp @@ -0,0 +1,362 @@ +//===--- int_misaligned_test.cc -- Testing unaligned integers ----- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file tests atomic operations on signed and unsigned integer types with +// unaligned memory addresses. +// +// The types tested are: uint32, uint64, int32, int64, uint128, int128. +// The ops tested are: add, sub, and, or, xor, xchg, xchg_n, cmpxchg, cmpxchg_n. +// The ALU operations are not tested on 128-bit integers. +// +// Please read the README before contributing. +// +//===----------------------------------------------------------------------===// + +#include + +#include +#include +#include +#include +#include + +#include "util.h" + +#ifdef SMALL_PROBLEM_SIZE +static constexpr int kIterations = 50'000; +#else +static constexpr int kIterations = 500'000; +#endif +static constexpr int kExpected = kThreads * kIterations; + +template +struct __attribute__((packed)) misaligned { + char byte; + T data; + char pad[(31 - sizeof(T)) % 16]; // Pad struct to 16 or 32 bytes. +}; + +template +void looper_int_misaligned_fetch_add(misaligned &astruct, int model) { + static constexpr T val = V >> right_shift(); + for (int n = 0; n < kIterations; ++n) + __atomic_fetch_add(&astruct.data, val, model); +} + +template +void test_int_misaligned_fetch_add() { + static constexpr T val = V >> right_shift(); + std::vector pool; + misaligned astruct; + for (int model : atomic_fetch_models) { + astruct.data = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_misaligned_fetch_add, std::ref(astruct), + model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (astruct.data != val * kExpected) + fail(); + } +} + +template +void looper_int_misaligned_fetch_sub(misaligned &astruct, int model) { + static constexpr T val = V >> right_shift(); + for (int n = 0; n < kIterations; ++n) + __atomic_fetch_sub(&astruct.data, val, model); +} + +template +void test_int_misaligned_fetch_sub() { + static constexpr T val = V >> right_shift(); + std::vector pool; + misaligned astruct; + for (int model : atomic_fetch_models) { + astruct.data = val * kExpected; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_misaligned_fetch_sub, std::ref(astruct), + model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (astruct.data != 0) + fail(); + } +} + +// See int_aligned_test.cc for an explanation of AND tests. +template +void __attribute__((optnone)) looper_int_misaligned_fetch_and( + const int id, misaligned &astruct, T *acnt, int model) { + T desired, expected = 0; + for (int n = 0; n < kIterations; ++n) { + const T mask = 1 << id; + __atomic_fetch_and(&astruct.data, ~mask, model); + if (~astruct.data & mask) { + __atomic_fetch_add(acnt, 1, model); + do { + desired = expected | mask; + } while (!__atomic_compare_exchange(&astruct.data, &expected, &desired, + true, model, model)); + } + } +} + +template +void test_int_misaligned_fetch_and() { + std::vector pool; + misaligned astruct; + for (int model : atomic_fetch_models) { + T acnt = 0; + astruct.data = ~0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_misaligned_fetch_and, n, + std::ref(astruct), &acnt, model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (acnt != kExpected) + fail(); + } +} + +// See int_aligned_test.cc for an explanation of OR tests. +template +void __attribute__((optnone)) looper_int_misaligned_fetch_or( + const int id, misaligned &astruct, T *acnt, int model) { + T desired, expected = 0; + for (int n = 0; n < kIterations; ++n) { + const T mask = 1 << id; + __atomic_fetch_or(&astruct.data, mask, model); + if (astruct.data & mask) { + __atomic_fetch_add(acnt, 1, model); + do { + desired = expected & ~mask; + } while (!__atomic_compare_exchange(&astruct.data, &expected, &desired, + true, model, model)); + } + } +} + +template +void test_int_misaligned_fetch_or() { + std::vector pool; + misaligned astruct; + for (int model : atomic_fetch_models) { + T acnt = 0; + astruct.data = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_misaligned_fetch_or, n, + std::ref(astruct), &acnt, model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (acnt != kExpected) + fail(); + } +} + +template +void looper_int_misaligned_fetch_xor(misaligned &astruct, int model) { + for (int n = 0; n < kIterations; ++n) + __atomic_fetch_xor(&astruct.data, n, model); +} + +template +void test_int_misaligned_fetch_xor() { + std::vector pool; + misaligned astruct; + for (int model : atomic_fetch_models) { + astruct.data = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_misaligned_fetch_xor, std::ref(astruct), + model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (astruct.data != 0) + fail(); + } +} + +// See int_aligned_test.cc for an explanation of xchg tests. +template +void looper_int_misaligned_xchg(misaligned &astruct, int model) { + static constexpr T val = V >> right_shift(); + __int128_t error = 0; + T next = astruct.data + val; + T result; + for (int n = 0; n < kIterations; ++n) { + __atomic_exchange(&astruct.data, &next, &result, model); + error += + static_cast<__int128_t>(next) - static_cast<__int128_t>(result + val); + next = result + val; + } + __atomic_fetch_sub(&astruct.data, static_cast(error), model); +} + +template +void test_int_misaligned_xchg() { + static constexpr T val = V >> right_shift(); + std::vector pool; + misaligned astruct; + for (int model : atomic_exchange_models) { + astruct.data = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_misaligned_xchg, std::ref(astruct), + model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (astruct.data != val * kExpected) + fail(); + } +} + +// See int_aligned_test.cc for an explanation of xchg tests. +template +void looper_int_misaligned_xchg_n(misaligned &astruct, int model) { + static constexpr T val = V >> right_shift(); + __int128_t error = 0; + T next = astruct.data + val; + for (int n = 0; n < kIterations; ++n) { + T result = __atomic_exchange_n(&astruct.data, next, model); + error += + static_cast<__int128_t>(next) - static_cast<__int128_t>(result + val); + next = result + val; + } + __atomic_fetch_sub(&astruct.data, static_cast(error), model); +} + +template +void test_int_misaligned_xchg_n() { + static constexpr T val = V >> right_shift(); + std::vector pool; + misaligned astruct; + for (int model : atomic_exchange_models) { + astruct.data = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_misaligned_xchg_n, std::ref(astruct), + model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (astruct.data != val * kExpected) + fail(); + } +} + +// See int_aligned_test.cc for an explanation of cmpxchg tests. +template +void looper_int_misaligned_cmpxchg(misaligned &astruct, int success_model, + int fail_model) { + static constexpr T val = V >> right_shift(); + for (int n = 0; n < kIterations; ++n) { + T desired, expected = 0; + do { + desired = expected + val; + } while (!__atomic_compare_exchange(&astruct.data, &expected, &desired, + true, success_model, fail_model)); + } +} + +template +void test_int_misaligned_cmpxchg() { + static constexpr T val = V >> right_shift(); + std::vector pool; + misaligned astruct; + for (int success_model : atomic_compare_exchange_models) { + for (int fail_model : atomic_compare_exchange_models) { + astruct.data = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_misaligned_cmpxchg, std::ref(astruct), + success_model, fail_model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (astruct.data != static_cast(val) * kExpected) + fail(); + } + } +} + +// See int_aligned_test.cc for an explanation of cmpxchg tests. +template +void looper_int_misaligned_cmpxchg_n(misaligned &astruct, int success_model, + int fail_model) { + static constexpr T val = V >> right_shift(); + for (int n = 0; n < kIterations; ++n) { + T desired, expected = 0; + do { + desired = expected + val; + } while (!__atomic_compare_exchange_n(&astruct.data, &expected, desired, + true, success_model, fail_model)); + } +} + +template +void test_int_misaligned_cmpxchg_n() { + static constexpr T val = V >> right_shift(); + std::vector pool; + misaligned astruct; + for (int success_model : atomic_compare_exchange_models) { + for (int fail_model : atomic_compare_exchange_models) { + astruct.data = 0; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_int_misaligned_cmpxchg_n, std::ref(astruct), + success_model, fail_model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + if (astruct.data != static_cast(val) * kExpected) + fail(); + } + } +} + +void test_misaligned_int() { +#define INT_SUITE(type) \ + { \ + std::cout << "Testing misaligned " #type "\n"; \ + test_int_misaligned_fetch_add(); \ + test_int_misaligned_fetch_sub(); \ + test_int_misaligned_fetch_and(); \ + test_int_misaligned_fetch_or(); \ + test_int_misaligned_fetch_xor(); \ + test_int_misaligned_xchg(); \ + test_int_misaligned_xchg_n(); \ + test_int_misaligned_cmpxchg(); \ + test_int_misaligned_cmpxchg_n(); \ + } + INT_SUITE(uint32_t); + INT_SUITE(uint64_t); + INT_SUITE(int32_t); + INT_SUITE(int64_t); +#undef INT_SUITE + +#if TEST16 +#define INT_SUITE(type) \ + { \ + std::cout << "Testing misaligned " #type "\n"; \ + test_int_misaligned_xchg(); \ + test_int_misaligned_xchg_n(); \ + test_int_misaligned_cmpxchg(); \ + test_int_misaligned_cmpxchg_n(); \ + } + INT_SUITE(__uint128_t); + INT_SUITE(__int128_t); +#undef INT_SUITE +#endif +} + +int main() { + test_misaligned_int(); + std::cout << "PASSED\n"; +} diff --git a/SingleSource/UnitTests/Atomic/int_misaligned_test.reference_output b/SingleSource/UnitTests/Atomic/int_misaligned_test.reference_output new file mode 100644 index 0000000000..89a1b7a69e --- /dev/null +++ b/SingleSource/UnitTests/Atomic/int_misaligned_test.reference_output @@ -0,0 +1,8 @@ +Testing misaligned uint32_t +Testing misaligned uint64_t +Testing misaligned int32_t +Testing misaligned int64_t +Testing misaligned __uint128_t +Testing misaligned __int128_t +PASSED +exit 0 diff --git a/SingleSource/UnitTests/Atomic/misshapen_test.cpp b/SingleSource/UnitTests/Atomic/misshapen_test.cpp new file mode 100644 index 0000000000..ac94d0c626 --- /dev/null +++ b/SingleSource/UnitTests/Atomic/misshapen_test.cpp @@ -0,0 +1,155 @@ +//===--- misshapen.cc -- Testing non-power-of-2 byte objects ------ C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file tests atomic operations on oddly sized objects with aligned +// memory addresses. +// +// The types tested are: 3 byte up to 15 byte objects (skipping 4 & 8). +// The ops tested are: xchg, cmpxchg. +// +// Please read the README before contributing. +// +//===----------------------------------------------------------------------===// + +#include +#include +#include + +#include "util.h" + +#ifdef SMALL_PROBLEM_SIZE +static constexpr int kIterations = 100'000; +#else +static constexpr int kIterations = 1'000'000; +#endif +static constexpr int kExpected = kThreads * kIterations; + +template +struct misshapen { + unsigned char v[N]; +}; + +// See int_aligned_test.cc for an explanation of xchg tests. +template +void looper_misshapen_xchg(misshapen *amis, int model) { + unsigned char error[N] = {}; + misshapen next, result; + __atomic_load(amis, &next, model); + for (int k = 0; k < N; ++k) + next.v[k]++; + for (int n = 0; n < kIterations; ++n) { + __atomic_exchange(amis, &next, &result, model); + for (int k = 0; k < N; ++k) { + error[k] += next.v[k] - (result.v[k] + 1); + next.v[k] = result.v[k] + 1; + } + } + // We can't use atomic_sub here; combining atomic operations on array members + // and the array as a whole is undefined. + misshapen desired, expected; + __atomic_load(amis, &expected, model); + do { + desired = expected; + for (int k = 0; k < N; ++k) + desired.v[k] -= error[k]; + } while (!__atomic_compare_exchange(amis, &expected, &desired, true, + model, model)); +} + +template +void test_misshapen_xchg() { + std::vector pool; + for (int model : atomic_exchange_models) { + misshapen amis = {}; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_misshapen_xchg, &amis, model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + for (int n = 0; n < N; ++n) + if (amis.v[n] != kExpected % (1 << (8 * sizeof(amis.v[0])))) + fail(); + } +} + +// See int_aligned_test.cc for an explanation of cmpxchg tests. +template +void looper_misshapen_cmpxchg(misshapen *amis, int success_model, + int fail_model) { + for (int n = 0; n < kIterations; ++n) { + misshapen desired, expected = {}; + do { + desired = expected; + for (int k = 0; k < N; ++k) + desired.v[k]++; + } while (!__atomic_compare_exchange(amis, &expected, &desired, true, + success_model, fail_model)); + } +} + +template +void test_misshapen_cmpxchg() { + std::vector pool; + for (int success_model : atomic_compare_exchange_models) { + for (int fail_model : atomic_compare_exchange_models) { + misshapen amis = {}; + for (int n = 0; n < kThreads; ++n) + pool.emplace_back(looper_misshapen_cmpxchg, &amis, success_model, + fail_model); + for (int n = 0; n < kThreads; ++n) + pool[n].join(); + pool.clear(); + for (int n = 0; n < N; ++n) + if (amis.v[n] != kExpected % (1 << (8 * sizeof(amis.v[0])))) + fail(); + } + } +} + +void test_misshapen() { + std::cout << "Testing misshapen 3 byte\n"; + test_misshapen_xchg<3>(); + test_misshapen_cmpxchg<3>(); + // Skip 4. + std::cout << "Testing misshapen 5 byte\n"; + test_misshapen_xchg<5>(); + test_misshapen_cmpxchg<5>(); + std::cout << "Testing misshapen 6 byte\n"; + test_misshapen_xchg<6>(); + test_misshapen_cmpxchg<6>(); + std::cout << "Testing misshapen 7 byte\n"; + test_misshapen_xchg<7>(); + test_misshapen_cmpxchg<7>(); + // Skip 8. + std::cout << "Testing misshapen 9 byte\n"; + test_misshapen_xchg<9>(); + test_misshapen_cmpxchg<9>(); + std::cout << "Testing misshapen 10 byte\n"; + test_misshapen_xchg<10>(); + test_misshapen_cmpxchg<10>(); + std::cout << "Testing misshapen 11 byte\n"; + test_misshapen_xchg<11>(); + test_misshapen_cmpxchg<11>(); + std::cout << "Testing misshapen 12 byte\n"; + test_misshapen_xchg<12>(); + test_misshapen_cmpxchg<12>(); + std::cout << "Testing misshapen 13 byte\n"; + test_misshapen_xchg<13>(); + test_misshapen_cmpxchg<13>(); + std::cout << "Testing misshapen 14 byte\n"; + test_misshapen_xchg<14>(); + test_misshapen_cmpxchg<14>(); + std::cout << "Testing misshapen 15 byte\n"; + test_misshapen_xchg<15>(); + test_misshapen_cmpxchg<15>(); +} + +int main() { + test_misshapen(); + std::cout << "PASSED\n"; +} diff --git a/SingleSource/UnitTests/Atomic/misshapen_test.reference_output b/SingleSource/UnitTests/Atomic/misshapen_test.reference_output new file mode 100644 index 0000000000..ff5a3a85dc --- /dev/null +++ b/SingleSource/UnitTests/Atomic/misshapen_test.reference_output @@ -0,0 +1,13 @@ +Testing misshapen 3 byte +Testing misshapen 5 byte +Testing misshapen 6 byte +Testing misshapen 7 byte +Testing misshapen 9 byte +Testing misshapen 10 byte +Testing misshapen 11 byte +Testing misshapen 12 byte +Testing misshapen 13 byte +Testing misshapen 14 byte +Testing misshapen 15 byte +PASSED +exit 0 diff --git a/SingleSource/UnitTests/Atomic/util.h b/SingleSource/UnitTests/Atomic/util.h new file mode 100644 index 0000000000..161f8fb5b4 --- /dev/null +++ b/SingleSource/UnitTests/Atomic/util.h @@ -0,0 +1,61 @@ +//===--- util.h -- Utility functions shared by tests -------------- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _UTIL_H_ +#define _UTIL_H_ + +#include +#include +#include + +#define TEST16 1 + +static constexpr uint64_t V = 0x4243444546474849; +static constexpr int kThreads = 10; + +// TODO: When support for other models is added, be sure to use constant values, +// perhaps using template arguments. +constexpr int atomic_fetch_models[] = { + // TODO: Figure out a way to test all of these in parallel. + // __ATOMIC_RELAXED, __ATOMIC_CONSUME, __ATOMIC_ACQUIRE, + // __ATOMIC_ACQ_REL, __ATOMIC_RELEASE, + __ATOMIC_SEQ_CST, +}; +constexpr int atomic_exchange_models[] = { + // TODO: Figure out a way to test all of these in parallel. + // __ATOMIC_RELAXED, __ATOMIC_ACQUIRE, __ATOMIC_RELEASE, + // __ATOMIC_ACQ_REL, + __ATOMIC_SEQ_CST, +}; +constexpr int atomic_compare_exchange_models[] = { + // TODO: Figure out a way to test all of these in parallel. + // __ATOMIC_RELAXED, __ATOMIC_CONSUME, __ATOMIC_ACQUIRE, + // __ATOMIC_ACQ_REL, __ATOMIC_RELEASE, + __ATOMIC_SEQ_CST, +}; + +inline void fail() { + std::cout << "FAILED\n"; + exit(1); +} + +template +constexpr uint8_t right_shift() { + switch (sizeof(T)) { + case 16: + return 0; + case 8: + return 24; + case 4: + return 32 + 24; + default: + return 62; + } +} + +#endif // _UTIL_H_ diff --git a/SingleSource/UnitTests/CMakeLists.txt b/SingleSource/UnitTests/CMakeLists.txt index c04bd2347b..95c7c8672f 100644 --- a/SingleSource/UnitTests/CMakeLists.txt +++ b/SingleSource/UnitTests/CMakeLists.txt @@ -1,5 +1,6 @@ include(CheckCCompilerFlag) +add_subdirectory(Atomic) add_subdirectory(C++11) add_subdirectory(Float) add_subdirectory(Matrix) From 9dc1b025d793f06594273549bf727eb5ea787c0b Mon Sep 17 00:00:00 2001 From: Sean Luchen Date: Tue, 27 Feb 2024 18:15:26 +0000 Subject: [PATCH 2/2] Conditionally test with libclang_rt.atomic; always test with -latomic. --- SingleSource/UnitTests/Atomic/CMakeLists.txt | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/SingleSource/UnitTests/Atomic/CMakeLists.txt b/SingleSource/UnitTests/Atomic/CMakeLists.txt index 9a341d187a..4431e1f0ee 100644 --- a/SingleSource/UnitTests/Atomic/CMakeLists.txt +++ b/SingleSource/UnitTests/Atomic/CMakeLists.txt @@ -2,8 +2,15 @@ execute_process(COMMAND ${CMAKE_C_COMPILER} --print-file-name=libclang_rt.atomic.so OUTPUT_VARIABLE _path_to_libatomic OUTPUT_STRIP_TRAILING_WHITESPACE) -get_filename_component(_libatomic_dir ${_path_to_libatomic} DIRECTORY) -add_link_options("LINKER:${_path_to_libatomic},-rpath=${_libatomic_dir}") +if(NOT _path_to_libatomic STREQUAL "libclang_rt.atomic.so") + get_directory_property(CURRENT_LINK_OPTIONS LINK_OPTIONS) + get_filename_component(_libatomic_dir ${_path_to_libatomic} DIRECTORY) + add_link_options("LINKER:${_path_to_libatomic},-rpath=${_libatomic_dir}") -llvm_singlesource() + llvm_singlesource(PREFIX "libclang_rt-atomic-") + set_directory_properties(LINK_OPTIONS ${CURRENT_LINK_OPTIONS}) +endif() + +add_link_options("LINKER:-latomic") +llvm_singlesource(PREFIX "latomic-")