From 237a9cf50803925e8e2c81a8981ad18fb1ca926c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Wed, 12 Aug 2026 14:37:05 -0700 Subject: [PATCH] Use native 8-bit PTX atomic loads and stores. --- libcudacxx/codegen/generators/definitions.h | 2 +- libcudacxx/codegen/generators/ld_st.h | 49 ++- .../std/__atomic/functions/cuda_ptx_derived.h | 33 -- .../__atomic/functions/cuda_ptx_generated.h | 403 ++++++++++++++++++ 4 files changed, 452 insertions(+), 35 deletions(-) diff --git a/libcudacxx/codegen/generators/definitions.h b/libcudacxx/codegen/generators/definitions.h index 2ae453f6812a..d1ba0e8a38ba 100644 --- a/libcudacxx/codegen/generators/definitions.h +++ b/libcudacxx/codegen/generators/definitions.h @@ -107,7 +107,7 @@ inline std::string constraints(Operand op, size_t sz) }}, }; - if (sz == 16) + if (sz <= 16) { return {"h"}; } diff --git a/libcudacxx/codegen/generators/ld_st.h b/libcudacxx/codegen/generators/ld_st.h index d836d80f78ad..b8bab245e071 100644 --- a/libcudacxx/codegen/generators/ld_st.h +++ b/libcudacxx/codegen/generators/ld_st.h @@ -97,8 +97,18 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( const _Type* __ptr, _Type& __dst, {3}, __atomic_cuda_operand_{0}{1}, {5}, {7}) {{ asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__dst) : "l"(__ptr) : "memory"); }})XXX"; + constexpr auto asm_intrinsic_format_8 = R"XXX( +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, {3}, __atomic_cuda_operand_{0}{1}, {5}, {7}) +{{ + uint16_t __tmp; + asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +}})XXX"; constexpr size_t supported_sizes[] = { + 8, 16, 32, 64, @@ -140,7 +150,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( { for (auto mm : mmio_states) { - if (size == 16 && type == Operand::Floating) + if (size <= 16 && type == Operand::Floating) { continue; } @@ -167,6 +177,20 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( /* 7 */ mmio_tag(mm), /* 8 */ mmio(mm)); } + else if (size == 8) + { + out << std::format( + asm_intrinsic_format_8, + /* 0 */ operand(type), + /* 1 */ size, + /* 2 */ constraints(type, size), + /* 3 */ semantic_tag(sem), + /* 4 */ semantic_ld_st(sem), + /* 5 */ scope_tag(sco), + /* 6 */ scope_ld_st(sem, sco), + /* 7 */ mmio_tag(mm), + /* 8 */ mmio(mm)); + } else { out << std::format( @@ -281,8 +305,17 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( _Type* __ptr, _Type& __val, {3}, __atomic_cuda_operand_{0}{1}, {5}, {7}) {{ asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__val) : "memory"); }})XXX"; + constexpr auto asm_intrinsic_format_8 = R"XXX( +template +static inline _CCCL_DEVICE void __cuda_atomic_store( + _Type* __ptr, _Type& __val, {3}, __atomic_cuda_operand_{0}{1}, {5}, {7}) +{{ + const uint16_t __tmp = static_cast(__val); + asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__tmp) : "memory"); +}})XXX"; constexpr size_t supported_sizes[] = { + 8, 16, 32, 64, @@ -348,6 +381,20 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( /* 7 */ mmio_tag(mm), /* 8 */ mmio(mm)); } + else if (size == 8) + { + out << std::format( + asm_intrinsic_format_8, + /* 0 */ operand(type), + /* 1 */ size, + /* 2 */ constraints(type, size), + /* 3 */ semantic_tag(sem), + /* 4 */ semantic_ld_st(sem), + /* 5 */ scope_tag(sco), + /* 6 */ scope_ld_st(sem, sco), + /* 7 */ mmio_tag(mm), + /* 8 */ mmio(mm)); + } else { out << std::format( diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_derived.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_derived.h index 759ba6db8a1d..e6d58ec178c1 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_derived.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_derived.h @@ -44,26 +44,6 @@ using __cuda_atomic_enable_non_native_bitwise = enable_if_t<_Operand::__size <= template using __cuda_atomic_enable_native_bitwise = enable_if_t<_Operand::__size >= 32, bool>; -template -using __cuda_atomic_enable_non_native_ld_st = enable_if_t<_Operand::__size <= 8, bool>; - -template -using __cuda_atomic_enable_native_ld_st = enable_if_t<_Operand::__size >= 16, bool>; - -template = 0> -_CCCL_DEVICE static void -__cuda_atomic_load(const _Type* __ptr, _Type& __dst, _Order, _Operand, _Sco, __atomic_cuda_mmio_disable) -{ - constexpr uint64_t __alignmask = (sizeof(uint16_t) - 1); - uint16_t* __aligned = (uint16_t*) ((intptr_t) __ptr & (~__alignmask)); // NOLINT(performance-no-int-to-ptr) - const uint8_t __offset = uint16_t((intptr_t) __ptr & __alignmask) * 8; - - uint16_t __value = 0; - __cuda_atomic_load(__aligned, __value, _Order{}, __atomic_cuda_operand_b16{}, _Sco{}, __atomic_cuda_mmio_disable{}); - - __dst = static_cast<_Type>(__value >> __offset); -} - template = 0> _CCCL_DEVICE static bool __cuda_atomic_compare_exchange(_Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, _Order, _Operand, _Sco) @@ -212,19 +192,6 @@ _CCCL_DEVICE_API _Type __cuda_atomic_fetch_update(_Type* __ptr, const _Fn& __op, return __expected; } -template = 0> -_CCCL_DEVICE static void -__cuda_atomic_store(_Type* __ptr, _Type __val, _Order, _Operand, _Sco, __atomic_cuda_mmio_disable) -{ - // Store requires cas on 8/16b types - __cuda_atomic_fetch_update( - __ptr, - __cuda_atomic_op_bind<_Type, ::cuda::std::__cuda_atomic_op_store>{__val}, - _Order{}, - __atomic_cuda_operand_tag<__atomic_cuda_operand::_b, _Operand::__size>{}, - _Sco{}); -} - template = 0> _CCCL_DEVICE static void __cuda_atomic_fetch_add(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco) { diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h index 479815f41366..afff5432af94 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h @@ -122,6 +122,318 @@ static inline _CCCL_DEVICE void __cuda_atomic_load_memory_order_dispatch(_Fn &__ ) } +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_acquire, __atomic_cuda_operand_b8, __thread_scope_block_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.acquire.cta.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_acquire, __atomic_cuda_operand_b8, __thread_scope_cluster_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.acquire.cluster.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_acquire, __atomic_cuda_operand_b8, __thread_scope_device_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.acquire.gpu.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_acquire, __atomic_cuda_operand_b8, __thread_scope_system_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.acquire.sys.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_relaxed, __atomic_cuda_operand_b8, __thread_scope_block_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.relaxed.cta.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_relaxed, __atomic_cuda_operand_b8, __thread_scope_cluster_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.relaxed.cluster.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_relaxed, __atomic_cuda_operand_b8, __thread_scope_device_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.relaxed.gpu.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_relaxed, __atomic_cuda_operand_b8, __thread_scope_system_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.relaxed.sys.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_relaxed, __atomic_cuda_operand_b8, __thread_scope_system_tag, __atomic_cuda_mmio_enable) +{ + uint16_t __tmp; + asm volatile("ld.mmio.relaxed.sys.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_volatile, __atomic_cuda_operand_b8, __thread_scope_block_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_volatile, __atomic_cuda_operand_b8, __thread_scope_cluster_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_volatile, __atomic_cuda_operand_b8, __thread_scope_device_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_volatile, __atomic_cuda_operand_b8, __thread_scope_system_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_acquire, __atomic_cuda_operand_u8, __thread_scope_block_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.acquire.cta.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_acquire, __atomic_cuda_operand_u8, __thread_scope_cluster_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.acquire.cluster.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_acquire, __atomic_cuda_operand_u8, __thread_scope_device_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.acquire.gpu.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_acquire, __atomic_cuda_operand_u8, __thread_scope_system_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.acquire.sys.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_relaxed, __atomic_cuda_operand_u8, __thread_scope_block_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.relaxed.cta.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_relaxed, __atomic_cuda_operand_u8, __thread_scope_cluster_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.relaxed.cluster.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_relaxed, __atomic_cuda_operand_u8, __thread_scope_device_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.relaxed.gpu.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_relaxed, __atomic_cuda_operand_u8, __thread_scope_system_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.relaxed.sys.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_relaxed, __atomic_cuda_operand_u8, __thread_scope_system_tag, __atomic_cuda_mmio_enable) +{ + uint16_t __tmp; + asm volatile("ld.mmio.relaxed.sys.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_volatile, __atomic_cuda_operand_u8, __thread_scope_block_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_volatile, __atomic_cuda_operand_u8, __thread_scope_cluster_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_volatile, __atomic_cuda_operand_u8, __thread_scope_device_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_volatile, __atomic_cuda_operand_u8, __thread_scope_system_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_acquire, __atomic_cuda_operand_s8, __thread_scope_block_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.acquire.cta.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_acquire, __atomic_cuda_operand_s8, __thread_scope_cluster_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.acquire.cluster.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_acquire, __atomic_cuda_operand_s8, __thread_scope_device_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.acquire.gpu.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_acquire, __atomic_cuda_operand_s8, __thread_scope_system_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.acquire.sys.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_relaxed, __atomic_cuda_operand_s8, __thread_scope_block_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.relaxed.cta.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_relaxed, __atomic_cuda_operand_s8, __thread_scope_cluster_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.relaxed.cluster.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_relaxed, __atomic_cuda_operand_s8, __thread_scope_device_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.relaxed.gpu.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_relaxed, __atomic_cuda_operand_s8, __thread_scope_system_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.relaxed.sys.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_relaxed, __atomic_cuda_operand_s8, __thread_scope_system_tag, __atomic_cuda_mmio_enable) +{ + uint16_t __tmp; + asm volatile("ld.mmio.relaxed.sys.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_volatile, __atomic_cuda_operand_s8, __thread_scope_block_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_volatile, __atomic_cuda_operand_s8, __thread_scope_cluster_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_volatile, __atomic_cuda_operand_s8, __thread_scope_device_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_load( + const _Type* __ptr, _Type& __dst, __atomic_cuda_volatile, __atomic_cuda_operand_s8, __thread_scope_system_tag, __atomic_cuda_mmio_disable) +{ + uint16_t __tmp; + asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); + __dst = static_cast<_Type>(__tmp); +} template static inline _CCCL_DEVICE void __cuda_atomic_load( const _Type* __ptr, _Type& __dst, __atomic_cuda_acquire, __atomic_cuda_operand_b16, __thread_scope_block_tag, __atomic_cuda_mmio_disable) @@ -971,6 +1283,97 @@ static inline _CCCL_DEVICE void __cuda_atomic_store_memory_order_dispatch(_Fn &_ ) } +template +static inline _CCCL_DEVICE void __cuda_atomic_store( + _Type* __ptr, _Type& __val, __atomic_cuda_release, __atomic_cuda_operand_b8, __thread_scope_block_tag, __atomic_cuda_mmio_disable) +{ + const uint16_t __tmp = static_cast(__val); + asm volatile("st.release.cta.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_store( + _Type* __ptr, _Type& __val, __atomic_cuda_release, __atomic_cuda_operand_b8, __thread_scope_cluster_tag, __atomic_cuda_mmio_disable) +{ + const uint16_t __tmp = static_cast(__val); + asm volatile("st.release.cluster.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_store( + _Type* __ptr, _Type& __val, __atomic_cuda_release, __atomic_cuda_operand_b8, __thread_scope_device_tag, __atomic_cuda_mmio_disable) +{ + const uint16_t __tmp = static_cast(__val); + asm volatile("st.release.gpu.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_store( + _Type* __ptr, _Type& __val, __atomic_cuda_release, __atomic_cuda_operand_b8, __thread_scope_system_tag, __atomic_cuda_mmio_disable) +{ + const uint16_t __tmp = static_cast(__val); + asm volatile("st.release.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_store( + _Type* __ptr, _Type& __val, __atomic_cuda_relaxed, __atomic_cuda_operand_b8, __thread_scope_block_tag, __atomic_cuda_mmio_disable) +{ + const uint16_t __tmp = static_cast(__val); + asm volatile("st.relaxed.cta.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_store( + _Type* __ptr, _Type& __val, __atomic_cuda_relaxed, __atomic_cuda_operand_b8, __thread_scope_cluster_tag, __atomic_cuda_mmio_disable) +{ + const uint16_t __tmp = static_cast(__val); + asm volatile("st.relaxed.cluster.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_store( + _Type* __ptr, _Type& __val, __atomic_cuda_relaxed, __atomic_cuda_operand_b8, __thread_scope_device_tag, __atomic_cuda_mmio_disable) +{ + const uint16_t __tmp = static_cast(__val); + asm volatile("st.relaxed.gpu.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_store( + _Type* __ptr, _Type& __val, __atomic_cuda_relaxed, __atomic_cuda_operand_b8, __thread_scope_system_tag, __atomic_cuda_mmio_disable) +{ + const uint16_t __tmp = static_cast(__val); + asm volatile("st.relaxed.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_store( + _Type* __ptr, _Type& __val, __atomic_cuda_relaxed, __atomic_cuda_operand_b8, __thread_scope_system_tag, __atomic_cuda_mmio_enable) +{ + const uint16_t __tmp = static_cast(__val); + asm volatile("st.mmio.relaxed.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_store( + _Type* __ptr, _Type& __val, __atomic_cuda_volatile, __atomic_cuda_operand_b8, __thread_scope_block_tag, __atomic_cuda_mmio_disable) +{ + const uint16_t __tmp = static_cast(__val); + asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_store( + _Type* __ptr, _Type& __val, __atomic_cuda_volatile, __atomic_cuda_operand_b8, __thread_scope_cluster_tag, __atomic_cuda_mmio_disable) +{ + const uint16_t __tmp = static_cast(__val); + asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_store( + _Type* __ptr, _Type& __val, __atomic_cuda_volatile, __atomic_cuda_operand_b8, __thread_scope_device_tag, __atomic_cuda_mmio_disable) +{ + const uint16_t __tmp = static_cast(__val); + asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); +} +template +static inline _CCCL_DEVICE void __cuda_atomic_store( + _Type* __ptr, _Type& __val, __atomic_cuda_volatile, __atomic_cuda_operand_b8, __thread_scope_system_tag, __atomic_cuda_mmio_disable) +{ + const uint16_t __tmp = static_cast(__val); + asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); +} template static inline _CCCL_DEVICE void __cuda_atomic_store( _Type* __ptr, _Type& __val, __atomic_cuda_release, __atomic_cuda_operand_b16, __thread_scope_block_tag, __atomic_cuda_mmio_disable)