libcu++: atomics SASS FileCheck suite for basic atomic operations

.github/actions/workflow-build/build-workflow.py

@@ -334,6 +334,19 @@ def get_job_type_info(job):
     return result
 
 
+@memoize_result
+def get_codegen_target(codegen_target):
+    if codegen_target not in matrix_yaml["codegen_targets"]:
+        raise Exception(
+            f"Unknown codegen target '{codegen_target}'. Valid options are: "
+            + ", ".join(matrix_yaml["codegen_targets"].keys())
+        )
+
+    result = matrix_yaml["codegen_targets"][codegen_target]
+    result["id"] = codegen_target
+    return result
+
+
 @memoize_result
 def get_tag_info(tag):
     if tag not in matrix_yaml["tags"].keys():
@@ -364,6 +377,7 @@ def get_all_matrix_job_tags_sorted():
     sorted_important_tags = [
         "project",
         "jobs",
+        "codegen_target",
         "cudacxx",
         "cxx",
         "ctk",
@@ -432,6 +446,10 @@ def generate_dispatch_group_name(matrix_job):
 
 def generate_dispatch_job_name(matrix_job, job_type):
     job_info = get_job_type_info(job_type)
+    job_name = job_info["name"]
+    if "codegen_target" in matrix_job:
+        codegen_target = get_codegen_target(matrix_job["codegen_target"])
+        job_name += f" {codegen_target['name']}"
     cpu_str = matrix_job["cpu"]
     if job_info["gpu"]:
         gpu = get_gpu(matrix_job["gpu"])
@@ -470,7 +488,7 @@ def generate_dispatch_job_name(matrix_job, job_type):
         else ""
     )
 
-    return f"[{config_tag}] {job_info['name']}({cpu_str}{gpu_str}){extra_info}"
+    return f"[{config_tag}] {job_name}({cpu_str}{gpu_str}){extra_info}"
 
 
 def generate_dispatch_job_runner(matrix_job, job_type):
@@ -556,6 +574,13 @@ def generate_dispatch_job_command(matrix_job, job_type):
         command += f' -py-version "{py_version}"'
     if py_ctk_mode:
         command += f' -ctk-mode "{py_ctk_mode}"'
+    if "codegen_target" in matrix_job:
+        codegen_target = get_codegen_target(matrix_job["codegen_target"])
+        command += f' -target "{codegen_target["cmake_target"]}"'
+        command += (
+            " -cmake-options "
+            f'"-DLIBCUDACXX_CODEGEN_FILECHECK_TESTS={codegen_target["id"]}"'
+        )
     if extra_args:
         command += f" {extra_args}"
 
@@ -599,6 +624,11 @@ def generate_dispatch_job_origin(matrix_job, job_type):
     if "args" in origin_job and not origin_job["args"]:
         del origin_job["args"]
 
+    if "codegen_target" in origin_job:
+        origin_job["codegen_target"] = get_codegen_target(origin_job["codegen_target"])[
+            "name"
+        ]
+
     origin["matrix_job"] = origin_job
 
     return origin
@@ -1054,6 +1084,33 @@ def validate_tags(matrix_job, ignore_required=False):
                 error_message_with_matrix_job(matrix_job, f"Unknown tag '{tag}'")
             )
 
+    jobs = matrix_job.get("jobs", [])
+    jobs = jobs if isinstance(jobs, list) else [jobs]
+    has_codegen_job = "codegen_filecheck" in jobs
+    if has_codegen_job and "codegen_target" not in matrix_job:
+        raise Exception(
+            error_message_with_matrix_job(
+                matrix_job,
+                "The codegen_filecheck job requires a codegen_target tag.",
+            )
+        )
+    if "codegen_target" in matrix_job and any(
+        job != "codegen_filecheck" for job in jobs
+    ):
+        raise Exception(
+            error_message_with_matrix_job(
+                matrix_job,
+                "The codegen_target tag is only valid for codegen_filecheck jobs.",
+            )
+        )
+    if "codegen_target" in matrix_job:
+        codegen_targets = matrix_job["codegen_target"]
+        codegen_targets = (
+            codegen_targets if isinstance(codegen_targets, list) else [codegen_targets]
+        )
+        for codegen_target in codegen_targets:
+            get_codegen_target(codegen_target)
+
     if "gpu" in matrix_job:
         gpus = (
             matrix_job["gpu"]

CMakePresets.json

@@ -145,6 +145,16 @@
         "LIBCUDACXX_ENABLE_LIBCUDACXX_TESTS": true
       }
     },
+    {
+      "name": "libcudacxx-codegen-filecheck",
+      "displayName": "libcu++: Codegen FileCheck",
+      "inherits": "libcudacxx",
+      "cacheVariables": {
+        "CMAKE_CUDA_ARCHITECTURES": "80",
+        "LIBCUDACXX_CODEGEN_FILECHECK_TESTS": "all",
+        "LIBCUDACXX_REQUIRE_CODEGEN_TEST_TOOLS": true
+      }
+    },
     {
       "name": "libcudacxx-cpp17",
       "displayName": "libcu++: C++17",
@@ -530,12 +540,20 @@
         "libcudacxx.test.public_headers_host_only",
         "libcudacxx.test.lit.precompile",
         "libcudacxx.test.nvtarget",
-        "libcudacxx.test.atomics.ptx",
-        "libcudacxx.test.simd.ptx",
         "libcudacxx.test.c2h_all",
         "libcudacxx.test.debugging"
       ]
     },
+    {
+      "name": "libcudacxx-codegen-filecheck",
+      "configurePreset": "libcudacxx-codegen-filecheck",
+      "targets": [
+        "libcudacxx.test.atomics.ptx",
+        "libcudacxx.test.atomics.sass",
+        "libcudacxx.test.simd.ptx",
+        "libcudacxx.test.simd.sass"
+      ]
+    },
     {
       "name": "libcudacxx-cpp17",
       "configurePreset": "libcudacxx-cpp17",

ci/build_libcudacxx.sh

@@ -4,14 +4,52 @@ set -euo pipefail
 
 ci_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
+build_target=""
+codegen_tests=false
+common_args=()
+while [[ $# -ne 0 ]]; do
+  case "$1" in
+    -codegen-tests)
+      codegen_tests=true
+      shift
+      ;;
+    -target)
+      if [[ $# -lt 2 ]]; then
+        echo "Error: -target requires a value" >&2
+        exit 1
+      fi
+      build_target="$2"
+      shift 2
+      ;;
+    *)
+      common_args+=("$1")
+      shift
+      ;;
+  esac
+done
+set -- "${common_args[@]}"
+
 # shellcheck source=ci/build_common.sh
 source "${ci_dir}/build_common.sh"
 
 print_environment_details
 
 PRESET="libcudacxx"
+if $codegen_tests; then
+  PRESET="libcudacxx-codegen-filecheck"
+fi
+
 CMAKE_OPTIONS=("-DCMAKE_CXX_STANDARD=${CXX_STANDARD}" "-DCMAKE_CUDA_STANDARD=${CXX_STANDARD}")
 
+if [[ -n "$build_target" ]]; then
+  configure_preset "$PRESET" "$PRESET" "${CMAKE_OPTIONS[@]}"
+  if ! $CONFIGURE_ONLY; then
+    build_preset "$PRESET" "$PRESET" --target "$build_target"
+  fi
+  print_time_summary
+  exit 0
+fi
+
 upload_test_artifacts=false
 if [[ -n "${GITHUB_ACTIONS:-}" ]] && "${ci_dir}/util/workflow/has_consumers.sh"; then
   upload_test_artifacts=true

ci/matrix.yaml

@@ -110,6 +110,18 @@ workflows:
     - {jobs: ['test_gpu'], project: 'thrust', cmake_options: '-DTHRUST_DISPATCH_TYPE=Force32bit', gpu: 'rtx4090'}
     - {jobs: ['nvrtc'], project: 'libcudacxx', std: 'all', gpu: 'rtx2080', sm: 'gpu'}
     - {jobs: ['verify_codegen'], project: 'libcudacxx'}
+    # libcu++ Codegen FileCheck: PRs use one GCC host compiler per CTK.
+    # Suite-specific architectures are added separately.
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '12.0', cxx: 'gcc12', codegen_target: ['atomics-ptx', 'atomics-sass'], sm: [75, 80, 90]}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '12.X', cxx: 'gcc14', codegen_target: ['atomics-ptx', 'atomics-sass', 'simd-ptx', 'simd-sass'], sm: [80, 90, 100, 120]}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '12.X', cxx: 'gcc14', codegen_target: ['atomics-ptx', 'atomics-sass'], sm: 75}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '12.X', cxx: 'gcc14', codegen_target: 'simd-sass', sm: [103, '120f']}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '13.0', cxx: 'gcc15', codegen_target: ['atomics-ptx', 'atomics-sass', 'simd-ptx', 'simd-sass'], sm: [80, 90, 100, 120]}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '13.0', cxx: 'gcc15', codegen_target: ['atomics-ptx', 'atomics-sass'], sm: 75}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '13.0', cxx: 'gcc15', codegen_target: 'simd-sass', sm: [103, '120f']}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', cxx: 'gcc15', codegen_target: ['atomics-ptx', 'atomics-sass', 'simd-ptx', 'simd-sass'], sm: [80, 90, 100, 120]}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', cxx: 'gcc15', codegen_target: ['atomics-ptx', 'atomics-sass'], sm: 75}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', cxx: 'gcc15', codegen_target: 'simd-sass', sm: [103, '120f']}
     # c.parallel -- pinned to gcc13 / msvc2022 to match python
     - {jobs: ['test'], project: 'cccl_c_parallel', ctk: '12.X', cxx: ['gcc13', 'msvc2022'], gpu: ['t4']}
     - {jobs: ['test'], project: 'cccl_c_parallel', ctk: '13.X', cxx: ['gcc13', 'msvc2022'], gpu: ['rtx2080', 'l4', 'h100']}
@@ -284,6 +296,18 @@ workflows:
     # NVRTC tests don't currently support 12.0:
     - {jobs: ['nvrtc'],          project: 'libcudacxx', ctk: [        '12.X', '13.0', '13.X'], cxx: 'gcc12', std: 'all', gpu: 'rtx2080', sm: 'gpu'}
     - {jobs: ['verify_codegen'], project: 'libcudacxx'}
+    # libcu++ Codegen FileCheck: nightly covers the GCC and Clang host compilers
+    # supported by each CTK. Suite-specific architectures are added separately.
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '12.0', cxx: ['gcc12', 'clang14'], codegen_target: ['atomics-ptx', 'atomics-sass'], sm: [75, 80, 90]}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '12.X', cxx: ['gcc14', 'clang19'], codegen_target: ['atomics-ptx', 'atomics-sass', 'simd-ptx', 'simd-sass'], sm: [80, 90, 100, 120]}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '12.X', cxx: ['gcc14', 'clang19'], codegen_target: ['atomics-ptx', 'atomics-sass'], sm: 75}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '12.X', cxx: ['gcc14', 'clang19'], codegen_target: 'simd-sass', sm: [103, '120f']}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '13.0', cxx: ['gcc15', 'clang20'], codegen_target: ['atomics-ptx', 'atomics-sass', 'simd-ptx', 'simd-sass'], sm: [80, 90, 100, 120]}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '13.0', cxx: ['gcc15', 'clang20'], codegen_target: ['atomics-ptx', 'atomics-sass'], sm: 75}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '13.0', cxx: ['gcc15', 'clang20'], codegen_target: 'simd-sass', sm: [103, '120f']}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', cxx: ['gcc15', 'clang21'], codegen_target: ['atomics-ptx', 'atomics-sass', 'simd-ptx', 'simd-sass'], sm: [80, 90, 100, 120]}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', cxx: ['gcc15', 'clang21'], codegen_target: ['atomics-ptx', 'atomics-sass'], sm: 75}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', cxx: ['gcc15', 'clang21'], codegen_target: 'simd-sass', sm: [103, '120f']}
     # c.parallel -- pinned to gcc13 / msvc2022 to match python
     - {jobs: ['test'],  project: ['cccl_c_parallel'], ctk: '12.X', cxx: ['gcc13', 'msvc2022'], gpu: ['t4']}
     - {jobs: ['test'],  project: ['cccl_c_parallel'], ctk: '13.X', cxx: ['gcc13', 'msvc2022'], gpu: ['rtx2080', 'l4', 'h100']}
@@ -393,6 +417,18 @@ workflows:
     # NVRTC tests don't currently support 12.0:
     - {jobs: ['nvrtc'],          project: 'libcudacxx', ctk: [        '12.X', '13.0', '13.X'], cxx: 'gcc12', std: 'all', gpu: 'rtx2080', sm: 'gpu'}
     - {jobs: ['verify_codegen'], project: 'libcudacxx'}
+    # libcu++ Codegen FileCheck: weekly covers the GCC and Clang host compilers
+    # supported by each CTK. Suite-specific architectures are added separately.
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '12.0', cxx: ['gcc12', 'clang14'], codegen_target: ['atomics-ptx', 'atomics-sass'], sm: [75, 80, 90]}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '12.X', cxx: ['gcc14', 'clang19'], codegen_target: ['atomics-ptx', 'atomics-sass', 'simd-ptx', 'simd-sass'], sm: [80, 90, 100, 120]}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '12.X', cxx: ['gcc14', 'clang19'], codegen_target: ['atomics-ptx', 'atomics-sass'], sm: 75}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '12.X', cxx: ['gcc14', 'clang19'], codegen_target: 'simd-sass', sm: [103, '120f']}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '13.0', cxx: ['gcc15', 'clang20'], codegen_target: ['atomics-ptx', 'atomics-sass', 'simd-ptx', 'simd-sass'], sm: [80, 90, 100, 120]}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '13.0', cxx: ['gcc15', 'clang20'], codegen_target: ['atomics-ptx', 'atomics-sass'], sm: 75}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', ctk: '13.0', cxx: ['gcc15', 'clang20'], codegen_target: 'simd-sass', sm: [103, '120f']}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', cxx: ['gcc15', 'clang21'], codegen_target: ['atomics-ptx', 'atomics-sass', 'simd-ptx', 'simd-sass'], sm: [80, 90, 100, 120]}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', cxx: ['gcc15', 'clang21'], codegen_target: ['atomics-ptx', 'atomics-sass'], sm: 75}
+    - {jobs: ['codegen_filecheck'], project: 'libcudacxx', std: 'max', cxx: ['gcc15', 'clang21'], codegen_target: 'simd-sass', sm: [103, '120f']}
     # c.parallel -- pinned to gcc13 / msvc2022 to match python
     - {jobs: ['test'],  project: ['cccl_c_parallel'], ctk: '12.X', cxx: ['gcc13', 'msvc2022'], gpu: ['t4']}
     - {jobs: ['test'],  project: ['cccl_c_parallel'], ctk: '13.X', cxx: ['gcc13', 'msvc2022'], gpu: ['rtx2080', 'l4', 'h100']}
@@ -607,6 +643,7 @@ jobs:
   # libcudacxx:
   nvrtc: { gpu: true, name: 'NVRTC' }
   verify_codegen: { gpu: false, name: 'VerifyCodegen' }
+  codegen_filecheck: { gpu: false, name: 'libcu++ Codegen FileCheck', invoke: { prefix: 'build', args: '-codegen-tests' } }
 
   # CUB:
   build_nolid: { name: 'BuildNoLaunch',     gpu: false, invoke: { prefix: 'build', args: '-no-lid'} }
@@ -656,6 +693,12 @@ jobs:
   dc:     { gpu: false }
   dc_ext: { gpu: false, cuda_ext: true }
 
+codegen_targets:
+  atomics-ptx: { name: 'Atomics PTX', cmake_target: 'libcudacxx.test.atomics.ptx' }
+  atomics-sass: { name: 'Atomics SASS', cmake_target: 'libcudacxx.test.atomics.sass' }
+  simd-ptx: { name: 'SIMD PTX', cmake_target: 'libcudacxx.test.simd.ptx' }
+  simd-sass: { name: 'SIMD SASS', cmake_target: 'libcudacxx.test.simd.sass' }
+
 # Projects have the following properties:
 #
 # Keys are project subdirectories names. These will also be used in script names.
@@ -783,9 +826,11 @@ gpus:
 #  - required: Whether the tag is required. Default is false.
 #  - default: The default value for the tag. Default is null.
 tags:
-   # An array of jobs (e.g. 'build', 'test', 'nvrtc', 'infra', 'verify_codegen', ...)
+   # An array of jobs (e.g. 'build', 'test', 'nvrtc', 'infra', 'verify_codegen', 'codegen_filecheck', ...)
    # See the `jobs` map.
   jobs: { required: true }
+  # FileCheck codegen suite selected by a codegen_filecheck job.
+  codegen_target: { required: false }
   # CUDA ToolKit version
   # See the `ctks` map.
   ctk: { default: '13.X' }

libcudacxx/include/cuda/__atomic/atomic.h

@@ -22,6 +22,7 @@
 #endif // no system header
 
 #include <cuda/std/__type_traits/copy_cv.h>
+#include <cuda/std/__type_traits/remove_cv.h>
 #include <cuda/std/atomic>
 
 #include <cuda/std/__cccl/prologue.h>
@@ -80,7 +81,7 @@ struct atomic : public ::cuda::std::__atomic_impl<_Tp, _Sco>
 template <class _Tp, thread_scope _Sco = thread_scope::thread_scope_system>
 struct atomic_ref : public ::cuda::std::__atomic_ref_impl<_Tp, _Sco>
 {
-  using value_type = _Tp;
+  using value_type = ::cuda::std::remove_cv_t<_Tp>;
 
   static constexpr size_t required_alignment = sizeof(_Tp);
 
@@ -90,7 +91,7 @@ struct atomic_ref : public ::cuda::std::__atomic_ref_impl<_Tp, _Sco>
       : ::cuda::std::__atomic_ref_impl<_Tp, _Sco>(__ref)
   {}
 
-  _CCCL_HOST_DEVICE_API inline _Tp operator=(_Tp __v) const noexcept
+  _CCCL_HOST_DEVICE_API inline value_type operator=(value_type __v) const noexcept
   {
     this->store(__v);
     return __v;
@@ -105,12 +106,14 @@ struct atomic_ref : public ::cuda::std::__atomic_ref_impl<_Tp, _Sco>
   atomic_ref& operator=(const atomic_ref&)                   = delete;
   atomic_ref& operator=(const atomic_ref&) const             = delete;
 
-  _CCCL_HOST_DEVICE_API inline _Tp fetch_max(const _Tp& __op, memory_order __m = memory_order_seq_cst) const noexcept
+  _CCCL_HOST_DEVICE_API inline value_type
+  fetch_max(const value_type& __op, memory_order __m = memory_order_seq_cst) const noexcept
   {
     return ::cuda::std::__atomic_fetch_max_dispatch(&this->__a, __op, __m, ::cuda::std::__scope_to_tag<_Sco>{});
   }
 
-  _CCCL_HOST_DEVICE_API inline _Tp fetch_min(const _Tp& __op, memory_order __m = memory_order_seq_cst) const noexcept
+  _CCCL_HOST_DEVICE_API inline value_type
+  fetch_min(const value_type& __op, memory_order __m = memory_order_seq_cst) const noexcept
   {
     return ::cuda::std::__atomic_fetch_min_dispatch(&this->__a, __op, __m, ::cuda::std::__scope_to_tag<_Sco>{});
   }