From 2f861590976fa49aa7cad618c87f537574c9bd7d Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 22 Mar 2026 16:16:02 +0100 Subject: [PATCH 01/73] Added MNIST example, applied formattin, and fixed some bugs. --- CMakeLists.txt | 2 +- examples/mnist.py | 139 +++++ python_lib/dl_lib/__init__.py | 4 +- readme.md | 4 +- src/CMakeLists.txt | 12 +- src/backend/data_modeling/dim_type.cpp | 4 +- src/backend/data_modeling/dim_type.h | 2 +- src/backend/data_modeling/tensor.cpp | 19 +- src/backend/data_modeling/tensor.h | 497 +++++++++--------- src/backend/module/layers/ff_layer.h | 4 +- .../training/optimizers/optimizer_base.cpp | 49 +- .../training/optimizers/optimizer_base.h | 1 + src/backend/training/optimizers/rmsprop.cpp | 2 +- src/backend/utility/initializers.cpp | 4 +- src/python/py_core/py_core.cpp | 8 + src/python/py_core/py_core_util.h | 148 +++++- src/python/py_train/py_train.cpp | 39 +- tests/python/test_autograd.py | 138 ++--- tests/python/test_tensorops.py | 38 +- tests/python/test_training.py | 10 +- 20 files changed, 735 insertions(+), 389 deletions(-) create mode 100644 examples/mnist.py diff --git a/CMakeLists.txt b/CMakeLists.txt index e317ea5..90dba31 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.24) set(CMAKE_OSX_SYSROOT "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk") project(dllib VERSION 1.0.0 LANGUAGES CXX) -set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD 23) set(CMAKE_CXX_STANDARD_REQUIRED ON) diff --git a/examples/mnist.py b/examples/mnist.py new file mode 100644 index 0000000..7628a47 --- /dev/null +++ b/examples/mnist.py @@ -0,0 +1,139 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "python_lib")) + +import math +import numpy as np +from sklearn.datasets import fetch_openml +from sklearn.model_selection import train_test_split + +from dl_lib import Tensor, fromNumpy, toNumpy +from dl_lib.nn import Sequential, FfLayer +from dl_lib.nn.activation import LeakyReLU +from dl_lib.train.loss import CrossEntropyWithSoftmax +from dl_lib.train.optim import RmsProp + + +# ─── data loading ──────────────────────────────────────────────────────────── + +def load_mnist(): + print("Loading MNIST...") + mnist = fetch_openml("mnist_784", version=1, as_frame=False) + x = mnist.data.astype(np.float32) / 255.0 # normalize to [0,1] + y = mnist.target.astype(np.int32) + return x, y + + +def to_one_hot(y, n_classes=10): + n = len(y) + one_hot = np.zeros((n, n_classes), dtype=np.float32) + one_hot[np.arange(n), y] = 1.0 + return one_hot + + +def make_batches(x, y, batch_size, shuffle=True): + n = x.shape[0] + indices = np.arange(n) + if shuffle: + np.random.shuffle(indices) + for start in range(0, n, batch_size): + batch_idx = indices[start : start + batch_size] + yield x[batch_idx], y[batch_idx] + + +# ─── network ───────────────────────────────────────────────────────────────── + +def make_net(): + net = Sequential() + net.append(FfLayer(784, 256)) + net.append(LeakyReLU(0.01)) + net.append(FfLayer(256, 128)) + net.append(LeakyReLU(0.01)) + net.append(FfLayer(128, 10)) + return net + + +# ─── debugging ──────────────────────────────────────────────────────────────── + +def print_weight_stats(net, batch_num): + for i, p in enumerate(net.parameters()): + p_np = toNumpy(p) + print( + f"Param {i}: min={p_np.min():.4f} max={p_np.max():.4f} " + f"mean={p_np.mean():.4f} std={p_np.std():.4f}" + ) + + +# ─── training ──────────────────────────────────────────────────────────────── + +def train_epoch(net, loss_fn, optim, x, y, batch_size=64): + #print_weight_stats(net, 0) + + total_loss = 0.0 + n_batches = 0 + max_batches = math.ceil(x.shape[0] / batch_size) + for xb, yb in make_batches(x, y, batch_size): + xTensor = fromNumpy(xb) + yTensor = fromNumpy(yb) + + pred = net.forward(xTensor) + loss = loss_fn(yTensor, pred) + loss.backward() + + optim.clipGradients(1.0) + optim.step() + optim.zeroGrad() + + total_loss += loss.getitem(0) + n_batches += 1 + if n_batches == 1 or n_batches % 10 == 0: + print(f"Batch {n_batches} / {max_batches}, loss {loss.getitem(0)}") + #print_weight_stats(net, n_batches) + + return total_loss / n_batches + + +def evaluate(net, x, y_int, batch_size=256): + correct = 0 + total = 0 + for xb, yb in make_batches(x, y_int, batch_size, shuffle=False): + xTensor = fromNumpy(xb) + pred = net.forward(xTensor) + pred_np = toNumpy(pred) + + predicted = np.argmax(pred_np, axis=1) + + correct += np.sum(predicted == np.argmax(yb, axis=1)) + total += len(yb) + return correct / total + + +# ─── main ──────────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + # load and split data + x, y_int = load_mnist() + y = to_one_hot(y_int) + + x_train, x_val, y_train, y_val = train_test_split( + x, y, test_size=0.1, random_state=42 + ) + + print(f"Train: {x_train.shape}, Val: {x_val.shape}") + + # setup + net = make_net() + loss_fn = CrossEntropyWithSoftmax() + optim = RmsProp(net.parameters(), 0.0001, 0.95) # lr and decay + + # training loop + n_epochs = 5 + for epoch in range(n_epochs): + train_loss = train_epoch(net, loss_fn, optim, x_train, y_train) + val_acc = evaluate(net, x_val, y_val) + print( + f"Epoch {epoch+1}/{n_epochs} " + f"loss={train_loss:.4f} " + f"val_acc={val_acc:.4f}" + ) diff --git a/python_lib/dl_lib/__init__.py b/python_lib/dl_lib/__init__.py index e7f6844..9558f9c 100644 --- a/python_lib/dl_lib/__init__.py +++ b/python_lib/dl_lib/__init__.py @@ -1,5 +1,5 @@ -from ._compiled._core import Tensor, Dimension, Device +from ._compiled._core import Tensor, Dimension, Device, fromNumpy, toNumpy __all__ = ['Tensor', 'Device', 'Dimension'] -__version__ = "0.2.0" \ No newline at end of file +__version__ = "1.0.0" \ No newline at end of file diff --git a/readme.md b/readme.md index ff83ee2..2a8152e 100644 --- a/readme.md +++ b/readme.md @@ -18,6 +18,7 @@ For some examples on Python interface, see tests/python. - Backpropagation engine - Neural network layers - Training framework (optimizers, loss functions, layers, and networks) +- **Example code**: Full MNIST dataset training example - **Python Interface**: Seamless integration via Boost.Python - **Clean Architecture**: Modular design, ~4K LOC - **CI/CD**: Automated testing with GTest and GitHub Actions @@ -38,6 +39,7 @@ For some examples on Python interface, see tests/python. Roadmap: - [x] Python Binding Unit Tests - [x] Optimizers and training framework +- [x] MNIST example - [ ] CUDA mode for operations - [ ] Additional layer types (Conv2D, Dropout, etc.) - [ ] AlexNet reference implementation @@ -68,12 +70,12 @@ ctest - Boost Python - Cmake > 3.28 - Python 3 (we test with 3.10, but it should work with any version) +- numpy 1.26.4 - pytest and GTest for unit tests (we use pytest=9.0.2) - Google Benchmark for benchmarking ## Troubleshooting - ### Building on Windows The implementation of the Python wrapper does not work on MSVC6/7 in its current form. This is due to an issue that arises from Boost Python in combination with these compilers. Workarounds are proposed, but not implemented. More information here [here](https://beta.boost.org/doc/libs/develop/libs/python/doc/html/tutorial/tutorial/exposing.html). diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5250607..ed4abf0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,14 +2,22 @@ add_subdirectory(backend) add_subdirectory(python) +execute_process( + COMMAND python3 -c "import numpy; print(numpy.get_include())" + OUTPUT_VARIABLE NUMPY_INCLUDE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE +) + target_link_libraries(_core PRIVATE ${Boost_LIBRARIES} - ${Python_LIBRARIES} + ${Python_LIBRARIES} BackendCore) target_include_directories(_core PRIVATE ${Python_INCLUDE_DIRS} - ${Boost_INCLUDE_DIRS}) + ${Boost_INCLUDE_DIRS} + ${NUMPY_INCLUDE_DIR} + ) target_link_libraries(_nn PRIVATE ${Boost_LIBRARIES} diff --git a/src/backend/data_modeling/dim_type.cpp b/src/backend/data_modeling/dim_type.cpp index f7d1804..684fde0 100644 --- a/src/backend/data_modeling/dim_type.cpp +++ b/src/backend/data_modeling/dim_type.cpp @@ -17,8 +17,8 @@ using namespace std; -tensorDim_t Dimension::multVector(const std::vector& dims) const noexcept { - tensorDim_t res = 1; +tensorSize_t Dimension::multVector(const std::vector& dims) const noexcept { + tensorSize_t res = 1; #ifndef NDEBUG utility::SafeArithmetics_t mult(1); diff --git a/src/backend/data_modeling/dim_type.h b/src/backend/data_modeling/dim_type.h index 6af6933..7b6bbed 100644 --- a/src/backend/data_modeling/dim_type.h +++ b/src/backend/data_modeling/dim_type.h @@ -23,7 +23,7 @@ class Dimension final { std::vector dims; tensorSize_t size = 0; - tensorDim_t multVector(const std::vector& dims) const noexcept; + tensorSize_t multVector(const std::vector& dims) const noexcept; public: /** diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index 1b5a9fd..e398c2c 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -16,6 +16,7 @@ #include #include +#include using namespace std; @@ -55,6 +56,20 @@ Tensor::tensorValues_t::~tensorValues_t() noexcept { } } +/** + * @brief Copy from pointer into this object. + */ +void Tensor::tensorValues_t::copyFromRaw(const ftype* src, tensorSize_t n) { + assert(n == size); + switch(device){ + case Device::CPU: + std::memcpy(values, src, n * sizeof(ftype)); + break; + case Device::CUDA: + __throw_runtime_error("copyFromRaw not implemented for CUDA"); + } +} + /** * @brief For convenience, since copy- and std::move-constructors and assigment operators * do not create a deepcopy, but construct another pointer pointing to the same piece @@ -70,7 +85,7 @@ void Tensor::tensorValues_t::copyValues(Tensor::tensorValues_t& target) const { } break; case Device::CUDA: - __throw_runtime_error("CUDA not implemented for deep copy"); + __throw_runtime_error("Deep copy not implemented for CUDA"); break; } } @@ -349,7 +364,7 @@ void Tensor::matMul2DCpu(Tensor& res, const Tensor& left, const Tensor& right, c leftIdx++; rightIdx += nColsRight; } - + (*res.values)[resIdx] = scalar; resIdx++; } diff --git a/src/backend/data_modeling/tensor.h b/src/backend/data_modeling/tensor.h index d8a59fc..01f4091 100644 --- a/src/backend/data_modeling/tensor.h +++ b/src/backend/data_modeling/tensor.h @@ -1,12 +1,12 @@ /** * @file tensor.h * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2025-12-07 - * + * * @copyright Copyright (c) 2025 - * + * */ #pragma once @@ -29,244 +29,259 @@ #include // break circular dependency -namespace cgraph { - class GraphNode; - class TopologicalSort; +namespace cgraph +{ + class GraphNode; + class TopologicalSort; } -class Tensor final : public std::enable_shared_from_this { - friend class cgraph::TopologicalSort; - - private: - /** - * @brief Here we encapsulate the tensor's values. - * Enables us to use a shared_ptr, as well as encapsulate all the - * memory management logic for different devices, like a GPU. - * Structured as a flat array, the logic for multiple dimensions - * encapsulated by surrounding tensor object. - */ - class tensorValues_t final { - private: - tensorSize_t size = 0; - ftype* values = nullptr; - - Device device; - inline static Device defaultDevice = Device::CPU; - - void addOtherCpu(const tensorValues_t& other) noexcept; - - public: - explicit tensorValues_t(); - explicit tensorValues_t(Device d); - ~tensorValues_t() noexcept; - - tensorValues_t(const tensorValues_t& other) = delete; - tensorValues_t& operator=(const tensorValues_t& other) = delete; - - tensorValues_t(tensorValues_t&& other) noexcept; - tensorValues_t& operator=(tensorValues_t&& other) noexcept; - - explicit operator bool() const noexcept; - ftype& operator[](const tensorSize_t idx); - ftype operator[](const tensorSize_t idx) const; - - void set(ftype v, tensorSize_t idx); - ftype get(tensorSize_t idx); - - tensorSize_t getSize() const noexcept; - - // needed for gradient descent - tensorValues_t& operator+=(const tensorValues_t& other); - - template - requires (std::is_integral_v< std::remove_const_t >) - void resize(const T size) { - this->size = static_cast(size); - switch(this->device){ - case Device::CPU: - values = static_cast( std::malloc(this->size * sizeof(ftype)) ); - break; - case Device::CUDA: - std::__throw_invalid_argument("Not implemented yet."); - break; - } - } - - void setDevice(const Device d) noexcept; - Device getDevice() const noexcept; - - void copyValues(tensorValues_t& target) const; - void copyValues(tensorValues_t& target, tensorSize_t low, tensorSize_t high, tensorSize_t targetOffset) const; - void copyValues(tensorValues_t& target, std::span indices, const tensorSize_t sizeOfDim) const; - - static void setDefaultDevice(const Device d) noexcept; - static Device getDefaultDevice() noexcept; - }; - - Dimension dims; - std::unique_ptr values = nullptr; // contained values of tensor - - bool requiresGrad = false; - std::shared_ptr grads = nullptr; // gradients - std::shared_ptr cgNode = nullptr; - - static Tensor multiplyScalar(const Tensor& scalar, const Tensor& other) noexcept; - - static Tensor matMulImpl(const Tensor& left, const Tensor& right); - static void matMul2DCpu(Tensor& res, const Tensor& left, const Tensor& right, - const tensorSize_t resOffset, const tensorSize_t leftOffset, - const tensorSize_t rightOffset); - - void transposeImpl2D(Tensor& target, const int dim1, const int dim2) const noexcept; - void transposeImpl(Tensor& target, const int dim1, const int dim2) const noexcept; - - // convenience functions that appear in multiple places - static tensorSize_t computeLinearIdx(const std::vector&& idx, const Dimension& dims); - static tensorSize_t computeLinearIdx(const std::vector& idx, const Dimension& dims); - - static tensorSize_t getDimOffset(const tensorDim_t dim, const Dimension& dims); - static tensorSize_t getDimOffset(const int dim, const Dimension& dims); - static tensorDim_t mapDim(const int dim, const Dimension& dims); - - friend void printValuesCpu(std::ostream& os, const Tensor& t); - - public: - template - requires (std::is_same_v, Dimension>) - explicit Tensor(T&& dims, Device d, bool requiresGrad=false) - : Tensor{dims.toVector(), tensorValues_t::getDefaultDevice(), requiresGrad} - { } - - explicit Tensor(const std::vector& dims, bool requiresGrad=false) : - dims{dims}, values{std::make_unique()}, requiresGrad{requiresGrad} { - values->resize(this->dims.getSize()); - } - - explicit Tensor(const std::vector& dims, Device d, bool requiresGrad=false) : - dims{dims}, values{std::make_unique(d)}, requiresGrad{requiresGrad} { - values->resize(this->dims.getSize()); - } - - explicit Tensor(const std::vector& dims, const std::vector& initValues, bool requiresGrad=false) : - Tensor{dims, std::move(initValues), Tensor::getDefaultDevice(), requiresGrad} { - } - - explicit Tensor(const std::vector& dims, const std::vector& initValues, Device d, bool requiresGrad=false) : - Tensor{dims, d, requiresGrad} { - for(tensorSize_t i=0; iset(initValues[i], i); - } - } - - /** - * Tensors can become very large. Deleting those two - * helps us to not accidentally copy something we do not - * intend to copy. We create extra methods for that. - */ - Tensor(const Tensor& other) = delete; - Tensor& operator=(const Tensor& other) = delete; - - Tensor createEmptyCopy() const; - Tensor createDeepCopy() const; - - /** - * @brief Moving. Move array of values - * to new instance as well. - */ - Tensor(Tensor&& other) noexcept; - Tensor& operator=(Tensor&& other) noexcept; - - void reset(const ftype x) noexcept; - void reset(const std::shared_ptr init) noexcept; - - const Dimension& getDims() const noexcept; - tensorSize_t getSize() const noexcept; - - //Tensor operator@(const Tensor& other) const; in higher C++ versions than 20 - Tensor matmul(const Tensor& other) const; - - Tensor operator+(const Tensor& other) const; - Tensor add(const Tensor& other) const; - - // TODO: Tensor operator-(const Tensor& other) const; - - Tensor operator*(const Tensor& t) const; - Tensor elementwiseMul(const Tensor& other) const; - - // TODO: Tensor operator/(const Tensor& other) const; - - // the following operators all broadcast - Tensor operator*(ftype scalar) const; - Tensor operator/(ftype scalar) const; - Tensor operator+(ftype scalar) const; - Tensor operator-(ftype scalar) const; - - // turn around the arguments as well: scalar *:+ tensor - friend Tensor operator*(ftype scalar, const Tensor& tensor); - friend Tensor operator+(ftype scalar, const Tensor& tensor); - - void backward(); - - std::shared_ptr getGrads() const; - void setGrads(std::shared_ptr grads) noexcept { - this->grads = std::move(grads); - } - bool hasGrads() const noexcept { return grads!=nullptr; } - - void transposeThis() noexcept; - void transposeThis(int dim1, int dim2) noexcept; - - Tensor transpose(int dim1, int dim2) const; - Tensor transpose(int dim1, int dim2, const bool requiresGrad) const; - - void permute(const std::vector&& newOrder) noexcept; - - friend std::ostream& operator<<(std::ostream& os, const Tensor& t) noexcept; - - // for convenience we provide some simple getters - ftype get(tensorSize_t idx) const; - ftype get(tensorDim_t idx0, tensorDim_t idx1) const; - ftype get(tensorDim_t idx0, tensorDim_t idx1, tensorDim_t idx2) const; - ftype get(tensorDim_t idx0, tensorDim_t idx1, tensorDim_t idx2, tensorDim_t idx3) const; - - // non-const version of operator[] does not exist because of CUDA - ftype operator[](tensorSize_t idx) const; - - ftype get(const std::vector& idx) const; - - // for convenience we provide some simple setters - void set(ftype item, tensorDim_t idx); - void set(ftype item, tensorDim_t idx0, tensorDim_t idx1); - void set(ftype item, tensorDim_t idx0, tensorDim_t idx1, tensorDim_t idx2); - void set(ftype item, tensorDim_t idx0, tensorDim_t idx1, tensorDim_t idx2, tensorDim_t idx3); - void set(ftype item, const std::vector& idx); - - void setDevice(const Device d) noexcept; - Device getDevice() const noexcept; - - bool getRequiresGrad() const noexcept { return requiresGrad; } - void setRequiresGrad(const bool requiresGrad) noexcept { this->requiresGrad=requiresGrad; } - - void setCgNode(std::shared_ptr node) noexcept { - cgNode = std::move(node); - requiresGrad = true; - } - - std::shared_ptr getSharedPtr() const { - try { - return std::const_pointer_cast(shared_from_this()); - } - catch (const std::bad_weak_ptr&) { - throw std::runtime_error( - "Tensor must be managed by shared_ptr for autograd operations" - ); - } - } - - Tensor getSlice(tensorSize_t low, tensorSize_t high) const; - Tensor getSlice(std::span indices) const; - - // these two should not be exposed to the python interface - static void setDefaultDevice(const Device d) noexcept; - static Device getDefaultDevice() noexcept; +class Tensor final : public std::enable_shared_from_this +{ + friend class cgraph::TopologicalSort; + +private: + /** + * @brief Here we encapsulate the tensor's values. + * Enables us to use a shared_ptr, as well as encapsulate all the + * memory management logic for different devices, like a GPU. + * Structured as a flat array, the logic for multiple dimensions + * encapsulated by surrounding tensor object. + */ + class tensorValues_t final + { + private: + tensorSize_t size = 0; + ftype *values = nullptr; + + Device device; + inline static Device defaultDevice = Device::CPU; + + void addOtherCpu(const tensorValues_t& other) noexcept; + + public: + explicit tensorValues_t(); + explicit tensorValues_t(Device d); + ~tensorValues_t() noexcept; + + tensorValues_t(const tensorValues_t& other) = delete; + tensorValues_t& operator=(const tensorValues_t& other) = delete; + + tensorValues_t(tensorValues_t&& other) noexcept; + tensorValues_t& operator=(tensorValues_t&& other) noexcept; + + void copyFromRaw(const ftype* src, tensorSize_t n); + + explicit operator bool() const noexcept; + ftype& operator[](const tensorSize_t idx); + ftype operator[](const tensorSize_t idx) const; + + void set(ftype v, tensorSize_t idx); + ftype get(tensorSize_t idx); + + tensorSize_t getSize() const noexcept; + + // needed for gradient descent + tensorValues_t& operator+=(const tensorValues_t& other); + + template + requires(std::is_integral_v>) + void resize(const T size) + { + this->size = static_cast(size); + switch (this->device) + { + case Device::CPU: + values = static_cast(std::malloc(this->size * sizeof(ftype))); + break; + case Device::CUDA: + std::__throw_invalid_argument("Not implemented yet."); + break; + } + } + + void setDevice(const Device d) noexcept; + Device getDevice() const noexcept; + + void copyValues(tensorValues_t& target) const; + void copyValues(tensorValues_t& target, tensorSize_t low, tensorSize_t high, tensorSize_t targetOffset) const; + void copyValues(tensorValues_t& target, std::span indices, const tensorSize_t sizeOfDim) const; + + static void setDefaultDevice(const Device d) noexcept; + static Device getDefaultDevice() noexcept; + }; + + Dimension dims; + std::unique_ptr values = nullptr; // contained values of tensor + + bool requiresGrad = false; + std::shared_ptr grads = nullptr; // gradients + std::shared_ptr cgNode = nullptr; + + static Tensor multiplyScalar(const Tensor& scalar, const Tensor& other) noexcept; + + static Tensor matMulImpl(const Tensor& left, const Tensor& right); + static void matMul2DCpu(Tensor& res, const Tensor& left, const Tensor& right, + const tensorSize_t resOffset, const tensorSize_t leftOffset, + const tensorSize_t rightOffset); + + void transposeImpl2D(Tensor& target, const int dim1, const int dim2) const noexcept; + void transposeImpl(Tensor& target, const int dim1, const int dim2) const noexcept; + + // convenience functions that appear in multiple places + static tensorSize_t computeLinearIdx(const std::vector&& idx, const Dimension& dims); + static tensorSize_t computeLinearIdx(const std::vector& idx, const Dimension& dims); + + static tensorSize_t getDimOffset(const tensorDim_t dim, const Dimension& dims); + static tensorSize_t getDimOffset(const int dim, const Dimension& dims); + static tensorDim_t mapDim(const int dim, const Dimension& dims); + + friend void printValuesCpu(std::ostream& os, const Tensor& t); + +public: + template + requires(std::is_same_v, Dimension>) + explicit Tensor(T&& dims, Device d, bool requiresGrad = false) + : Tensor{dims.toVector(), tensorValues_t::getDefaultDevice(), requiresGrad} + { } + + explicit Tensor(const std::vector& dims, bool requiresGrad = false) : dims{dims}, values{std::make_unique()}, requiresGrad{requiresGrad} + { + values->resize(this->dims.getSize()); + } + + explicit Tensor(const std::vector& dims, Device d, bool requiresGrad = false) : dims{dims}, values{std::make_unique(d)}, requiresGrad{requiresGrad} + { + values->resize(this->dims.getSize()); + } + + explicit Tensor(const std::vector& dims, const std::vector& initValues, bool requiresGrad = false) : Tensor{dims, std::move(initValues), Tensor::getDefaultDevice(), requiresGrad} + { + } + + explicit Tensor(const std::vector& dims, const std::vector& initValues, Device d, bool requiresGrad = false) : Tensor{dims, d, requiresGrad} + { + for (tensorSize_t i=0; iset(initValues[i], i); + } + } + + Tensor(const std::vector& dims, const ftype *data, tensorSize_t dataSize, + Device d = Device::CPU, bool requiresGrad = false) + : Tensor(dims, d, requiresGrad) + { + assert(values->getSize() == dataSize); + values->copyFromRaw(data, dataSize); + } + + /** + * Tensors can become very large. Deleting those two + * helps us to not accidentally copy something we do not + * intend to copy. We create extra methods for that. + */ + Tensor(const Tensor& other) = delete; + Tensor& operator=(const Tensor& other) = delete; + + Tensor createEmptyCopy() const; + Tensor createDeepCopy() const; + + /** + * @brief Moving. Move array of values + * to new instance as well. + */ + Tensor(Tensor&& other) noexcept; + Tensor& operator=(Tensor&& other) noexcept; + + void reset(const ftype x) noexcept; + void reset(const std::shared_ptr init) noexcept; + + const Dimension& getDims() const noexcept; + tensorSize_t getSize() const noexcept; + + // Tensor operator@(const Tensor& other) const; in higher C++ versions than 20 + Tensor matmul(const Tensor& other) const; + + Tensor operator+(const Tensor& other) const; + Tensor add(const Tensor& other) const; + + // TODO: Tensor operator-(const Tensor& other) const; + + Tensor operator*(const Tensor& t) const; + Tensor elementwiseMul(const Tensor& other) const; + + // TODO: Tensor operator/(const Tensor& other) const; + + // the following operators all broadcast + Tensor operator*(ftype scalar) const; + Tensor operator/(ftype scalar) const; + Tensor operator+(ftype scalar) const; + Tensor operator-(ftype scalar) const; + + // turn around the arguments as well: scalar *:+ tensor + friend Tensor operator*(ftype scalar, const Tensor& tensor); + friend Tensor operator+(ftype scalar, const Tensor& tensor); + + void backward(); + + std::shared_ptr getGrads() const; + void setGrads(std::shared_ptr grads) noexcept{ + this->grads = std::move(grads); + } + bool hasGrads() const noexcept { return grads!=nullptr; } + + void transposeThis() noexcept; + void transposeThis(int dim1, int dim2) noexcept; + + Tensor transpose(int dim1, int dim2) const; + Tensor transpose(int dim1, int dim2, const bool requiresGrad) const; + + void permute(const std::vector&& newOrder) noexcept; + + friend std::ostream& operator<<(std::ostream& os, const Tensor& t) noexcept; + + // for convenience we provide some simple getters + ftype get(tensorSize_t idx) const; + ftype get(tensorDim_t idx0, tensorDim_t idx1) const; + ftype get(tensorDim_t idx0, tensorDim_t idx1, tensorDim_t idx2) const; + ftype get(tensorDim_t idx0, tensorDim_t idx1, tensorDim_t idx2, tensorDim_t idx3) const; + + // non-const version of operator[] does not exist because of CUDA + ftype operator[](tensorSize_t idx) const; + + ftype get(const std::vector& idx) const; + + // for convenience we provide some simple setters + void set(ftype item, tensorDim_t idx); + void set(ftype item, tensorDim_t idx0, tensorDim_t idx1); + void set(ftype item, tensorDim_t idx0, tensorDim_t idx1, tensorDim_t idx2); + void set(ftype item, tensorDim_t idx0, tensorDim_t idx1, tensorDim_t idx2, tensorDim_t idx3); + void set(ftype item, const std::vector& idx); + + void setDevice(const Device d) noexcept; + Device getDevice() const noexcept; + + bool getRequiresGrad() const noexcept { return requiresGrad; } + void setRequiresGrad(const bool requiresGrad) noexcept { this->requiresGrad=requiresGrad; } + + void setCgNode(std::shared_ptr node) noexcept { + cgNode = std::move(node); + requiresGrad = true; + } + + std::shared_ptr getSharedPtr() const + { + try{ + return std::const_pointer_cast(shared_from_this()); + } + catch (const std::bad_weak_ptr&) { + throw std::runtime_error( + "Tensor must be managed by shared_ptr for autograd operations"); + } + } + + Tensor getSlice(tensorSize_t low, tensorSize_t high) const; + Tensor getSlice(std::span indices) const; + + // these two should not be exposed to the python interface + static void setDefaultDevice(const Device d) noexcept; + static Device getDefaultDevice() noexcept; }; \ No newline at end of file diff --git a/src/backend/module/layers/ff_layer.h b/src/backend/module/layers/ff_layer.h index 8c58dc2..2c99057 100644 --- a/src/backend/module/layers/ff_layer.h +++ b/src/backend/module/layers/ff_layer.h @@ -26,10 +26,10 @@ namespace module { public: FfLayer(tensorDim_t inSize, tensorDim_t outSize, - bool useBias=true, bool requiresGrad=false, std::shared_ptr init=nullptr); + bool useBias=true, bool requiresGrad=true, std::shared_ptr init=nullptr); FfLayer(tensorDim_t inSize, tensorDim_t outSize, Device d, - bool useBias=true, bool requiresGrad=false, std::shared_ptr init=nullptr); + bool useBias=true, bool requiresGrad=true, std::shared_ptr init=nullptr); Tensor operator()(const Tensor& input) const override; std::shared_ptr operator()(const std::shared_ptr& input) const override; diff --git a/src/backend/training/optimizers/optimizer_base.cpp b/src/backend/training/optimizers/optimizer_base.cpp index e2a6d8d..04b3827 100644 --- a/src/backend/training/optimizers/optimizer_base.cpp +++ b/src/backend/training/optimizers/optimizer_base.cpp @@ -1,25 +1,62 @@ /** * @file optimizer_base.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-14 - * + * * @copyright Copyright (c) 2026 - * + * */ #include "optimizer_base.h" #include "data_modeling/tensor_functions.h" +#include + using namespace train; -void OptimizerBase::zeroGrad() noexcept{ - for(auto& p: params){ +void OptimizerBase::zeroGrad() noexcept +{ + for (auto& p: params){ auto grads = p->getGrads(); - + if(grads) TensorFunctions::ToZeros(*grads); } +} + +void OptimizerBase::clipGradients(ftype maxNorm) noexcept +{ + // compute global L2 norm across all parameters + ftype totalNorm = 0.0f; + for (const auto ¶m : params) + { + auto grads = param->getGrads(); + if (!grads) + continue; + for (tensorSize_t i = 0; i < grads->getSize(); i++) + { + auto g = (*grads)[i]; + totalNorm += g * g; + } + } + totalNorm = std::sqrt(totalNorm); + + if (totalNorm > maxNorm) + { + constexpr ftype eps = 1e-6; + const ftype scale = maxNorm / (totalNorm + eps); + for (const auto ¶m : params) + { + auto grads = param->getGrads(); + if (!grads) + continue; + for (tensorSize_t i = 0; i < grads->getSize(); i++) + { + grads->set((*grads)[i] * scale, i); + } + } + } } \ No newline at end of file diff --git a/src/backend/training/optimizers/optimizer_base.h b/src/backend/training/optimizers/optimizer_base.h index af4f6ef..54324fe 100644 --- a/src/backend/training/optimizers/optimizer_base.h +++ b/src/backend/training/optimizers/optimizer_base.h @@ -45,5 +45,6 @@ namespace train { virtual void step() = 0; void zeroGrad() noexcept; + void clipGradients(ftype maxNorm) noexcept; }; } \ No newline at end of file diff --git a/src/backend/training/optimizers/rmsprop.cpp b/src/backend/training/optimizers/rmsprop.cpp index c5a93a8..6878f56 100644 --- a/src/backend/training/optimizers/rmsprop.cpp +++ b/src/backend/training/optimizers/rmsprop.cpp @@ -15,7 +15,7 @@ using namespace std; using namespace train; void RmsPropOptimizer::step() { - constexpr ftype eps = 1e-9; + constexpr ftype eps = 1e-8; for(const auto& param: params){ auto tPtr = param.get(); const auto gPtr = tPtr->getGrads().get(); diff --git a/src/backend/utility/initializers.cpp b/src/backend/utility/initializers.cpp index 0fad81c..b618023 100644 --- a/src/backend/utility/initializers.cpp +++ b/src/backend/utility/initializers.cpp @@ -21,7 +21,7 @@ ftype GaussianInitializer::drawNumber() const { } ftype UniformXavierInitializer::computeRange(ftype nInputs, ftype nOutputs) { - return sqrt(6/nInputs + nOutputs); + return sqrt(6 / (nInputs + nOutputs)); } ftype UniformXavierInitializer::drawNumber() const { @@ -29,7 +29,7 @@ ftype UniformXavierInitializer::drawNumber() const { } ftype NormalXavierInitializer::computeSigma(ftype nInputs, ftype nOutputs) { - return sqrt(6/nInputs + nOutputs); + return sqrt(2/ (nInputs + nOutputs)); } ftype NormalXavierInitializer::drawNumber() const { diff --git a/src/python/py_core/py_core.cpp b/src/python/py_core/py_core.cpp index 6e13ab6..9c7fab2 100644 --- a/src/python/py_core/py_core.cpp +++ b/src/python/py_core/py_core.cpp @@ -23,6 +23,9 @@ #include #include +#define PY_ARRAY_UNIQUE_SYMBOL MY_ARRAY_API +#include + BOOST_PYTHON_MODULE(_core) { using namespace boost::python; @@ -237,4 +240,9 @@ BOOST_PYTHON_MODULE(_core) def("Gaussian", WRAP_FREE_FUNC_3(Py_DataModeling::Gaussian1, std::vector, Device, ftype)); def("Gaussian", WRAP_FREE_FUNC_3(Py_DataModeling::Gaussian2, std::vector, ftype, const bool)); def("Gaussian", WRAP_FREE_FUNC_8(Py_DataModeling::Gaussian3, std::vector, Device, ftype, const bool)); + + // must call this before using numpy C API + Py_DataModeling::initNumpy(); // numpy initialization + def("fromNumpy", &Py_DataModeling::fromNumpy); + def("toNumpy", &Py_DataModeling::toNumpy); } \ No newline at end of file diff --git a/src/python/py_core/py_core_util.h b/src/python/py_core/py_core_util.h index 7aa01d0..4da52cf 100644 --- a/src/python/py_core/py_core_util.h +++ b/src/python/py_core/py_core_util.h @@ -4,9 +4,9 @@ * @brief Helper and wrapper functions * @version 0.1 * @date 2026-02-21 - * + * * @copyright Copyright (c) 2026 - * + * */ #pragma once @@ -21,50 +21,65 @@ #include #include +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#include + #include +#include -namespace Py_DataModeling { +namespace Py_DataModeling +{ /********************************************************************************************************* ********************************************** Dimension ************************************************* *********************************************************************************************************/ - inline bool (Dimension::*dimEquals1)(const Dimension&) const = &Dimension::operator==; - inline bool (Dimension::*dimEquals2)(const std::vector&) const = &Dimension::operator==; + inline bool (Dimension::*dimEquals1)(const Dimension &) const = &Dimension::operator==; + inline bool (Dimension::*dimEquals2)(const std::vector &) const = &Dimension::operator==; - inline bool (Dimension::*nDimEquals1)(const Dimension&) const = &Dimension::operator!=; - inline bool (Dimension::*nDimEquals2)(const std::vector&) const = &Dimension::operator!=; + inline bool (Dimension::*nDimEquals1)(const Dimension &) const = &Dimension::operator!=; + inline bool (Dimension::*nDimEquals2)(const std::vector &) const = &Dimension::operator!=; /********************************************************************************************************* *********************************************** Tensor *************************************************** *********************************************************************************************************/ - ftype tensorGetItem(const Tensor& self, boost::python::object index); - void tensorSetItem(Tensor& self, boost::python::object index, ftype value); + ftype tensorGetItem(const Tensor &self, boost::python::object index); + void tensorSetItem(Tensor &self, boost::python::object index, ftype value); + + template + std::shared_ptr fromNumpy(boost::python::object npArray); + + inline boost::python::object toNumpy(const Tensor& t); + + static bool initNumpy() { + import_array1(false); // variant that returns bool + return true; + } // need wrappers for default arguments, see // https://beta.boost.org/doc/libs/develop/libs/python/doc/html/tutorial/tutorial/functions.html inline auto OnesWrapper0(std::vector dims) { - return TensorFunctions::Ones(std::move(dims)); + return TensorFunctions::Ones(std::move(dims)); } inline auto OnesWrapper1(std::vector dims, Device d) { - return TensorFunctions::Ones(std::move(dims), d); + return TensorFunctions::Ones(std::move(dims), d); } inline auto ZerosWrapper0(std::vector dims) { - return TensorFunctions::Zeros(std::move(dims)); + return TensorFunctions::Zeros(std::move(dims)); } inline auto ZerosWrapper1(std::vector dims, Device d) { - return TensorFunctions::Zeros(std::move(dims), d); + return TensorFunctions::Zeros(std::move(dims), d); } inline auto GaussianWrapper0(std::vector dims, ftype stddev) { - return TensorFunctions::Gaussian(std::move(dims), stddev); + return TensorFunctions::Gaussian(std::move(dims), stddev); } inline auto GaussianWrapper1(std::vector dims, Device d, ftype stddev) { - return TensorFunctions::Gaussian(std::move(dims), d, stddev); + return TensorFunctions::Gaussian(std::move(dims), d, stddev); } inline Tensor (*Ones0)(std::vector) = &OnesWrapper0; @@ -119,7 +134,7 @@ namespace Py_DataModeling { // matmul inline std::shared_ptr (*matmul) (const std::shared_ptr left, const std::shared_ptr right) = &(cgraph::matmul); - + // sub, div inline std::shared_ptr (*scalarsub) (const std::shared_ptr, ftype) = &(cgraph::sub); @@ -133,4 +148,105 @@ namespace Py_DataModeling { inline std::shared_ptr (*getItemAsTensor2) (const std::shared_ptr& t, const std::vector& idx) = &(cgraph::get); +} + + +template +constexpr int numpyTypeCode() +{ + if constexpr (std::is_same_v || std::is_same_v) + return NPY_FLOAT32; + else if constexpr (std::is_same_v || std::is_same_v) + return NPY_FLOAT64; + else if constexpr (std::is_same_v) + return NPY_FLOAT16; + else + static_assert(false, "Unexpected ftype"); +} + +template +std::shared_ptr Py_DataModeling::fromNumpy(boost::python::object npArray) +{ + using namespace boost::python; + + if (!PyArray_API){ + throw std::runtime_error("Numpy C API not initialized. Call import_array() first."); + } + + // ensure we have a contiguous float32/double array + PyObject *obj = npArray.ptr(); + + if (!PyArray_Check(obj)) + throw std::invalid_argument("Expected numpy array"); + + PyArrayObject *arr = reinterpret_cast(obj); + + // ensure contiguous C order + if (!PyArray_IS_C_CONTIGUOUS(arr)){ + arr = reinterpret_cast( + PyArray_GETCONTIGUOUS(arr)); + } + + // get shape + int ndim = PyArray_NDIM(arr); + std::vector dims(ndim); + npy_intp *shape = PyArray_SHAPE(arr); + tensorSize_t totalSize = 1; + for (int i = 0; i < ndim; i++){ + dims[i] = static_cast(shape[i]); + totalSize *= dims[i]; + } + + PyArrayObject *floatArr = arr; + // cast if needed + if (PyArray_TYPE(arr) != numpyTypeCode()){ + floatArr = reinterpret_cast( + PyArray_Cast(arr, numpyTypeCode())); + } + + const ftype *data = static_cast(PyArray_DATA(floatArr)); + auto result = std::make_shared(dims, data, totalSize); + + // cleanup if we created new arrays + if (floatArr != arr) + Py_DECREF(floatArr); + if (arr != reinterpret_cast(obj)) + Py_DECREF(arr); + + return result; +} + +boost::python::object Py_DataModeling::toNumpy(const Tensor &t) +{ + using namespace boost::python; + + if (!PyArray_API){ + throw std::runtime_error("Numpy C API not initialized. Call import_array() first."); + } + + // get shape + const auto &dims = t.getDims(); + int ndim = static_cast(dims.nDims()); + + std::vector shape(ndim); + for (int i = 0; i < ndim; i++) + shape[i] = static_cast(dims[i]); + + // determine numpy dtype from ftype + constexpr int dtype = numpyTypeCode(); + + // allocate numpy array + PyObject *arr = PyArray_SimpleNew(ndim, shape.data(), dtype); + if (!arr) + throw std::runtime_error("Failed to allocate numpy array"); + + // copy data + ftype *dst = static_cast(PyArray_DATA( + reinterpret_cast(arr))); + + const tensorSize_t size = t.getSize(); + for (tensorSize_t i = 0; i < size; i++) + dst[i] = t[i]; + + return object(handle<>(arr)); } \ No newline at end of file diff --git a/src/python/py_train/py_train.cpp b/src/python/py_train/py_train.cpp index b4ca60c..19a52ec 100644 --- a/src/python/py_train/py_train.cpp +++ b/src/python/py_train/py_train.cpp @@ -1,12 +1,12 @@ /** * @file py_train.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-14 - * + * * @copyright Copyright (c) 2026 - * + * */ #include @@ -18,6 +18,7 @@ #include "training/loss_functions/crossentropy_loss.h" #include "training/loss_functions/crossentropy_softmax_loss.h" +#include "training/optimizers/optimizer_base.h" #include "training/optimizers/sgd.h" #include "training/optimizers/rmsprop.h" @@ -25,7 +26,7 @@ BOOST_PYTHON_MODULE(_train) { - // enable conversion from Tensor registered in _core + // enable conversion from Tensor registered in _core boost::python::object coreModule = boost::python::import("dl_lib._compiled._core"); boost::python::register_ptr_to_python>(); @@ -37,7 +38,7 @@ BOOST_PYTHON_MODULE(_train) ; class_, boost::noncopyable>("BceWithSigmoid") - .def("__call__", &train::BceSigmoidLoss::operator()) + .def("__call__", &train::BceSigmoidLoss::operator()) ; class_, boost::noncopyable>("CrossEntropy") @@ -45,26 +46,30 @@ BOOST_PYTHON_MODULE(_train) ; class_, boost::noncopyable>("CrossEntropyWithSoftmax") - .def("__call__", &train::CrossEntropySoftmaxLoss::operator()) + .def("__call__", &train::CrossEntropySoftmaxLoss::operator()) ; // Optimizers - class_, boost::noncopyable>("SGD", no_init) - .def(init >, ftype>()) - .def("step", &train::SgdOptimizer::step) - .def("zeroGrad", &train::SgdOptimizer::zeroGrad) + class_("_OptimizerBase", no_init) + .def("step", pure_virtual(&train::OptimizerBase::step)) + .def("zeroGrad", &train::OptimizerBase::zeroGrad) + .def("clipGradients", &train::OptimizerBase::clipGradients) ; - class_, boost::noncopyable>("RmsProp", no_init) - .def(init >, ftype, ftype>()) - .def("step", &train::RmsPropOptimizer::step) - .def("zeroGrad", &train::RmsPropOptimizer::zeroGrad) + class_, std::shared_ptr, boost::noncopyable>("SGD", no_init) + .def(init>, ftype>()) + .def("step", &train::SgdOptimizer::step) + ; + + class_, std::shared_ptr, boost::noncopyable>("RmsProp", no_init) + .def(init>, ftype, ftype>()) + .def("step", &train::RmsPropOptimizer::step) ; // Trainers class_, boost::noncopyable>("TrainLoop", no_init) - .def(init&, std::shared_ptr, - std::shared_ptr, size_t, tensorDim_t>()) - .def("run", &train::BaseTrainLoop::run) + .def(init&, std::shared_ptr, + std::shared_ptr, size_t, tensorDim_t>()) + .def("run", &train::BaseTrainLoop::run) ; } \ No newline at end of file diff --git a/tests/python/test_autograd.py b/tests/python/test_autograd.py index aa4b975..a886726 100644 --- a/tests/python/test_autograd.py +++ b/tests/python/test_autograd.py @@ -12,100 +12,100 @@ import pytest class TestAutograd: - def test_backward(self): - t = Tensor([2, 2], True) + def test_backward(self): + t = Tensor([2, 2], True) - loss = (t * 2).sum() - loss.backward() + loss = (t * 2).sum() + loss.backward() - assert t.grads is not None + assert t.grads is not None - def test_nograd_throws(self): - t1 = Tensor([1], [3.0], False) - t2 = Tensor([1], [3.0], False) + def test_nograd_throws(self): + t1 = Tensor([1], [3.0], False) + t2 = Tensor([1], [3.0], False) - t3 = t1*t2 + t3 = t1*t2 - assert not t3.requiresGrad - with pytest.raises(RuntimeError): - t3.backward() + assert not t3.requiresGrad + with pytest.raises(RuntimeError): + t3.backward() - def test_add(self): - t1 = Tensor([1], [3.0], True) - t2 = Tensor([1], [2.0], True) + def test_add(self): + t1 = Tensor([1], [3.0], True) + t2 = Tensor([1], [2.0], True) - t3 = t1+t2 - loss = t3*t3 + t3 = t1+t2 + loss = t3*t3 - loss.backward() + loss.backward() - assert t1.grads.getitem(0) == pytest.approx(10.0) - assert t2.grads.getitem(0) == pytest.approx(10.0) + assert t1.grads.getitem(0) == pytest.approx(10.0) + assert t2.grads.getitem(0) == pytest.approx(10.0) - def test_scalar_mul(self): - t1 = Tensor([1], [2.0], True) - t2 = Tensor([1], [3.0], True) + def test_scalar_mul(self): + t1 = Tensor([1], [2.0], True) + t2 = Tensor([1], [3.0], True) - t3 = t1*t2 - loss = t3*t3 + t3 = t1*t2 + loss = t3*t3 - loss.backward() + loss.backward() - assert t1.grads.getitem(0) == pytest.approx(36.0) - assert t2.grads.getitem(0) == pytest.approx(24.0) + assert t1.grads.getitem(0) == pytest.approx(36.0) + assert t2.grads.getitem(0) == pytest.approx(24.0) - def test_matmul(self): - t1 = Tensor([2, 3], [1, 2, 3, 4, 5, 6], True) - t2 = Tensor([3, 2], [1, 2, 3, 4, 5, 6], True) + def test_matmul(self): + t1 = Tensor([2, 3], [1, 2, 3, 4, 5, 6], True) + t2 = Tensor([3, 2], [1, 2, 3, 4, 5, 6], True) - t3 = t1@t2 - loss = t3.sum() + t3 = t1@t2 + loss = t3.sum() - loss.backward() + loss.backward() - # dL/dt1 = dloss/dt3 @ t2^t = Ones({2, 2}) @ t2^t - assert t1.grads.getitem([0, 0]) == pytest.approx(3.0) - assert t1.grads.getitem([0, 1]) == pytest.approx(7.0) - assert t1.grads.getitem([0, 2]) == pytest.approx(11.0) - assert t1.grads.getitem([1, 0]) == pytest.approx(3.0) - assert t1.grads.getitem([1, 1]) == pytest.approx(7.0) - assert t1.grads.getitem([1, 2]) == pytest.approx(11.0) + # dL/dt1 = dloss/dt3 @ t2^t = Ones({2, 2}) @ t2^t + assert t1.grads.getitem([0, 0]) == pytest.approx(3.0) + assert t1.grads.getitem([0, 1]) == pytest.approx(7.0) + assert t1.grads.getitem([0, 2]) == pytest.approx(11.0) + assert t1.grads.getitem([1, 0]) == pytest.approx(3.0) + assert t1.grads.getitem([1, 1]) == pytest.approx(7.0) + assert t1.grads.getitem([1, 2]) == pytest.approx(11.0) - # dL/dt2 = t1^t @ dloss/dt3 = t1^t @ Ones({2, 2}) - assert t2.grads.getitem([0, 0]) == pytest.approx(5.0) - assert t2.grads.getitem([0, 1]) == pytest.approx(5.0) - assert t2.grads.getitem([1, 0]) == pytest.approx(7.0) - assert t2.grads.getitem([1, 1]) == pytest.approx(7.0) - assert t2.grads.getitem([2, 0]) == pytest.approx(9.0) - assert t2.grads.getitem([2, 1]) == pytest.approx(9.0) + # dL/dt2 = t1^t @ dloss/dt3 = t1^t @ Ones({2, 2}) + assert t2.grads.getitem([0, 0]) == pytest.approx(5.0) + assert t2.grads.getitem([0, 1]) == pytest.approx(5.0) + assert t2.grads.getitem([1, 0]) == pytest.approx(7.0) + assert t2.grads.getitem([1, 1]) == pytest.approx(7.0) + assert t2.grads.getitem([2, 0]) == pytest.approx(9.0) + assert t2.grads.getitem([2, 1]) == pytest.approx(9.0) - def test_chainrule(self): - x = Tensor([1], [2.0], True) + def test_chainrule(self): + x = Tensor([1], [2.0], True) - y = x * x - z = x + y - loss = z * z + y = x * x + z = x + y + loss = z * z - loss.backward() + loss.backward() - # dloss/dx = 2(x^2 + x) * (2x + 1) - # At x=2: 2(4 + 2) * (4 + 1) = 2 * 6 * 5 = 60 - assert x.grads.getitem(0) == pytest.approx(60.0) + # dloss/dx = 2(x^2 + x) * (2x + 1) + # At x=2: 2(4 + 2) * (4 + 1) = 2 * 6 * 5 = 60 + assert x.grads.getitem(0) == pytest.approx(60.0) - def test_multivariate_chainrule(self): - x = Tensor([2], [1.0, 2.0], True) - y = x * 3 + def test_multivariate_chainrule(self): + x = Tensor([2], [1.0, 2.0], True) + y = x * 3 - loss = Tensor([1], [0.0], True) - for i in range(len(y)): - loss = loss + y[i] - loss.backward() + loss = Tensor([1], [0.0], True) + for i in range(len(y)): + loss = loss + y[i] + loss.backward() - assert x.grads.getitem(0) == pytest.approx(3.0) - assert x.grads.getitem(1) == pytest.approx(3.0) + assert x.grads.getitem(0) == pytest.approx(3.0) + assert x.grads.getitem(1) == pytest.approx(3.0) - assert y.grads.getitem(0) == pytest.approx(1.0) - assert y.grads.getitem(1) == pytest.approx(1.0) + assert y.grads.getitem(0) == pytest.approx(1.0) + assert y.grads.getitem(1) == pytest.approx(1.0) if __name__ == '__main__': - raise RuntimeError("Not a standalone script") \ No newline at end of file + raise RuntimeError("Not a standalone script") \ No newline at end of file diff --git a/tests/python/test_tensorops.py b/tests/python/test_tensorops.py index 3bcce8b..f3f6602 100644 --- a/tests/python/test_tensorops.py +++ b/tests/python/test_tensorops.py @@ -10,29 +10,29 @@ from dl_lib import Tensor, Device class TestTensorOps(): - def test_ones(self): - t = Tensor.ones([2, 2]) - assert t.dims == [2, 2] + def test_ones(self): + t = Tensor.ones([2, 2]) + assert t.dims == [2, 2] - def test_ctor(self): - t = Tensor([2], [1.0, 2.0], Device.CPU, False) + def test_ctor(self): + t = Tensor([2], [1.0, 2.0], Device.CPU, False) - assert t.getitem(0) == 1.0 - assert t.getitem(1) == 2.0 + assert t.getitem(0) == 1.0 + assert t.getitem(1) == 2.0 - assert t.device == Device.CPU - assert t.requiresGrad == False + assert t.device == Device.CPU + assert t.requiresGrad == False - def test_multiplication(self): - a = Tensor.ones([2, 2]) * 3 - b = Tensor.ones([2, 2]) * 0.5 - c = a * b + def test_multiplication(self): + a = Tensor.ones([2, 2]) * 3 + b = Tensor.ones([2, 2]) * 0.5 + c = a * b - assert c.dims == [2, 2] - assert c.getitem([0, 0]) == 1.5 - assert c.getitem([0, 1]) == 1.5 - assert c.getitem([1, 0]) == 1.5 - assert c.getitem([1, 1]) == 1.5 + assert c.dims == [2, 2] + assert c.getitem([0, 0]) == 1.5 + assert c.getitem([0, 1]) == 1.5 + assert c.getitem([1, 0]) == 1.5 + assert c.getitem([1, 1]) == 1.5 if __name__ == '__main__': - raise RuntimeError("Not a standalone script") \ No newline at end of file + raise RuntimeError("Not a standalone script") \ No newline at end of file diff --git a/tests/python/test_training.py b/tests/python/test_training.py index 7d7bf98..822c386 100644 --- a/tests/python/test_training.py +++ b/tests/python/test_training.py @@ -11,7 +11,7 @@ from dl_lib import Tensor from dl_lib.nn import FfLayer, Sequential from dl_lib.nn.activation import LeakyReLU -from dl_lib.train.loss import BCE, BceWithSigmoid, CrossEntropyWithSoftmax +from dl_lib.train.loss import BceWithSigmoid, CrossEntropyWithSoftmax from dl_lib.train.optim import SGD, RmsProp from dl_lib.sys import setSeed @@ -32,16 +32,16 @@ def train(net, loss_fn, optim, x, y, epochs): def make_binary_net(): net = Sequential() - net.append(FfLayer(2, 4, True, True)) + net.append(FfLayer(2, 4)) net.append(LeakyReLU(0.01)) - net.append(FfLayer(4, 1, True, True)) + net.append(FfLayer(4, 1)) return net def make_multiclass_net(): net = Sequential() - net.append(FfLayer(2, 8, True, True)) + net.append(FfLayer(2, 8)) net.append(LeakyReLU(0.01)) - net.append(FfLayer(8, 3, True, True)) + net.append(FfLayer(8, 3)) return net def make_xor_data(): From ce702ff8d1d7a061bf65b6eadc04ca2c7b42a6b3 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 22 Mar 2026 17:26:04 +0100 Subject: [PATCH 02/73] Enabled CUDA compilation --- .gitignore | 4 ++++ CMakeLists.txt | 17 +++++++++++++- src/backend/CMakeLists.txt | 26 +++++++++++++++++++++- src/backend/data_modeling/tensor.cpp | 17 ++++++++++++-- src/backend/data_modeling/tensor.h | 15 ++++++++++--- src/backend/utility/cuda/cuda_common.cu | 28 ++++++++++++++++++++++++ src/backend/utility/cuda/cuda_common.cuh | 24 ++++++++++++++++++++ 7 files changed, 124 insertions(+), 7 deletions(-) create mode 100644 src/backend/utility/cuda/cuda_common.cu create mode 100644 src/backend/utility/cuda/cuda_common.cuh diff --git a/.gitignore b/.gitignore index 2744889..38485c5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,9 @@ python_lib/dl_lib/_compiled *__pycache__* *_cache +perf.data* + +docs + # TODO: remove later benchmarks \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 90dba31..f4c3db7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,22 @@ endif() add_compile_options("$<$:/utf-8>") add_compile_options("$<$:/utf-8>") -# TODO: add flag for double precision? +option(CUDA "Enable CUDA execution for some faster data structures" OFF) + +if (CUDA) + include(CheckLanguage) + check_language(CUDA) + + if(CMAKE_CUDA_COMPILER) + add_definitions(-D__CUDA) + enable_language(CUDA) + + set(CMAKE_CUDA_STANDARD 17) + set(CMAKE_CUDA_STANDARD_REQUIRED ON) + else() + message(WARNING "Could not find CUDA on system. Exiting.") + endif() +endif() # include python libs if(APPLE) diff --git a/src/backend/CMakeLists.txt b/src/backend/CMakeLists.txt index ed6bade..7420a99 100644 --- a/src/backend/CMakeLists.txt +++ b/src/backend/CMakeLists.txt @@ -7,6 +7,18 @@ file(GLOB_RECURSE CORE_SOURCES utility/*.cpp ) +if(CMAKE_CUDA_COMPILER) + file(GLOB_RECURSE CUDA_SOURCES + computational_graph/*.cu + data_modeling/*.cu + module/*.cu + system/*.cu + training/*.cu + utility/*.cu + ) + list(APPEND CORE_SOURCES ${CUDA_SOURCES}) +endif() + add_library(BackendCore SHARED ${CORE_SOURCES}) target_include_directories(BackendCore PUBLIC @@ -15,4 +27,16 @@ target_include_directories(BackendCore PUBLIC set_target_properties(BackendCore PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${PYTHON_MODULE_DIR}" # make sure Python-modules see backend -) \ No newline at end of file +) + +if(CMAKE_CUDA_COMPILER) + set_target_properties(BackendCore PROPERTIES + CUDA_SEPARABLE_COMPILATION ON + # nvidia-smi --query-gpu=compute_cap --format=csv,noheader + # I get 12.0, hence 120 + CUDA_ARCHITECTURES 120 + ) + + find_library(CUDART_LIBRARY cudart ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES}) + target_link_libraries(BackendCore ${CUDART_LIBRARY}) +endif() \ No newline at end of file diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index e398c2c..68ee754 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -18,6 +18,11 @@ #include #include +#ifdef __CUDA +#include "cuda_runtime.h" +#include "device_launch_parameters.h" +#endif + using namespace std; /******************************************************************** @@ -51,7 +56,11 @@ Tensor::tensorValues_t::~tensorValues_t() noexcept { free(values); break; case Device::CUDA: - std::__throw_invalid_argument("Cuda destructor not implemented yet."); + #ifdef __CUDA + gpuErrchk(cudaFree(values)); + #else + std::__throw_invalid_argument("Not compiled with CUDA."); + #endif break; } } @@ -66,7 +75,11 @@ void Tensor::tensorValues_t::copyFromRaw(const ftype* src, tensorSize_t n) { std::memcpy(values, src, n * sizeof(ftype)); break; case Device::CUDA: - __throw_runtime_error("copyFromRaw not implemented for CUDA"); + #ifdef __CUDA + gpuErrchk(cudaMemcpy(values, src, n*sizeof(ftype), cudaMemcpyHostToDevice)); + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif } } diff --git a/src/backend/data_modeling/tensor.h b/src/backend/data_modeling/tensor.h index 01f4091..0627837 100644 --- a/src/backend/data_modeling/tensor.h +++ b/src/backend/data_modeling/tensor.h @@ -28,6 +28,11 @@ #include #include +#ifdef __CUDA +#include "cuda.h" +#include "utility/cuda/cuda_common.cuh" +#endif + // break circular dependency namespace cgraph { @@ -88,13 +93,17 @@ class Tensor final : public std::enable_shared_from_this void resize(const T size) { this->size = static_cast(size); - switch (this->device) + switch (device) { case Device::CPU: values = static_cast(std::malloc(this->size * sizeof(ftype))); break; - case Device::CUDA: - std::__throw_invalid_argument("Not implemented yet."); + case Device::CUDA + #ifdef __CUDA + gpuErrchk(cudaMalloc((void**) &values, this->size * sizeof(ftype))); + #else + std::__throw_invalid_argument("Not compiled with CUDA."); + #endif break; } } diff --git a/src/backend/utility/cuda/cuda_common.cu b/src/backend/utility/cuda/cuda_common.cu new file mode 100644 index 0000000..86e766e --- /dev/null +++ b/src/backend/utility/cuda/cuda_common.cu @@ -0,0 +1,28 @@ +/** + * @file cuda_common.cu + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-03-22 + * + * @copyright Copyright (c) 2026 + * + */ + +#ifndef __CUDA +static_assert(false, "Should not inlcude this file without CUDA compile options"); +#endif + +#include "cuda_common.cuh" + +#include + +void utility::gpuAssert(cudaError_t code, const char *file, int line, bool abort) +{ + if (code != cudaSuccess) + { + //fprintf(stderr,"GPUassert: %s %s %d\n", cudaGetErrorString(code), file, line); + std::cerr << "GPUassert: " << cudaGetErrorString(code) << " " << file << " " << line; + if (abort) exit(code); + } +} \ No newline at end of file diff --git a/src/backend/utility/cuda/cuda_common.cuh b/src/backend/utility/cuda/cuda_common.cuh new file mode 100644 index 0000000..baa67b5 --- /dev/null +++ b/src/backend/utility/cuda/cuda_common.cuh @@ -0,0 +1,24 @@ +/** + * @file cuda_common.cuh + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-03-22 + * + * @copyright Copyright (c) 2026 + * + */ + +#pragma once + +#ifdef __CUDA + +#include "cuda.h" + +namespace utility { + void gpuAssert(cudaError_t code, const char *file, int line, bool abort=true); +} + +#define gpuErrchk(ans) { utility::gpuAssert((ans), __FILE__, __LINE__); } + +#endif // __CUDA \ No newline at end of file From 120b36d6d4bb001fe616453151d1223e60117714 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Mon, 23 Mar 2026 12:35:20 +0100 Subject: [PATCH 03/73] Better mem-access for matmul --- src/backend/data_modeling/tensor.cpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index 68ee754..fbdf6c9 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -364,22 +364,22 @@ void Tensor::matMul2DCpu(Tensor& res, const Tensor& left, const Tensor& right, c const auto nRowsRight = static_cast(right.dims.get(-2)); const auto nColsRight = static_cast(right.dims.get(-1)); - tensorSize_t resIdx = resOffset; for(tensorSize_t lrow=0; lrow Date: Mon, 23 Mar 2026 13:17:01 +0100 Subject: [PATCH 04/73] Optimized matmul for better cache alignment/lower cache misses --- examples/mnist.py | 2 +- readme.md | 9 +++++++++ src/backend/data_modeling/tensor.cpp | 13 +++++++------ src/backend/data_modeling/tensor.h | 4 ++-- src/backend/utility/cuda/cuda_common.cu | 2 +- src/backend/utility/cuda/cuda_common.cuh | 7 ++++--- tests/python/__init__.py | 0 7 files changed, 24 insertions(+), 13 deletions(-) delete mode 100644 tests/python/__init__.py diff --git a/examples/mnist.py b/examples/mnist.py index 7628a47..868f925 100644 --- a/examples/mnist.py +++ b/examples/mnist.py @@ -125,7 +125,7 @@ def evaluate(net, x, y_int, batch_size=256): # setup net = make_net() loss_fn = CrossEntropyWithSoftmax() - optim = RmsProp(net.parameters(), 0.0001, 0.95) # lr and decay + optim = RmsProp(net.parameters(), 0.00001, 0.9) # lr and decay # training loop n_epochs = 5 diff --git a/readme.md b/readme.md index 2a8152e..906b7d2 100644 --- a/readme.md +++ b/readme.md @@ -53,6 +53,15 @@ make ctest ``` +### Building with CUDA + +To build with CUDA, a flag has to be supplied to CMake. + +```bash +cmake --DCUDA=On .. +``` + + ## Running Unit Tests Compile with building tests enabled: diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index fbdf6c9..d38d668 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -57,7 +57,7 @@ Tensor::tensorValues_t::~tensorValues_t() noexcept { break; case Device::CUDA: #ifdef __CUDA - gpuErrchk(cudaFree(values)); + cudaErrchk(cudaFree(values)); #else std::__throw_invalid_argument("Not compiled with CUDA."); #endif @@ -76,7 +76,7 @@ void Tensor::tensorValues_t::copyFromRaw(const ftype* src, tensorSize_t n) { break; case Device::CUDA: #ifdef __CUDA - gpuErrchk(cudaMemcpy(values, src, n*sizeof(ftype), cudaMemcpyHostToDevice)); + cudaErrchk(cudaMemcpy(values, src, n*sizeof(ftype), cudaMemcpyHostToDevice)); #else __throw_runtime_error("Not compiled with CUDA"); #endif @@ -370,15 +370,16 @@ void Tensor::matMul2DCpu(Tensor& res, const Tensor& left, const Tensor& right, c tensorSize_t rightIdx = rightOffset; // res likely has undefined memory content - for(tensorSize_t rrow=0; rrow case Device::CPU: values = static_cast(std::malloc(this->size * sizeof(ftype))); break; - case Device::CUDA + case Device::CUDA: #ifdef __CUDA - gpuErrchk(cudaMalloc((void**) &values, this->size * sizeof(ftype))); + cudaErrchk(cudaMalloc((void**) &values, this->size * sizeof(ftype))); #else std::__throw_invalid_argument("Not compiled with CUDA."); #endif diff --git a/src/backend/utility/cuda/cuda_common.cu b/src/backend/utility/cuda/cuda_common.cu index 86e766e..101ea3f 100644 --- a/src/backend/utility/cuda/cuda_common.cu +++ b/src/backend/utility/cuda/cuda_common.cu @@ -10,7 +10,7 @@ */ #ifndef __CUDA -static_assert(false, "Should not inlcude this file without CUDA compile options"); +static_assert(false, "File should not be included without CUDA enabled"); #endif #include "cuda_common.cuh" diff --git a/src/backend/utility/cuda/cuda_common.cuh b/src/backend/utility/cuda/cuda_common.cuh index baa67b5..c325aa4 100644 --- a/src/backend/utility/cuda/cuda_common.cuh +++ b/src/backend/utility/cuda/cuda_common.cuh @@ -11,7 +11,9 @@ #pragma once -#ifdef __CUDA +#ifndef __CUDA +static_assert(false, "File should not be included without CUDA enabled"); +#endif // __CUDA #include "cuda.h" @@ -19,6 +21,5 @@ namespace utility { void gpuAssert(cudaError_t code, const char *file, int line, bool abort=true); } -#define gpuErrchk(ans) { utility::gpuAssert((ans), __FILE__, __LINE__); } +#define cudaErrchk(ans) { utility::gpuAssert((ans), __FILE__, __LINE__); } -#endif // __CUDA \ No newline at end of file diff --git a/tests/python/__init__.py b/tests/python/__init__.py deleted file mode 100644 index e69de29..0000000 From 3ed4515a2c49d6ca6bdba18d18e926e6e68db2c7 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Mon, 23 Mar 2026 13:52:45 +0100 Subject: [PATCH 05/73] MNIST and gitignore update --- .gitignore | 2 +- examples/mnist.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 38485c5..df79fe8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ python_lib/dl_lib/_compiled perf.data* -docs +optimization # TODO: remove later benchmarks \ No newline at end of file diff --git a/examples/mnist.py b/examples/mnist.py index 868f925..a54ad4d 100644 --- a/examples/mnist.py +++ b/examples/mnist.py @@ -125,7 +125,7 @@ def evaluate(net, x, y_int, batch_size=256): # setup net = make_net() loss_fn = CrossEntropyWithSoftmax() - optim = RmsProp(net.parameters(), 0.00001, 0.9) # lr and decay + optim = RmsProp(net.parameters(), 0.00001, 0.95) # lr and decay # training loop n_epochs = 5 From 6bb06f07a18e581d3eb9b39a72f2b21dfc406d8b Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sat, 28 Mar 2026 13:57:06 +0100 Subject: [PATCH 06/73] Tensor backend for CUDA ready --- readme.md | 5 +- src/backend/CMakeLists.txt | 15 +- .../cuda/activation_nodes.cu | 18 + .../cuda/activation_nodes.cuh | 26 + .../loss_functions/cuda/loss_nodes.cu | 18 + .../loss_functions/cuda/loss_nodes.cuh | 28 ++ src/backend/data_modeling/cuda/tensorops.cu | 85 ++++ src/backend/data_modeling/cuda/tensorops.cuh | 38 ++ src/backend/data_modeling/tensor.cpp | 464 +++++++++++++----- src/backend/data_modeling/tensor.h | 51 +- .../activation_functions/cuda/activations.cu | 0 .../activation_functions/cuda/activations.cuh | 0 src/backend/utility/cuda/cuda_common.cuh | 2 +- 13 files changed, 591 insertions(+), 159 deletions(-) create mode 100644 src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu create mode 100644 src/backend/computational_graph/activation_functions/cuda/activation_nodes.cuh create mode 100644 src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu create mode 100644 src/backend/computational_graph/loss_functions/cuda/loss_nodes.cuh create mode 100644 src/backend/data_modeling/cuda/tensorops.cu create mode 100644 src/backend/data_modeling/cuda/tensorops.cuh create mode 100644 src/backend/module/activation_functions/cuda/activations.cu create mode 100644 src/backend/module/activation_functions/cuda/activations.cuh diff --git a/readme.md b/readme.md index 906b7d2..a2a846c 100644 --- a/readme.md +++ b/readme.md @@ -20,7 +20,7 @@ For some examples on Python interface, see tests/python. - Training framework (optimizers, loss functions, layers, and networks) - **Example code**: Full MNIST dataset training example - **Python Interface**: Seamless integration via Boost.Python -- **Clean Architecture**: Modular design, ~4K LOC +- **Clean Architecture**: Modular design, maintainable and extensible - **CI/CD**: Automated testing with GTest and GitHub Actions ## Tech Stack @@ -50,7 +50,6 @@ Roadmap: mkdir build && cd build cmake .. make -ctest ``` ### Building with CUDA @@ -70,7 +69,7 @@ Compile with building tests enabled: mkdir build && cd build cmake -DBUILD_TESTS=On .. make -ctest +ctest . ``` ## Required diff --git a/src/backend/CMakeLists.txt b/src/backend/CMakeLists.txt index 7420a99..a4f89ed 100644 --- a/src/backend/CMakeLists.txt +++ b/src/backend/CMakeLists.txt @@ -34,9 +34,16 @@ if(CMAKE_CUDA_COMPILER) CUDA_SEPARABLE_COMPILATION ON # nvidia-smi --query-gpu=compute_cap --format=csv,noheader # I get 12.0, hence 120 - CUDA_ARCHITECTURES 120 + #set(CMAKE_CUDA_ARCHITECTURES "75;86;89;100;120") + CMAKE_CUDA_ARCHITECTURES native ) - find_library(CUDART_LIBRARY cudart ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES}) - target_link_libraries(BackendCore ${CUDART_LIBRARY}) -endif() \ No newline at end of file + find_package(CUDAToolkit REQUIRED) + target_include_directories(BackendCore PRIVATE + ${CUDAToolkit_INCLUDE_DIRS} + ) + target_link_libraries(BackendCore + CUDA::cudart + ) +endif() + diff --git a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu new file mode 100644 index 0000000..6ba9039 --- /dev/null +++ b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu @@ -0,0 +1,18 @@ +/** + * @file activation_nodes.cu + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-03-23 + * + * @copyright Copyright (c) 2026 + * + */ + +#ifndef __CUDA +static_assert(false, "File should not be included without CUDA enabled"); +#endif // __CUDA + +#include "activation_nodes.cuh" + +using namespace cuda; \ No newline at end of file diff --git a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cuh b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cuh new file mode 100644 index 0000000..3e2c7bc --- /dev/null +++ b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cuh @@ -0,0 +1,26 @@ +/** + * @file activation_nodes.cuh + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-03-23 + * + * @copyright Copyright (c) 2026 + * + */ + +#pragma once + +#ifndef __CUDA +static_assert(false, "File should not be included without CUDA enabled"); +#endif // __CUDA + +#include "utility/global_params.h" + +namespace cuda { + __global__ void reluBackward(ftype* res, const ftype* const upstreamGrad, tensorSize_t size); + __global__ void leakyReluBackward(ftype* res, const ftype* const upstreamGrad, ftype eps, tensorSize_t size); + + __global__ void sigmoidBackward(ftype* res, const ftype* const sigmoids, const ftype* const upstreamGrad, tensorSize_t size); + __global__ void softmaxBackward(ftype* res, const ftype* const softmax, const ftype* const upstreamGrad, tensorSize_t size); +} \ No newline at end of file diff --git a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu new file mode 100644 index 0000000..dfb251c --- /dev/null +++ b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu @@ -0,0 +1,18 @@ +/** + * @file loss_nodes.cu + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-03-23 + * + * @copyright Copyright (c) 2026 + * + */ + +#ifndef __CUDA +static_assert(false, "File should not be included without CUDA enabled"); +#endif // __CUDA + +#include "loss_nodes.cuh" + +using namespace cuda; \ No newline at end of file diff --git a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cuh b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cuh new file mode 100644 index 0000000..773de53 --- /dev/null +++ b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cuh @@ -0,0 +1,28 @@ +/** + * @file loss_nodes.cuh + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-03-23 + * + * @copyright Copyright (c) 2026 + * + */ + +#pragma once + +#ifndef __CUDA +static_assert(false, "File should not be included without CUDA enabled"); +#endif // __CUDA + +#include "utility/global_params.h" + +namespace cuda { + __global__ void bceBackward(ftype* res, const ftype* const upstreamGrad, tensorSize_t size); + __global__ void bceWithSigmoidBackward(ftype* res, const ftype* const upstreamGrad, ftype eps, tensorSize_t size); + + __global__ void crossEntropyBackward(ftype* res, const ftype* const sigmoids, const ftype* const upstreamGrad, tensorSize_t size); + __global__ void crossEntropyWithSoftmaxBackward(ftype* res, const ftype* const softmax, const ftype* const upstreamGrad, tensorSize_t size); + + __global__ void rmseBackward(ftype* res, const ftype* const sigmoids, const ftype* const upstreamGrad, tensorSize_t size); +} \ No newline at end of file diff --git a/src/backend/data_modeling/cuda/tensorops.cu b/src/backend/data_modeling/cuda/tensorops.cu new file mode 100644 index 0000000..421355c --- /dev/null +++ b/src/backend/data_modeling/cuda/tensorops.cu @@ -0,0 +1,85 @@ +/** + * @file tensorops.cu + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-03-23 + * + * @copyright Copyright (c) 2026 + * + */ + +#ifndef __CUDA +static_assert(false, "File should not be included without CUDA enabled"); +#endif // __CUDA + +#include "tensorops.cuh" +#include "utility/cuda/cuda_common.cuh" + +#include "cuda_runtime.h" + +namespace{ + __global__ void elementwiseaddKernel(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size) { + int gid = blockDim.x * blockIdx.x + threadIdx.x; + res[gid] = left[gid] + right[gid]; + } + + __global__ void elementwisemulKernel(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size) { + int gid = blockDim.x * blockIdx.x + threadIdx.x; + res[gid] = left[gid] * right[gid]; + } + + __global__ void scalaraddKernel(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size){ + int gid = blockDim.x * blockIdx.x + threadIdx.x; + res[gid] = left[gid] + scalar; + } + + __global__ void scalarmulKernel(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size) { + int gid = blockDim.x * blockIdx.x + threadIdx.x; + res[gid] = left[gid] + scalar; + } +} + +namespace cuda { + void scalaradd(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size) { + int threadsPerBlock = 256; + int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; + scalaraddKernel<<>>(res, left, scalar, size); + cudaErrchk(cudaDeviceSynchronize()); + } + + void scalarmul(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size) { + int threadsPerBlock = 256; + int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; + scalarmulKernel<<>>(res, left, scalar, size); + cudaErrchk(cudaDeviceSynchronize()); + } + + void elementwiseadd(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size) { + int threadsPerBlock = 256; + int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; + elementwiseaddKernel<<>>(res, left, right, size); + cudaErrchk(cudaDeviceSynchronize()); + } + + void elementwisemul(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size) { + int threadsPerBlock = 256; + int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; + elementwisemulKernel<<>>(res, left, right, size); + cudaErrchk(cudaDeviceSynchronize()); + } + + void matmul(ftype* res, const ftype* const left, const ftype* const right) { + // TODO + } + + ftype get(const ftype* const t, tensorSize_t idx) { + ftype res; + cudaErrchk(cudaMemcpy(&res, t+idx, sizeof(ftype), cudaMemcpyDeviceToHost)); + return res; + } + + ftype set(ftype value, const ftype* t, tensorSize_t idx) { + cudaErrchk(cudaMemcpy((void*)t+idx, &value, sizeof(ftype), cudaMemcpyHostToDevice)); + } +} diff --git a/src/backend/data_modeling/cuda/tensorops.cuh b/src/backend/data_modeling/cuda/tensorops.cuh new file mode 100644 index 0000000..15572d8 --- /dev/null +++ b/src/backend/data_modeling/cuda/tensorops.cuh @@ -0,0 +1,38 @@ +/** + * @file tensorops.cuh + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-03-23 + * + * @copyright Copyright (c) 2026 + * + */ + +#pragma once + +#ifndef __CUDA +static_assert(false, "File should not be included without CUDA enabled"); +#endif // __CUDA + +#include "utility/global_params.h" +#include "data_modeling/dim_type.h" + +namespace cuda { + // scalar ops + void scalaradd(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size); + void scalarmul(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size); + + // matrix ops + void elementwiseadd(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size); + void broadcastedadd(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size); + void elementwisemul(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size); + void matmul(ftype* res, const ftype* const left, const ftype* const right); + + void transpose2D(ftype* res, const ftype* const src, Dimension dims, tensorDim_t dim1, tensorDim_t dim2); + void transpose(ftype* res, const ftype* const src, Dimension dims, tensorDim_t dim1, tensorDim_t dim2); + + // other + ftype get(const ftype* const t, tensorSize_t idx); + ftype set(ftype value, const ftype* t, tensorSize_t idx); +} diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index d38d668..be2ba0e 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -18,9 +18,11 @@ #include #include +#include + #ifdef __CUDA -#include "cuda_runtime.h" -#include "device_launch_parameters.h" +#include "utility/cuda/cuda_common.cuh" +#include "data_modeling/cuda/tensorops.cuh" #endif using namespace std; @@ -65,6 +67,33 @@ Tensor::tensorValues_t::~tensorValues_t() noexcept { } } +void Tensor::tensorValues_t::resize(const tensorSize_t size) { + this->size = size; + switch (device) { + case Device::CPU: + values = static_cast(std::malloc(this->size * sizeof(ftype))); + break; + case Device::CUDA: + #ifdef __CUDA + cudaErrchk(cudaMalloc((void**) &values, this->size * sizeof(ftype))); + #else + std::__throw_invalid_argument("Not compiled with CUDA."); + #endif + break; + } +} + +ftype* Tensor::tensorValues_t::getData() const noexcept { +#ifndef __CUDA + static_assert(false, "Should only be callable with CUDA enabled"); +#endif + + if(device==Device::CPU){ + __throw_runtime_error("Should only be called on CUDA tensor."); + } + return values; +} + /** * @brief Copy from pointer into this object. */ @@ -76,10 +105,11 @@ void Tensor::tensorValues_t::copyFromRaw(const ftype* src, tensorSize_t n) { break; case Device::CUDA: #ifdef __CUDA - cudaErrchk(cudaMemcpy(values, src, n*sizeof(ftype), cudaMemcpyHostToDevice)); + cudaErrchk(cudaMemcpy(values, src, n*sizeof(ftype), cudaMemcpyDeviceToDevice)); #else __throw_runtime_error("Not compiled with CUDA"); #endif + break; } } @@ -93,12 +123,14 @@ void Tensor::tensorValues_t::copyValues(Tensor::tensorValues_t& target) const { switch(device){ case Device::CPU: - for(tensorSize_t i=0; i= high - low); + if(highsize * sizeof(ftype))); + cudaErrchk(cudaMemcpy(tmp, values, this->size * sizeof(ftype), cudaMemcpyHostToDevice)); + free(values); + values = tmp; + break; + } + case Device::CUDA:{ + ftype* tmp = static_cast(std::malloc(this->size * sizeof(ftype))); + cudaErrchk(cudaMemcpy(tmp, values, this->size * sizeof(ftype), cudaMemcpyDeviceToHost)); + cudaErrchk(cudaFree(values)); + values = tmp; + break; + } + } + + device = d; + #endif } Device Tensor::tensorValues_t::getDevice() const noexcept { @@ -188,7 +264,11 @@ Tensor::tensorValues_t::operator+=(const Tensor::tensorValues_t& other) { addOtherCpu(other); break; case Device::CUDA: - __throw_invalid_argument("CUDA not supported yet for += operation"); + #ifdef __CUDA + cuda::elementwiseadd(values, values, other.values, size); + #else + __throw_invalid_argument("Not compiled with CUDA"); + #endif break; } @@ -200,7 +280,7 @@ ftype& Tensor::tensorValues_t::operator[](const tensorSize_t idx) { throw std::out_of_range("Out of range for tensor"); if(device!=Device::CPU){ - __throw_invalid_argument("Operator [] only implemented for CPU"); + __throw_invalid_argument("'ftype& operator[] const' only implemented for CPU"); } return values[idx]; } @@ -209,10 +289,20 @@ ftype Tensor::tensorValues_t::operator[](const tensorSize_t idx) const { if(idx >= size) throw std::out_of_range("Out of range for tensor"); - if(device!=Device::CPU){ - __throw_invalid_argument("Operator [] only implemented for CPU"); + switch(device){ + case Device::CPU: + return values[idx]; + case Device::CUDA: + #ifdef __CUDA + return cuda::get(values, idx); + #else + __throw_invalid_argument("Not compiled with CUDA"); + break; + #endif } - return values[idx]; + + __throw_runtime_error("Should never reach here."); + return 0; // suppress warnings } void Tensor::tensorValues_t::set(ftype v, tensorSize_t idx) { @@ -222,12 +312,15 @@ void Tensor::tensorValues_t::set(ftype v, tensorSize_t idx) { switch(device){ case Device::CPU: values[idx] = v; - return; + break; case Device::CUDA: - __throw_runtime_error("Not implemented for CUDA yet"); + #ifdef __CUDA + cuda::set(v, values, idx); + #else + __throw_invalid_argument("Not compiled with CUDA"); + #endif + break; } - - __throw_runtime_error("Should never reach here."); } ftype Tensor::tensorValues_t::get(tensorSize_t idx) { @@ -238,7 +331,12 @@ ftype Tensor::tensorValues_t::get(tensorSize_t idx) { case Device::CPU: return values[idx]; case Device::CUDA: - __throw_runtime_error("Not implemented for CUDA yet"); + #ifdef __CUDA + return cuda::get(values, idx); + #else + __throw_invalid_argument("Not compiled with CUDA"); + break; + #endif } __throw_runtime_error("Should never reach here."); @@ -297,17 +395,12 @@ Tensor Tensor::createDeepCopy() const { return res; } - /** - * @brief Scalar multiplication, capable of broadcasting. First argument assumed - * to be the scalar tensor. + * @brief For convenience. Needed for other classes to perform their CUDA operations. + * Avoids moving all CUDA code into tensor. */ -Tensor Tensor::multiplyScalar(const Tensor& scalar, const Tensor& right) noexcept { - Tensor res(right.dims, right.values->getDevice(), false); - for(int i=0; igetData(); } /** @@ -338,16 +431,29 @@ Tensor Tensor::matMulImpl(const Tensor& left, const Tensor& right) { const tensorSize_t rightSize = right.dims.get(-1) * right.dims.get(-2); const tensorSize_t resSize = left.dims.get(-2) * right.dims.get(-1); - tensorSize_t leftOffset = 0; - tensorSize_t rightOffset = 0; - tensorSize_t resOffset = 0; - - while(leftOffset < left.getSize()){ - matMul2DCpu(res, left, right, resOffset, leftOffset, rightOffset); + switch(left.values->getDevice()){ + case Device::CPU: + { + tensorSize_t leftOffset = 0; + tensorSize_t rightOffset = 0; + tensorSize_t resOffset = 0; + + while(leftOffset < left.getSize()){ + matMul2DCpu(res, left, right, resOffset, leftOffset, rightOffset); + leftOffset += leftSize; + rightOffset += rightSize; + resOffset += resSize; + } - leftOffset += leftSize; - rightOffset += rightSize; - resOffset += resSize; + break; + } + case Device::CUDA: + #ifdef __CUDA + cuda::matmul(res.getData(), left.values->getData(), right.values->getData()); + #else + __throw_invalid_argument("Not compiled with CUDA"); + #endif + break; } return res; @@ -390,14 +496,10 @@ void Tensor::matMul2DCpu(Tensor& res, const Tensor& left, const Tensor& right, c */ Tensor Tensor::matmul(const Tensor& other) const { assert(values->getDevice()==other.values->getDevice()); - if(values->getDevice()==Device::CUDA){ - __throw_invalid_argument("Multiplication not implemented on CUDA"); - } if(values->getDevice()!=other.values->getDevice()){ __throw_runtime_error("Tensors on different devices."); } - return matMulImpl(*this, other); } @@ -409,10 +511,6 @@ Tensor Tensor::matmul(const Tensor& other) const { * other.dims == (dimN) && this->dims == (dim0, dim1,..., dimN). */ Tensor Tensor::operator+(const Tensor& other) const { - if(values->getDevice()==Device::CUDA){ - __throw_invalid_argument("Addition not implemented on CUDA"); - } - if(this->dims != other.dims && !(other.dims.nDims() == 1 && other.dims.get(0) == dims.get(-1))){ __throw_invalid_argument("Tensors need matching dimensions"); @@ -422,23 +520,37 @@ Tensor Tensor::operator+(const Tensor& other) const { } Tensor res(dims, values->getDevice()); - - if(dims==other.dims){ - // elementwise add - for(tensorSize_t i=0; igetSize(); i++){ - (*res.values)[i] = (*values)[i] + (*other.values)[i]; - } - } - else { [[likely]] - // broadcasted add - const auto stride = static_cast(other.dims.get(0)); - for(tensorSize_t offset=0; offsetgetSize(); offset+=stride){ - for(tensorSize_t i=0; igetDevice()){ + case Device::CPU: + if(dims==other.dims){ + // elementwise add + for(tensorSize_t i=0; igetSize(); i++){ + (*res.values)[i] = (*values)[i] + (*other.values)[i]; + } } - } + else [[likely]] { + // broadcasted add + const auto stride = static_cast(other.dims.get(0)); + for(tensorSize_t offset=0; offsetgetSize(); offset+=stride){ + for(tensorSize_t i=0; igetData(), other.getData(), values->getSize()); + } + else [[likely]] { + cuda::broadcastedadd(res.getData(), values->getData(), other.getData(), values->getSize()); + } + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif + break; } - return res; } @@ -453,19 +565,6 @@ Tensor Tensor::add(const Tensor& other) const { * @brief Elementwise multiplication. */ Tensor Tensor::operator*(const Tensor& other) const { - assert(values->getDevice()==other.values->getDevice()); - if(values->getDevice()==Device::CUDA){ - __throw_invalid_argument("Multiplication not implemented on CUDA"); - } - - // TODO: check what to do about these two gradients and if you want broadcasting here at all -/* if(other.dims.getSize()==1){ - return multiplyScalar(other, *this); - } - else if(dims.getSize()==1){ - return multiplyScalar(*this, other); - } */ - if(this->dims != other.dims){ __throw_invalid_argument("Tensors need same dimensions"); } @@ -473,10 +572,21 @@ Tensor Tensor::operator*(const Tensor& other) const { __throw_runtime_error("Tensors on different devices."); } - assert(values->getSize()==other.values->getSize()); Tensor res(dims, values->getDevice(), false); - for(tensorSize_t i=0; igetSize(); i++){ - (*res.values)[i] = (*values)[i] * (*other.values)[i]; + switch(values->getDevice()){ + case Device::CPU: + for(tensorSize_t i=0; igetSize(); i++){ + (*res.values)[i] = (*values)[i] * (*other.values)[i]; + } + break; + case Device::CUDA: + #ifdef __CUDA + cuda::elementwisemul(res.getData(), values->getData(), + other.values->getData(), values->getSize()); + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif + break; } return res; @@ -491,8 +601,19 @@ Tensor Tensor::elementwiseMul(const Tensor& other) const { Tensor Tensor::operator*(ftype scalar) const { Tensor res(dims, values->getDevice(), false); - for (tensorSize_t i = 0; i < values->getSize(); ++i) { - (*res.values)[i] = (*values)[i] * scalar; + switch(values->getDevice()){ + case Device::CPU: + for (tensorSize_t i = 0; i < values->getSize(); ++i) { + (*res.values)[i] = (*values)[i] * scalar; + } + break; + case Device::CUDA: + #ifdef __CUDA + cuda::scalarmul(res.getData(), values->getData(), scalar, values->getSize()); + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif + break; } return res; @@ -504,8 +625,19 @@ Tensor Tensor::operator/(ftype scalar) const { } Tensor res(dims, values->getDevice(), false); - for (tensorSize_t i = 0; i < values->getSize(); ++i) { - (*res.values)[i] = (*values)[i] / scalar; + switch(values->getDevice()){ + case Device::CPU: + for (tensorSize_t i = 0; i < values->getSize(); ++i) { + (*res.values)[i] = (*values)[i] / scalar; + } + break; + case Device::CUDA: + #ifdef __CUDA + cuda::scalarmul(res.getData(), values->getData(), 1 / scalar, values->getSize()); + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif + break; } return res; @@ -513,8 +645,19 @@ Tensor Tensor::operator/(ftype scalar) const { Tensor Tensor::operator+(ftype scalar) const { Tensor res(dims, values->getDevice(), false); - for (tensorSize_t i = 0; i < values->getSize(); ++i) { - (*res.values)[i] = (*values)[i] + scalar; + switch(values->getDevice()){ + case Device::CPU: + for (tensorSize_t i = 0; i < values->getSize(); ++i) { + (*res.values)[i] = (*values)[i] + scalar; + } + break; + case Device::CUDA: + #ifdef __CUDA + cuda::scalaradd(res.getData(), values->getData(), scalar, values->getSize()); + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif + break; } return res; @@ -522,8 +665,19 @@ Tensor Tensor::operator+(ftype scalar) const { Tensor Tensor::operator-(ftype scalar) const { Tensor res(dims, values->getDevice(), false); - for (tensorSize_t i = 0; i < values->getSize(); ++i) { - (*res.values)[i] = (*values)[i] - scalar; + switch(values->getDevice()){ + case Device::CPU: + for (tensorSize_t i = 0; i < values->getSize(); ++i) { + (*res.values)[i] = (*values)[i] - scalar; + } + break; + case Device::CUDA: + #ifdef __CUDA + cuda::scalaradd(res.getData(), values->getData(), -scalar, values->getSize()); + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif + break; } return res; @@ -600,11 +754,8 @@ tensorDim_t Tensor::mapDim(const int dim, const Dimension& dims) { return dims.nDims() + dim; } -void Tensor::transposeImpl(Tensor& target, const int dim1, const int dim2) const noexcept { +void Tensor::transposeImplCpu(Tensor& target, const int dim1, const int dim2) const noexcept { assert(values->getSize() == target.values->getSize() && dims.nDims()==target.dims.nDims()); - if(target.getDevice() == Device::CUDA) { - __throw_runtime_error("Transposition for CUDA not implemented yet"); - } if(dim1 == dim2) { return; } @@ -667,14 +818,11 @@ void Tensor::transposeImpl(Tensor& target, const int dim1, const int dim2) const } /** - * @brief Transposes 2D tensor. A little sleeker than transposeImpl. + * @brief Transposes 2D tensor. A little sleeker than transposeImplCpu. */ -void Tensor::transposeImpl2D(Tensor& target, const int dim1, const int dim2) const noexcept { +void Tensor::transposeImpl2DCpu(Tensor& target, const int dim1, const int dim2) const noexcept { assert(values->getSize()==target.values->getSize() && target.dims.nDims()==2 && dims.nDims()==2); - if(target.getDevice()==Device::CUDA){ - __throw_runtime_error("Transposition for CUDA not implemented yet"); - } - else if(dim1==dim2){ + if(dim1==dim2){ return; } @@ -709,10 +857,6 @@ void Tensor::transposeImpl2D(Tensor& target, const int dim1, const int dim2) con target.values = std::move(transposedValues); target.dims.swap(dim1Mapped, dim2Mapped); - - /* if(source.grads){ - source.grads->transposeImpl(*target.grads, dim1, dim2); // TODO: do we need this? - } */ } @@ -722,11 +866,29 @@ void Tensor::transposeImpl2D(Tensor& target, const int dim1, const int dim2) con * Out of place operation. */ void Tensor::transposeThis(int dim1, int dim2) noexcept { - if(dims.nDims()>2){ - transposeImpl(*this, dim1, dim2); - } - else{ - transposeImpl2D(*this, dim1, dim2); + switch(values->getDevice()){ + case Device::CPU: + if(dims.nDims()>2){ + transposeImplCpu(*this, dim1, dim2); + } + else{ + transposeImpl2DCpu(*this, dim1, dim2); + } + break; + case Device::CUDA: + #ifdef __CUDA + if(dims.nDims()>2){ + // TODO: can make this better inplace? + cuda::transpose(values->getData(), values->getData(), dims, dim1, dim2); + } + else{ + // TODO: can make this better inplace? + cuda::transpose2D(values->getData(), values->getData(), dims, dim1, dim2); + } + #else + __throw_runtime_error("Not implemented with CUDA"); + #endif + break; } } @@ -758,11 +920,27 @@ Tensor Tensor::transpose(int dim1, int dim2) const { Tensor Tensor::transpose(int dim1, int dim2, const bool requiresGrad) const { Tensor res(dims, values->getDevice(), requiresGrad); - if(dims.nDims()>2){ - transposeImpl(res, dim1, dim2); - } - else{ - transposeImpl2D(res, dim1, dim2); + switch(values->getDevice()){ + case Device::CPU: + if(dims.nDims()>2){ + transposeImplCpu(res, dim1, dim2); + } + else{ + transposeImpl2DCpu(res, dim1, dim2); + } + break; + case Device::CUDA: + #ifdef __CUDA + if(dims.nDims()>2){ + cuda::transpose(res.values->getData(), values->getData(), res.dims, dim1, dim2); + } + else{ + cuda::transpose2D(res.values->getData(), values->getData(), res.dims, dim1, dim2); + } + #else + __throw_runtime_error("Not implemented with CUDA"); + #endif + break; } return res; @@ -794,8 +972,25 @@ void Tensor::reset(const ftype x) noexcept { * @brief Populates the tensor with values drawn according to initializer. */ void Tensor::reset(const shared_ptr init) noexcept { - for(tensorSize_t i=0; igetSize(); i++){ - (*values)[i] = init->drawNumber(); + switch(values->getDevice()){ + case Device::CPU: + for(tensorSize_t i=0; igetSize(); i++){ + (*values)[i] = init->drawNumber(); + } + break; + case Device::CUDA: + #ifdef __CUDA + auto newValues = static_cast(std::malloc(values->getSize() * sizeof(ftype))); + for(tensorSize_t i=0; igetSize(); i++){ + newValues[i] = init->drawNumber(); + } + cudaErrchk(cudaMemcpy(values->getData(), newValues, values->getSize() * sizeof(ftype), cudaMemcpyHostToDevice)); + free(newValues); + // TODO: better initialize directly on GPU + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif + break; } } @@ -892,6 +1087,35 @@ void printValuesCpu(std::ostream& os, const Tensor& t) { } } +/** + * @brief Print out the first few values of the flattened array. + */ +#ifdef __CUDA +void printValuesCuda(std::ostream& os, const Tensor& t) { + __throw_logic_error("printValuesCuda should not be reachable when not compiled with CUDA"); + auto printVals = [&os](const Tensor& t){ + constexpr auto MAX_IDX = static_cast(10); + + const auto maxIdx = min(MAX_IDX, t.values->getSize()); + auto tmp = static_cast(std::malloc(t.getSize() * sizeof(ftype))); + cudaErrchk(cudaMemcpy(tmp, t.getData(), maxIdx*sizeof(ftype), cudaMemcpyDeviceToHost)); + + for(tensorSize_t i=0; igetDevice()); @@ -902,7 +1126,11 @@ ostream& operator<<(ostream& os, const Tensor& t) noexcept { printValuesCpu(os, t); break; case Device::CUDA: - __throw_invalid_argument("CUDA not supported yet in printing"); + #ifdef __CUDA + printValuesCuda(os, t); + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif break; } @@ -1004,7 +1232,7 @@ ftype Tensor::get(tensorDim_t idx0, tensorDim_t idx1, tensorDim_t idx2, tensorDi * @brief No explanation needed. */ void Tensor::set(ftype item, const std::vector& idx) { - (*values)[computeLinearIdx(idx, dims)] = item; + values->set(item, computeLinearIdx(idx, dims)); } /** @@ -1012,7 +1240,7 @@ void Tensor::set(ftype item, const std::vector& idx) { * Can lead to unexpected results in multidimensional tensors. */ void Tensor::set(ftype item, tensorDim_t idx) { - (*values)[idx] = item; + values->set(item, idx); } void Tensor::set(ftype item, tensorDim_t idx0, tensorDim_t idx1) { diff --git a/src/backend/data_modeling/tensor.h b/src/backend/data_modeling/tensor.h index b8abe64..5133ebb 100644 --- a/src/backend/data_modeling/tensor.h +++ b/src/backend/data_modeling/tensor.h @@ -28,11 +28,6 @@ #include #include -#ifdef __CUDA -#include "cuda.h" -#include "utility/cuda/cuda_common.cuh" -#endif - // break circular dependency namespace cgraph { @@ -56,7 +51,7 @@ class Tensor final : public std::enable_shared_from_this { private: tensorSize_t size = 0; - ftype *values = nullptr; + ftype* values = nullptr; Device device; inline static Device defaultDevice = Device::CPU; @@ -76,6 +71,7 @@ class Tensor final : public std::enable_shared_from_this void copyFromRaw(const ftype* src, tensorSize_t n); + ftype* getData() const noexcept; explicit operator bool() const noexcept; ftype& operator[](const tensorSize_t idx); ftype operator[](const tensorSize_t idx) const; @@ -88,25 +84,7 @@ class Tensor final : public std::enable_shared_from_this // needed for gradient descent tensorValues_t& operator+=(const tensorValues_t& other); - template - requires(std::is_integral_v>) - void resize(const T size) - { - this->size = static_cast(size); - switch (device) - { - case Device::CPU: - values = static_cast(std::malloc(this->size * sizeof(ftype))); - break; - case Device::CUDA: - #ifdef __CUDA - cudaErrchk(cudaMalloc((void**) &values, this->size * sizeof(ftype))); - #else - std::__throw_invalid_argument("Not compiled with CUDA."); - #endif - break; - } - } + void resize(const tensorSize_t size); void setDevice(const Device d) noexcept; Device getDevice() const noexcept; @@ -126,15 +104,13 @@ class Tensor final : public std::enable_shared_from_this std::shared_ptr grads = nullptr; // gradients std::shared_ptr cgNode = nullptr; - static Tensor multiplyScalar(const Tensor& scalar, const Tensor& other) noexcept; - static Tensor matMulImpl(const Tensor& left, const Tensor& right); static void matMul2DCpu(Tensor& res, const Tensor& left, const Tensor& right, const tensorSize_t resOffset, const tensorSize_t leftOffset, const tensorSize_t rightOffset); - void transposeImpl2D(Tensor& target, const int dim1, const int dim2) const noexcept; - void transposeImpl(Tensor& target, const int dim1, const int dim2) const noexcept; + void transposeImpl2DCpu(Tensor& target, const int dim1, const int dim2) const noexcept; + void transposeImplCpu(Tensor& target, const int dim1, const int dim2) const noexcept; // convenience functions that appear in multiple places static tensorSize_t computeLinearIdx(const std::vector&& idx, const Dimension& dims); @@ -145,6 +121,9 @@ class Tensor final : public std::enable_shared_from_this static tensorDim_t mapDim(const int dim, const Dimension& dims); friend void printValuesCpu(std::ostream& os, const Tensor& t); + #ifdef __CUDA + friend void printValuesCuda(std::ostream& os, const Tensor& t); + #endif public: template @@ -153,21 +132,25 @@ class Tensor final : public std::enable_shared_from_this : Tensor{dims.toVector(), tensorValues_t::getDefaultDevice(), requiresGrad} { } - explicit Tensor(const std::vector& dims, bool requiresGrad = false) : dims{dims}, values{std::make_unique()}, requiresGrad{requiresGrad} + explicit Tensor(const std::vector& dims, bool requiresGrad = false) + : dims{dims}, values{std::make_unique()}, requiresGrad{requiresGrad} { values->resize(this->dims.getSize()); } - explicit Tensor(const std::vector& dims, Device d, bool requiresGrad = false) : dims{dims}, values{std::make_unique(d)}, requiresGrad{requiresGrad} + explicit Tensor(const std::vector& dims, Device d, bool requiresGrad = false) + : dims{dims}, values{std::make_unique(d)}, requiresGrad{requiresGrad} { values->resize(this->dims.getSize()); } - explicit Tensor(const std::vector& dims, const std::vector& initValues, bool requiresGrad = false) : Tensor{dims, std::move(initValues), Tensor::getDefaultDevice(), requiresGrad} + explicit Tensor(const std::vector& dims, const std::vector& initValues, bool requiresGrad = false) + : Tensor{dims, std::move(initValues), Tensor::getDefaultDevice(), requiresGrad} { } - explicit Tensor(const std::vector& dims, const std::vector& initValues, Device d, bool requiresGrad = false) : Tensor{dims, d, requiresGrad} + explicit Tensor(const std::vector& dims, const std::vector& initValues, Device d, bool requiresGrad = false) + : Tensor{dims, d, requiresGrad} { for (tensorSize_t i=0; iset(initValues[i], i); @@ -200,6 +183,8 @@ class Tensor final : public std::enable_shared_from_this Tensor(Tensor&& other) noexcept; Tensor& operator=(Tensor&& other) noexcept; + ftype* getData() const noexcept; + void reset(const ftype x) noexcept; void reset(const std::shared_ptr init) noexcept; diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/module/activation_functions/cuda/activations.cuh b/src/backend/module/activation_functions/cuda/activations.cuh new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/utility/cuda/cuda_common.cuh b/src/backend/utility/cuda/cuda_common.cuh index c325aa4..46023a2 100644 --- a/src/backend/utility/cuda/cuda_common.cuh +++ b/src/backend/utility/cuda/cuda_common.cuh @@ -15,7 +15,7 @@ static_assert(false, "File should not be included without CUDA enabled"); #endif // __CUDA -#include "cuda.h" +#include "cuda_runtime.h" namespace utility { void gpuAssert(cudaError_t code, const char *file, int line, bool abort=true); From d9483d601c87a979b6529ec79ddd9d932a54f7b4 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 29 Mar 2026 10:34:02 +0200 Subject: [PATCH 07/73] Minor optimization using memcopy --- src/backend/data_modeling/tensor.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index be2ba0e..7f9275c 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -963,8 +963,17 @@ void Tensor::permute(const std::vector&& newOrder) noexcept { * @brief Populates the tensor with value. */ void Tensor::reset(const ftype x) noexcept { - for(tensorSize_t i=0; igetSize(); i++){ - (*values)[i] = x; + switch(values->getDevice()){ + case Device::CPU: + memset(values->getData(), x, values->getSize()); + break; + case Device::CUDA: + #ifdef __CUDA + cudaErrchk(cudaMemset(values->getData(), x, values->getSize() * sizeof(ftype))); + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif + break; } } From 6ba1018f57d858f00294921b55abe2f8c9ffeb14 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 29 Mar 2026 11:28:27 +0200 Subject: [PATCH 08/73] Laying foundation for easy transpose --- src/backend/data_modeling/dim_type.cpp | 60 ++++++++++++++++++++++++-- src/backend/data_modeling/dim_type.h | 20 ++++++--- src/backend/utility/global_params.h | 2 + 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/src/backend/data_modeling/dim_type.cpp b/src/backend/data_modeling/dim_type.cpp index 684fde0..514ffa3 100644 --- a/src/backend/data_modeling/dim_type.cpp +++ b/src/backend/data_modeling/dim_type.cpp @@ -46,37 +46,80 @@ void Dimension::resize(const std::vector& dims) { * @brief Swap along the two given dimensions. */ void Dimension::swap(const tensorDim_t dim1, const tensorDim_t dim2) { + auto swapArr = [dim1, dim2](array& arr){ + auto tmp = arr[dim1]; + arr[dim1] = arr[dim2]; + arr[dim2] = tmp; + }; + auto tmp = dims[dim1]; dims[dim1] = dims[dim2]; dims[dim2] = tmp; + + swapArr(strides); } -Dimension::Dimension(const vector& dims) : dims{dims} { +Dimension::Dimension(const vector& dims) : dims{dims}, strides{} { size = multVector(dims); + lastDimIdx = dims.size()-1; + resetStrides(); assert(size>0); } -Dimension::Dimension(const Dimension& other) : dims{other.dims}, size{other.size} { } +Dimension::Dimension(vector&& dims, array&& strides) + : dims{dims}, strides{strides} +{ + size = multVector(dims); + lastDimIdx = dims.size()-1; +} + +Dimension::Dimension(const Dimension& other) + : dims{other.dims}, strides{other.strides}, size{other.size}, lastDimIdx{other.lastDimIdx} +{} Dimension& Dimension::operator=(const Dimension& other) { if(this==&other) return *this; dims = other.dims; + strides = other.strides; size = other.size; + lastDimIdx = other.lastDimIdx; return *this; } -Dimension::Dimension(Dimension&& other) noexcept : dims{move(other.dims)}, size{other.size} {} +Dimension::Dimension(Dimension&& other) noexcept + : dims{move(other.dims)}, strides{std::move(other.strides)}, size{other.size}, lastDimIdx{other.lastDimIdx} +{} Dimension& Dimension::operator=(Dimension&& other) noexcept { if(this==&other) return *this; dims = move(other.dims); + strides = std::move(other.strides); size = other.size; + lastDimIdx = other.lastDimIdx; + return *this; } +/** + * @brief Computes the strides as they are; + */ +void Dimension::resetStrides() noexcept { + tensorSize_t res=1; + strides[lastDimIdx] = res; + for(tensorDim_t i=lastDimIdx-1; i>=0; i++){ + strides[i] = strides[i+1] * dims[i+1]; + } +} + +tensorSize_t Dimension::getStride(const int i) const noexcept { + if(i<0) + return strides[lastDimIdx + i + 1]; + return strides[i]; +} + /** * @brief This method gets interesting when we want to get a copy of * this dimension instance, but we collapsed one of the dimensions. @@ -96,7 +139,16 @@ Dimension Dimension::collapseDimension(int idx) const { newDims.insert(newDims.end(), dims.begin(), dims.begin() + idx); newDims.insert(newDims.end(), dims.begin() + idx + 1, dims.end()); - return Dimension(newDims); + std::array newStrides{}; + tensorDim_t strideIdx = 0; + for(tensorDim_t i=0; i +#include #include #include @@ -21,15 +22,18 @@ class Dimension final { private: std::vector dims; - tensorSize_t size = 0; + std::array strides; + tensorDim_t lastDimIdx; // look up end in strides/dims + tensorSize_t size = 0; // total size of tensor + + void resetStrides() noexcept; tensorSize_t multVector(const std::vector& dims) const noexcept; + + Dimension(std::vector&& dims, std::array&& strides); public: - /** - * @brief Explicit default ctor, so that dims is zero initialized. - * Otherwise we will encounter undefined behavior. - */ + Dimension(const std::vector& dims); Dimension(const Dimension& other); @@ -55,17 +59,19 @@ class Dimension final { tensorDim_t operator[](int idx) const { assert(size>0); if(idx<0){ - idx = dims.size() + idx; // -1 is last idx, -2 second last and so forth + return dims[lastDimIdx + idx + 1]; // -1 is last idx, -2 second last and so forth } return dims[idx]; } + tensorSize_t getStride(int i) const noexcept; + std::vector toVector() const noexcept{ return dims; } - void swap(const tensorDim_t dim1, const tensorDim_t dim2); + void swap(tensorDim_t dim1, tensorDim_t dim2); size_t nDims() const noexcept { return dims.size(); diff --git a/src/backend/utility/global_params.h b/src/backend/utility/global_params.h index 3d6edcb..ac7550f 100644 --- a/src/backend/utility/global_params.h +++ b/src/backend/utility/global_params.h @@ -29,6 +29,8 @@ using ftype = float; // TODO: make compiler flag? using tensorDim_t = std::uint16_t; using tensorSize_t = std::uint32_t; +constexpr tensorDim_t MAX_NDIMS = 6; + // we assert this here so during conversions of tensorDim_t to // tensorSize_t we do not need to cast explicitely static_assert(sizeof(tensorDim_t)<=sizeof(tensorSize_t)); From 4f74be978312f14a2385c76f618b8989dc10ee75 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 29 Mar 2026 12:05:57 +0200 Subject: [PATCH 09/73] Further enabling of resetting dims --- src/backend/data_modeling/dim_type.cpp | 51 ++++++++++---------------- src/backend/data_modeling/dim_type.h | 9 +++-- 2 files changed, 26 insertions(+), 34 deletions(-) diff --git a/src/backend/data_modeling/dim_type.cpp b/src/backend/data_modeling/dim_type.cpp index 514ffa3..469983c 100644 --- a/src/backend/data_modeling/dim_type.cpp +++ b/src/backend/data_modeling/dim_type.cpp @@ -59,59 +59,48 @@ void Dimension::swap(const tensorDim_t dim1, const tensorDim_t dim2) { swapArr(strides); } -Dimension::Dimension(const vector& dims) : dims{dims}, strides{} { +Dimension::Dimension(const vector& dims) + : creationDims{dims}, creationStrides{createStrides(dims)}, dims{dims}, strides{creationStrides} { size = multVector(dims); lastDimIdx = dims.size()-1; - resetStrides(); assert(size>0); } Dimension::Dimension(vector&& dims, array&& strides) - : dims{dims}, strides{strides} + : creationDims{dims}, dims{creationDims}, creationStrides{strides}, strides{creationStrides} { size = multVector(dims); lastDimIdx = dims.size()-1; + assert(size>0); } Dimension::Dimension(const Dimension& other) - : dims{other.dims}, strides{other.strides}, size{other.size}, lastDimIdx{other.lastDimIdx} + : creationDims{other.creationDims}, creationStrides{other.creationStrides}, dims{other.dims}, + strides{other.strides}, size{other.size}, lastDimIdx{other.lastDimIdx} {} -Dimension& Dimension::operator=(const Dimension& other) { - if(this==&other) return *this; - - dims = other.dims; - strides = other.strides; - size = other.size; - lastDimIdx = other.lastDimIdx; - - return *this; -} - Dimension::Dimension(Dimension&& other) noexcept - : dims{move(other.dims)}, strides{std::move(other.strides)}, size{other.size}, lastDimIdx{other.lastDimIdx} + : creationDims{std::move(other.creationDims)}, creationStrides{std::move(other.creationStrides)}, + dims{std::move(other.dims)}, strides{std::move(other.strides)}, size{other.size}, lastDimIdx{other.lastDimIdx} {} -Dimension& Dimension::operator=(Dimension&& other) noexcept { - if(this==&other) return *this; - - dims = move(other.dims); - strides = std::move(other.strides); - size = other.size; - lastDimIdx = other.lastDimIdx; - - return *this; -} - /** * @brief Computes the strides as they are; */ -void Dimension::resetStrides() noexcept { - tensorSize_t res=1; - strides[lastDimIdx] = res; +array Dimension::createStrides(const vector& dims) const noexcept { + array res; + const auto lastDimIdx = dims.size()-1; + + tensorSize_t stride=1; + res[lastDimIdx] = stride; + stride *= dims[lastDimIdx]; + for(tensorDim_t i=lastDimIdx-1; i>=0; i++){ - strides[i] = strides[i+1] * dims[i+1]; + res[i] = stride; + stride *= dims[i]; } + + return res; } tensorSize_t Dimension::getStride(const int i) const noexcept { diff --git a/src/backend/data_modeling/dim_type.h b/src/backend/data_modeling/dim_type.h index b890b9d..9bf8b44 100644 --- a/src/backend/data_modeling/dim_type.h +++ b/src/backend/data_modeling/dim_type.h @@ -21,13 +21,16 @@ class Dimension final { private: + const std::vector creationDims; + const std::array creationStrides; + std::vector dims; std::array strides; tensorDim_t lastDimIdx; // look up end in strides/dims tensorSize_t size = 0; // total size of tensor - void resetStrides() noexcept; + std::array Dimension::createStrides(const std::vector& dims) const noexcept; tensorSize_t multVector(const std::vector& dims) const noexcept; Dimension(std::vector&& dims, std::array&& strides); @@ -37,10 +40,10 @@ class Dimension final { Dimension(const std::vector& dims); Dimension(const Dimension& other); - Dimension& operator=(const Dimension& other); + Dimension& operator=(const Dimension& other) = delete; Dimension(Dimension&& other) noexcept; - Dimension& operator=(Dimension&& other) noexcept; + Dimension& operator=(Dimension&& other) noexcept = delete; ~Dimension() noexcept = default; From 638d129057dc37edc9ddaa26addf0d2927bab95e Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 29 Mar 2026 14:30:41 +0200 Subject: [PATCH 10/73] Progress on view transpose/lazy transpose --- src/backend/data_modeling/cuda/tensorops.cu | 39 ++++++ src/backend/data_modeling/cuda/tensorops.cuh | 4 + src/backend/data_modeling/dim_type.cpp | 35 +++++- src/backend/data_modeling/dim_type.h | 25 ++-- src/backend/data_modeling/tensor.cpp | 124 ++++++++++++------- src/backend/data_modeling/tensor.h | 13 +- src/backend/utility/global_params.h | 2 + 7 files changed, 182 insertions(+), 60 deletions(-) diff --git a/src/backend/data_modeling/cuda/tensorops.cu b/src/backend/data_modeling/cuda/tensorops.cu index 421355c..1647227 100644 --- a/src/backend/data_modeling/cuda/tensorops.cu +++ b/src/backend/data_modeling/cuda/tensorops.cu @@ -18,6 +18,8 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "cuda_runtime.h" +#include "data_modeling/tensor.h" + namespace{ __global__ void elementwiseaddKernel(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size) { int gid = blockDim.x * blockIdx.x + threadIdx.x; @@ -38,6 +40,25 @@ namespace{ int gid = blockDim.x * blockIdx.x + threadIdx.x; res[gid] = left[gid] + scalar; } + + __global__ void createLinearCopyKernel( + float* dst, const float* const src, + const tensorSize_t* const strides, // original strides + const tensorSize_t* const contiguousStrides, // new linear strides + int ndim, tensorSize_t size) + { + tensorSize_t flatIdx = blockIdx.x * blockDim.x + threadIdx.x; + if (flatIdx >= size) return; + + tensorSize_t remainder = flatIdx; + tensorSize_t srcOffset = 0; + for (int i = 0; i < ndim; ++i) { + tensorSize_t coord = remainder / contiguousStrides[i]; + remainder %= contiguousStrides[i]; + srcOffset += coord * strides[i]; + } + dst[flatIdx] = src[srcOffset]; + } } namespace cuda { @@ -82,4 +103,22 @@ namespace cuda { ftype set(ftype value, const ftype* t, tensorSize_t idx) { cudaErrchk(cudaMemcpy((void*)t+idx, &value, sizeof(ftype), cudaMemcpyHostToDevice)); } + + void createLinearCopy(Tensor& res, const Tensor& src) { + assert(res.getSize()==src.getSize()); + + ftype* dst = res.getData() + const ftype* const srcData = src.getData(); + + auto oldStrides = src.getDims().getCreationStrides().data(); + auto newStrides = src.getDims().getStrides().data(); + + int threadsPerBlock = 256; + int blocksPerGrid = (nBytes + threadsPerBlock - 1) / threadsPerBlock; + + cudaErrchk(createLinearCopyKernel<<>>( + dst, srcData, oldStrides, newStrides, dims.nDims(), nBytes)); + + cudaErrchk(cudaDeviceSynchronize()); + } } diff --git a/src/backend/data_modeling/cuda/tensorops.cuh b/src/backend/data_modeling/cuda/tensorops.cuh index 15572d8..4254734 100644 --- a/src/backend/data_modeling/cuda/tensorops.cuh +++ b/src/backend/data_modeling/cuda/tensorops.cuh @@ -19,6 +19,8 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "data_modeling/dim_type.h" namespace cuda { + class Tensor; + // scalar ops void scalaradd(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size); void scalarmul(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size); @@ -35,4 +37,6 @@ namespace cuda { // other ftype get(const ftype* const t, tensorSize_t idx); ftype set(ftype value, const ftype* t, tensorSize_t idx); + + void createLinearCopy(Tensor& res, const Tensor& src); } diff --git a/src/backend/data_modeling/dim_type.cpp b/src/backend/data_modeling/dim_type.cpp index 469983c..e372716 100644 --- a/src/backend/data_modeling/dim_type.cpp +++ b/src/backend/data_modeling/dim_type.cpp @@ -46,7 +46,7 @@ void Dimension::resize(const std::vector& dims) { * @brief Swap along the two given dimensions. */ void Dimension::swap(const tensorDim_t dim1, const tensorDim_t dim2) { - auto swapArr = [dim1, dim2](array& arr){ + auto swapArr = [dim1, dim2](dim_t& arr){ auto tmp = arr[dim1]; arr[dim1] = arr[dim2]; arr[dim2] = tmp; @@ -60,13 +60,13 @@ void Dimension::swap(const tensorDim_t dim1, const tensorDim_t dim2) { } Dimension::Dimension(const vector& dims) - : creationDims{dims}, creationStrides{createStrides(dims)}, dims{dims}, strides{creationStrides} { + : creationDims{dims}, creationStrides{makeStrides(dims)}, dims{dims}, strides{creationStrides} { size = multVector(dims); lastDimIdx = dims.size()-1; assert(size>0); } -Dimension::Dimension(vector&& dims, array&& strides) +Dimension::Dimension(vector&& dims, dim_t&& strides) : creationDims{dims}, dims{creationDims}, creationStrides{strides}, strides{creationStrides} { size = multVector(dims); @@ -79,16 +79,39 @@ Dimension::Dimension(const Dimension& other) strides{other.strides}, size{other.size}, lastDimIdx{other.lastDimIdx} {} +Dimension& Dimension::operator=(const Dimension& other) { + creationDims = other.creationDims; + creationStrides = other.creationStrides; + + dims = other.dims; + strides = other.strides; + + size = other.size; + lastDimIdx = other.lastDimIdx; +} + Dimension::Dimension(Dimension&& other) noexcept : creationDims{std::move(other.creationDims)}, creationStrides{std::move(other.creationStrides)}, dims{std::move(other.dims)}, strides{std::move(other.strides)}, size{other.size}, lastDimIdx{other.lastDimIdx} {} +Dimension& Dimension::operator=(Dimension&& other) noexcept { + creationDims = std::move(other.creationDims); + creationStrides = std::move(other.creationStrides); + + dims = std::move(other.dims); + strides = std::move(other.strides); + + size = other.size; + lastDimIdx = other.lastDimIdx; +} + + /** * @brief Computes the strides as they are; */ -array Dimension::createStrides(const vector& dims) const noexcept { - array res; +dim_t Dimension::makeStrides(const vector& dims) const noexcept { + dim_t res; const auto lastDimIdx = dims.size()-1; tensorSize_t stride=1; @@ -128,7 +151,7 @@ Dimension Dimension::collapseDimension(int idx) const { newDims.insert(newDims.end(), dims.begin(), dims.begin() + idx); newDims.insert(newDims.end(), dims.begin() + idx + 1, dims.end()); - std::array newStrides{}; + dim_t newStrides{}; tensorDim_t strideIdx = 0; for(tensorDim_t i=0; i class Dimension final { + using dim_t = std::array; + private: - const std::vector creationDims; - const std::array creationStrides; + // creationDims and creationStrides should only be set in constructors, otherwise + // bugs will emerge. Made non-const so we can use move-assignment operator + std::vector creationDims; + dim_t creationStrides; std::vector dims; - std::array strides; + dim_t strides; tensorDim_t lastDimIdx; // look up end in strides/dims tensorSize_t size = 0; // total size of tensor - std::array Dimension::createStrides(const std::vector& dims) const noexcept; + dim_t Dimension::makeStrides(const std::vector& dims) const noexcept; tensorSize_t multVector(const std::vector& dims) const noexcept; - Dimension(std::vector&& dims, std::array&& strides); + Dimension(std::vector&& dims, dim_t&& strides); public: Dimension(const std::vector& dims); Dimension(const Dimension& other); - Dimension& operator=(const Dimension& other) = delete; + Dimension& operator=(const Dimension& other); Dimension(Dimension&& other) noexcept; - Dimension& operator=(Dimension&& other) noexcept = delete; + Dimension& operator=(Dimension&& other) noexcept; ~Dimension() noexcept = default; Dimension collapseDimension(int idx) const; + bool inOriginalState() const noexcept { + return creationDims == dims && creationStrides == strides; + } + void resize(const std::vector& dims); tensorSize_t getSize() const noexcept { @@ -68,6 +76,9 @@ class Dimension final { return dims[idx]; } + const auto getStrides() const noexcept { return strides; } + const auto getOriginalStrides() const noexcept { return creationStrides; } + tensorSize_t getStride(int i) const noexcept; std::vector toVector() const noexcept{ diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index 7f9275c..3420e74 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -368,6 +368,17 @@ Tensor& Tensor::operator=(Tensor&& other) noexcept { return *this; } +/** + * @brief Only createShallowCopy is supposed to access this ctor. + * Like a copy ctor, shares recourses (grad, cgNode, values) with + * original (other) tensor. + */ +Tensor::Tensor(const Tensor& other, [[maybe_unused]] shallowCopyToken) + : dims(other.dims), cgNode{other.cgNode}, values{other.values}, + grads{other.grads}, requiresGrad{other.requiresGrad} +{ +} + /** * @brief Creates an empty copy of this tensor. * Metadata all filled, but gradients not initialized, and @@ -377,6 +388,56 @@ Tensor Tensor::createEmptyCopy() const { auto res = Tensor(dims, values->getDevice(), requiresGrad); return res; } + +/** + * @brief Creates a shallow copy. New tensor object, but all resources + * (grads, cgNode, values) are shared with this tensor. Useful for optimization. + */ +Tensor Tensor::createShallowCopy() const { + auto res = Tensor(*this, shallowCopyToken{}); + return res; +} + +/** + * @brief Creates a linearized copy of this tensor. New memory allocations for + * values, and values will be copied so that the memory layout reflects the + * current shape of the tensor. + */ +Tensor Tensor::createLinearCopy() const { + auto res = createEmptyCopy(); + + switch(values->getDevice()){ + case Device::CPU: + { + for (tensorSize_t flatIdx = 0; flatIdx < values->getSize(); ++flatIdx) { + tensorSize_t remainder = flatIdx; + tensorSize_t srcOffset = 0; + + for (int i=dims.nDims()-1; i>=0; i--) { + tensorSize_t coord = remainder % dims[i]; + remainder /= dims[i]; + srcOffset += coord * dims.getStride(i); + } + + res.values->set((*values)[srcOffset], flatIdx); + } + break; + } + case Device::CUDA: + { + #ifdef __CUDA + cuda::createLinearCopy(res, *this); + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif + break; + } + } + + + return res; +} + /** * @brief Does a deep copy, but omits gradient and computational graph information. */ @@ -386,10 +447,6 @@ Tensor Tensor::createDeepCopy() const { auto res = Tensor(dims, values->getDevice(), requiresGrad); values->copyValues(*res.values); - /* if(grads){ - res.grads = make_shared( grads->createDeepCopy() ); // TODO: do we want this? - } */ - assert(!res.grads); // TODO: check if this makes sense assert(!res.cgNode); // TODO: do we want to give it pointer of same node? return res; @@ -449,7 +506,7 @@ Tensor Tensor::matMulImpl(const Tensor& left, const Tensor& right) { } case Device::CUDA: #ifdef __CUDA - cuda::matmul(res.getData(), left.values->getData(), right.values->getData()); + cuda::matmul(res, left, right); #else __throw_invalid_argument("Not compiled with CUDA"); #endif @@ -699,12 +756,10 @@ void Tensor::backward() { __throw_runtime_error("Invoking backward on Tensor not created by a differentiable operation"); } - // check this one out for sure + // last node has no incoming gradients -> factor 1 if (!grads) { - grads = make_unique(dims, values->getDevice(), false); - for(tensorSize_t i=0; igetSize(); i++){ - (*grads->values)[i] = 1; - } + grads = make_shared(dims, values->getDevice(), false); + grads->reset(1); } vector sortedTensors = cgraph::TopologicalSort::reverseSort(this); @@ -836,8 +891,8 @@ void Tensor::transposeImpl2DCpu(Tensor& target, const int dim1, const int dim2) const auto smallDim = dim1Mapped < dim2Mapped ? dim2Mapped : dim1Mapped; // largeDimSize >= smallDimSize - const auto largeDimOffset = getDimOffset(largeDim, dims); - const auto smallDimOffset = getDimOffset(smallDim, dims); + const auto largeDimOffset = dims.getStride(largeDim); + const auto smallDimOffset = dims.getStride(smallDim); auto transposedValues = make_unique(source.values->getDevice()); transposedValues->resize(source.values->getSize()); @@ -859,6 +914,14 @@ void Tensor::transposeImpl2DCpu(Tensor& target, const int dim1, const int dim2) target.dims.swap(dim1Mapped, dim2Mapped); } +/** + * @brief Get a tensor that is linear in memory. Useful for coalesced memory access patterns. + */ +Tensor Tensor::getLinear() const { + if(dims.inOriginalState()) + return createShallowCopy(); + return createLinearCopy(); +} /** * @brief Swap dim1 and dim2, modify this tensor. @@ -1063,7 +1126,7 @@ Tensor Tensor::getSlice(span indices) const { resDims[0] = indices.size(); Tensor res(std::move(resDims), values->getDevice(), false); - values->copyValues(*res.values, indices, getDimOffset(0, resDims)); + values->copyValues(*res.values, indices, res.getDims().getStride(0)); return res; } @@ -1165,43 +1228,16 @@ tensorSize_t Tensor::computeLinearIdx(const std::vector& idx, const __throw_invalid_argument("Number of idxs must match number of dimensions."); } else if(idx.size()==0){ - return 0; // TODO: this was 1. What is going on here? + return 0; } - const auto lastIdx = idx.size()-1; - tensorSize_t offsetFactor = dims.get(lastIdx); - - tensorSize_t res = idx[lastIdx]; - for(int i=lastIdx-1; i>=0; i--){ - res += idx[i] * offsetFactor; - offsetFactor *= dims.get(i); + tensorSize_t res = 0; + for(tensorDim_t i=0; idim; idx--){ - res *= dims.get(idx); - } - - assert(res!=0); - return res; -} - -/** - * @brief Like overload, but accepts negative dims. - */ -tensorSize_t Tensor::getDimOffset(const int dim, const Dimension& dims) { - return getDimOffset(mapDim(dim, dims), dims); -} - /** * @brief No explanation needed. */ diff --git a/src/backend/data_modeling/tensor.h b/src/backend/data_modeling/tensor.h index 5133ebb..c3b8992 100644 --- a/src/backend/data_modeling/tensor.h +++ b/src/backend/data_modeling/tensor.h @@ -40,6 +40,8 @@ class Tensor final : public std::enable_shared_from_this friend class cgraph::TopologicalSort; private: + struct shallowCopyToken{}; // we only want createShallowCopy to do shallow copies + /** * @brief Here we encapsulate the tensor's values. * Enables us to use a shared_ptr, as well as encapsulate all the @@ -98,7 +100,7 @@ class Tensor final : public std::enable_shared_from_this }; Dimension dims; - std::unique_ptr values = nullptr; // contained values of tensor + std::shared_ptr values = nullptr; // contained values of tensor bool requiresGrad = false; std::shared_ptr grads = nullptr; // gradients @@ -116,8 +118,6 @@ class Tensor final : public std::enable_shared_from_this static tensorSize_t computeLinearIdx(const std::vector&& idx, const Dimension& dims); static tensorSize_t computeLinearIdx(const std::vector& idx, const Dimension& dims); - static tensorSize_t getDimOffset(const tensorDim_t dim, const Dimension& dims); - static tensorSize_t getDimOffset(const int dim, const Dimension& dims); static tensorDim_t mapDim(const int dim, const Dimension& dims); friend void printValuesCpu(std::ostream& os, const Tensor& t); @@ -125,11 +125,14 @@ class Tensor final : public std::enable_shared_from_this friend void printValuesCuda(std::ostream& os, const Tensor& t); #endif + Tensor(const Tensor& other, shallowCopyToken); + public: template requires(std::is_same_v, Dimension>) explicit Tensor(T&& dims, Device d, bool requiresGrad = false) : Tensor{dims.toVector(), tensorValues_t::getDefaultDevice(), requiresGrad} + // !!!needs dims.toVector() to not trigger the copy ctors!!! { } explicit Tensor(const std::vector& dims, bool requiresGrad = false) @@ -174,6 +177,8 @@ class Tensor final : public std::enable_shared_from_this Tensor& operator=(const Tensor& other) = delete; Tensor createEmptyCopy() const; + Tensor createShallowCopy() const; + Tensor createLinearCopy() const; Tensor createDeepCopy() const; /** @@ -228,6 +233,8 @@ class Tensor final : public std::enable_shared_from_this Tensor transpose(int dim1, int dim2) const; Tensor transpose(int dim1, int dim2, const bool requiresGrad) const; + std::shared_ptr getLinear() const; + void permute(const std::vector&& newOrder) noexcept; friend std::ostream& operator<<(std::ostream& os, const Tensor& t) noexcept; diff --git a/src/backend/utility/global_params.h b/src/backend/utility/global_params.h index ac7550f..5fef2ce 100644 --- a/src/backend/utility/global_params.h +++ b/src/backend/utility/global_params.h @@ -13,6 +13,8 @@ #include +#include + using ftype = float; // TODO: make compiler flag? /** From 5e9b304f79e6749939d3c5997adeedb78c277958 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 29 Mar 2026 15:40:28 +0200 Subject: [PATCH 11/73] Finished implementing transpose logic --- src/backend/data_modeling/cuda/tensorops.cu | 60 ++++- src/backend/data_modeling/cuda/tensorops.cuh | 4 +- src/backend/data_modeling/dim_type.cpp | 3 + src/backend/data_modeling/tensor.cpp | 244 ++++--------------- src/backend/data_modeling/tensor.h | 23 +- src/python/py_core/py_core.cpp | 13 +- src/python/py_core/py_core_util.h | 8 +- 7 files changed, 117 insertions(+), 238 deletions(-) diff --git a/src/backend/data_modeling/cuda/tensorops.cu b/src/backend/data_modeling/cuda/tensorops.cu index 1647227..9c86490 100644 --- a/src/backend/data_modeling/cuda/tensorops.cu +++ b/src/backend/data_modeling/cuda/tensorops.cu @@ -21,34 +21,62 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "data_modeling/tensor.h" namespace{ - __global__ void elementwiseaddKernel(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size) { + __global__ void elementwiseaddKernel(ftype* res, const ftype* const left, const ftype* const right, const tensorSize_t size) { int gid = blockDim.x * blockIdx.x + threadIdx.x; + if(gid>=size) + return; + res[gid] = left[gid] + right[gid]; } - __global__ void elementwisemulKernel(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size) { + __global__ void broadcastaddKernel(ftype* res, const ftype* const matrix, const ftype* const vec, + const tensorSize_t vectorSize, const tensorSize_t matrixSize) { int gid = blockDim.x * blockIdx.x + threadIdx.x; + if(gid>=matrixSize) + return; + + const int vectorIdx = gid % vectorSize; + res[gid] = matrix[gid] + vec[vectorIdx]; + } + + __global__ void elementwisemulKernel(ftype* res, const ftype* const left, const ftype* const right, const tensorSize_t size) { + int gid = blockDim.x * blockIdx.x + threadIdx.x; + if(gid>=size) + return; + res[gid] = left[gid] * right[gid]; } __global__ void scalaraddKernel(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size){ int gid = blockDim.x * blockIdx.x + threadIdx.x; + if(gid>=size) + return; + res[gid] = left[gid] + scalar; } __global__ void scalarmulKernel(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size) { int gid = blockDim.x * blockIdx.x + threadIdx.x; + if(gid>=size) + return; + res[gid] = left[gid] + scalar; } - __global__ void createLinearCopyKernel( - float* dst, const float* const src, - const tensorSize_t* const strides, // original strides - const tensorSize_t* const contiguousStrides, // new linear strides - int ndim, tensorSize_t size) + /** + * @brief Create a contiguous copy of src and copy it into dst. Used for reshaping, transposing, etc. + * + * @param strides The original strides of src. With these strides src would be contiguous. + * @param contiguousStrides The new, contiguous strides of dst. We rearrange to match those in memory. + * @param ndim Number of dimension of both src and dst. + * @param size Total size of both src and dst. + */ + __global__ void createContiguousCopyKernel(float* dst, const float* const src, const tensorSize_t* const strides, + const tensorSize_t* const contiguousStrides, int ndim, tensorSize_t size) { tensorSize_t flatIdx = blockIdx.x * blockDim.x + threadIdx.x; - if (flatIdx >= size) return; + if (flatIdx >= size) + return; tensorSize_t remainder = flatIdx; tensorSize_t srcOffset = 0; @@ -76,6 +104,16 @@ namespace cuda { cudaErrchk(cudaDeviceSynchronize()); } + void broadcastadd(Tensor& res, const Tensor& matrix, const Tensor& vec){ + const auto nBytes = src.getSize(); + + int threadsPerBlock = 256; + int blocksPerGrid = (nBytes + threadsPerBlock - 1) / threadsPerBlock; + + broadcastaddKernel<<>>( + res.getData(), matrix.getData(), vec.getData(), vec.getDims()[0], matrix.getSize()); + } + void elementwiseadd(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size) { int threadsPerBlock = 256; int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; @@ -104,7 +142,7 @@ namespace cuda { cudaErrchk(cudaMemcpy((void*)t+idx, &value, sizeof(ftype), cudaMemcpyHostToDevice)); } - void createLinearCopy(Tensor& res, const Tensor& src) { + void createContiguousCopy(Tensor& res, const Tensor& src) { assert(res.getSize()==src.getSize()); ftype* dst = res.getData() @@ -113,10 +151,12 @@ namespace cuda { auto oldStrides = src.getDims().getCreationStrides().data(); auto newStrides = src.getDims().getStrides().data(); + const auto nBytes = src.getSize(); + int threadsPerBlock = 256; int blocksPerGrid = (nBytes + threadsPerBlock - 1) / threadsPerBlock; - cudaErrchk(createLinearCopyKernel<<>>( + cudaErrchk(createContiguousCopyKernel<<>>( dst, srcData, oldStrides, newStrides, dims.nDims(), nBytes)); cudaErrchk(cudaDeviceSynchronize()); diff --git a/src/backend/data_modeling/cuda/tensorops.cuh b/src/backend/data_modeling/cuda/tensorops.cuh index 4254734..7f7ff23 100644 --- a/src/backend/data_modeling/cuda/tensorops.cuh +++ b/src/backend/data_modeling/cuda/tensorops.cuh @@ -27,7 +27,7 @@ namespace cuda { // matrix ops void elementwiseadd(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size); - void broadcastedadd(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size); + void broadcastadd(Tensor& res, const Tensor& matrix, const Tensor& vec); void elementwisemul(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size); void matmul(ftype* res, const ftype* const left, const ftype* const right); @@ -38,5 +38,5 @@ namespace cuda { ftype get(const ftype* const t, tensorSize_t idx); ftype set(ftype value, const ftype* t, tensorSize_t idx); - void createLinearCopy(Tensor& res, const Tensor& src); + void createContiguousCopy(Tensor& res, const Tensor& src); } diff --git a/src/backend/data_modeling/dim_type.cpp b/src/backend/data_modeling/dim_type.cpp index e372716..b18d4da 100644 --- a/src/backend/data_modeling/dim_type.cpp +++ b/src/backend/data_modeling/dim_type.cpp @@ -46,6 +46,9 @@ void Dimension::resize(const std::vector& dims) { * @brief Swap along the two given dimensions. */ void Dimension::swap(const tensorDim_t dim1, const tensorDim_t dim2) { + if(dim1==dim2) + return; + auto swapArr = [dim1, dim2](dim_t& arr){ auto tmp = arr[dim1]; arr[dim1] = arr[dim2]; diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index 3420e74..785edf5 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -403,7 +403,7 @@ Tensor Tensor::createShallowCopy() const { * values, and values will be copied so that the memory layout reflects the * current shape of the tensor. */ -Tensor Tensor::createLinearCopy() const { +Tensor Tensor::createContiguousCopy() const { auto res = createEmptyCopy(); switch(values->getDevice()){ @@ -426,7 +426,7 @@ Tensor Tensor::createLinearCopy() const { case Device::CUDA: { #ifdef __CUDA - cuda::createLinearCopy(res, *this); + cuda::createContiguousCopy(res, *this); #else __throw_runtime_error("Not compiled with CUDA"); #endif @@ -434,10 +434,26 @@ Tensor Tensor::createLinearCopy() const { } } - return res; } +void Tensor::makeContiguous(){ + if(isContiguous()) + return; + + auto tmp = createContiguousCopy(); + values = std::move(tmp.values); + dims = std::move(tmp.dims); + + // TODO: perhaps we do this one lazily too? + if(grads){ + if(grads->hasGrads()) + __throw_runtime_error("Grads should never have grads themselves"); + + grads->makeContiguous(); + } +} + /** * @brief Does a deep copy, but omits gradient and computational graph information. */ @@ -557,7 +573,7 @@ Tensor Tensor::matmul(const Tensor& other) const { if(values->getDevice()!=other.values->getDevice()){ __throw_runtime_error("Tensors on different devices."); } - return matMulImpl(*this, other); + return matMulImpl(getContiguous(), other.getContiguous()); } /** @@ -577,6 +593,7 @@ Tensor Tensor::operator+(const Tensor& other) const { } Tensor res(dims, values->getDevice()); + switch(values->getDevice()){ case Device::CPU: if(dims==other.dims){ @@ -584,13 +601,15 @@ Tensor Tensor::operator+(const Tensor& other) const { for(tensorSize_t i=0; igetSize(); i++){ (*res.values)[i] = (*values)[i] + (*other.values)[i]; } + return res; } else [[likely]] { // broadcasted add - const auto stride = static_cast(other.dims.get(0)); + Tensor src = getContiguous(); + const auto stride = static_cast(other.dims.get(0)); // other is a vector for(tensorSize_t offset=0; offsetgetSize(); offset+=stride){ for(tensorSize_t i=0; igetData(), other.getData(), values->getSize()); } else [[likely]] { - cuda::broadcastedadd(res.getData(), values->getData(), other.getData(), values->getSize()); + Tensor src = getContiguous(); + cuda::broadcastadd(res, src, other); } #else __throw_runtime_error("Not compiled with CUDA"); @@ -809,204 +829,20 @@ tensorDim_t Tensor::mapDim(const int dim, const Dimension& dims) { return dims.nDims() + dim; } -void Tensor::transposeImplCpu(Tensor& target, const int dim1, const int dim2) const noexcept { - assert(values->getSize() == target.values->getSize() && dims.nDims()==target.dims.nDims()); - if(dim1 == dim2) { - return; - } - - const auto& source = *this; // easier to read - - auto dim1Mapped = mapDim(dim1, source.dims); - auto dim2Mapped = mapDim(dim2, source.dims); - - const int numDims = source.dims.nDims(); - std::vector indices(numDims, 0); - std::vector dimSizes(numDims); - std::vector sourceStrides(numDims); - - // strides for source - tensorSize_t stride = 1; - for(int d = numDims - 1; d >= 0; d--) { - dimSizes[d] = source.dims.get(d); - sourceStrides[d] = stride; - stride *= dimSizes[d]; - } - - // strides for target - std::vector targetDimSizes = dimSizes; - std::swap(targetDimSizes[dim1Mapped], targetDimSizes[dim2Mapped]); - - std::vector targetStrides(numDims); - stride = 1; - for(int d = numDims - 1; d >= 0; d--) { - targetStrides[d] = stride; - stride *= targetDimSizes[d]; - } - - auto transposedValues = make_unique(source.values->getDevice()); - transposedValues->resize(source.values->getSize()); - - tensorSize_t totalSize = source.values->getSize(); - for(tensorSize_t targetIdx = 0; targetIdx < totalSize; targetIdx++) { - // linear target index to multi-dimensional idx - tensorSize_t tmp = targetIdx; - for(int d = 0; d < numDims; d++) { - indices[d] = tmp / targetStrides[d]; - tmp %= targetStrides[d]; - } - - // Swap the transposed dimensions to get source indices - std::swap(indices[dim1Mapped], indices[dim2Mapped]); - - // Convert multi-dimensional source indices to linear index - tensorSize_t sourceIdx = 0; - for(int d = 0; d < numDims; d++) { - sourceIdx += indices[d] * sourceStrides[d]; - } - - (*transposedValues)[targetIdx] = (*source.values)[sourceIdx]; - } - - target.values = std::move(transposedValues); - target.dims.swap(dim1Mapped, dim2Mapped); -} - -/** - * @brief Transposes 2D tensor. A little sleeker than transposeImplCpu. - */ -void Tensor::transposeImpl2DCpu(Tensor& target, const int dim1, const int dim2) const noexcept { - assert(values->getSize()==target.values->getSize() && target.dims.nDims()==2 && dims.nDims()==2); - if(dim1==dim2){ - return; - } - - const auto& source = *this; // easier to read - - auto dim1Mapped = mapDim(dim1, source.dims); - auto dim2Mapped = mapDim(dim2, source.dims); - - // large dim wraps small dim - const auto largeDim = dim1Mapped < dim2Mapped ? dim1Mapped : dim2Mapped; - const auto smallDim = dim1Mapped < dim2Mapped ? dim2Mapped : dim1Mapped; - - // largeDimSize >= smallDimSize - const auto largeDimOffset = dims.getStride(largeDim); - const auto smallDimOffset = dims.getStride(smallDim); - - auto transposedValues = make_unique(source.values->getDevice()); - transposedValues->resize(source.values->getSize()); - - tensorSize_t resIdx = 0; - for(tensorSize_t smallDimCount=0; smallDimCountgetDevice()){ - case Device::CPU: - if(dims.nDims()>2){ - transposeImplCpu(*this, dim1, dim2); - } - else{ - transposeImpl2DCpu(*this, dim1, dim2); - } - break; - case Device::CUDA: - #ifdef __CUDA - if(dims.nDims()>2){ - // TODO: can make this better inplace? - cuda::transpose(values->getData(), values->getData(), dims, dim1, dim2); - } - else{ - // TODO: can make this better inplace? - cuda::transpose2D(values->getData(), values->getData(), dims, dim1, dim2); - } - #else - __throw_runtime_error("Not implemented with CUDA"); - #endif - break; - } -} - -/** - * @brief Out of place transposition of last two axes. - * - */ -void Tensor::transposeThis() noexcept { - if(dims.nDims()<2){ - return; - } - transposeThis(-1, -2); -} - -/** - * @brief Like overloaded transpose with requiresGrad==false. - */ -Tensor Tensor::transpose(int dim1, int dim2) const { - return transpose(dim1, dim2, false); -} - -/** - * @brief Like transposeThis, but returns a new tensor. - * We give requiresGrad as an optional argument to give more control of - * what this tensor is intended to do. E.g. in backprop sometimes we do - * need to create transposed tensors to multiply with, and with - * requiresGrad==false we avoid unnecessary memory allocation overhead. - */ -Tensor Tensor::transpose(int dim1, int dim2, const bool requiresGrad) const { - Tensor res(dims, values->getDevice(), requiresGrad); - - switch(values->getDevice()){ - case Device::CPU: - if(dims.nDims()>2){ - transposeImplCpu(res, dim1, dim2); - } - else{ - transposeImpl2DCpu(res, dim1, dim2); - } - break; - case Device::CUDA: - #ifdef __CUDA - if(dims.nDims()>2){ - cuda::transpose(res.values->getData(), values->getData(), res.dims, dim1, dim2); - } - else{ - cuda::transpose2D(res.values->getData(), values->getData(), res.dims, dim1, dim2); - } - #else - __throw_runtime_error("Not implemented with CUDA"); - #endif - break; - } - - return res; +Tensor Tensor::transpose(int dim1, int dim2) { + dims.swap(dim1, dim2); } /** @@ -1014,11 +850,11 @@ Tensor Tensor::transpose(int dim1, int dim2, const bool requiresGrad) const { * * New order aligns axes newly. E.g. (2, 3, 1, 0) */ -void Tensor::permute(const std::vector&& newOrder) noexcept { - // TODO: highly inefficient -> refactor - assert(newOrder.size()<=std::numeric_limits::max()); +void Tensor::permute(const std::vector& newOrder) noexcept { + assert(newOrder.size()==dims.nDims()); + for(tensorDim_t i=0; i(newOrder.size()); i++){ - transpose(i, newOrder[i]); + dims.swap(i, newOrder[i]); } } @@ -1100,11 +936,13 @@ Device Tensor::getDevice() const noexcept { * @param high Upper idx, non-inclusive bound. * @return Tensor The slices tensor. */ -Tensor Tensor::getSlice(tensorSize_t low, tensorSize_t high) const { +Tensor Tensor::getSlice(tensorSize_t low, tensorSize_t high) { if(high<=low){ __throw_invalid_argument("Upper bound most be larger than lower bound."); } + makeContiguous(); + auto resDims = dims.toVector(); resDims[0] = high-low; Tensor res(std::move(resDims), values->getDevice(), false); @@ -1119,8 +957,10 @@ Tensor Tensor::getSlice(tensorSize_t low, tensorSize_t high) const { * @param indices A list of indices * @return Tensor The result. */ -Tensor Tensor::getSlice(span indices) const { +Tensor Tensor::getSlice(span indices) { assert(indices.size()>0); + + makeContiguous(); auto resDims = dims.toVector(); resDims[0] = indices.size(); diff --git a/src/backend/data_modeling/tensor.h b/src/backend/data_modeling/tensor.h index c3b8992..7ccad85 100644 --- a/src/backend/data_modeling/tensor.h +++ b/src/backend/data_modeling/tensor.h @@ -111,11 +111,10 @@ class Tensor final : public std::enable_shared_from_this const tensorSize_t resOffset, const tensorSize_t leftOffset, const tensorSize_t rightOffset); - void transposeImpl2DCpu(Tensor& target, const int dim1, const int dim2) const noexcept; - void transposeImplCpu(Tensor& target, const int dim1, const int dim2) const noexcept; + void makeContiguous(); // convenience functions that appear in multiple places - static tensorSize_t computeLinearIdx(const std::vector&& idx, const Dimension& dims); + static tensorSize_t computeLinearIdx(std::vector&& idx, const Dimension& dims); static tensorSize_t computeLinearIdx(const std::vector& idx, const Dimension& dims); static tensorDim_t mapDim(const int dim, const Dimension& dims); @@ -178,7 +177,7 @@ class Tensor final : public std::enable_shared_from_this Tensor createEmptyCopy() const; Tensor createShallowCopy() const; - Tensor createLinearCopy() const; + Tensor createContiguousCopy() const; Tensor createDeepCopy() const; /** @@ -227,15 +226,11 @@ class Tensor final : public std::enable_shared_from_this } bool hasGrads() const noexcept { return grads!=nullptr; } - void transposeThis() noexcept; - void transposeThis(int dim1, int dim2) noexcept; + Tensor transpose(int dim1=-1, int dim2=-2); + void permute(const std::vector& newOrder) noexcept; - Tensor transpose(int dim1, int dim2) const; - Tensor transpose(int dim1, int dim2, const bool requiresGrad) const; - - std::shared_ptr getLinear() const; - - void permute(const std::vector&& newOrder) noexcept; + bool isContiguous() const noexcept { return dims.inOriginalState(); } + Tensor getContiguous() const; friend std::ostream& operator<<(std::ostream& os, const Tensor& t) noexcept; @@ -279,8 +274,8 @@ class Tensor final : public std::enable_shared_from_this } } - Tensor getSlice(tensorSize_t low, tensorSize_t high) const; - Tensor getSlice(std::span indices) const; + Tensor getSlice(tensorSize_t low, tensorSize_t high); + Tensor getSlice(std::span indices); // these two should not be exposed to the python interface static void setDefaultDevice(const Device d) noexcept; diff --git a/src/python/py_core/py_core.cpp b/src/python/py_core/py_core.cpp index 9c7fab2..91a3936 100644 --- a/src/python/py_core/py_core.cpp +++ b/src/python/py_core/py_core.cpp @@ -47,8 +47,13 @@ BOOST_PYTHON_MODULE(_core) } // different, since those are not methods anymore + #define WRAP_FREE_MEMBER_FUNC_0(fPtr) \ + +[](const Tensor& self) -> std::shared_ptr { \ + return std::make_shared((self.*fPtr)()); \ + } + #define WRAP_FREE_MEMBER_FUNC_1(fPtr, T1, T2) \ - +[](const Tensor& self, int v1, int v2) -> std::shared_ptr { \ + +[](const Tensor& self, T1 v1, T2 v2) -> std::shared_ptr { \ return std::make_shared((self.*fPtr)(v1, v2)); \ } @@ -217,10 +222,8 @@ BOOST_PYTHON_MODULE(_core) return t->hasGrads(); }) - .def("transpose", WRAP_FREE_MEMBER_FUNC_1(Py_DataModeling::transpose1, int, int)) - .def("transpose", WRAP_FREE_MEMBER_FUNC_2(Py_DataModeling::transpose2, int, int, bool)) - .def("transposeThis", Py_DataModeling::transposeThis1) - .def("transposeThis", Py_DataModeling::transposeThis2) + .def("transpose", WRAP_FREE_MEMBER_FUNC_1(Py_DataModeling::transpose1)) + .def("transpose", WRAP_FREE_MEMBER_FUNC_1(Py_DataModeling::transpose2, int, int)) .def("backward", &Tensor::backward) ; diff --git a/src/python/py_core/py_core_util.h b/src/python/py_core/py_core_util.h index 4da52cf..a2d5b7e 100644 --- a/src/python/py_core/py_core_util.h +++ b/src/python/py_core/py_core_util.h @@ -100,12 +100,10 @@ namespace Py_DataModeling inline void (Tensor::*reset1)(const ftype) = &Tensor::reset; inline void (Tensor::*reset2)(const std::shared_ptr) = &Tensor::reset; - inline void (Tensor::*transposeThis1)() = &Tensor::transposeThis; - inline void (Tensor::*transposeThis2)(int, int) = &Tensor::transposeThis; - inline Tensor (Tensor::*transpose1)(int, int) const = &Tensor::transpose; - inline Tensor (Tensor::*transpose2)(int, int, bool) const = &Tensor::transpose; + inline Tensor (Tensor::*transpose1)() = &Tensor::transpose; + inline Tensor (Tensor::*transpose2)(int, int) = &Tensor::transpose; - inline ftype (Tensor::*getItemVector)(const std::vector&) const = &Tensor::get; + inline ftype (Tensor::*getItemVector)(const std::vector&) const = &Tensor::get; /********************************************************************************************************* ***************************************** Graph creation ************************************************* From e6f646c093359f9de16710e33cc884274f6c4c6b Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Fri, 3 Apr 2026 14:39:29 +0200 Subject: [PATCH 12/73] Some more CUDA and compiler hints, clean up values_t, fix leaky-relu --- .../loss_functions/bce_node.cpp | 2 +- .../loss_functions/crossentropy_node.cpp | 2 +- src/backend/data_modeling/cuda/tensorops.cu | 35 +++---- src/backend/data_modeling/tensor.cpp | 96 ++++++++++--------- src/backend/data_modeling/tensor.h | 25 ++--- .../activation_functions/cuda/activations.cu | 64 +++++++++++++ .../activation_functions/cuda/activations.cuh | 27 ++++++ .../activation_functions/leaky_relu.cpp | 5 +- .../module/activation_functions/leaky_relu.h | 2 +- .../training/loss_functions/bce_loss.cpp | 2 +- .../loss_functions/crossentropy_loss.cpp | 2 +- src/backend/utility/global_params.h | 8 +- 12 files changed, 188 insertions(+), 82 deletions(-) diff --git a/src/backend/computational_graph/loss_functions/bce_node.cpp b/src/backend/computational_graph/loss_functions/bce_node.cpp index add016f..c0f30be 100644 --- a/src/backend/computational_graph/loss_functions/bce_node.cpp +++ b/src/backend/computational_graph/loss_functions/bce_node.cpp @@ -27,7 +27,7 @@ vector< shared_ptr > BceNode::backward(const Tensor& upstreamGrad) { auto yi = (*yTrue)[i]; auto yiHat = (*yPred)[i]; - auto g = -yi/std::max(yiHat, epsBce) + (1-yi)/std::max(1-yiHat, epsBce); + auto g = -yi/std::max(yiHat, EPS_BCE) + (1-yi)/std::max(1-yiHat, EPS_BCE); res->set(g/bSize, i); } diff --git a/src/backend/computational_graph/loss_functions/crossentropy_node.cpp b/src/backend/computational_graph/loss_functions/crossentropy_node.cpp index 249de43..8d74229 100644 --- a/src/backend/computational_graph/loss_functions/crossentropy_node.cpp +++ b/src/backend/computational_graph/loss_functions/crossentropy_node.cpp @@ -28,7 +28,7 @@ vector< shared_ptr > CrossEntropyNode::backward(const Tensor& upstreamGr auto yij = yTrue->get(i, j); auto yijHat = yPred->get(i, j); - auto g = -yij/std::max(yijHat, epsCrossentropy); + auto g = -yij/std::max(yijHat, EPS_CROSSENTROPY); res->set(g/bSize, i, j); } } diff --git a/src/backend/data_modeling/cuda/tensorops.cu b/src/backend/data_modeling/cuda/tensorops.cu index 9c86490..80048e0 100644 --- a/src/backend/data_modeling/cuda/tensorops.cu +++ b/src/backend/data_modeling/cuda/tensorops.cu @@ -10,7 +10,7 @@ */ #ifndef __CUDA -static_assert(false, "File should not be included without CUDA enabled"); +static_assert(false, "File should not be compiled without CUDA enabled"); #endif // __CUDA #include "tensorops.cuh" @@ -91,15 +91,17 @@ namespace{ namespace cuda { void scalaradd(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size) { - int threadsPerBlock = 256; - int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; + constexpr int threadsPerBlock = 256; + const int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; + scalaraddKernel<<>>(res, left, scalar, size); cudaErrchk(cudaDeviceSynchronize()); } void scalarmul(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size) { - int threadsPerBlock = 256; - int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; + constexpr int threadsPerBlock = 256; + const int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; + scalarmulKernel<<>>(res, left, scalar, size); cudaErrchk(cudaDeviceSynchronize()); } @@ -107,29 +109,31 @@ namespace cuda { void broadcastadd(Tensor& res, const Tensor& matrix, const Tensor& vec){ const auto nBytes = src.getSize(); - int threadsPerBlock = 256; - int blocksPerGrid = (nBytes + threadsPerBlock - 1) / threadsPerBlock; + constexpr int threadsPerBlock = 256; + const int blocksPerGrid = (nBytes + threadsPerBlock - 1) / threadsPerBlock; broadcastaddKernel<<>>( res.getData(), matrix.getData(), vec.getData(), vec.getDims()[0], matrix.getSize()); + cudaErrchk(cudaDeviceSynchronize()); } void elementwiseadd(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size) { - int threadsPerBlock = 256; + constexpr int threadsPerBlock = 256; int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; elementwiseaddKernel<<>>(res, left, right, size); cudaErrchk(cudaDeviceSynchronize()); } void elementwisemul(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size) { - int threadsPerBlock = 256; - int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; + constexpr int threadsPerBlock = 256; + const int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; + elementwisemulKernel<<>>(res, left, right, size); cudaErrchk(cudaDeviceSynchronize()); } void matmul(ftype* res, const ftype* const left, const ftype* const right) { - // TODO + static_assert(false); } ftype get(const ftype* const t, tensorSize_t idx) { @@ -148,17 +152,16 @@ namespace cuda { ftype* dst = res.getData() const ftype* const srcData = src.getData(); - auto oldStrides = src.getDims().getCreationStrides().data(); - auto newStrides = src.getDims().getStrides().data(); + const auto oldStrides = src.getDims().getCreationStrides().data(); + const auto newStrides = src.getDims().getStrides().data(); const auto nBytes = src.getSize(); - int threadsPerBlock = 256; - int blocksPerGrid = (nBytes + threadsPerBlock - 1) / threadsPerBlock; + constexpr int threadsPerBlock = 256; + const int blocksPerGrid = (nBytes + threadsPerBlock - 1) / threadsPerBlock; cudaErrchk(createContiguousCopyKernel<<>>( dst, srcData, oldStrides, newStrides, dims.nDims(), nBytes)); - cudaErrchk(cudaDeviceSynchronize()); } } diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index 785edf5..e1ac41e 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -210,15 +210,15 @@ void Tensor::tensorValues_t::setDevice(const Device d) noexcept { switch(device){ case Device::CPU:{ ftype* tmp; - cudaErrchk(cudaMalloc((void**) &tmp, this->size * sizeof(ftype))); - cudaErrchk(cudaMemcpy(tmp, values, this->size * sizeof(ftype), cudaMemcpyHostToDevice)); + cudaErrchk(cudaMalloc((void**) &tmp, size * sizeof(ftype))); + cudaErrchk(cudaMemcpy(tmp, values, size * sizeof(ftype), cudaMemcpyHostToDevice)); free(values); values = tmp; break; } case Device::CUDA:{ - ftype* tmp = static_cast(std::malloc(this->size * sizeof(ftype))); - cudaErrchk(cudaMemcpy(tmp, values, this->size * sizeof(ftype), cudaMemcpyDeviceToHost)); + ftype* tmp = static_cast(std::malloc(size * sizeof(ftype))); + cudaErrchk(cudaMemcpy(tmp, values, size * sizeof(ftype), cudaMemcpyDeviceToHost)); cudaErrchk(cudaFree(values)); values = tmp; break; @@ -229,6 +229,44 @@ void Tensor::tensorValues_t::setDevice(const Device d) noexcept { #endif } +void Tensor::tensorValues_t::reset(const ftype x) noexcept { + switch(device){ + case Device::CPU: + memset(values, x, size); + break; + case Device::CUDA: + #ifdef __CUDA + cudaErrchk(cudaMemset(values, x, size * sizeof(ftype))); + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif + break; + } +} + +void Tensor::tensorValues_t::reset(const std::shared_ptr init) noexcept { + switch(device){ + case Device::CPU: + for(tensorSize_t i=0; idrawNumber(); + } + break; + case Device::CUDA: + #ifdef __CUDA + auto newValues = static_cast(std::malloc(size * sizeof(ftype))); + for(tensorSize_t i=0; idrawNumber(); + } + cudaErrchk(cudaMemcpy(values, newValues, size * sizeof(ftype), cudaMemcpyHostToDevice)); + free(newValues); + // TODO: better initialize directly on GPU + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif + break; + } +} + Device Tensor::tensorValues_t::getDevice() const noexcept { return device; } @@ -676,7 +714,7 @@ Tensor Tensor::elementwiseMul(const Tensor& other) const { return *this * other; } -Tensor Tensor::operator*(ftype scalar) const { +Tensor Tensor::operator*(const ftype scalar) const { Tensor res(dims, values->getDevice(), false); switch(values->getDevice()){ case Device::CPU: @@ -696,7 +734,7 @@ Tensor Tensor::operator*(ftype scalar) const { return res; } -Tensor Tensor::operator/(ftype scalar) const { +Tensor Tensor::operator/(const ftype scalar) const { if(scalar==0.0){ __throw_runtime_error("Cannot divide by zero."); } @@ -720,7 +758,7 @@ Tensor Tensor::operator/(ftype scalar) const { return res; } -Tensor Tensor::operator+(ftype scalar) const { +Tensor Tensor::operator+(const ftype scalar) const { Tensor res(dims, values->getDevice(), false); switch(values->getDevice()){ case Device::CPU: @@ -740,7 +778,7 @@ Tensor Tensor::operator+(ftype scalar) const { return res; } -Tensor Tensor::operator-(ftype scalar) const { +Tensor Tensor::operator-(const ftype scalar) const { Tensor res(dims, values->getDevice(), false); switch(values->getDevice()){ case Device::CPU: @@ -760,11 +798,11 @@ Tensor Tensor::operator-(ftype scalar) const { return res; } -Tensor operator*(ftype scalar, const Tensor& tensor) { +Tensor operator*(const ftype scalar, const Tensor& tensor) { return tensor * scalar; } -Tensor operator+(ftype scalar, const Tensor& tensor) { +Tensor operator+(const ftype scalar, const Tensor& tensor) { return tensor + scalar; } @@ -862,44 +900,14 @@ void Tensor::permute(const std::vector& newOrder) noexcept { * @brief Populates the tensor with value. */ void Tensor::reset(const ftype x) noexcept { - switch(values->getDevice()){ - case Device::CPU: - memset(values->getData(), x, values->getSize()); - break; - case Device::CUDA: - #ifdef __CUDA - cudaErrchk(cudaMemset(values->getData(), x, values->getSize() * sizeof(ftype))); - #else - __throw_runtime_error("Not compiled with CUDA"); - #endif - break; - } + values->reset(x); } /** * @brief Populates the tensor with values drawn according to initializer. */ -void Tensor::reset(const shared_ptr init) noexcept { - switch(values->getDevice()){ - case Device::CPU: - for(tensorSize_t i=0; igetSize(); i++){ - (*values)[i] = init->drawNumber(); - } - break; - case Device::CUDA: - #ifdef __CUDA - auto newValues = static_cast(std::malloc(values->getSize() * sizeof(ftype))); - for(tensorSize_t i=0; igetSize(); i++){ - newValues[i] = init->drawNumber(); - } - cudaErrchk(cudaMemcpy(values->getData(), newValues, values->getSize() * sizeof(ftype), cudaMemcpyHostToDevice)); - free(newValues); - // TODO: better initialize directly on GPU - #else - __throw_runtime_error("Not compiled with CUDA"); - #endif - break; - } +void Tensor::reset(shared_ptr init) noexcept { + values->reset(std::move(init)); } const Dimension& Tensor::getDims() const noexcept { @@ -936,7 +944,7 @@ Device Tensor::getDevice() const noexcept { * @param high Upper idx, non-inclusive bound. * @return Tensor The slices tensor. */ -Tensor Tensor::getSlice(tensorSize_t low, tensorSize_t high) { +Tensor Tensor::getSlice(const tensorSize_t low, const tensorSize_t high) { if(high<=low){ __throw_invalid_argument("Upper bound most be larger than lower bound."); } diff --git a/src/backend/data_modeling/tensor.h b/src/backend/data_modeling/tensor.h index 7ccad85..f46eae0 100644 --- a/src/backend/data_modeling/tensor.h +++ b/src/backend/data_modeling/tensor.h @@ -75,8 +75,8 @@ class Tensor final : public std::enable_shared_from_this ftype* getData() const noexcept; explicit operator bool() const noexcept; - ftype& operator[](const tensorSize_t idx); - ftype operator[](const tensorSize_t idx) const; + ftype& operator[](tensorSize_t idx); + ftype operator[](tensorSize_t idx) const; void set(ftype v, tensorSize_t idx); ftype get(tensorSize_t idx); @@ -86,16 +86,19 @@ class Tensor final : public std::enable_shared_from_this // needed for gradient descent tensorValues_t& operator+=(const tensorValues_t& other); - void resize(const tensorSize_t size); + void resize(tensorSize_t size); - void setDevice(const Device d) noexcept; + void setDevice(Device d) noexcept; Device getDevice() const noexcept; + void reset(ftype x) noexcept; + void reset(std::shared_ptr init) noexcept; + void copyValues(tensorValues_t& target) const; void copyValues(tensorValues_t& target, tensorSize_t low, tensorSize_t high, tensorSize_t targetOffset) const; - void copyValues(tensorValues_t& target, std::span indices, const tensorSize_t sizeOfDim) const; + void copyValues(tensorValues_t& target, std::span indices, tensorSize_t sizeOfDim) const; - static void setDefaultDevice(const Device d) noexcept; + static void setDefaultDevice(Device d) noexcept; static Device getDefaultDevice() noexcept; }; @@ -108,8 +111,8 @@ class Tensor final : public std::enable_shared_from_this static Tensor matMulImpl(const Tensor& left, const Tensor& right); static void matMul2DCpu(Tensor& res, const Tensor& left, const Tensor& right, - const tensorSize_t resOffset, const tensorSize_t leftOffset, - const tensorSize_t rightOffset); + tensorSize_t resOffset, tensorSize_t leftOffset, + tensorSize_t rightOffset); void makeContiguous(); @@ -117,7 +120,7 @@ class Tensor final : public std::enable_shared_from_this static tensorSize_t computeLinearIdx(std::vector&& idx, const Dimension& dims); static tensorSize_t computeLinearIdx(const std::vector& idx, const Dimension& dims); - static tensorDim_t mapDim(const int dim, const Dimension& dims); + static tensorDim_t mapDim(int dim, const Dimension& dims); friend void printValuesCpu(std::ostream& os, const Tensor& t); #ifdef __CUDA @@ -189,8 +192,8 @@ class Tensor final : public std::enable_shared_from_this ftype* getData() const noexcept; - void reset(const ftype x) noexcept; - void reset(const std::shared_ptr init) noexcept; + void reset(ftype x) noexcept; + void reset(std::shared_ptr init) noexcept; const Dimension& getDims() const noexcept; tensorSize_t getSize() const noexcept; diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index e69de29..fbf20b1 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -0,0 +1,64 @@ +/** + * @file activations.cu + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-03-31 + * + * @copyright Copyright (c) 2026 + * + */ + +#ifndef __CUDA +static_assert(false, "File should not be compiled without CUDA enabled"); +#endif // __CUDA + +#include "activations.cuh" + +#include "utility/cuda/cuda_common.cuh" + +namespace { + __global__ void reluKernel(ftype* res, const ftype* const input, const tensorSize_t size) { + int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid>=size) + return; + + constexpr ftype zero = 0; + res[gid] = fmaxf(input[gid], zero); + } + + __global__ void leakyReluKernel(ftype* res, const ftype* const input, const ftype eps, const tensorSize_t size) { + int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid>=size) + return; + + res[gid] = fmaxf(input[gid], eps*input[gid]); // eps < 1 + } +} + + +namespace cuda { + void relu(Tensor& res, const Tensor& in) { + constexpr int threadsPerBlock = 256; + const int blocks = (in.getSize()+threadsPerBlock-1) / threadsPerBlock; + + reluKernel<<>>(res.getData(), in.getData(), in.getSize()); + cudaErrchk(cudaDeviceSynchronize()); + } + + void leakyRelu(Tensor& res, const Tensor& in, ftype eps) { + constexpr int threadsPerBlock = 256; + const int blocks = (in.getSize()+threadsPerBlock-1) / threadsPerBlock; + + leakyReluKernel<<>>(res.getData(), in.getData(), eps, in.getSize()); + cudaErrchk(cudaDeviceSynchronize()); + } + + void sigmoid(Tensor& res, const Tensor& in) { + static_assert(false); + } + + void softmax(Tensor& res, const Tensor& in) { + static_assert(false); + } +} \ No newline at end of file diff --git a/src/backend/module/activation_functions/cuda/activations.cuh b/src/backend/module/activation_functions/cuda/activations.cuh index e69de29..94002c0 100644 --- a/src/backend/module/activation_functions/cuda/activations.cuh +++ b/src/backend/module/activation_functions/cuda/activations.cuh @@ -0,0 +1,27 @@ +/** + * @file activations.cuh + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-03-31 + * + * @copyright Copyright (c) 2026 + * + */ + +#pragma once + +#ifndef __CUDA +static_assert(false, "File should not be included without CUDA enabled"); +#endif // __CUDA + +#include "utility/global_params.h" +#include "data_modeling/tensor.h" + +namespace cuda { + void relu(Tensor& res, const Tensor& in); + void leakyRelu(Tensor& res, const Tensor& in, ftype eps); + + void sigmoid(Tensor& res, const Tensor& in); + void softmax(Tensor& res, const Tensor& in); +} diff --git a/src/backend/module/activation_functions/leaky_relu.cpp b/src/backend/module/activation_functions/leaky_relu.cpp index 687e2bf..0cd46bc 100644 --- a/src/backend/module/activation_functions/leaky_relu.cpp +++ b/src/backend/module/activation_functions/leaky_relu.cpp @@ -19,10 +19,7 @@ Tensor LeakyReLu::operator()(const Tensor& t) const { auto res = t.createDeepCopy(); for(tensorSize_t i=0; i BceLoss::operator()(const shared_ptr y, const shared_ } auto bce = [](ftype y, ftype ypred){ - return y*log(std::max(ypred, epsBce)) + (1-y)*log(std::max(1-ypred, epsBce)); + return y*log(std::max(ypred, EPS_BCE)) + (1-y)*log(std::max(1-ypred, EPS_BCE)); }; const auto nBatches = y->getDims()[0]; diff --git a/src/backend/training/loss_functions/crossentropy_loss.cpp b/src/backend/training/loss_functions/crossentropy_loss.cpp index d1a5291..ee6b07c 100644 --- a/src/backend/training/loss_functions/crossentropy_loss.cpp +++ b/src/backend/training/loss_functions/crossentropy_loss.cpp @@ -36,7 +36,7 @@ shared_ptr CrossEntropyLoss::operator()(const shared_ptr y, cons auto ce = [&y, &ypred](const tensorDim_t b){ ftype res = 0; for(tensorDim_t i=0; igetDims()[-1]; i++){ - res += y->get(b, i) * log(std::max(ypred->get(b, i), epsCrossentropy)); + res += y->get(b, i) * log(std::max(ypred->get(b, i), EPS_CROSSENTROPY)); } return res; }; diff --git a/src/backend/utility/global_params.h b/src/backend/utility/global_params.h index 5fef2ce..98af143 100644 --- a/src/backend/utility/global_params.h +++ b/src/backend/utility/global_params.h @@ -39,5 +39,9 @@ static_assert(sizeof(tensorDim_t)<=sizeof(tensorSize_t)); // ----------------- Numerical stability ------------------- -constexpr ftype epsCrossentropy = 1e-5; -constexpr ftype epsBce = 1e-5; \ No newline at end of file +constexpr ftype EPS_CROSSENTROPY = 1e-5; +constexpr ftype EPS_BCE = 1e-5; + +// ----------------- Default values ------------------------ + +constexpr ftype EPS_LEAKY_RELU = 0.01; From 582e299a4dae6f727a35a813af32f90af16ce34f Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Fri, 3 Apr 2026 18:17:40 +0200 Subject: [PATCH 13/73] Sigmoid kernel --- .gitignore | 13 ----------- .../activation_functions/cuda/activations.cu | 22 +++++++++++++++++-- 2 files changed, 20 insertions(+), 15 deletions(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index df79fe8..0000000 --- a/.gitignore +++ /dev/null @@ -1,13 +0,0 @@ -build -.vscode -*.txt -python_lib/dl_lib/_compiled -*__pycache__* -*_cache - -perf.data* - -optimization - -# TODO: remove later -benchmarks \ No newline at end of file diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index fbf20b1..32e350d 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -34,6 +34,20 @@ namespace { res[gid] = fmaxf(input[gid], eps*input[gid]); // eps < 1 } + + __device__ __forceinline__ ftype sigmoid(ftype x) { + ftype z = expf(-fabsf(x)); + ftype s = 1.0f / (1.0f + z); + return (x >= 0.f) ? s : z * s; // x < 0 => e^x/(e^x+1) + } + + __global__ void sigmoidKernel(ftype* res, const ftype* const input, const tensorSize_t size) { + int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid>=size) + return; + + res[gid] = sigmoid(input[gid]); + } } @@ -55,8 +69,12 @@ namespace cuda { } void sigmoid(Tensor& res, const Tensor& in) { - static_assert(false); - } + constexpr int threadsPerBlock = 256; + const int blocks = (in.getSize()+threadsPerBlock-1) / threadsPerBlock; + + sigmoidKernel<<>>(res.getData(), in.getData(), in.getSize()); + cudaErrchk(cudaDeviceSynchronize()); +} void softmax(Tensor& res, const Tensor& in) { static_assert(false); From 9d18a0ce13a5bfff15aa589c2c54e2ce779d6217 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sat, 2 May 2026 19:49:35 +0200 Subject: [PATCH 14/73] Fixed issues with new transposition logic --- src/backend/data_modeling/tensor.cpp | 33 +++++++++++++++++----------- src/backend/data_modeling/tensor.h | 10 ++++----- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index e1ac41e..c61bc3b 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -444,7 +444,7 @@ Tensor Tensor::createShallowCopy() const { Tensor Tensor::createContiguousCopy() const { auto res = createEmptyCopy(); - switch(values->getDevice()){ + switch(values->getDevice()) { case Device::CPU: { for (tensorSize_t flatIdx = 0; flatIdx < values->getSize(); ++flatIdx) { @@ -475,7 +475,7 @@ Tensor Tensor::createContiguousCopy() const { return res; } -void Tensor::makeContiguous(){ +void Tensor::makeContiguous() { if(isContiguous()) return; @@ -605,9 +605,12 @@ void Tensor::matMul2DCpu(Tensor& res, const Tensor& left, const Tensor& right, c /** * @brief Matrix multiplication. */ -Tensor Tensor::matmul(const Tensor& other) const { +Tensor Tensor::matmul(Tensor& other) { assert(values->getDevice()==other.values->getDevice()); + makeContiguous(); + other.makeContiguous(); + if(values->getDevice()!=other.values->getDevice()){ __throw_runtime_error("Tensors on different devices."); } @@ -621,7 +624,7 @@ Tensor Tensor::matmul(const Tensor& other) const { * 2. The second tensor is a vector. In this case broadcast it. We assume * other.dims == (dimN) && this->dims == (dim0, dim1,..., dimN). */ -Tensor Tensor::operator+(const Tensor& other) const { +Tensor Tensor::operator+(Tensor& other) { if(this->dims != other.dims && !(other.dims.nDims() == 1 && other.dims.get(0) == dims.get(-1))){ __throw_invalid_argument("Tensors need matching dimensions"); @@ -630,11 +633,13 @@ Tensor Tensor::operator+(const Tensor& other) const { __throw_runtime_error("Tensors on different devices."); } + makeContiguous(); + other.makeContiguous(); Tensor res(dims, values->getDevice()); switch(values->getDevice()){ case Device::CPU: - if(dims==other.dims){ + if(dims==other.dims) [[unlikely]] { // elementwise add for(tensorSize_t i=0; igetSize(); i++){ (*res.values)[i] = (*values)[i] + (*other.values)[i]; @@ -643,23 +648,21 @@ Tensor Tensor::operator+(const Tensor& other) const { } else [[likely]] { // broadcasted add - Tensor src = getContiguous(); const auto stride = static_cast(other.dims.get(0)); // other is a vector for(tensorSize_t offset=0; offsetgetSize(); offset+=stride){ for(tensorSize_t i=0; igetData(), other.getData(), values->getSize()); } else [[likely]] { - Tensor src = getContiguous(); - cuda::broadcastadd(res, src, other); + cuda::broadcastadd(res, *this, other); } #else __throw_runtime_error("Not compiled with CUDA"); @@ -672,14 +675,14 @@ Tensor Tensor::operator+(const Tensor& other) const { /** * @brief Named version of operator +. */ -Tensor Tensor::add(const Tensor& other) const { +Tensor Tensor::add(Tensor& other) { return *this + other; } /** * @brief Elementwise multiplication. */ -Tensor Tensor::operator*(const Tensor& other) const { +Tensor Tensor::operator*(Tensor& other) { if(this->dims != other.dims){ __throw_invalid_argument("Tensors need same dimensions"); } @@ -687,7 +690,10 @@ Tensor Tensor::operator*(const Tensor& other) const { __throw_runtime_error("Tensors on different devices."); } + makeContiguous(); + other.makeContiguous(); Tensor res(dims, values->getDevice(), false); + switch(values->getDevice()){ case Device::CPU: for(tensorSize_t i=0; igetSize(); i++){ @@ -710,7 +716,7 @@ Tensor Tensor::operator*(const Tensor& other) const { /** * @brief Named version of operator *. */ -Tensor Tensor::elementwiseMul(const Tensor& other) const { +Tensor Tensor::elementwiseMul(Tensor& other) { return *this * other; } @@ -823,6 +829,7 @@ void Tensor::backward() { vector sortedTensors = cgraph::TopologicalSort::reverseSort(this); for(auto tPtr: sortedTensors){ auto& tensor = *tPtr; + tensor.makeContiguous(); assert(tensor.grads && !tensor.grads->requiresGrad); // gradient should not require grad auto incomingGrads = tensor.cgNode->backward(*tensor.grads); diff --git a/src/backend/data_modeling/tensor.h b/src/backend/data_modeling/tensor.h index f46eae0..923b508 100644 --- a/src/backend/data_modeling/tensor.h +++ b/src/backend/data_modeling/tensor.h @@ -199,15 +199,15 @@ class Tensor final : public std::enable_shared_from_this tensorSize_t getSize() const noexcept; // Tensor operator@(const Tensor& other) const; in higher C++ versions than 20 - Tensor matmul(const Tensor& other) const; + Tensor matmul(Tensor& other); - Tensor operator+(const Tensor& other) const; - Tensor add(const Tensor& other) const; + Tensor operator+(Tensor& other); + Tensor add(Tensor& other); // TODO: Tensor operator-(const Tensor& other) const; - Tensor operator*(const Tensor& t) const; - Tensor elementwiseMul(const Tensor& other) const; + Tensor operator*(Tensor& t); + Tensor elementwiseMul(Tensor& other); // TODO: Tensor operator/(const Tensor& other) const; From 1d072cb608448a15050acb75f2368758c6f58acb Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Wed, 6 May 2026 12:39:11 +0200 Subject: [PATCH 15/73] skeleton for softmax --- .../activation_functions/cuda/activations.cu | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index 32e350d..583f8f4 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -46,6 +46,17 @@ namespace { if(gid>=size) return; + res[gid] = sigmoid(input[gid]); + } + + __global__ void softmaxKernel(ftype* res, const ftype* const input, const tensorSize_t size, const tensorSize_t stride) { + int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid>=size) + return; + + res[gid] = expf(input[gid]); + + res[gid] = sigmoid(input[gid]); } } @@ -77,6 +88,10 @@ namespace cuda { } void softmax(Tensor& res, const Tensor& in) { - static_assert(false); + constexpr int threadsPerBlock = 256; + const int blocks = (in.getSize()+threadsPerBlock-1) / threadsPerBlock; + + //sigmoidKernel<<>>(res.getData(), in.getData(), in.getSize()); + cudaErrchk(cudaDeviceSynchronize()); } } \ No newline at end of file From e42498db11c8d5f2a79c1694ad8982469eb4318e Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 10 May 2026 11:17:44 +0200 Subject: [PATCH 16/73] First softmax untested --- .../activation_functions/cuda/activations.cu | 149 ++++++++++++++++-- src/backend/utility/cuda/cuda_common.cuh | 5 +- 2 files changed, 140 insertions(+), 14 deletions(-) diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index 583f8f4..a00e20f 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -14,13 +14,16 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #endif // __CUDA #include "activations.cuh" - #include "utility/cuda/cuda_common.cuh" +#include + +using namespace std; + namespace { __global__ void reluKernel(ftype* res, const ftype* const input, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; - if(gid>=size) + if(gid >= size) return; constexpr ftype zero = 0; @@ -29,7 +32,7 @@ namespace { __global__ void leakyReluKernel(ftype* res, const ftype* const input, const ftype eps, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; - if(gid>=size) + if(gid >= size) return; res[gid] = fmaxf(input[gid], eps*input[gid]); // eps < 1 @@ -43,21 +46,116 @@ namespace { __global__ void sigmoidKernel(ftype* res, const ftype* const input, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; - if(gid>=size) + if(gid >= size) return; res[gid] = sigmoid(input[gid]); } - __global__ void softmaxKernel(ftype* res, const ftype* const input, const tensorSize_t size, const tensorSize_t stride) { + // TODO: use shared memory + __global__ void findMaxKernel(ftype* res, ftype* const input, const tensorSize_t stride, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; - if(gid>=size) + if(gid >= size) return; - res[gid] = expf(input[gid]); + int tid = threadIdx.x; + const tensorSize_t baseOffset = gid % stride; + for(tensorSize_t offset = (stride + 1) / 2; offset > 32; offset = (offset + 1) / 2) { // ceil-division + // ceil-division over two -> last thread crosses boundary iff stride % 2 == 1 + if(baseOffset + offset < stride) { + input[gid] = max(input[gid], input[gid + offset]); + } + __syncthreads(); + } + + // loop unrolling + if(baseOffset + 16 < stride) { + input[gid] = max(input[gid], input[gid + 16]); + __syncthreads(); + } + if(baseOffset + 8 < stride) { + input[gid] = max(input[gid], input[gid + 8]); + __syncthreads(); + } + if(baseOffset + 4 < stride) { + input[gid] = max(input[gid], input[gid + 4]); + __syncthreads(); + } + if(baseOffset + 2 < stride) { + input[gid] = max(input[gid], input[gid + 2]); + __syncthreads(); + } + if(baseOffset + 1 < stride) { + input[gid] = max(input[gid], input[gid + 1]); + __syncthreads(); + } + + // write to result + if(baseOffset == 0) { + res[gid / stride] = input[gid]; + } + } - - res[gid] = sigmoid(input[gid]); + // TODO: use shared memory + __global__ void expAndSumKernel(ftype* res, ftype* const tmp, const ftype* const input, const ftype* const maxValues, + const tensorSize_t stride, const tensorSize_t size) { + int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) + return; + + const auto strideOffset = gid / stride; + const auto maxValue = maxValues[strideOffset]; + if constexpr (std::is_same_v) { + res[gid] = expf(input[gid] - maxValue); + } + else if constexpr (std::is_same_v) { + res[gid] = exp(input[gid] - maxValue); + } + else { + static_assert(always_false, "ftype encountered unexpected type"); + } + + __syncthreads(); + + const tensorSize_t baseOffset = gid % stride; + tensorSize_t offset = (stride + 1) / 2; // ceil-division + while(offset > 32) { + if(baseOffset + offset >= stride) continue; // ceil-division -> last thread can cross boundary + + tmp[gid] = res[gid] + res[gid + offset]; + offset = (offset + 1) / 2; + } + + // loop unrolling + if(offset <= 32 && baseOffset + 16 < stride) { + tmp[gid] = res[gid] + res[gid + 16]; + } + if(offset <= 16 && baseOffset + 8 < stride) { + tmp[gid] = res[gid] + res[gid + 8]; + } + if(offset <= 8 && baseOffset + 4 < stride) { + tmp[gid] = res[gid] + res[gid + 4]; + } + if(offset <= 4 && baseOffset + 2 < stride) { + tmp[gid] = res[gid] + res[gid + 2]; + } + if(offset <= 2 && baseOffset + 1 < stride) { + tmp[gid] = res[gid] + res[gid + 1]; + } + + // write to result + if(baseOffset == 0) { + tmp[strideOffset] = tmp[gid]; + } + } + + __global__ void softmaxDivisionKernel(ftype* res, const ftype* const tmp, const tensorSize_t stride, const tensorSize_t size) { + int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) + return; + + const auto strideOffset = gid / stride; + res[gid] = res[gid] / tmp[strideOffset]; } } @@ -88,10 +186,35 @@ namespace cuda { } void softmax(Tensor& res, const Tensor& in) { - constexpr int threadsPerBlock = 256; - const int blocks = (in.getSize()+threadsPerBlock-1) / threadsPerBlock; + const tensorSize_t stride = static_cast(in.getDims().get(-1)); - //sigmoidKernel<<>>(res.getData(), in.getData(), in.getSize()); - cudaErrchk(cudaDeviceSynchronize()); + if (stride <= 1 << 10) { + // threads per block needs to be multiple of stride + int threadsPerBlock = 256 / stride * stride; // 256 >= threadsPerBlock >= 256 - stride + 1 + const int blocks = (in.getSize()+threadsPerBlock-1) / threadsPerBlock; + + ftype* tmp; + cudaErrchk(cudaMalloc(&tmp, in.getSize() * sizeof(ftype))); + cudaErrchk(cudaMemcpy(tmp, in.getData(), in.getSize() * sizeof(ftype), cudaMemcpyDeviceToDevice)); + + ftype* maxValues; + const tensorSize_t nMaxValues = in.getSize() / stride; + cudaErrchk(cudaMalloc(&maxValues, nMaxValues * sizeof(ftype))); + + findMaxKernel<<>>(maxValues, tmp, stride, in.getSize()); + cudaErrchk(cudaDeviceSynchronize()); + + expAndSumKernel<<>>(res.getData(), tmp, in.getData(), maxValues, stride, in.getSize()); + cudaErrchk(cudaDeviceSynchronize()); + + softmaxDivisionKernel(res.getData(), tmp, stride, in.getSize()); + cudaErrchk(cudaDeviceSynchronize()); + + cudaErrchk(cudaFree(tmp)); + cudaErrchk(cudaFree(maxValues)); + } + else { + __throw_runtime_error("Softmax kernels not yet implemented at inter-block level"); + } } } \ No newline at end of file diff --git a/src/backend/utility/cuda/cuda_common.cuh b/src/backend/utility/cuda/cuda_common.cuh index 46023a2..96adcbd 100644 --- a/src/backend/utility/cuda/cuda_common.cuh +++ b/src/backend/utility/cuda/cuda_common.cuh @@ -17,9 +17,12 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "cuda_runtime.h" -namespace utility { +namespace utility { void gpuAssert(cudaError_t code, const char *file, int line, bool abort=true); } +template +constexpr bool always_false = false; + #define cudaErrchk(ans) { utility::gpuAssert((ans), __FILE__, __LINE__); } From e214e54b20659c99300ac8b24d7a0fc5cf9de2c6 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 10 May 2026 15:03:23 +0200 Subject: [PATCH 17/73] Skeleton of cuda bce loss --- .../training/loss_functions/bce_loss.cpp | 41 +++++--- .../loss_functions/cuda/loss_functions.cu | 96 +++++++++++++++++++ .../loss_functions/cuda/loss_functions.cuh | 28 ++++++ 3 files changed, 154 insertions(+), 11 deletions(-) create mode 100644 src/backend/training/loss_functions/cuda/loss_functions.cu create mode 100644 src/backend/training/loss_functions/cuda/loss_functions.cuh diff --git a/src/backend/training/loss_functions/bce_loss.cpp b/src/backend/training/loss_functions/bce_loss.cpp index c2af731..00d61fc 100644 --- a/src/backend/training/loss_functions/bce_loss.cpp +++ b/src/backend/training/loss_functions/bce_loss.cpp @@ -10,9 +10,12 @@ */ #include "bce_loss.h" - #include "computational_graph/loss_functions/bce_node.h" +#ifdef CUDA +#include "training/loss_functions/cuda/loss_functions.cuh" +#endif + #include using namespace std; @@ -33,20 +36,36 @@ shared_ptr BceLoss::operator()(const shared_ptr y, const shared_ __throw_invalid_argument("Tensors must be of same shape"); } - auto bce = [](ftype y, ftype ypred){ - return y*log(std::max(ypred, EPS_BCE)) + (1-y)*log(std::max(1-ypred, EPS_BCE)); - }; + shared_ptr res = nullptr; - const auto nBatches = y->getDims()[0]; + switch(y->getDevice()) { + case Device::CUDA: + #ifdef CUDA + res = cuda::bceLoss(*y, *ypred); + break; + #else + __throw_runtime_error("Should not reach this line"); + #endif + case Device::CPU: + { + auto bce = [](ftype y, ftype ypred){ + return y * log(std::max(ypred, EPS_BCE)) + (1 - y) * log(std::max(1-ypred, EPS_BCE)); + }; - ftype loss = 0; - for(tensorSize_t i=0; igetDims()[0]; + ftype loss = 0; + for(tensorSize_t i = 0; i < nBatches; i++){ + loss += bce((*y)[i], (*ypred)[i]); + } + res = make_shared(std::vector{1}, std::vector{-loss / nBatches}, Device::CPU, true); + + break; + } + default: + __throw_invalid_argument("Unexpected device encountered"); } - auto res = make_shared(std::vector{1}, std::vector{-loss / nBatches}, y->getDevice(), true); res->setCgNode(make_shared(y, ypred)); assert(res->getRequiresGrad()); - - return res; + return res; } \ No newline at end of file diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cu b/src/backend/training/loss_functions/cuda/loss_functions.cu new file mode 100644 index 0000000..3d13e3f --- /dev/null +++ b/src/backend/training/loss_functions/cuda/loss_functions.cu @@ -0,0 +1,96 @@ +/** + * @file loss_functions.cu + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-05-10 + * + * @copyright Copyright (c) 2026 + * + */ + + +#ifndef __CUDA +static_assert(false, "File should not be compiled without CUDA enabled"); +#endif // __CUDA + +#include "loss_functions.cuh" +#include "utility/cuda/cuda_common.cuh" + +#include + +using namespace std; + +namespace { + __forceinline__ __device__ ftype bce(ftype y, ftype ypred) { + if constexpr (std::is_same_v) { + return y * __logf(max(ypred, EPS_BCE)) + (1 - y) * __logf(max(1-ypred, EPS_BCE)); + } + else if constexpr (std::is_same_v) { + return y * log(max(ypred, EPS_BCE)) + (1 - y) * log(max(1-ypred, EPS_BCE)); + } + else { + static_assert(always_false, "Unexpected value for ftype"); + } + } + + __global__ void bceKernel(ftype* res, const ftype* const y, const ftype* const ypred, tensorSize_t size) { + int gid = blockDim.x * blockIdx.x + threadIdx.x; + if(gid >= size) + return; + + int tid = threadIdx.x; + extern __shared__ ftype sdata[]; + + // pre-load first round + { + int i = blockIdx.x * (blockDim.x * 2) + threadIdx.x; + sdata[tid] = bce(y[i], ypred[i]) + bce(y[i + blockDim.x], ypred[i + blockDim.x]); + __syncthreads(); + } + + for(tensorSize_t i = blockDim.x / 2; i >= 64; i >>= 1){ + if(tid < i) { + sdata[tid] += sdata[tid + i]; + } + __syncthreads; + } + + if(tid > 16) { + + } + + // TODO: divide by size + } +} + +namespace cuda { + Tensor&& bceLoss(const Tensor& y, const Tensor& yPred) { + + constexpr int threadsPerBlock = 256; + const int blocks = (in.getSize() + threadsPerBlock - 1) / (threadsPerBlock * 2); + + auto res = Tensor(vector{1}, Device::CUDA, true); + + bceKernel<<>>(res.getData(), y.getData(), yPred.getData(), y.getDims()[0]); + cudaErrchk(cudaDeviceSynchronize()); + + return std::move(res); + } + + Tensor&& bceSigmoidLoss(const Tensor& y, const Tensor& yPred) { + + } + + Tensor&& crossEntropyLoss(const Tensor& y, const Tensor& yPred) { + + } + + Tensor&& crossEntropySoftmaxLoss(const Tensor& y, const Tensor& yPred) { + + } + + Tensor&& rmseLoss(const Tensor& y, const Tensor& yPred) { + + } +} \ No newline at end of file diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cuh b/src/backend/training/loss_functions/cuda/loss_functions.cuh new file mode 100644 index 0000000..9bee53e --- /dev/null +++ b/src/backend/training/loss_functions/cuda/loss_functions.cuh @@ -0,0 +1,28 @@ +/** + * @file loss_functions.cuh + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-05-10 + * + * @copyright Copyright (c) 2026 + * + */ + +#pragma once + +#ifndef __CUDA +static_assert(false, "File should not be included without CUDA enabled"); +#endif // __CUDA + +#include "data_modeling/tensor.h" + +namespace cuda { + Tensor&& bceLoss(const Tensor& y, const Tensor& yPred); + Tensor&& bceSigmoidLoss(const Tensor& y, const Tensor& yPred); + + Tensor&& crossEntropyLoss(const Tensor& y, const Tensor& yPred); + Tensor&& crossEntropySoftmaxLoss(const Tensor& y, const Tensor& yPred); + + Tensor&& rmseLoss(const Tensor& y, const Tensor& yPred); +} \ No newline at end of file From 563a1412eabb392ddb97470be301593c5b22cc48 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Mon, 11 May 2026 11:53:11 +0200 Subject: [PATCH 18/73] Renaming of cuda namespace --- .../cuda/activation_nodes.cu | 2 +- .../cuda/activation_nodes.cuh | 2 +- src/backend/data_modeling/tensor.cpp | 26 +++++++------- .../activation_functions/cuda/activations.cuh | 2 +- .../training/loss_functions/bce_loss.cpp | 2 +- .../loss_functions/cuda/loss_functions.cu | 35 +++++++++++++++---- .../loss_functions/cuda/loss_functions.cuh | 2 +- 7 files changed, 47 insertions(+), 24 deletions(-) diff --git a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu index 6ba9039..94bee47 100644 --- a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu +++ b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu @@ -15,4 +15,4 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "activation_nodes.cuh" -using namespace cuda; \ No newline at end of file +using namespace cuda_impl; \ No newline at end of file diff --git a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cuh b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cuh index 3e2c7bc..abb1d5c 100644 --- a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cuh +++ b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cuh @@ -17,7 +17,7 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "utility/global_params.h" -namespace cuda { +namespace cuda_impl { __global__ void reluBackward(ftype* res, const ftype* const upstreamGrad, tensorSize_t size); __global__ void leakyReluBackward(ftype* res, const ftype* const upstreamGrad, ftype eps, tensorSize_t size); diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index c61bc3b..e1f090a 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -303,7 +303,7 @@ Tensor::tensorValues_t::operator+=(const Tensor::tensorValues_t& other) { break; case Device::CUDA: #ifdef __CUDA - cuda::elementwiseadd(values, values, other.values, size); + cuda_impl::elementwiseadd(values, values, other.values, size); #else __throw_invalid_argument("Not compiled with CUDA"); #endif @@ -332,7 +332,7 @@ ftype Tensor::tensorValues_t::operator[](const tensorSize_t idx) const { return values[idx]; case Device::CUDA: #ifdef __CUDA - return cuda::get(values, idx); + return cuda_impl::get(values, idx); #else __throw_invalid_argument("Not compiled with CUDA"); break; @@ -353,7 +353,7 @@ void Tensor::tensorValues_t::set(ftype v, tensorSize_t idx) { break; case Device::CUDA: #ifdef __CUDA - cuda::set(v, values, idx); + cuda_impl::set(v, values, idx); #else __throw_invalid_argument("Not compiled with CUDA"); #endif @@ -370,7 +370,7 @@ ftype Tensor::tensorValues_t::get(tensorSize_t idx) { return values[idx]; case Device::CUDA: #ifdef __CUDA - return cuda::get(values, idx); + return cuda_impl::get(values, idx); #else __throw_invalid_argument("Not compiled with CUDA"); break; @@ -464,7 +464,7 @@ Tensor Tensor::createContiguousCopy() const { case Device::CUDA: { #ifdef __CUDA - cuda::createContiguousCopy(res, *this); + cuda_impl::createContiguousCopy(res, *this); #else __throw_runtime_error("Not compiled with CUDA"); #endif @@ -560,7 +560,7 @@ Tensor Tensor::matMulImpl(const Tensor& left, const Tensor& right) { } case Device::CUDA: #ifdef __CUDA - cuda::matmul(res, left, right); + cuda_impl::matmul(res, left, right); #else __throw_invalid_argument("Not compiled with CUDA"); #endif @@ -659,10 +659,10 @@ Tensor Tensor::operator+(Tensor& other) { case Device::CUDA: #ifdef __CUDA if(dims==other.dims) [[unlikely]] { - cuda::elementwiseadd(res.getData(), values->getData(), other.getData(), values->getSize()); + cuda_impl::elementwiseadd(res.getData(), values->getData(), other.getData(), values->getSize()); } else [[likely]] { - cuda::broadcastadd(res, *this, other); + cuda_impl::broadcastadd(res, *this, other); } #else __throw_runtime_error("Not compiled with CUDA"); @@ -702,7 +702,7 @@ Tensor Tensor::operator*(Tensor& other) { break; case Device::CUDA: #ifdef __CUDA - cuda::elementwisemul(res.getData(), values->getData(), + cuda_impl::elementwisemul(res.getData(), values->getData(), other.values->getData(), values->getSize()); #else __throw_runtime_error("Not compiled with CUDA"); @@ -730,7 +730,7 @@ Tensor Tensor::operator*(const ftype scalar) const { break; case Device::CUDA: #ifdef __CUDA - cuda::scalarmul(res.getData(), values->getData(), scalar, values->getSize()); + cuda_impl::scalarmul(res.getData(), values->getData(), scalar, values->getSize()); #else __throw_runtime_error("Not compiled with CUDA"); #endif @@ -754,7 +754,7 @@ Tensor Tensor::operator/(const ftype scalar) const { break; case Device::CUDA: #ifdef __CUDA - cuda::scalarmul(res.getData(), values->getData(), 1 / scalar, values->getSize()); + cuda_impl::scalarmul(res.getData(), values->getData(), 1 / scalar, values->getSize()); #else __throw_runtime_error("Not compiled with CUDA"); #endif @@ -774,7 +774,7 @@ Tensor Tensor::operator+(const ftype scalar) const { break; case Device::CUDA: #ifdef __CUDA - cuda::scalaradd(res.getData(), values->getData(), scalar, values->getSize()); + cuda_impl::scalaradd(res.getData(), values->getData(), scalar, values->getSize()); #else __throw_runtime_error("Not compiled with CUDA"); #endif @@ -794,7 +794,7 @@ Tensor Tensor::operator-(const ftype scalar) const { break; case Device::CUDA: #ifdef __CUDA - cuda::scalaradd(res.getData(), values->getData(), -scalar, values->getSize()); + cuda_impl::scalaradd(res.getData(), values->getData(), -scalar, values->getSize()); #else __throw_runtime_error("Not compiled with CUDA"); #endif diff --git a/src/backend/module/activation_functions/cuda/activations.cuh b/src/backend/module/activation_functions/cuda/activations.cuh index 94002c0..6bfd89e 100644 --- a/src/backend/module/activation_functions/cuda/activations.cuh +++ b/src/backend/module/activation_functions/cuda/activations.cuh @@ -18,7 +18,7 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "utility/global_params.h" #include "data_modeling/tensor.h" -namespace cuda { +namespace cuda_impl { void relu(Tensor& res, const Tensor& in); void leakyRelu(Tensor& res, const Tensor& in, ftype eps); diff --git a/src/backend/training/loss_functions/bce_loss.cpp b/src/backend/training/loss_functions/bce_loss.cpp index 00d61fc..48eaf4f 100644 --- a/src/backend/training/loss_functions/bce_loss.cpp +++ b/src/backend/training/loss_functions/bce_loss.cpp @@ -41,7 +41,7 @@ shared_ptr BceLoss::operator()(const shared_ptr y, const shared_ switch(y->getDevice()) { case Device::CUDA: #ifdef CUDA - res = cuda::bceLoss(*y, *ypred); + res = cuda_impl::bceLoss(*y, *ypred); break; #else __throw_runtime_error("Should not reach this line"); diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cu b/src/backend/training/loss_functions/cuda/loss_functions.cu index 3d13e3f..55ee006 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cu +++ b/src/backend/training/loss_functions/cuda/loss_functions.cu @@ -53,22 +53,45 @@ namespace { if(tid < i) { sdata[tid] += sdata[tid + i]; } - __syncthreads; + __syncthreads(); } - if(tid > 16) { - + if(tid < 32 && gid + 32 < size) { + sdata[tid] += sdata[tid + 32]; + } + __syncthreads(); + + if(tid < 16 && gid + 16 < size) { + sdata[tid] += sdata[tid + 16]; + } + __syncthreads(); + + if(tid < 8 && gid + 8 < size) { + sdata[tid] += sdata[tid + 8]; } + __syncthreads(); + + if(tid < 4 && gid + 4 < size) { + sdata[tid] += sdata[tid + 4]; + } + __syncthreads(); + + if(tid < 2 && gid + 2 < size) { + sdata[tid] += sdata[tid + 2]; + } + __syncthreads(); - // TODO: divide by size + if(tid == 0 && gid + 1 < size) { + sdata[0] = (sdata[0] + sdata[1]) / size; + } } } -namespace cuda { +namespace cuda_impl { Tensor&& bceLoss(const Tensor& y, const Tensor& yPred) { constexpr int threadsPerBlock = 256; - const int blocks = (in.getSize() + threadsPerBlock - 1) / (threadsPerBlock * 2); + const int blocks = (y.getSize() + threadsPerBlock - 1) / (threadsPerBlock * 2); auto res = Tensor(vector{1}, Device::CUDA, true); diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cuh b/src/backend/training/loss_functions/cuda/loss_functions.cuh index 9bee53e..8c43f09 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cuh +++ b/src/backend/training/loss_functions/cuda/loss_functions.cuh @@ -17,7 +17,7 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "data_modeling/tensor.h" -namespace cuda { +namespace cuda_impl { Tensor&& bceLoss(const Tensor& y, const Tensor& yPred); Tensor&& bceSigmoidLoss(const Tensor& y, const Tensor& yPred); From 8de5740abe9f0ae793e8c8914bb8fadc4b083c3a Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Mon, 11 May 2026 15:09:02 +0200 Subject: [PATCH 19/73] Fixed several compile-time and run-time bugs --- src/backend/data_modeling/dim_type.cpp | 59 ++++---------------- src/backend/data_modeling/dim_type.h | 26 +++++---- src/backend/data_modeling/tensor.cpp | 34 +++++++---- src/backend/data_modeling/tensor.h | 18 +++--- src/backend/data_modeling/tensor_functions.h | 12 ++-- src/backend/utility/macros.h | 25 +++++++++ src/python/py_core/py_core.cpp | 8 ++- src/python/py_core/py_core_util.h | 1 - tests/backend/test_data_modeling.cpp | 58 +++++-------------- tests/backend/test_module.cpp | 5 +- tests/backend/test_train_loop.cpp | 4 +- 11 files changed, 111 insertions(+), 139 deletions(-) create mode 100644 src/backend/utility/macros.h diff --git a/src/backend/data_modeling/dim_type.cpp b/src/backend/data_modeling/dim_type.cpp index b18d4da..43b5612 100644 --- a/src/backend/data_modeling/dim_type.cpp +++ b/src/backend/data_modeling/dim_type.cpp @@ -45,21 +45,15 @@ void Dimension::resize(const std::vector& dims) { /** * @brief Swap along the two given dimensions. */ -void Dimension::swap(const tensorDim_t dim1, const tensorDim_t dim2) { +void Dimension::swap(int dim1, int dim2) { if(dim1==dim2) return; - - auto swapArr = [dim1, dim2](dim_t& arr){ - auto tmp = arr[dim1]; - arr[dim1] = arr[dim2]; - arr[dim2] = tmp; - }; - auto tmp = dims[dim1]; - dims[dim1] = dims[dim2]; - dims[dim2] = tmp; - - swapArr(strides); + auto d1 = mapSignedIdx(dim1); + auto d2 = mapSignedIdx(dim2); + + std::swap(dims[d1], dims[d2]); + std::swap(strides[d1], strides[d2]); } Dimension::Dimension(const vector& dims) @@ -77,51 +71,18 @@ Dimension::Dimension(vector&& dims, dim_t&& strides) assert(size>0); } -Dimension::Dimension(const Dimension& other) - : creationDims{other.creationDims}, creationStrides{other.creationStrides}, dims{other.dims}, - strides{other.strides}, size{other.size}, lastDimIdx{other.lastDimIdx} -{} - -Dimension& Dimension::operator=(const Dimension& other) { - creationDims = other.creationDims; - creationStrides = other.creationStrides; - - dims = other.dims; - strides = other.strides; - - size = other.size; - lastDimIdx = other.lastDimIdx; -} - -Dimension::Dimension(Dimension&& other) noexcept - : creationDims{std::move(other.creationDims)}, creationStrides{std::move(other.creationStrides)}, - dims{std::move(other.dims)}, strides{std::move(other.strides)}, size{other.size}, lastDimIdx{other.lastDimIdx} -{} - -Dimension& Dimension::operator=(Dimension&& other) noexcept { - creationDims = std::move(other.creationDims); - creationStrides = std::move(other.creationStrides); - - dims = std::move(other.dims); - strides = std::move(other.strides); - - size = other.size; - lastDimIdx = other.lastDimIdx; -} - - /** * @brief Computes the strides as they are; */ -dim_t Dimension::makeStrides(const vector& dims) const noexcept { +Dimension::dim_t Dimension::makeStrides(const vector& dims) const noexcept { dim_t res; - const auto lastDimIdx = dims.size()-1; + const int lastDimIdx = dims.size() - 1; - tensorSize_t stride=1; + tensorSize_t stride = 1; res[lastDimIdx] = stride; stride *= dims[lastDimIdx]; - for(tensorDim_t i=lastDimIdx-1; i>=0; i++){ + for(int i = lastDimIdx - 1; i >= 0; i--){ res[i] = stride; stride *= dims[i]; } diff --git a/src/backend/data_modeling/dim_type.h b/src/backend/data_modeling/dim_type.h index 22dc305..f15fb47 100644 --- a/src/backend/data_modeling/dim_type.h +++ b/src/backend/data_modeling/dim_type.h @@ -34,20 +34,28 @@ class Dimension final { tensorDim_t lastDimIdx; // look up end in strides/dims tensorSize_t size = 0; // total size of tensor - dim_t Dimension::makeStrides(const std::vector& dims) const noexcept; + dim_t makeStrides(const std::vector& dims) const noexcept; tensorSize_t multVector(const std::vector& dims) const noexcept; Dimension(std::vector&& dims, dim_t&& strides); + tensorDim_t mapSignedIdx(int idx) const { + if(idx < 0) { + // -1 is last idx, -2 second last and so forth + return lastDimIdx + idx + 1; + } + return idx; + } + public: Dimension(const std::vector& dims); - Dimension(const Dimension& other); - Dimension& operator=(const Dimension& other); + Dimension(const Dimension& other) = default; + Dimension& operator=(const Dimension& other) = default; - Dimension(Dimension&& other) noexcept; - Dimension& operator=(Dimension&& other) noexcept; + Dimension(Dimension&& other) noexcept = default; + Dimension& operator=(Dimension&& other) noexcept = default; ~Dimension() noexcept = default; @@ -69,11 +77,7 @@ class Dimension final { tensorDim_t operator[](int idx) const { assert(size>0); - if(idx<0){ - return dims[lastDimIdx + idx + 1]; // -1 is last idx, -2 second last and so forth - } - - return dims[idx]; + return dims[mapSignedIdx(idx)]; } const auto getStrides() const noexcept { return strides; } @@ -85,7 +89,7 @@ class Dimension final { return dims; } - void swap(tensorDim_t dim1, tensorDim_t dim2); + void swap(int dim1, int dim2); size_t nDims() const noexcept { return dims.size(); diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index e1f090a..aa0d9f3 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -14,6 +14,8 @@ #include "computational_graph/graph_node.h" #include "computational_graph/topological_sort.h" +#include "utility/macros.h" + #include #include #include @@ -85,7 +87,7 @@ void Tensor::tensorValues_t::resize(const tensorSize_t size) { ftype* Tensor::tensorValues_t::getData() const noexcept { #ifndef __CUDA - static_assert(false, "Should only be callable with CUDA enabled"); + assert_debug(false, "Should only be called with CUDA enabled"); #endif if(device==Device::CPU){ @@ -232,11 +234,14 @@ void Tensor::tensorValues_t::setDevice(const Device d) noexcept { void Tensor::tensorValues_t::reset(const ftype x) noexcept { switch(device){ case Device::CPU: - memset(values, x, size); + std::fill(values, values + size, x); break; case Device::CUDA: #ifdef __CUDA - cudaErrchk(cudaMemset(values, x, size * sizeof(ftype))); + cudaErrchk( + thrust::fill(thrust::device_pointer_cast(values), + thrust::device_pointer_cast(values + size), x) + ); #else __throw_runtime_error("Not compiled with CUDA"); #endif @@ -247,14 +252,14 @@ void Tensor::tensorValues_t::reset(const ftype x) noexcept { void Tensor::tensorValues_t::reset(const std::shared_ptr init) noexcept { switch(device){ case Device::CPU: - for(tensorSize_t i=0; idrawNumber(); } break; case Device::CUDA: #ifdef __CUDA auto newValues = static_cast(std::malloc(size * sizeof(ftype))); - for(tensorSize_t i=0; idrawNumber(); } cudaErrchk(cudaMemcpy(values, newValues, size * sizeof(ftype), cudaMemcpyHostToDevice)); @@ -475,7 +480,7 @@ Tensor Tensor::createContiguousCopy() const { return res; } -void Tensor::makeContiguous() { +void Tensor::makeContiguous() const { if(isContiguous()) return; @@ -605,7 +610,7 @@ void Tensor::matMul2DCpu(Tensor& res, const Tensor& left, const Tensor& right, c /** * @brief Matrix multiplication. */ -Tensor Tensor::matmul(Tensor& other) { +Tensor Tensor::matmul(const Tensor& other) const { assert(values->getDevice()==other.values->getDevice()); makeContiguous(); @@ -624,7 +629,7 @@ Tensor Tensor::matmul(Tensor& other) { * 2. The second tensor is a vector. In this case broadcast it. We assume * other.dims == (dimN) && this->dims == (dim0, dim1,..., dimN). */ -Tensor Tensor::operator+(Tensor& other) { +Tensor Tensor::operator+(const Tensor& other) const { if(this->dims != other.dims && !(other.dims.nDims() == 1 && other.dims.get(0) == dims.get(-1))){ __throw_invalid_argument("Tensors need matching dimensions"); @@ -675,14 +680,14 @@ Tensor Tensor::operator+(Tensor& other) { /** * @brief Named version of operator +. */ -Tensor Tensor::add(Tensor& other) { +Tensor Tensor::add(const Tensor& other) const { return *this + other; } /** * @brief Elementwise multiplication. */ -Tensor Tensor::operator*(Tensor& other) { +Tensor Tensor::operator*(const Tensor& other) const { if(this->dims != other.dims){ __throw_invalid_argument("Tensors need same dimensions"); } @@ -716,7 +721,7 @@ Tensor Tensor::operator*(Tensor& other) { /** * @brief Named version of operator *. */ -Tensor Tensor::elementwiseMul(Tensor& other) { +Tensor Tensor::elementwiseMul(const Tensor& other) const { return *this * other; } @@ -887,7 +892,9 @@ Tensor Tensor::getContiguous() const { * @brief Quick transpose operation. */ Tensor Tensor::transpose(int dim1, int dim2) { - dims.swap(dim1, dim2); + Tensor result = createShallowCopy(); + result.dims.swap(dim1, dim2); + return result; } /** @@ -1048,6 +1055,7 @@ ostream& operator<<(ostream& os, const Tensor& t) noexcept { os << "\nDevice: " << DeviceToString(t.values->getDevice()); os << "\nrequiresGrad: " << t.requiresGrad << "\n\n"; + t.makeContiguous(); switch(t.values->getDevice()){ case Device::CPU: printValuesCpu(os, t); @@ -1097,6 +1105,7 @@ tensorSize_t Tensor::computeLinearIdx(const std::vector& idx, const * @brief No explanation needed. */ ftype Tensor::get(const std::vector& idx) const { + makeContiguous(); return (*values)[computeLinearIdx(idx, dims)]; } @@ -1132,6 +1141,7 @@ ftype Tensor::get(tensorDim_t idx0, tensorDim_t idx1, tensorDim_t idx2, tensorDi * @brief No explanation needed. */ void Tensor::set(ftype item, const std::vector& idx) { + makeContiguous(); values->set(item, computeLinearIdx(idx, dims)); } diff --git a/src/backend/data_modeling/tensor.h b/src/backend/data_modeling/tensor.h index 923b508..6109fde 100644 --- a/src/backend/data_modeling/tensor.h +++ b/src/backend/data_modeling/tensor.h @@ -102,8 +102,8 @@ class Tensor final : public std::enable_shared_from_this static Device getDefaultDevice() noexcept; }; - Dimension dims; - std::shared_ptr values = nullptr; // contained values of tensor + mutable Dimension dims; + mutable std::shared_ptr values = nullptr; // contained values of tensor bool requiresGrad = false; std::shared_ptr grads = nullptr; // gradients @@ -114,10 +114,10 @@ class Tensor final : public std::enable_shared_from_this tensorSize_t resOffset, tensorSize_t leftOffset, tensorSize_t rightOffset); - void makeContiguous(); + void makeContiguous() const; // convenience functions that appear in multiple places - static tensorSize_t computeLinearIdx(std::vector&& idx, const Dimension& dims); + static tensorSize_t computeLinearIdx(const std::vector&& idx, const Dimension& dims); static tensorSize_t computeLinearIdx(const std::vector& idx, const Dimension& dims); static tensorDim_t mapDim(int dim, const Dimension& dims); @@ -199,15 +199,15 @@ class Tensor final : public std::enable_shared_from_this tensorSize_t getSize() const noexcept; // Tensor operator@(const Tensor& other) const; in higher C++ versions than 20 - Tensor matmul(Tensor& other); + Tensor matmul(const Tensor& other) const; - Tensor operator+(Tensor& other); - Tensor add(Tensor& other); + Tensor operator+(const Tensor& other) const; + Tensor add(const Tensor& other) const; // TODO: Tensor operator-(const Tensor& other) const; - Tensor operator*(Tensor& t); - Tensor elementwiseMul(Tensor& other); + Tensor operator*(const Tensor& t) const; + Tensor elementwiseMul(const Tensor& other) const; // TODO: Tensor operator/(const Tensor& other) const; diff --git a/src/backend/data_modeling/tensor_functions.h b/src/backend/data_modeling/tensor_functions.h index 2d93811..8403b47 100644 --- a/src/backend/data_modeling/tensor_functions.h +++ b/src/backend/data_modeling/tensor_functions.h @@ -26,14 +26,14 @@ */ namespace TensorFunctions { // class name acts as namespace for us // Tensor creation - Tensor Zeros(std::vector dims, Device d, const bool requiresGrad=false); - Tensor Zeros(std::vector dims, const bool requiresGrad=false); + Tensor Zeros(std::vector dims, Device d, bool requiresGrad=false); + Tensor Zeros(std::vector dims, bool requiresGrad=false); - Tensor Ones(std::vector dims, Device d, const bool requiresGrad=false); - Tensor Ones(std::vector dims, const bool requiresGrad=false); + Tensor Ones(std::vector dims, Device d, bool requiresGrad=false); + Tensor Ones(std::vector dims, bool requiresGrad=false); - Tensor Gaussian(std::vector dims, Device d, ftype stddev, const bool requiresGrad=false); - Tensor Gaussian(std::vector dims, ftype stddev=1, const bool requiresGrad=false); + Tensor Gaussian(std::vector dims, Device d, ftype stddev, bool requiresGrad=false); + Tensor Gaussian(std::vector dims, ftype stddev=1, bool requiresGrad=false); std::shared_ptr makeSharedTensor(const std::vector& dims, bool requiresGrad=false); diff --git a/src/backend/utility/macros.h b/src/backend/utility/macros.h new file mode 100644 index 0000000..e541a8a --- /dev/null +++ b/src/backend/utility/macros.h @@ -0,0 +1,25 @@ +/** + * @file macros.h + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-05-11 + * + * @copyright Copyright (c) 2026 + * + */ + +#pragma once + +#ifdef NDEBUG + #define assert_debug(cond, msg) ((void)0) +#else + #define assert_debug(cond, msg) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "Assertion failed: %s\n Message: %s\n File: %s, Line: %d\n", \ + #cond, msg, __FILE__, __LINE__); \ + abort(); \ + } \ + } while(0) +#endif \ No newline at end of file diff --git a/src/python/py_core/py_core.cpp b/src/python/py_core/py_core.cpp index 91a3936..0e000b6 100644 --- a/src/python/py_core/py_core.cpp +++ b/src/python/py_core/py_core.cpp @@ -222,8 +222,12 @@ BOOST_PYTHON_MODULE(_core) return t->hasGrads(); }) - .def("transpose", WRAP_FREE_MEMBER_FUNC_1(Py_DataModeling::transpose1)) - .def("transpose", WRAP_FREE_MEMBER_FUNC_1(Py_DataModeling::transpose2, int, int)) + .def("transpose", +[](Tensor& self) -> std::shared_ptr { + return std::make_shared(self.transpose()); + }) + .def("transpose", +[](Tensor& self, int dim1, int dim2) -> std::shared_ptr { + return std::make_shared(self.transpose(dim1, dim2)); + }) .def("backward", &Tensor::backward) ; diff --git a/src/python/py_core/py_core_util.h b/src/python/py_core/py_core_util.h index a2d5b7e..230e498 100644 --- a/src/python/py_core/py_core_util.h +++ b/src/python/py_core/py_core_util.h @@ -100,7 +100,6 @@ namespace Py_DataModeling inline void (Tensor::*reset1)(const ftype) = &Tensor::reset; inline void (Tensor::*reset2)(const std::shared_ptr) = &Tensor::reset; - inline Tensor (Tensor::*transpose1)() = &Tensor::transpose; inline Tensor (Tensor::*transpose2)(int, int) = &Tensor::transpose; inline ftype (Tensor::*getItemVector)(const std::vector&) const = &Tensor::get; diff --git a/tests/backend/test_data_modeling.cpp b/tests/backend/test_data_modeling.cpp index 69f585b..512c55b 100644 --- a/tests/backend/test_data_modeling.cpp +++ b/tests/backend/test_data_modeling.cpp @@ -16,6 +16,10 @@ #include +#include + +using namespace std; + TEST(TensorOpsTest, TestCtor) { auto t = Tensor({2, 2}, {2.0, 3.0, 4.0, 5.0}, Device::CPU, false); @@ -206,9 +210,11 @@ TEST(TensorOpsTest, MatMulThrowsWhenDimensionsNotMatched) { } TEST(TensorOpsTest, TransposeWorksAsIntended1) { - auto t = TensorFunctions::Gaussian({3, 2}, false); - auto transposed = t.transpose(-1, -2); - + auto t = TensorFunctions::Gaussian({3, 2}, 1.0, false); + + auto transposed = t.createDeepCopy(); + transposed = transposed.transpose(-1, -2); + ASSERT_EQ(t.getDims().get(-1), transposed.getDims().get(-2)); ASSERT_EQ(t.getDims().get(-2), transposed.getDims().get(-1)); ASSERT_EQ(t.getDims().nDims(), transposed.getDims().nDims()); @@ -224,8 +230,8 @@ TEST(TensorOpsTest, TransposeWorksAsIntended1) { * @brief Swap first two dimensions. */ TEST(TensorOpsTest, TransposeWorksAsIntended2) { - auto t = TensorFunctions::Gaussian({3, 2, 5}, false); - auto transposed = t.transpose(0, 1); + auto t = TensorFunctions::Gaussian({3, 2, 5}, 1.0, false); + auto transposed = t.createDeepCopy().transpose(0, 1); ASSERT_EQ(t.getDims().get(0), transposed.getDims().get(1)); ASSERT_EQ(t.getDims().get(1), transposed.getDims().get(0)); @@ -246,8 +252,8 @@ TEST(TensorOpsTest, TransposeWorksAsIntended2) { * @brief Swap first and last dimension. */ TEST(TensorOpsTest, TransposeWorksAsIntended3) { - auto t = TensorFunctions::Gaussian({3, 2, 5}, false); - auto transposed = t.transpose(0, -1); + auto t = TensorFunctions::Gaussian({3, 2, 5}, 1.0, false); + auto transposed = t.createDeepCopy().transpose(0, -1); ASSERT_EQ(t.getDims().get(0), transposed.getDims().get(-1)); ASSERT_EQ(t.getDims().get(-1), transposed.getDims().get(0)); @@ -263,41 +269,3 @@ TEST(TensorOpsTest, TransposeWorksAsIntended3) { } } } - -TEST(TensorOpsTest, TransposeThisWorksAsIntended1) { - auto t = TensorFunctions::Gaussian({3, 2}, false); - auto tCopy = t.createDeepCopy(); - - t.transposeThis(); - - ASSERT_EQ(t.getDims().get(-1), tCopy.getDims().get(-2)); - ASSERT_EQ(t.getDims().get(-2), tCopy.getDims().get(-1)); - ASSERT_EQ(t.getDims().nDims(), tCopy.getDims().nDims()); - - for(auto row=0; row(); auto optim = make_shared( - net->parameters(), /*lr=*/0.0003, /*decay=*/0.95); + net->parameters(), /*lr=*/0.0001, /*decay=*/0.95); auto trainLoop = train::BaseTrainLoop( - net, loss, optim, /*epochs=*/10000, /*bsize=*/6); + net, loss, optim, /*epochs=*/2000, /*bsize=*/6); trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); From 21483d2e399f5d8c542669b3a38c6ba563f89b26 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Tue, 12 May 2026 09:50:23 +0200 Subject: [PATCH 20/73] Fixing some cuda errors in tensor.cpp --- src/backend/data_modeling/tensor.cpp | 37 +++------------------------- src/backend/data_modeling/tensor.h | 3 --- 2 files changed, 3 insertions(+), 37 deletions(-) diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index aa0d9f3..70aba5b 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -25,6 +25,7 @@ #ifdef __CUDA #include "utility/cuda/cuda_common.cuh" #include "data_modeling/cuda/tensorops.cuh" + #endif using namespace std; @@ -238,10 +239,7 @@ void Tensor::tensorValues_t::reset(const ftype x) noexcept { break; case Device::CUDA: #ifdef __CUDA - cudaErrchk( - thrust::fill(thrust::device_pointer_cast(values), - thrust::device_pointer_cast(values + size), x) - ); + cuda_impl::scalarFill(values, x, size); #else __throw_runtime_error("Not compiled with CUDA"); #endif @@ -1021,35 +1019,6 @@ void printValuesCpu(std::ostream& os, const Tensor& t) { } } -/** - * @brief Print out the first few values of the flattened array. - */ -#ifdef __CUDA -void printValuesCuda(std::ostream& os, const Tensor& t) { - __throw_logic_error("printValuesCuda should not be reachable when not compiled with CUDA"); - auto printVals = [&os](const Tensor& t){ - constexpr auto MAX_IDX = static_cast(10); - - const auto maxIdx = min(MAX_IDX, t.values->getSize()); - auto tmp = static_cast(std::malloc(t.getSize() * sizeof(ftype))); - cudaErrchk(cudaMemcpy(tmp, t.getData(), maxIdx*sizeof(ftype), cudaMemcpyDeviceToHost)); - - for(tensorSize_t i=0; igetDevice()); @@ -1062,7 +1031,7 @@ ostream& operator<<(ostream& os, const Tensor& t) noexcept { break; case Device::CUDA: #ifdef __CUDA - printValuesCuda(os, t); + cuda_impl::printValues(os, t); #else __throw_runtime_error("Not compiled with CUDA"); #endif diff --git a/src/backend/data_modeling/tensor.h b/src/backend/data_modeling/tensor.h index 6109fde..2bc1191 100644 --- a/src/backend/data_modeling/tensor.h +++ b/src/backend/data_modeling/tensor.h @@ -123,9 +123,6 @@ class Tensor final : public std::enable_shared_from_this static tensorDim_t mapDim(int dim, const Dimension& dims); friend void printValuesCpu(std::ostream& os, const Tensor& t); - #ifdef __CUDA - friend void printValuesCuda(std::ostream& os, const Tensor& t); - #endif Tensor(const Tensor& other, shallowCopyToken); From 82b3c382b4f10371b6eed6f75cf84c767a490a2a Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Tue, 12 May 2026 10:02:46 +0200 Subject: [PATCH 21/73] Fix some more compile time bugs in cuda backend --- CMakeLists.txt | 2 +- .../loss_functions/cuda/loss_nodes.cuh | 2 +- src/backend/data_modeling/cuda/tensorops.cu | 62 ++++++++++++++++--- src/backend/data_modeling/cuda/tensorops.cuh | 13 +++- .../activation_functions/cuda/activations.cu | 15 ++--- src/backend/module/module_base.cpp | 8 ++- .../loss_functions/cuda/loss_functions.cu | 11 ++-- 7 files changed, 83 insertions(+), 30 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f4c3db7..08f8d3b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,7 @@ if (CUDA) add_definitions(-D__CUDA) enable_language(CUDA) - set(CMAKE_CUDA_STANDARD 17) + set(CMAKE_CUDA_STANDARD 20) set(CMAKE_CUDA_STANDARD_REQUIRED ON) else() message(WARNING "Could not find CUDA on system. Exiting.") diff --git a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cuh b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cuh index 773de53..95e313d 100644 --- a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cuh +++ b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cuh @@ -17,7 +17,7 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "utility/global_params.h" -namespace cuda { +namespace cuda_impl { __global__ void bceBackward(ftype* res, const ftype* const upstreamGrad, tensorSize_t size); __global__ void bceWithSigmoidBackward(ftype* res, const ftype* const upstreamGrad, ftype eps, tensorSize_t size); diff --git a/src/backend/data_modeling/cuda/tensorops.cu b/src/backend/data_modeling/cuda/tensorops.cu index 80048e0..c59ccf4 100644 --- a/src/backend/data_modeling/cuda/tensorops.cu +++ b/src/backend/data_modeling/cuda/tensorops.cu @@ -17,6 +17,8 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include "utility/cuda/cuda_common.cuh" #include "cuda_runtime.h" +#include +#include #include "data_modeling/tensor.h" @@ -63,6 +65,12 @@ namespace{ res[gid] = left[gid] + scalar; } + __global__ void matMulKernel(ftype* res, const ftype* const left, const ftype* const right, + const tensorDim_t leftRows, const tensorDim_t leftCols, tensorDim_t rightRows, tensorDim_t rightCols) { + int gid = blockDim.x * blockIdx.x + threadIdx.x; + if() + } + /** * @brief Create a contiguous copy of src and copy it into dst. Used for reshaping, transposing, etc. * @@ -89,7 +97,7 @@ namespace{ } } -namespace cuda { +namespace cuda_impl { void scalaradd(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size) { constexpr int threadsPerBlock = 256; const int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; @@ -106,11 +114,12 @@ namespace cuda { cudaErrchk(cudaDeviceSynchronize()); } - void broadcastadd(Tensor& res, const Tensor& matrix, const Tensor& vec){ - const auto nBytes = src.getSize(); + // TODO: fix this one + void broadcastadd(Tensor& res, const Tensor& matrix, const Tensor& vec) { + const auto size = res.getSize(); constexpr int threadsPerBlock = 256; - const int blocksPerGrid = (nBytes + threadsPerBlock - 1) / threadsPerBlock; + const int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; broadcastaddKernel<<>>( res.getData(), matrix.getData(), vec.getData(), vec.getDims()[0], matrix.getSize()); @@ -132,8 +141,41 @@ namespace cuda { cudaErrchk(cudaDeviceSynchronize()); } - void matmul(ftype* res, const ftype* const left, const ftype* const right) { - static_assert(false); + void matmul(ftype* res, const ftype* const left, const ftype* const right, + const tensorDim_t resRows, const tensorDim_t resCols, const tensorSize_t resSize) { +/* constexpr int threadsPerBlock = 256; + const int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; + + matMulKernel<<>>(res, left, right, size); + cudaErrchk(cudaDeviceSynchronize()); */ + } + + void scalarFill(ftype* ptr, ftype value, tensorSize_t size) { + thrust::fill(thrust::device_pointer_cast(ptr), + thrust::device_pointer_cast(ptr + size), value); + cudaErrchk(cudaDeviceSynchronize()); + } + + void printValues(std::ostream& os, const Tensor& t) { + auto printVals = [&os](const Tensor& t) { + constexpr auto MAX_IDX = static_cast(10); + const auto maxIdx = min(MAX_IDX, t.getSize()); + + auto tmp = static_cast(std::malloc(t.getSize() * sizeof(ftype))); + cudaErrchk(cudaMemcpy(tmp, t.getData(), maxIdx * sizeof(ftype), cudaMemcpyDeviceToHost)); + + for(tensorSize_t i = 0; i < maxIdx; i++) + os << tmp[i]; + os << "\n\n"; + + free(tmp); + }; + + printVals(t); + if(t.hasGrads()) { + os << "\n\nGrads:\n"; + printVals(*t.getGrads()); + } } ftype get(const ftype* const t, tensorSize_t idx) { @@ -149,10 +191,10 @@ namespace cuda { void createContiguousCopy(Tensor& res, const Tensor& src) { assert(res.getSize()==src.getSize()); - ftype* dst = res.getData() + ftype* dst = res.getData(); const ftype* const srcData = src.getData(); - const auto oldStrides = src.getDims().getCreationStrides().data(); + const auto oldStrides = src.getDims().getOriginalStrides().data(); const auto newStrides = src.getDims().getStrides().data(); const auto nBytes = src.getSize(); @@ -160,8 +202,8 @@ namespace cuda { constexpr int threadsPerBlock = 256; const int blocksPerGrid = (nBytes + threadsPerBlock - 1) / threadsPerBlock; - cudaErrchk(createContiguousCopyKernel<<>>( - dst, srcData, oldStrides, newStrides, dims.nDims(), nBytes)); + createContiguousCopyKernel<<>>( + dst, srcData, oldStrides, newStrides, src.getDims().nDims(), nBytes); cudaErrchk(cudaDeviceSynchronize()); } } diff --git a/src/backend/data_modeling/cuda/tensorops.cuh b/src/backend/data_modeling/cuda/tensorops.cuh index 7f7ff23..41bbd7e 100644 --- a/src/backend/data_modeling/cuda/tensorops.cuh +++ b/src/backend/data_modeling/cuda/tensorops.cuh @@ -18,8 +18,11 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "utility/global_params.h" #include "data_modeling/dim_type.h" -namespace cuda { - class Tensor; +#include + +class Tensor; + +namespace cuda_impl { // scalar ops void scalaradd(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size); @@ -29,11 +32,15 @@ namespace cuda { void elementwiseadd(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size); void broadcastadd(Tensor& res, const Tensor& matrix, const Tensor& vec); void elementwisemul(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size); - void matmul(ftype* res, const ftype* const left, const ftype* const right); + void matmul(Tensor& res, const Tensor& left, const Tensor& right); void transpose2D(ftype* res, const ftype* const src, Dimension dims, tensorDim_t dim1, tensorDim_t dim2); void transpose(ftype* res, const ftype* const src, Dimension dims, tensorDim_t dim1, tensorDim_t dim2); + void scalarFill(ftype* ptr, ftype value, tensorSize_t size); + + void printValues(std::ostream& os, const Tensor& t); + // other ftype get(const ftype* const t, tensorSize_t idx); ftype set(ftype value, const ftype* t, tensorSize_t idx); diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index a00e20f..d9af1b1 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -97,7 +97,8 @@ namespace { } // TODO: use shared memory - __global__ void expAndSumKernel(ftype* res, ftype* const tmp, const ftype* const input, const ftype* const maxValues, + template + __global__ void expAndSumKernel(T* res, T* const tmp, const T* const input, const T* const maxValues, const tensorSize_t stride, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) @@ -105,14 +106,14 @@ namespace { const auto strideOffset = gid / stride; const auto maxValue = maxValues[strideOffset]; - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { res[gid] = expf(input[gid] - maxValue); } - else if constexpr (std::is_same_v) { + else if constexpr (std::is_same_v) { res[gid] = exp(input[gid] - maxValue); } else { - static_assert(always_false, "ftype encountered unexpected type"); + static_assert(always_false, "ftype encountered unexpected type"); } __syncthreads(); @@ -160,7 +161,7 @@ namespace { } -namespace cuda { +namespace cuda_impl { void relu(Tensor& res, const Tensor& in) { constexpr int threadsPerBlock = 256; const int blocks = (in.getSize()+threadsPerBlock-1) / threadsPerBlock; @@ -204,10 +205,10 @@ namespace cuda { findMaxKernel<<>>(maxValues, tmp, stride, in.getSize()); cudaErrchk(cudaDeviceSynchronize()); - expAndSumKernel<<>>(res.getData(), tmp, in.getData(), maxValues, stride, in.getSize()); + expAndSumKernel<<>>(res.getData(), tmp, in.getData(), maxValues, stride, in.getSize()); cudaErrchk(cudaDeviceSynchronize()); - softmaxDivisionKernel(res.getData(), tmp, stride, in.getSize()); + softmaxDivisionKernel<<>>(res.getData(), tmp, stride, in.getSize()); cudaErrchk(cudaDeviceSynchronize()); cudaErrchk(cudaFree(tmp)); diff --git a/src/backend/module/module_base.cpp b/src/backend/module/module_base.cpp index 951f96c..fdbcbbb 100644 --- a/src/backend/module/module_base.cpp +++ b/src/backend/module/module_base.cpp @@ -15,7 +15,9 @@ using namespace std; -ostream& module::operator<<(ostream& os, const module::ModuleBase& l) noexcept { - l.print(os); // calling vtable - return os; +namespace module { + ostream& operator<<(ostream& os, const ModuleBase& l) noexcept { + l.print(os); // calling vtable + return os; + } } \ No newline at end of file diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cu b/src/backend/training/loss_functions/cuda/loss_functions.cu index 55ee006..68dd776 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cu +++ b/src/backend/training/loss_functions/cuda/loss_functions.cu @@ -22,15 +22,16 @@ static_assert(false, "File should not be compiled without CUDA enabled"); using namespace std; namespace { - __forceinline__ __device__ ftype bce(ftype y, ftype ypred) { - if constexpr (std::is_same_v) { + template + __forceinline__ __device__ T bce(T y, T ypred) { + if constexpr (std::is_same_v) { return y * __logf(max(ypred, EPS_BCE)) + (1 - y) * __logf(max(1-ypred, EPS_BCE)); } - else if constexpr (std::is_same_v) { + else if constexpr (std::is_same_v) { return y * log(max(ypred, EPS_BCE)) + (1 - y) * log(max(1-ypred, EPS_BCE)); } else { - static_assert(always_false, "Unexpected value for ftype"); + static_assert(always_false, "Unexpected value for ftype"); } } @@ -45,7 +46,7 @@ namespace { // pre-load first round { int i = blockIdx.x * (blockDim.x * 2) + threadIdx.x; - sdata[tid] = bce(y[i], ypred[i]) + bce(y[i + blockDim.x], ypred[i + blockDim.x]); + sdata[tid] = bce(y[i], ypred[i]) + bce(y[i + blockDim.x], ypred[i + blockDim.x]); __syncthreads(); } From c23ef9a7389edac3d5e1218a9a589f0200da8ee2 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Tue, 12 May 2026 10:46:49 +0200 Subject: [PATCH 22/73] Resolved compile time errors --- src/backend/data_modeling/cuda/tensorops.cu | 40 ++--- src/backend/data_modeling/cuda/tensorops.cuh | 14 +- src/backend/data_modeling/tensor.cpp | 152 ++++++++---------- src/backend/data_modeling/tensor.h | 12 +- .../loss_functions/cuda/loss_functions.cuh | 10 +- 5 files changed, 102 insertions(+), 126 deletions(-) diff --git a/src/backend/data_modeling/cuda/tensorops.cu b/src/backend/data_modeling/cuda/tensorops.cu index c59ccf4..00be34f 100644 --- a/src/backend/data_modeling/cuda/tensorops.cu +++ b/src/backend/data_modeling/cuda/tensorops.cu @@ -68,7 +68,7 @@ namespace{ __global__ void matMulKernel(ftype* res, const ftype* const left, const ftype* const right, const tensorDim_t leftRows, const tensorDim_t leftCols, tensorDim_t rightRows, tensorDim_t rightCols) { int gid = blockDim.x * blockIdx.x + threadIdx.x; - if() + //if() } /** @@ -98,19 +98,19 @@ namespace{ } namespace cuda_impl { - void scalaradd(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size) { + void scalaradd(Tensor& res, const Tensor& src, ftype scalar) { constexpr int threadsPerBlock = 256; - const int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; + const int blocksPerGrid = (src.getSize() + threadsPerBlock - 1) / threadsPerBlock; - scalaraddKernel<<>>(res, left, scalar, size); + scalaraddKernel<<>>(res.getData(), src.getData(), scalar, src.getSize()); cudaErrchk(cudaDeviceSynchronize()); } - void scalarmul(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size) { + void scalarmul(Tensor& res, const Tensor& src, ftype scalar) { constexpr int threadsPerBlock = 256; - const int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; + const int blocksPerGrid = (src.getSize() + threadsPerBlock - 1) / threadsPerBlock; - scalarmulKernel<<>>(res, left, scalar, size); + scalarmulKernel<<>>(res.getData(), src.getData(), scalar, src.getSize()); cudaErrchk(cudaDeviceSynchronize()); } @@ -126,23 +126,22 @@ namespace cuda_impl { cudaErrchk(cudaDeviceSynchronize()); } - void elementwiseadd(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size) { + void elementwiseadd(Tensor& res, const Tensor& left, const Tensor& right) { constexpr int threadsPerBlock = 256; - int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; - elementwiseaddKernel<<>>(res, left, right, size); + const int blocksPerGrid = (left.getSize() + threadsPerBlock - 1) / threadsPerBlock; + elementwiseaddKernel<<>>(res.getData(), left.getData(), right.getData(), left.getSize()); cudaErrchk(cudaDeviceSynchronize()); } - void elementwisemul(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size) { + void elementwisemul(Tensor& res, const Tensor& left, const Tensor& right) { constexpr int threadsPerBlock = 256; - const int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; - - elementwisemulKernel<<>>(res, left, right, size); + const int blocksPerGrid = (left.getSize() + threadsPerBlock - 1) / threadsPerBlock; + + elementwisemulKernel<<>>(res.getData(), left.getData(), right.getData(), left.getSize()); cudaErrchk(cudaDeviceSynchronize()); } - void matmul(ftype* res, const ftype* const left, const ftype* const right, - const tensorDim_t resRows, const tensorDim_t resCols, const tensorSize_t resSize) { + void matmul(Tensor& res, const Tensor& left, const Tensor& right) { /* constexpr int threadsPerBlock = 256; const int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; @@ -150,9 +149,10 @@ namespace cuda_impl { cudaErrchk(cudaDeviceSynchronize()); */ } - void scalarFill(ftype* ptr, ftype value, tensorSize_t size) { + void scalarFill(Tensor& t, ftype value) { + ftype* ptr = t.getData(); thrust::fill(thrust::device_pointer_cast(ptr), - thrust::device_pointer_cast(ptr + size), value); + thrust::device_pointer_cast(ptr + t.getSize()), value); cudaErrchk(cudaDeviceSynchronize()); } @@ -184,8 +184,8 @@ namespace cuda_impl { return res; } - ftype set(ftype value, const ftype* t, tensorSize_t idx) { - cudaErrchk(cudaMemcpy((void*)t+idx, &value, sizeof(ftype), cudaMemcpyHostToDevice)); + void set(ftype value, const ftype* t, tensorSize_t idx) { + cudaErrchk(cudaMemcpy((void*)(t + idx), &value, sizeof(ftype), cudaMemcpyHostToDevice)); } void createContiguousCopy(Tensor& res, const Tensor& src) { diff --git a/src/backend/data_modeling/cuda/tensorops.cuh b/src/backend/data_modeling/cuda/tensorops.cuh index 41bbd7e..6c8f395 100644 --- a/src/backend/data_modeling/cuda/tensorops.cuh +++ b/src/backend/data_modeling/cuda/tensorops.cuh @@ -25,25 +25,25 @@ class Tensor; namespace cuda_impl { // scalar ops - void scalaradd(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size); - void scalarmul(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size); + void scalaradd(Tensor& res, const Tensor& src, ftype scalar); + void scalarmul(Tensor& res, const Tensor& src, ftype scalar); // matrix ops - void elementwiseadd(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size); + void elementwiseadd(Tensor& res, const Tensor& left, const Tensor& right); void broadcastadd(Tensor& res, const Tensor& matrix, const Tensor& vec); - void elementwisemul(ftype* res, const ftype* const left, const ftype* const right, tensorSize_t size); + void elementwisemul(Tensor& res, const Tensor& left, const Tensor& right); void matmul(Tensor& res, const Tensor& left, const Tensor& right); void transpose2D(ftype* res, const ftype* const src, Dimension dims, tensorDim_t dim1, tensorDim_t dim2); void transpose(ftype* res, const ftype* const src, Dimension dims, tensorDim_t dim1, tensorDim_t dim2); - void scalarFill(ftype* ptr, ftype value, tensorSize_t size); + void scalarFill(Tensor& t, ftype value); void printValues(std::ostream& os, const Tensor& t); // other - ftype get(const ftype* const t, tensorSize_t idx); - ftype set(ftype value, const ftype* t, tensorSize_t idx); + [[nodiscard]] ftype get(const ftype* const t, tensorSize_t idx); + void set(ftype value, const ftype* t, tensorSize_t idx); void createContiguousCopy(Tensor& res, const Tensor& src); } diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index 70aba5b..02c64a2 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -86,16 +86,6 @@ void Tensor::tensorValues_t::resize(const tensorSize_t size) { } } -ftype* Tensor::tensorValues_t::getData() const noexcept { -#ifndef __CUDA - assert_debug(false, "Should only be called with CUDA enabled"); -#endif - - if(device==Device::CPU){ - __throw_runtime_error("Should only be called on CUDA tensor."); - } - return values; -} /** * @brief Copy from pointer into this object. @@ -232,43 +222,6 @@ void Tensor::tensorValues_t::setDevice(const Device d) noexcept { #endif } -void Tensor::tensorValues_t::reset(const ftype x) noexcept { - switch(device){ - case Device::CPU: - std::fill(values, values + size, x); - break; - case Device::CUDA: - #ifdef __CUDA - cuda_impl::scalarFill(values, x, size); - #else - __throw_runtime_error("Not compiled with CUDA"); - #endif - break; - } -} - -void Tensor::tensorValues_t::reset(const std::shared_ptr init) noexcept { - switch(device){ - case Device::CPU: - for(tensorSize_t i = 0; i < size; i++){ - values[i] = init->drawNumber(); - } - break; - case Device::CUDA: - #ifdef __CUDA - auto newValues = static_cast(std::malloc(size * sizeof(ftype))); - for(tensorSize_t i = 0; i < size; i++){ - newValues[i] = init->drawNumber(); - } - cudaErrchk(cudaMemcpy(values, newValues, size * sizeof(ftype), cudaMemcpyHostToDevice)); - free(newValues); - // TODO: better initialize directly on GPU - #else - __throw_runtime_error("Not compiled with CUDA"); - #endif - break; - } -} Device Tensor::tensorValues_t::getDevice() const noexcept { return device; @@ -290,32 +243,6 @@ Tensor::tensorValues_t::operator bool() const noexcept { return values != nullptr; } -void Tensor::tensorValues_t::addOtherCpu(const Tensor::tensorValues_t& other) noexcept { - for(tensorSize_t i=0; isize; i++){ - this->values[i] += other.values[i]; - } -} - -Tensor::tensorValues_t& -Tensor::tensorValues_t::operator+=(const Tensor::tensorValues_t& other) { - assert(this->size==other.size && this->device == other.device); - - switch(device) { - case Device::CPU: - addOtherCpu(other); - break; - case Device::CUDA: - #ifdef __CUDA - cuda_impl::elementwiseadd(values, values, other.values, size); - #else - __throw_invalid_argument("Not compiled with CUDA"); - #endif - break; - } - - return *this; -} - ftype& Tensor::tensorValues_t::operator[](const tensorSize_t idx) { if(idx >= size) throw std::out_of_range("Out of range for tensor"); @@ -514,7 +441,9 @@ Tensor Tensor::createDeepCopy() const { * Avoids moving all CUDA code into tensor. */ ftype* Tensor::getData() const noexcept { - return values->getData(); + if(values->getDevice() == Device::CPU) + __throw_runtime_error("Should only be called on CUDA tensor."); + return values->data(); } /** @@ -662,7 +591,7 @@ Tensor Tensor::operator+(const Tensor& other) const { case Device::CUDA: #ifdef __CUDA if(dims==other.dims) [[unlikely]] { - cuda_impl::elementwiseadd(res.getData(), values->getData(), other.getData(), values->getSize()); + cuda_impl::elementwiseadd(res, *this, other); } else [[likely]] { cuda_impl::broadcastadd(res, *this, other); @@ -705,8 +634,7 @@ Tensor Tensor::operator*(const Tensor& other) const { break; case Device::CUDA: #ifdef __CUDA - cuda_impl::elementwisemul(res.getData(), values->getData(), - other.values->getData(), values->getSize()); + cuda_impl::elementwisemul(res, *this, other); #else __throw_runtime_error("Not compiled with CUDA"); #endif @@ -723,6 +651,25 @@ Tensor Tensor::elementwiseMul(const Tensor& other) const { return *this * other; } +Tensor& Tensor::operator+=(const Tensor& other) { + assert(dims == other.dims && values->getDevice() == other.values->getDevice()); + + switch(values->getDevice()) { + case Device::CPU: + for(tensorSize_t i = 0; i < values->getSize(); i++) + values->data()[i] += other.values->data()[i]; + break; + case Device::CUDA: + #ifdef __CUDA + cuda_impl::elementwiseadd(*this, *this, other); + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif + break; + } + return *this; +} + Tensor Tensor::operator*(const ftype scalar) const { Tensor res(dims, values->getDevice(), false); switch(values->getDevice()){ @@ -733,8 +680,8 @@ Tensor Tensor::operator*(const ftype scalar) const { break; case Device::CUDA: #ifdef __CUDA - cuda_impl::scalarmul(res.getData(), values->getData(), scalar, values->getSize()); - #else + cuda_impl::scalarmul(res, *this, scalar); + #else __throw_runtime_error("Not compiled with CUDA"); #endif break; @@ -757,7 +704,7 @@ Tensor Tensor::operator/(const ftype scalar) const { break; case Device::CUDA: #ifdef __CUDA - cuda_impl::scalarmul(res.getData(), values->getData(), 1 / scalar, values->getSize()); + cuda_impl::scalarmul(res, *this, 1 / scalar); #else __throw_runtime_error("Not compiled with CUDA"); #endif @@ -777,8 +724,8 @@ Tensor Tensor::operator+(const ftype scalar) const { break; case Device::CUDA: #ifdef __CUDA - cuda_impl::scalaradd(res.getData(), values->getData(), scalar, values->getSize()); - #else + cuda_impl::scalaradd(res, *this, scalar); + #else __throw_runtime_error("Not compiled with CUDA"); #endif break; @@ -797,7 +744,7 @@ Tensor Tensor::operator-(const ftype scalar) const { break; case Device::CUDA: #ifdef __CUDA - cuda_impl::scalaradd(res.getData(), values->getData(), -scalar, values->getSize()); + cuda_impl::scalaradd(res, *this, -scalar); #else __throw_runtime_error("Not compiled with CUDA"); #endif @@ -848,7 +795,7 @@ void Tensor::backward() { parent->grads = incomingGrads[i]; } else{ - *parent->grads->values += *incomingGrads[i]->values; + *parent->grads += *incomingGrads[i]; } } } @@ -912,14 +859,47 @@ void Tensor::permute(const std::vector& newOrder) noexcept { * @brief Populates the tensor with value. */ void Tensor::reset(const ftype x) noexcept { - values->reset(x); + switch(values->getDevice()) { + case Device::CPU: + std::fill(values->data(), values->data() + values->getSize(), x); + break; + case Device::CUDA: + #ifdef __CUDA + cuda_impl::scalarFill(*this, x); + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif + break; + } } /** * @brief Populates the tensor with values drawn according to initializer. */ void Tensor::reset(shared_ptr init) noexcept { - values->reset(std::move(init)); + const auto size = values->getSize(); + switch(values->getDevice()) { + case Device::CPU: + for(tensorSize_t i = 0; i < size; i++){ + values->data()[i] = init->drawNumber(); + } + break; + case Device::CUDA: + #ifdef __CUDA + { + auto newValues = static_cast(std::malloc(size * sizeof(ftype))); + for(tensorSize_t i = 0; i < size; i++){ + newValues[i] = init->drawNumber(); + } + cudaErrchk(cudaMemcpy(values->data(), newValues, size * sizeof(ftype), cudaMemcpyHostToDevice)); + free(newValues); + // TODO: better initialize directly on GPU + } + #else + __throw_runtime_error("Not compiled with CUDA"); + #endif + break; + } } const Dimension& Tensor::getDims() const noexcept { diff --git a/src/backend/data_modeling/tensor.h b/src/backend/data_modeling/tensor.h index 2bc1191..aa2a0a9 100644 --- a/src/backend/data_modeling/tensor.h +++ b/src/backend/data_modeling/tensor.h @@ -58,8 +58,6 @@ class Tensor final : public std::enable_shared_from_this Device device; inline static Device defaultDevice = Device::CPU; - void addOtherCpu(const tensorValues_t& other) noexcept; - public: explicit tensorValues_t(); explicit tensorValues_t(Device d); @@ -73,7 +71,8 @@ class Tensor final : public std::enable_shared_from_this void copyFromRaw(const ftype* src, tensorSize_t n); - ftype* getData() const noexcept; + ftype* data() noexcept { return values; } + const ftype* data() const noexcept { return values; } explicit operator bool() const noexcept; ftype& operator[](tensorSize_t idx); ftype operator[](tensorSize_t idx) const; @@ -83,16 +82,11 @@ class Tensor final : public std::enable_shared_from_this tensorSize_t getSize() const noexcept; - // needed for gradient descent - tensorValues_t& operator+=(const tensorValues_t& other); - void resize(tensorSize_t size); void setDevice(Device d) noexcept; Device getDevice() const noexcept; - void reset(ftype x) noexcept; - void reset(std::shared_ptr init) noexcept; void copyValues(tensorValues_t& target) const; void copyValues(tensorValues_t& target, tensorSize_t low, tensorSize_t high, tensorSize_t targetOffset) const; @@ -206,6 +200,8 @@ class Tensor final : public std::enable_shared_from_this Tensor operator*(const Tensor& t) const; Tensor elementwiseMul(const Tensor& other) const; + Tensor& operator+=(const Tensor& other); + // TODO: Tensor operator/(const Tensor& other) const; // the following operators all broadcast diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cuh b/src/backend/training/loss_functions/cuda/loss_functions.cuh index 8c43f09..f88fdad 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cuh +++ b/src/backend/training/loss_functions/cuda/loss_functions.cuh @@ -18,11 +18,11 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "data_modeling/tensor.h" namespace cuda_impl { - Tensor&& bceLoss(const Tensor& y, const Tensor& yPred); - Tensor&& bceSigmoidLoss(const Tensor& y, const Tensor& yPred); + [[nodiscard]] Tensor&& bceLoss(const Tensor& y, const Tensor& yPred); + [[nodiscard]] Tensor&& bceSigmoidLoss(const Tensor& y, const Tensor& yPred); - Tensor&& crossEntropyLoss(const Tensor& y, const Tensor& yPred); - Tensor&& crossEntropySoftmaxLoss(const Tensor& y, const Tensor& yPred); + [[nodiscard]] Tensor&& crossEntropyLoss(const Tensor& y, const Tensor& yPred); + [[nodiscard]] Tensor&& crossEntropySoftmaxLoss(const Tensor& y, const Tensor& yPred); - Tensor&& rmseLoss(const Tensor& y, const Tensor& yPred); + [[nodiscard]] Tensor&& rmseLoss(const Tensor& y, const Tensor& yPred); } \ No newline at end of file From 4f0065fa6a7ddcb60d4f01ccc758f60f1dbdb5b1 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Tue, 12 May 2026 11:27:36 +0200 Subject: [PATCH 23/73] Resolved compile time and runtime issues of unit tests --- src/backend/data_modeling/cuda/tensorops.cu | 2 +- src/backend/data_modeling/device.h | 8 +++- src/backend/data_modeling/tensor.cpp | 45 ++++++++------------- src/backend/data_modeling/tensor.h | 6 +-- tests/CMakeLists.txt | 24 +++++++++++ tests/backend/cuda/test_tensorops_cuda.cu | 28 +++++++++++++ 6 files changed, 80 insertions(+), 33 deletions(-) create mode 100644 tests/backend/cuda/test_tensorops_cuda.cu diff --git a/src/backend/data_modeling/cuda/tensorops.cu b/src/backend/data_modeling/cuda/tensorops.cu index 00be34f..8456e07 100644 --- a/src/backend/data_modeling/cuda/tensorops.cu +++ b/src/backend/data_modeling/cuda/tensorops.cu @@ -180,7 +180,7 @@ namespace cuda_impl { ftype get(const ftype* const t, tensorSize_t idx) { ftype res; - cudaErrchk(cudaMemcpy(&res, t+idx, sizeof(ftype), cudaMemcpyDeviceToHost)); + cudaErrchk(cudaMemcpy((void*)&res, t + idx, sizeof(ftype), cudaMemcpyDeviceToHost)); return res; } diff --git a/src/backend/data_modeling/device.h b/src/backend/data_modeling/device.h index 65f83a5..ea50669 100644 --- a/src/backend/data_modeling/device.h +++ b/src/backend/data_modeling/device.h @@ -11,9 +11,15 @@ #pragma once +#include + enum class Device { CPU, CUDA }; -const char* DeviceToString(Device d); \ No newline at end of file +const char* DeviceToString(Device d); + +inline std::ostream& operator<<(std::ostream& os, Device d) { + return os << DeviceToString(d); +} \ No newline at end of file diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index 02c64a2..cb731a6 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -25,7 +25,6 @@ #ifdef __CUDA #include "utility/cuda/cuda_common.cuh" #include "data_modeling/cuda/tensorops.cuh" - #endif using namespace std; @@ -243,15 +242,6 @@ Tensor::tensorValues_t::operator bool() const noexcept { return values != nullptr; } -ftype& Tensor::tensorValues_t::operator[](const tensorSize_t idx) { - if(idx >= size) - throw std::out_of_range("Out of range for tensor"); - - if(device!=Device::CPU){ - __throw_invalid_argument("'ftype& operator[] const' only implemented for CPU"); - } - return values[idx]; -} ftype Tensor::tensorValues_t::operator[](const tensorSize_t idx) const { if(idx >= size) @@ -513,21 +503,21 @@ void Tensor::matMul2DCpu(Tensor& res, const Tensor& left, const Tensor& right, c const auto nRowsRight = static_cast(right.dims.get(-2)); const auto nColsRight = static_cast(right.dims.get(-1)); - for(tensorSize_t lrow=0; lrowdata()[resRowOffset + rrow] = left.values->data()[leftRowOffset] * right.values->data()[rightIdx]; rightIdx++; } for(tensorSize_t lcol=1; lcoldata()[resRowOffset + rrow] += left.values->data()[leftIdx] * right.values->data()[rightIdx]; rightIdx++; } } @@ -573,17 +563,17 @@ Tensor Tensor::operator+(const Tensor& other) const { case Device::CPU: if(dims==other.dims) [[unlikely]] { // elementwise add - for(tensorSize_t i=0; igetSize(); i++){ - (*res.values)[i] = (*values)[i] + (*other.values)[i]; + for(tensorSize_t i = 0; i < values->getSize(); i++){ + res.values->data()[i] = values->data()[i] + other.values->data()[i]; } return res; } - else [[likely]] { + else [[likely]] { // broadcasted add const auto stride = static_cast(other.dims.get(0)); // other is a vector - for(tensorSize_t offset=0; offsetgetSize(); offset+=stride){ - for(tensorSize_t i=0; igetSize(); offset += stride){ + for(tensorSize_t i = 0; i < stride; i++){ + res.values->data()[offset + i] = values->data()[offset + i] + other.values->data()[i]; } } } @@ -628,8 +618,8 @@ Tensor Tensor::operator*(const Tensor& other) const { switch(values->getDevice()){ case Device::CPU: - for(tensorSize_t i=0; igetSize(); i++){ - (*res.values)[i] = (*values)[i] * (*other.values)[i]; + for(tensorSize_t i = 0; i < values->getSize(); i++){ + res.values->data()[i] = values->data()[i] * other.values->data()[i]; } break; case Device::CUDA: @@ -675,7 +665,7 @@ Tensor Tensor::operator*(const ftype scalar) const { switch(values->getDevice()){ case Device::CPU: for (tensorSize_t i = 0; i < values->getSize(); ++i) { - (*res.values)[i] = (*values)[i] * scalar; + res.values->data()[i] = values->data()[i] * scalar; } break; case Device::CUDA: @@ -699,7 +689,7 @@ Tensor Tensor::operator/(const ftype scalar) const { switch(values->getDevice()){ case Device::CPU: for (tensorSize_t i = 0; i < values->getSize(); ++i) { - (*res.values)[i] = (*values)[i] / scalar; + res.values->data()[i] = values->data()[i] / scalar; } break; case Device::CUDA: @@ -719,7 +709,7 @@ Tensor Tensor::operator+(const ftype scalar) const { switch(values->getDevice()){ case Device::CPU: for (tensorSize_t i = 0; i < values->getSize(); ++i) { - (*res.values)[i] = (*values)[i] + scalar; + res.values->data()[i] = values->data()[i] + scalar; } break; case Device::CUDA: @@ -739,7 +729,7 @@ Tensor Tensor::operator-(const ftype scalar) const { switch(values->getDevice()){ case Device::CPU: for (tensorSize_t i = 0; i < values->getSize(); ++i) { - (*res.values)[i] = (*values)[i] - scalar; + res.values->data()[i] = values->data()[i] - scalar; } break; case Device::CUDA: @@ -1073,7 +1063,6 @@ ftype Tensor::operator[](tensorSize_t idx) const { return (*values)[idx]; } - ftype Tensor::get(tensorDim_t idx0, tensorDim_t idx1) const { return get({idx0, idx1}); } diff --git a/src/backend/data_modeling/tensor.h b/src/backend/data_modeling/tensor.h index aa2a0a9..b37d853 100644 --- a/src/backend/data_modeling/tensor.h +++ b/src/backend/data_modeling/tensor.h @@ -73,10 +73,10 @@ class Tensor final : public std::enable_shared_from_this ftype* data() noexcept { return values; } const ftype* data() const noexcept { return values; } + explicit operator bool() const noexcept; - ftype& operator[](tensorSize_t idx); - ftype operator[](tensorSize_t idx) const; + ftype operator[](tensorSize_t idx) const; void set(ftype v, tensorSize_t idx); ftype get(tensorSize_t idx); @@ -124,7 +124,7 @@ class Tensor final : public std::enable_shared_from_this template requires(std::is_same_v, Dimension>) explicit Tensor(T&& dims, Device d, bool requiresGrad = false) - : Tensor{dims.toVector(), tensorValues_t::getDefaultDevice(), requiresGrad} + : Tensor{dims.toVector(), d, requiresGrad} // !!!needs dims.toVector() to not trigger the copy ctors!!! { } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c8ca76b..28e961e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,4 +1,5 @@ include(FetchContent) + FetchContent_Declare( googletest GIT_REPOSITORY https://github.com/google/googletest.git @@ -22,6 +23,29 @@ target_link_libraries(unit_tests_backend PRIVATE include(GoogleTest) gtest_discover_tests(unit_tests_backend) +if(CMAKE_CUDA_COMPILER) + file(GLOB_RECURSE CUDA_TEST_SOURCES backend/cuda/*.cu) + add_executable(unit_tests_cuda ${CUDA_TEST_SOURCES}) + + target_link_libraries(unit_tests_cuda PRIVATE + gtest_main + BackendCore + CUDA::cudart + ) + + set_target_properties(unit_tests_cuda PROPERTIES + CUDA_SEPARABLE_COMPILATION ON + CMAKE_CUDA_ARCHITECTURES native + ) + + find_package(CUDAToolkit REQUIRED) + target_include_directories(unit_tests_cuda PRIVATE + ${CUDAToolkit_INCLUDE_DIRS} + ) + + gtest_discover_tests(unit_tests_cuda) +endif() + find_package(Python3 COMPONENTS Interpreter) if(Python3_FOUND) # replace the placeholder variables and copy resulting file in .py file diff --git a/tests/backend/cuda/test_tensorops_cuda.cu b/tests/backend/cuda/test_tensorops_cuda.cu new file mode 100644 index 0000000..095b2f7 --- /dev/null +++ b/tests/backend/cuda/test_tensorops_cuda.cu @@ -0,0 +1,28 @@ +/** + * @file test_tensorops_cuda.cu + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-05-12 + * + * @copyright Copyright (c) 2026 + * + */ + +#include + +#include "data_modeling/tensor.h" +#include "data_modeling/tensor_functions.h" + +TEST(TensorOpsTest_CUDA, ScalarAddWorks) { + auto t1 = TensorFunctions::Ones({2, 2}, Device::CUDA, false); + + auto res = t1 + 1.5; + + constexpr ftype sum = 2.5; + for(auto i = 0; i < t1.getDims().get(0); i++) { + for(auto j = 0; j < t1.getDims().get(1); j++) { + ASSERT_DOUBLE_EQ(res.get(i, j), sum); + } + } +} \ No newline at end of file From d224845cabe8eacc18d00820d7d324e034fa6f39 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Tue, 12 May 2026 11:30:29 +0200 Subject: [PATCH 24/73] Fixed python unit tests --- examples/mnist.py | 2 +- tests/python/test_training.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/mnist.py b/examples/mnist.py index a54ad4d..5a3ca9a 100644 --- a/examples/mnist.py +++ b/examples/mnist.py @@ -125,7 +125,7 @@ def evaluate(net, x, y_int, batch_size=256): # setup net = make_net() loss_fn = CrossEntropyWithSoftmax() - optim = RmsProp(net.parameters(), 0.00001, 0.95) # lr and decay + optim = RmsProp(net.parameters(), 0.000001, 0.999) # lr and decay # training loop n_epochs = 5 diff --git a/tests/python/test_training.py b/tests/python/test_training.py index 822c386..1cef568 100644 --- a/tests/python/test_training.py +++ b/tests/python/test_training.py @@ -97,9 +97,9 @@ def test_multiclass_rmsprop_overfits(self): x, y = make_multiclass_data() net = make_multiclass_net() loss_fn = CrossEntropyWithSoftmax() - optim = RmsProp(net.parameters(), 0.0003, 0.95) + optim = RmsProp(net.parameters(), 0.0001, 0.95) - final_loss = train(net, loss_fn, optim, x, y, epochs=10000) + final_loss = train(net, loss_fn, optim, x, y, epochs=2000) assert final_loss.getitem(0) < 0.05, \ f"RmsProp failed to overfit multiclass, loss={final_loss.getitem(0)}" From be9b17dd1e3d310d40c3c23ac10cce2cfddf5e3d Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Tue, 12 May 2026 12:15:47 +0200 Subject: [PATCH 25/73] First CUDA unit tests, plus some fixes and checks based on them --- src/backend/data_modeling/cuda/tensorops.cu | 2 +- src/backend/data_modeling/tensor.cpp | 35 ++++++++++-------- tests/CMakeLists.txt | 4 ++- tests/backend/cuda/main.cu | 29 +++++++++++++++ tests/backend/cuda/test_tensorops_cuda.cu | 39 ++++++++++++++++++--- tests/backend/test_data_modeling.cpp | 26 +++++++------- 6 files changed, 102 insertions(+), 33 deletions(-) create mode 100644 tests/backend/cuda/main.cu diff --git a/src/backend/data_modeling/cuda/tensorops.cu b/src/backend/data_modeling/cuda/tensorops.cu index 8456e07..a85c302 100644 --- a/src/backend/data_modeling/cuda/tensorops.cu +++ b/src/backend/data_modeling/cuda/tensorops.cu @@ -62,7 +62,7 @@ namespace{ if(gid>=size) return; - res[gid] = left[gid] + scalar; + res[gid] = left[gid] * scalar; } __global__ void matMulKernel(ftype* res, const ftype* const left, const ftype* const right, diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index cb731a6..d4bf98c 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -200,23 +200,30 @@ void Tensor::tensorValues_t::setDevice(const Device d) noexcept { } switch(device){ - case Device::CPU:{ - ftype* tmp; - cudaErrchk(cudaMalloc((void**) &tmp, size * sizeof(ftype))); - cudaErrchk(cudaMemcpy(tmp, values, size * sizeof(ftype), cudaMemcpyHostToDevice)); - free(values); - values = tmp; + case Device::CPU: + if(d == Device::CUDA) { + ftype* tmp; + cudaErrchk(cudaMalloc((void**) &tmp, size * sizeof(ftype))); + cudaErrchk(cudaMemcpy(tmp, values, size * sizeof(ftype), cudaMemcpyHostToDevice)); + free(values); + values = tmp; + } + else { + __throw_runtime_error((string("Requested unexpected device ") + DeviceToString(d)).c_str()); + } break; - } - case Device::CUDA:{ - ftype* tmp = static_cast(std::malloc(size * sizeof(ftype))); - cudaErrchk(cudaMemcpy(tmp, values, size * sizeof(ftype), cudaMemcpyDeviceToHost)); - cudaErrchk(cudaFree(values)); - values = tmp; + case Device::CUDA: + if(d == Device::CPU) { + ftype* tmp = static_cast(std::malloc(size * sizeof(ftype))); + cudaErrchk(cudaMemcpy(tmp, values, size * sizeof(ftype), cudaMemcpyDeviceToHost)); + cudaErrchk(cudaFree(values)); + values = tmp; + } + else { + __throw_runtime_error((string("Requested unexpected device ") + DeviceToString(d)).c_str()); + } break; - } } - device = d; #endif } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 28e961e..7d880d8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -27,8 +27,10 @@ if(CMAKE_CUDA_COMPILER) file(GLOB_RECURSE CUDA_TEST_SOURCES backend/cuda/*.cu) add_executable(unit_tests_cuda ${CUDA_TEST_SOURCES}) + find_package(Threads REQUIRED) target_link_libraries(unit_tests_cuda PRIVATE - gtest_main + gtest + Threads::Threads # platform agnostic BackendCore CUDA::cudart ) diff --git a/tests/backend/cuda/main.cu b/tests/backend/cuda/main.cu new file mode 100644 index 0000000..c567981 --- /dev/null +++ b/tests/backend/cuda/main.cu @@ -0,0 +1,29 @@ +/** + * @file main.cu + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-05-12 + * + * @copyright Copyright (c) 2026 + * + */ + + #include + +class CudaEnvironment : public ::testing::Environment { +public: + void SetUp() override { + // cuda warmup to avoid context initialization costs + void* tmp; + cudaMalloc(&tmp, 1); + cudaFree(tmp); + cudaDeviceSynchronize(); + } +}; + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + ::testing::AddGlobalTestEnvironment(new CudaEnvironment()); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/tests/backend/cuda/test_tensorops_cuda.cu b/tests/backend/cuda/test_tensorops_cuda.cu index 095b2f7..b95ee81 100644 --- a/tests/backend/cuda/test_tensorops_cuda.cu +++ b/tests/backend/cuda/test_tensorops_cuda.cu @@ -14,15 +14,46 @@ #include "data_modeling/tensor.h" #include "data_modeling/tensor_functions.h" -TEST(TensorOpsTest_CUDA, ScalarAddWorks) { - auto t1 = TensorFunctions::Ones({2, 2}, Device::CUDA, false); +TEST(CudaTensorOpsTest, ScalarAddWorks) { + auto t1 = TensorFunctions::Ones({10000, 10000}, Device::CUDA, false); auto res = t1 + 1.5; + res.setDevice(Device::CPU); constexpr ftype sum = 2.5; for(auto i = 0; i < t1.getDims().get(0); i++) { for(auto j = 0; j < t1.getDims().get(1); j++) { - ASSERT_DOUBLE_EQ(res.get(i, j), sum); + ASSERT_NEAR(res.get(i, j), sum, 1e-5); } } -} \ No newline at end of file +} + +TEST(CudaTensorOpsTest, ScalarMulWorks) { + auto t1 = TensorFunctions::Ones({10000, 10000}, Device::CUDA, false); + + constexpr ftype f = 2.5; + auto res = t1 * f; + res.setDevice(Device::CPU); + + for(auto i=0; i Date: Tue, 12 May 2026 12:28:46 +0200 Subject: [PATCH 26/73] More unit tests and some unit test cleanup --- tests/backend/cuda/test_tensorops_cuda.cu | 92 ++++++++++++++++++++++- tests/backend/test_data_modeling.cpp | 58 +++++++------- 2 files changed, 117 insertions(+), 33 deletions(-) diff --git a/tests/backend/cuda/test_tensorops_cuda.cu b/tests/backend/cuda/test_tensorops_cuda.cu index b95ee81..2747f01 100644 --- a/tests/backend/cuda/test_tensorops_cuda.cu +++ b/tests/backend/cuda/test_tensorops_cuda.cu @@ -15,7 +15,7 @@ #include "data_modeling/tensor_functions.h" TEST(CudaTensorOpsTest, ScalarAddWorks) { - auto t1 = TensorFunctions::Ones({10000, 10000}, Device::CUDA, false); + auto t1 = TensorFunctions::Ones({10000, 10000}, Device::CUDA); auto res = t1 + 1.5; res.setDevice(Device::CPU); @@ -29,7 +29,7 @@ TEST(CudaTensorOpsTest, ScalarAddWorks) { } TEST(CudaTensorOpsTest, ScalarMulWorks) { - auto t1 = TensorFunctions::Ones({10000, 10000}, Device::CUDA, false); + auto t1 = TensorFunctions::Ones({10000, 10000}, Device::CUDA); constexpr ftype f = 2.5; auto res = t1 * f; @@ -43,8 +43,8 @@ TEST(CudaTensorOpsTest, ScalarMulWorks) { } TEST(CudaTensorOpsTest, TensorAddWorks) { - auto t1 = TensorFunctions::Ones({10000, 10000}, Device::CUDA, false); - auto t2 = TensorFunctions::Ones({10000, 10000}, Device::CUDA, false) * 4; + auto t1 = TensorFunctions::Ones({10000, 10000}, Device::CUDA); + auto t2 = TensorFunctions::Ones({10000, 10000}, Device::CUDA) * 4; auto res = t1 + t2; res.setDevice(Device::CPU); @@ -57,3 +57,87 @@ TEST(CudaTensorOpsTest, TensorAddWorks) { } } +TEST(CudaTensorOpsTest, TensorAddCanBroadCast) { + auto t1 = TensorFunctions::Ones({3, 2, 2}, Device::CUDA); + auto t2 = Tensor({2}, {2, 3}, Device::CUDA); + + auto res = t1 + t2; + + ASSERT_EQ(res.getDims(), t1.getDims()); + + for(auto i=0; i{3, 6}; @@ -173,10 +173,10 @@ TEST(TensorOpsTest, MatMulGivesCorrectValues1) { } TEST(TensorOpsTest, MatMulGivesCorrectValues2) { - auto t1 = Tensor({2, 2}, false); - auto t2 = Tensor({2, 2}, false); + auto t1 = Tensor({2, 2}); + auto t2 = Tensor({2, 2}); - auto cmpRes = Tensor({2, 2}, false); + auto cmpRes = Tensor({2, 2}); auto populateTensor = [](Tensor& t, ftype v1, ftype v2, ftype v3, ftype v4) { t.set(v1, {0, 0}); @@ -203,14 +203,14 @@ TEST(TensorOpsTest, MatMulGivesCorrectValues2) { } TEST(TensorOpsTest, MatMulThrowsWhenDimensionsNotMatched) { - auto t1 = TensorFunctions::Ones({2, 2}, false); - auto t2 = TensorFunctions::Ones({3, 2}, false); + auto t1 = TensorFunctions::Ones({2, 2}); + auto t2 = TensorFunctions::Ones({3, 2}); EXPECT_THROW(t1.matmul(t2), std::runtime_error); } TEST(TensorOpsTest, TransposeWorksAsIntended1) { - auto t = TensorFunctions::Gaussian({3, 2}, 1.0, false); + auto t = TensorFunctions::Gaussian({3, 2}, 1.0); auto transposed = t.createDeepCopy(); transposed = transposed.transpose(-1, -2); @@ -230,7 +230,7 @@ TEST(TensorOpsTest, TransposeWorksAsIntended1) { * @brief Swap first two dimensions. */ TEST(TensorOpsTest, TransposeWorksAsIntended2) { - auto t = TensorFunctions::Gaussian({3, 2, 5}, 1.0, false); + auto t = TensorFunctions::Gaussian({3, 2, 5}, 1.0); auto transposed = t.createDeepCopy().transpose(0, 1); ASSERT_EQ(t.getDims().get(0), transposed.getDims().get(1)); @@ -252,7 +252,7 @@ TEST(TensorOpsTest, TransposeWorksAsIntended2) { * @brief Swap first and last dimension. */ TEST(TensorOpsTest, TransposeWorksAsIntended3) { - auto t = TensorFunctions::Gaussian({3, 2, 5}, 1.0, false); + auto t = TensorFunctions::Gaussian({3, 2, 5}, 1.0); auto transposed = t.createDeepCopy().transpose(0, -1); ASSERT_EQ(t.getDims().get(0), transposed.getDims().get(-1)); From 21b24a81ec2c30db6e0d745f99ee3721dd4922b3 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Tue, 12 May 2026 14:05:24 +0200 Subject: [PATCH 27/73] Fixing Gaussian interface, using expect_near in unit tests instead of assert_double_eq for better test stability --- src/backend/data_modeling/tensor.cpp | 1 + .../data_modeling/tensor_functions.cpp | 6 +- src/backend/data_modeling/tensor_functions.h | 2 +- src/python/py_core/py_core.cpp | 8 +- src/python/py_core/py_core_util.h | 8 +- tests/backend/cuda/test_tensorops_cuda.cu | 82 ++++++++++++++++--- tests/backend/test_data_modeling.cpp | 44 +++++----- tests/backend/test_module.cpp | 10 +-- 8 files changed, 112 insertions(+), 49 deletions(-) diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index d4bf98c..5a7e2f5 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -916,6 +916,7 @@ Device Tensor::getDefaultDevice() noexcept { } void Tensor::setDevice(const Device d) noexcept { + makeContiguous(); values->setDevice(d); } diff --git a/src/backend/data_modeling/tensor_functions.cpp b/src/backend/data_modeling/tensor_functions.cpp index 5f06d2d..66defff 100644 --- a/src/backend/data_modeling/tensor_functions.cpp +++ b/src/backend/data_modeling/tensor_functions.cpp @@ -33,8 +33,8 @@ Tensor TensorFunctions::Ones(vector dims, const bool requiresGrad) return Ones(std::move(dims), Tensor::getDefaultDevice(), requiresGrad); } -Tensor TensorFunctions::Gaussian(vector dims, const Device d, - const ftype stddev, const bool requiresGrad) { +Tensor TensorFunctions::Gaussian(vector dims, const ftype stddev, + const Device d, const bool requiresGrad) { auto res = Tensor(std::move(dims), d, requiresGrad); res.reset(std::make_shared(stddev)); return res; @@ -42,7 +42,7 @@ Tensor TensorFunctions::Gaussian(vector dims, const Device d, Tensor TensorFunctions::Gaussian(vector dims, const ftype stddev, const bool requiresGrad) { - return Gaussian(std::move(dims), Tensor::getDefaultDevice(), stddev, requiresGrad); + return Gaussian(std::move(dims), stddev, Tensor::getDefaultDevice(), requiresGrad); } // Tensor manipulation diff --git a/src/backend/data_modeling/tensor_functions.h b/src/backend/data_modeling/tensor_functions.h index 8403b47..c5fdf0e 100644 --- a/src/backend/data_modeling/tensor_functions.h +++ b/src/backend/data_modeling/tensor_functions.h @@ -32,7 +32,7 @@ namespace TensorFunctions { // class name acts as namespace for us Tensor Ones(std::vector dims, Device d, bool requiresGrad=false); Tensor Ones(std::vector dims, bool requiresGrad=false); - Tensor Gaussian(std::vector dims, Device d, ftype stddev, bool requiresGrad=false); + Tensor Gaussian(std::vector dims, ftype stddev, Device d, bool requiresGrad=false); Tensor Gaussian(std::vector dims, ftype stddev=1, bool requiresGrad=false); std::shared_ptr makeSharedTensor(const std::vector& dims, bool requiresGrad=false); diff --git a/src/python/py_core/py_core.cpp b/src/python/py_core/py_core.cpp index 0e000b6..545ccc4 100644 --- a/src/python/py_core/py_core.cpp +++ b/src/python/py_core/py_core.cpp @@ -172,9 +172,9 @@ BOOST_PYTHON_MODULE(_core) .staticmethod("zeros") .def("gauss", WRAP_FREE_FUNC_2(Py_DataModeling::Gaussian0, std::vector, ftype)) - .def("gauss", WRAP_FREE_FUNC_3(Py_DataModeling::Gaussian1, std::vector, Device, ftype)) + .def("gauss", WRAP_FREE_FUNC_3(Py_DataModeling::Gaussian1, std::vector, ftype, Device)) .def("gauss", WRAP_FREE_FUNC_3(Py_DataModeling::Gaussian2, std::vector, ftype, const bool)) - .def("gauss", WRAP_FREE_FUNC_8(Py_DataModeling::Gaussian3, std::vector, Device, ftype, const bool)) + .def("gauss", WRAP_FREE_FUNC_8(Py_DataModeling::Gaussian3, std::vector, ftype, Device, const bool)) .staticmethod("gauss") // properties @@ -244,9 +244,9 @@ BOOST_PYTHON_MODULE(_core) def("Zeros", WRAP_FREE_FUNC_3(Py_DataModeling::Zeros3, std::vector, Device, const bool)); def("Gaussian", WRAP_FREE_FUNC_2(Py_DataModeling::Gaussian0, std::vector, ftype)); - def("Gaussian", WRAP_FREE_FUNC_3(Py_DataModeling::Gaussian1, std::vector, Device, ftype)); + def("Gaussian", WRAP_FREE_FUNC_3(Py_DataModeling::Gaussian1, std::vector, ftype, Device)); def("Gaussian", WRAP_FREE_FUNC_3(Py_DataModeling::Gaussian2, std::vector, ftype, const bool)); - def("Gaussian", WRAP_FREE_FUNC_8(Py_DataModeling::Gaussian3, std::vector, Device, ftype, const bool)); + def("Gaussian", WRAP_FREE_FUNC_8(Py_DataModeling::Gaussian3, std::vector, ftype, Device, const bool)); // must call this before using numpy C API Py_DataModeling::initNumpy(); // numpy initialization diff --git a/src/python/py_core/py_core_util.h b/src/python/py_core/py_core_util.h index 230e498..e6c7b21 100644 --- a/src/python/py_core/py_core_util.h +++ b/src/python/py_core/py_core_util.h @@ -78,8 +78,8 @@ namespace Py_DataModeling return TensorFunctions::Gaussian(std::move(dims), stddev); } - inline auto GaussianWrapper1(std::vector dims, Device d, ftype stddev) { - return TensorFunctions::Gaussian(std::move(dims), d, stddev); + inline auto GaussianWrapper1(std::vector dims, ftype stddev, Device d) { + return TensorFunctions::Gaussian(std::move(dims), stddev, d); } inline Tensor (*Ones0)(std::vector) = &OnesWrapper0; @@ -93,9 +93,9 @@ namespace Py_DataModeling inline Tensor (*Zeros3)(std::vector, Device, const bool) = &(TensorFunctions::Zeros); inline Tensor (*Gaussian0)(std::vector, ftype) = &GaussianWrapper0; - inline Tensor (*Gaussian1)(std::vector, Device, ftype) = &GaussianWrapper1; + inline Tensor (*Gaussian1)(std::vector, ftype, Device) = &GaussianWrapper1; inline Tensor (*Gaussian2)(std::vector, ftype, const bool) = &(TensorFunctions::Gaussian); - inline Tensor (*Gaussian3)(std::vector, Device, ftype, const bool) = &(TensorFunctions::Gaussian); + inline Tensor (*Gaussian3)(std::vector, ftype, Device, const bool) = &(TensorFunctions::Gaussian); inline void (Tensor::*reset1)(const ftype) = &Tensor::reset; inline void (Tensor::*reset2)(const std::shared_ptr) = &Tensor::reset; diff --git a/tests/backend/cuda/test_tensorops_cuda.cu b/tests/backend/cuda/test_tensorops_cuda.cu index 2747f01..ba14f54 100644 --- a/tests/backend/cuda/test_tensorops_cuda.cu +++ b/tests/backend/cuda/test_tensorops_cuda.cu @@ -67,8 +67,8 @@ TEST(CudaTensorOpsTest, TensorAddCanBroadCast) { for(auto i=0; i 0) ASSERT_DOUBLE_EQ(x->getGrads()->get(0), 0.0); ASSERT_DOUBLE_EQ(x->getGrads()->get(1), 0.0); - ASSERT_DOUBLE_EQ(x->getGrads()->get(2), 1.0); + EXPECT_NEAR(x->getGrads()->get(2), 1.0, 1e-5); } TEST(ActivationTest, LeakyReluForward) { @@ -70,7 +70,7 @@ TEST(ActivationTest, LeakyReluForward) { auto res = f(t1); for(size_t i=0; i 0) ASSERT_DOUBLE_EQ(x->getGrads()->get(0), eps); ASSERT_DOUBLE_EQ(x->getGrads()->get(1), eps); // by convention - ASSERT_DOUBLE_EQ(x->getGrads()->get(2), 1.0); + EXPECT_NEAR(x->getGrads()->get(2), 1.0, 1e-5); } TEST(ActivationTest, SigmoidForward) { From 332fb9760633141103d2c538c291f931d16ba659 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Tue, 12 May 2026 15:55:18 +0200 Subject: [PATCH 28/73] Minor refactor --- src/backend/data_modeling/cuda/tensorops.cu | 38 ++------------------ src/backend/data_modeling/cuda/tensorops.cuh | 8 ----- src/backend/data_modeling/tensor.cpp | 34 +++++++++++++++--- 3 files changed, 32 insertions(+), 48 deletions(-) diff --git a/src/backend/data_modeling/cuda/tensorops.cu b/src/backend/data_modeling/cuda/tensorops.cu index a85c302..9fbff17 100644 --- a/src/backend/data_modeling/cuda/tensorops.cu +++ b/src/backend/data_modeling/cuda/tensorops.cu @@ -13,15 +13,14 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #endif // __CUDA +#include "data_modeling/tensor.h" + #include "tensorops.cuh" #include "utility/cuda/cuda_common.cuh" -#include "cuda_runtime.h" #include #include -#include "data_modeling/tensor.h" - namespace{ __global__ void elementwiseaddKernel(ftype* res, const ftype* const left, const ftype* const right, const tensorSize_t size) { int gid = blockDim.x * blockIdx.x + threadIdx.x; @@ -129,6 +128,7 @@ namespace cuda_impl { void elementwiseadd(Tensor& res, const Tensor& left, const Tensor& right) { constexpr int threadsPerBlock = 256; const int blocksPerGrid = (left.getSize() + threadsPerBlock - 1) / threadsPerBlock; + elementwiseaddKernel<<>>(res.getData(), left.getData(), right.getData(), left.getSize()); cudaErrchk(cudaDeviceSynchronize()); } @@ -156,38 +156,6 @@ namespace cuda_impl { cudaErrchk(cudaDeviceSynchronize()); } - void printValues(std::ostream& os, const Tensor& t) { - auto printVals = [&os](const Tensor& t) { - constexpr auto MAX_IDX = static_cast(10); - const auto maxIdx = min(MAX_IDX, t.getSize()); - - auto tmp = static_cast(std::malloc(t.getSize() * sizeof(ftype))); - cudaErrchk(cudaMemcpy(tmp, t.getData(), maxIdx * sizeof(ftype), cudaMemcpyDeviceToHost)); - - for(tensorSize_t i = 0; i < maxIdx; i++) - os << tmp[i]; - os << "\n\n"; - - free(tmp); - }; - - printVals(t); - if(t.hasGrads()) { - os << "\n\nGrads:\n"; - printVals(*t.getGrads()); - } - } - - ftype get(const ftype* const t, tensorSize_t idx) { - ftype res; - cudaErrchk(cudaMemcpy((void*)&res, t + idx, sizeof(ftype), cudaMemcpyDeviceToHost)); - return res; - } - - void set(ftype value, const ftype* t, tensorSize_t idx) { - cudaErrchk(cudaMemcpy((void*)(t + idx), &value, sizeof(ftype), cudaMemcpyHostToDevice)); - } - void createContiguousCopy(Tensor& res, const Tensor& src) { assert(res.getSize()==src.getSize()); diff --git a/src/backend/data_modeling/cuda/tensorops.cuh b/src/backend/data_modeling/cuda/tensorops.cuh index 6c8f395..595f564 100644 --- a/src/backend/data_modeling/cuda/tensorops.cuh +++ b/src/backend/data_modeling/cuda/tensorops.cuh @@ -18,8 +18,6 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "utility/global_params.h" #include "data_modeling/dim_type.h" -#include - class Tensor; namespace cuda_impl { @@ -39,11 +37,5 @@ namespace cuda_impl { void scalarFill(Tensor& t, ftype value); - void printValues(std::ostream& os, const Tensor& t); - - // other - [[nodiscard]] ftype get(const ftype* const t, tensorSize_t idx); - void set(ftype value, const ftype* t, tensorSize_t idx); - void createContiguousCopy(Tensor& res, const Tensor& src); } diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index 5a7e2f5..726253d 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -259,7 +259,11 @@ ftype Tensor::tensorValues_t::operator[](const tensorSize_t idx) const { return values[idx]; case Device::CUDA: #ifdef __CUDA - return cuda_impl::get(values, idx); + { + ftype res; + cudaErrchk(cudaMemcpy(&res, values + idx, sizeof(ftype), cudaMemcpyDeviceToHost)); + return res; + } #else __throw_invalid_argument("Not compiled with CUDA"); break; @@ -280,7 +284,7 @@ void Tensor::tensorValues_t::set(ftype v, tensorSize_t idx) { break; case Device::CUDA: #ifdef __CUDA - cuda_impl::set(v, values, idx); + cudaErrchk(cudaMemcpy(values + idx, &v, sizeof(ftype), cudaMemcpyHostToDevice)); #else __throw_invalid_argument("Not compiled with CUDA"); #endif @@ -291,13 +295,17 @@ void Tensor::tensorValues_t::set(ftype v, tensorSize_t idx) { ftype Tensor::tensorValues_t::get(tensorSize_t idx) { if(idx >= size) throw std::out_of_range("Out of range for tensor"); - + switch(device){ case Device::CPU: return values[idx]; case Device::CUDA: #ifdef __CUDA - return cuda_impl::get(values, idx); + { + ftype res; + cudaErrchk(cudaMemcpy(&res, values + idx, sizeof(ftype), cudaMemcpyDeviceToHost)); + return res; + } #else __throw_invalid_argument("Not compiled with CUDA"); break; @@ -1009,7 +1017,23 @@ ostream& operator<<(ostream& os, const Tensor& t) noexcept { break; case Device::CUDA: #ifdef __CUDA - cuda_impl::printValues(os, t); + { + auto printVals = [&os](const Tensor& t) { + constexpr auto MAX_IDX = static_cast(10); + const auto maxIdx = min(MAX_IDX, t.getSize()); + auto tmp = static_cast(std::malloc(t.getSize() * sizeof(ftype))); + cudaErrchk(cudaMemcpy(tmp, t.getData(), maxIdx * sizeof(ftype), cudaMemcpyDeviceToHost)); + for(tensorSize_t i = 0; i < maxIdx; i++) + os << tmp[i]; + os << "\n\n"; + free(tmp); + }; + printVals(t); + if(t.hasGrads()) { + os << "\n\nGrads:\n"; + printVals(*t.getGrads()); + } + } #else __throw_runtime_error("Not compiled with CUDA"); #endif From 9c9a9774f79a5d2707b1c3fe6061dd5f03615d43 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Wed, 13 May 2026 10:08:26 +0200 Subject: [PATCH 29/73] Remove cuda branch from printing --- src/backend/data_modeling/cuda/tensorops.cu | 8 +-- src/backend/data_modeling/tensor.cpp | 58 ++++++--------------- 2 files changed, 19 insertions(+), 47 deletions(-) diff --git a/src/backend/data_modeling/cuda/tensorops.cu b/src/backend/data_modeling/cuda/tensorops.cu index 9fbff17..b86c915 100644 --- a/src/backend/data_modeling/cuda/tensorops.cu +++ b/src/backend/data_modeling/cuda/tensorops.cu @@ -79,7 +79,7 @@ namespace{ * @param size Total size of both src and dst. */ __global__ void createContiguousCopyKernel(float* dst, const float* const src, const tensorSize_t* const strides, - const tensorSize_t* const contiguousStrides, int ndim, tensorSize_t size) + const tensorSize_t* const contiguousStrides, const int ndims, const tensorSize_t size) { tensorSize_t flatIdx = blockIdx.x * blockDim.x + threadIdx.x; if (flatIdx >= size) @@ -87,9 +87,9 @@ namespace{ tensorSize_t remainder = flatIdx; tensorSize_t srcOffset = 0; - for (int i = 0; i < ndim; ++i) { - tensorSize_t coord = remainder / contiguousStrides[i]; - remainder %= contiguousStrides[i]; + for (int i = ndims - 1; i >= 0; i--) { + tensorSize_t coord = remainder % contiguousStrides[i]; + remainder /= contiguousStrides[i]; srcOffset += coord * strides[i]; } dst[flatIdx] = src[srcOffset]; diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index 726253d..654ce79 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -382,7 +382,7 @@ Tensor Tensor::createContiguousCopy() const { switch(values->getDevice()) { case Device::CPU: { - for (tensorSize_t flatIdx = 0; flatIdx < values->getSize(); ++flatIdx) { + for (tensorSize_t flatIdx = 0; flatIdx < values->getSize(); flatIdx++) { tensorSize_t remainder = flatIdx; tensorSize_t srcOffset = 0; @@ -392,7 +392,7 @@ Tensor Tensor::createContiguousCopy() const { srcOffset += coord * dims.getStride(i); } - res.values->set((*values)[srcOffset], flatIdx); + res.values->data()[flatIdx] = (*values)[srcOffset]; } break; } @@ -980,66 +980,38 @@ Tensor Tensor::getSlice(span indices) { * @brief Prints only sample of up to 2D tensors. */ void printValuesCpu(std::ostream& os, const Tensor& t) { + +} + +ostream& operator<<(ostream& os, const Tensor& t) noexcept { + os << "Dims: " << t.getDims(); + os << "\nDevice: " << DeviceToString(t.values->getDevice()); + os << "\nrequiresGrad: " << t.requiresGrad << "\n\n"; + auto printVals = [&os](const Tensor& t){ constexpr auto MAX_IDX = static_cast(10); - if(t.dims.nDims()==2){ - for(tensorDim_t i=0; i(t.values->getSize())); i++){ + for(tensorDim_t i = 0; i < min(MAX_IDX, static_cast(t.values->getSize())); i++){ os << (*t.values)[i] << " "; } } }; + t.makeContiguous(); printVals(t); if(t.grads){ os << "\n\nGrads:\n"; printVals(*t.grads); } -} - -ostream& operator<<(ostream& os, const Tensor& t) noexcept { - os << "Dims: " << t.getDims(); - os << "\nDevice: " << DeviceToString(t.values->getDevice()); - os << "\nrequiresGrad: " << t.requiresGrad << "\n\n"; - - t.makeContiguous(); - switch(t.values->getDevice()){ - case Device::CPU: - printValuesCpu(os, t); - break; - case Device::CUDA: - #ifdef __CUDA - { - auto printVals = [&os](const Tensor& t) { - constexpr auto MAX_IDX = static_cast(10); - const auto maxIdx = min(MAX_IDX, t.getSize()); - auto tmp = static_cast(std::malloc(t.getSize() * sizeof(ftype))); - cudaErrchk(cudaMemcpy(tmp, t.getData(), maxIdx * sizeof(ftype), cudaMemcpyDeviceToHost)); - for(tensorSize_t i = 0; i < maxIdx; i++) - os << tmp[i]; - os << "\n\n"; - free(tmp); - }; - printVals(t); - if(t.hasGrads()) { - os << "\n\nGrads:\n"; - printVals(*t.getGrads()); - } - } - #else - __throw_runtime_error("Not compiled with CUDA"); - #endif - break; - } - return os; } From 70a9fab7cc7e0d86deb6a98e1b12220f3c295ce9 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Wed, 13 May 2026 11:10:21 +0200 Subject: [PATCH 30/73] Fixed CUDA transpose --- src/backend/data_modeling/cuda/tensorops.cu | 57 ++++++++++----------- src/backend/data_modeling/dim_type.cpp | 4 +- src/backend/data_modeling/dim_type.h | 14 ++--- src/backend/data_modeling/tensor.cpp | 7 --- src/backend/data_modeling/tensor.h | 2 - 5 files changed, 37 insertions(+), 47 deletions(-) diff --git a/src/backend/data_modeling/cuda/tensorops.cu b/src/backend/data_modeling/cuda/tensorops.cu index b86c915..5dc9614 100644 --- a/src/backend/data_modeling/cuda/tensorops.cu +++ b/src/backend/data_modeling/cuda/tensorops.cu @@ -22,7 +22,8 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include namespace{ - __global__ void elementwiseaddKernel(ftype* res, const ftype* const left, const ftype* const right, const tensorSize_t size) { + __global__ void elementwiseaddKernel(ftype* res, const ftype* const left, const ftype* const right, const tensorSize_t size) + { int gid = blockDim.x * blockIdx.x + threadIdx.x; if(gid>=size) return; @@ -31,7 +32,8 @@ namespace{ } __global__ void broadcastaddKernel(ftype* res, const ftype* const matrix, const ftype* const vec, - const tensorSize_t vectorSize, const tensorSize_t matrixSize) { + const tensorSize_t vectorSize, const tensorSize_t matrixSize) + { int gid = blockDim.x * blockIdx.x + threadIdx.x; if(gid>=matrixSize) return; @@ -40,7 +42,8 @@ namespace{ res[gid] = matrix[gid] + vec[vectorIdx]; } - __global__ void elementwisemulKernel(ftype* res, const ftype* const left, const ftype* const right, const tensorSize_t size) { + __global__ void elementwisemulKernel(ftype* res, const ftype* const left, const ftype* const right, const tensorSize_t size) + { int gid = blockDim.x * blockIdx.x + threadIdx.x; if(gid>=size) return; @@ -48,7 +51,8 @@ namespace{ res[gid] = left[gid] * right[gid]; } - __global__ void scalaraddKernel(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size){ + __global__ void scalaraddKernel(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size) + { int gid = blockDim.x * blockIdx.x + threadIdx.x; if(gid>=size) return; @@ -56,7 +60,8 @@ namespace{ res[gid] = left[gid] + scalar; } - __global__ void scalarmulKernel(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size) { + __global__ void scalarmulKernel(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size) + { int gid = blockDim.x * blockIdx.x + threadIdx.x; if(gid>=size) return; @@ -65,7 +70,8 @@ namespace{ } __global__ void matMulKernel(ftype* res, const ftype* const left, const ftype* const right, - const tensorDim_t leftRows, const tensorDim_t leftCols, tensorDim_t rightRows, tensorDim_t rightCols) { + const tensorDim_t leftRows, const tensorDim_t leftCols, tensorDim_t rightRows, tensorDim_t rightCols) + { int gid = blockDim.x * blockIdx.x + threadIdx.x; //if() } @@ -78,21 +84,21 @@ namespace{ * @param ndim Number of dimension of both src and dst. * @param size Total size of both src and dst. */ - __global__ void createContiguousCopyKernel(float* dst, const float* const src, const tensorSize_t* const strides, - const tensorSize_t* const contiguousStrides, const int ndims, const tensorSize_t size) + __global__ void createContiguousCopyKernel(ftype* dst, const ftype* const src, const tensorSize_t* const strides, + const tensorDim_t* const dims, const int ndims, const tensorSize_t size) { - tensorSize_t flatIdx = blockIdx.x * blockDim.x + threadIdx.x; - if (flatIdx >= size) - return; + tensorSize_t flatIdx = blockIdx.x * blockDim.x + threadIdx.x; + if(flatIdx >= size) + return; - tensorSize_t remainder = flatIdx; - tensorSize_t srcOffset = 0; - for (int i = ndims - 1; i >= 0; i--) { - tensorSize_t coord = remainder % contiguousStrides[i]; - remainder /= contiguousStrides[i]; - srcOffset += coord * strides[i]; - } - dst[flatIdx] = src[srcOffset]; + tensorSize_t remainder = flatIdx; + tensorSize_t srcOffset = 0; + for (int i = ndims - 1; i >= 0; i--) { + tensorSize_t coord = remainder % dims[i]; + remainder /= dims[i]; + srcOffset += coord * strides[i]; + } + dst[flatIdx] = src[srcOffset]; } } @@ -159,19 +165,12 @@ namespace cuda_impl { void createContiguousCopy(Tensor& res, const Tensor& src) { assert(res.getSize()==src.getSize()); - ftype* dst = res.getData(); - const ftype* const srcData = src.getData(); - - const auto oldStrides = src.getDims().getOriginalStrides().data(); - const auto newStrides = src.getDims().getStrides().data(); - - const auto nBytes = src.getSize(); - + const auto size = src.getSize(); constexpr int threadsPerBlock = 256; - const int blocksPerGrid = (nBytes + threadsPerBlock - 1) / threadsPerBlock; + const int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; createContiguousCopyKernel<<>>( - dst, srcData, oldStrides, newStrides, src.getDims().nDims(), nBytes); + res.getData(), src.getData(), src.getDims().getStrides().data(), src.getDims().data(), src.getDims().nDims(), size); cudaErrchk(cudaDeviceSynchronize()); } } diff --git a/src/backend/data_modeling/dim_type.cpp b/src/backend/data_modeling/dim_type.cpp index 43b5612..2eae45c 100644 --- a/src/backend/data_modeling/dim_type.cpp +++ b/src/backend/data_modeling/dim_type.cpp @@ -57,14 +57,14 @@ void Dimension::swap(int dim1, int dim2) { } Dimension::Dimension(const vector& dims) - : creationDims{dims}, creationStrides{makeStrides(dims)}, dims{dims}, strides{creationStrides} { + : contiguousDims{dims}, contiguousStrides{makeStrides(dims)}, dims{dims}, strides{contiguousStrides} { size = multVector(dims); lastDimIdx = dims.size()-1; assert(size>0); } Dimension::Dimension(vector&& dims, dim_t&& strides) - : creationDims{dims}, dims{creationDims}, creationStrides{strides}, strides{creationStrides} + : contiguousDims{dims}, dims{contiguousDims}, contiguousStrides{strides}, strides{contiguousStrides} { size = multVector(dims); lastDimIdx = dims.size()-1; diff --git a/src/backend/data_modeling/dim_type.h b/src/backend/data_modeling/dim_type.h index f15fb47..8d5783f 100644 --- a/src/backend/data_modeling/dim_type.h +++ b/src/backend/data_modeling/dim_type.h @@ -23,10 +23,10 @@ class Dimension final { using dim_t = std::array; private: - // creationDims and creationStrides should only be set in constructors, otherwise - // bugs will emerge. Made non-const so we can use move-assignment operator - std::vector creationDims; - dim_t creationStrides; + // those two indicate the structure of the contiguous data that lies underneath + // WARNING: Should only be set in ctor. We did not make them constant, so that we can use copy c'tor + std::vector contiguousDims; + dim_t contiguousStrides; std::vector dims; dim_t strides; @@ -48,7 +48,6 @@ class Dimension final { } public: - Dimension(const std::vector& dims); Dimension(const Dimension& other) = default; @@ -62,7 +61,7 @@ class Dimension final { Dimension collapseDimension(int idx) const; bool inOriginalState() const noexcept { - return creationDims == dims && creationStrides == strides; + return contiguousDims == dims && contiguousStrides == strides; } void resize(const std::vector& dims); @@ -80,8 +79,9 @@ class Dimension final { return dims[mapSignedIdx(idx)]; } + const tensorDim_t* data() const noexcept { return dims.data(); } const auto getStrides() const noexcept { return strides; } - const auto getOriginalStrides() const noexcept { return creationStrides; } + const auto getContiguousStrides() const noexcept { return contiguousStrides; } tensorSize_t getStride(int i) const noexcept; diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index 654ce79..90934d1 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -976,13 +976,6 @@ Tensor Tensor::getSlice(span indices) { return res; } -/** - * @brief Prints only sample of up to 2D tensors. - */ -void printValuesCpu(std::ostream& os, const Tensor& t) { - -} - ostream& operator<<(ostream& os, const Tensor& t) noexcept { os << "Dims: " << t.getDims(); os << "\nDevice: " << DeviceToString(t.values->getDevice()); diff --git a/src/backend/data_modeling/tensor.h b/src/backend/data_modeling/tensor.h index b37d853..0e0145b 100644 --- a/src/backend/data_modeling/tensor.h +++ b/src/backend/data_modeling/tensor.h @@ -116,8 +116,6 @@ class Tensor final : public std::enable_shared_from_this static tensorDim_t mapDim(int dim, const Dimension& dims); - friend void printValuesCpu(std::ostream& os, const Tensor& t); - Tensor(const Tensor& other, shallowCopyToken); public: From c4e25146e1a226d6be55d768e6f84a14391e1007 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Wed, 13 May 2026 12:53:48 +0200 Subject: [PATCH 31/73] Implemented naive matmul in CUDA --- src/backend/data_modeling/cuda/tensorops.cu | 70 ++++++++++-- src/backend/data_modeling/tensor.cpp | 33 +++--- tests/backend/cuda/test_tensorops_cuda.cu | 116 ++++++++++++++++---- 3 files changed, 167 insertions(+), 52 deletions(-) diff --git a/src/backend/data_modeling/cuda/tensorops.cu b/src/backend/data_modeling/cuda/tensorops.cu index 5dc9614..f0f6229 100644 --- a/src/backend/data_modeling/cuda/tensorops.cu +++ b/src/backend/data_modeling/cuda/tensorops.cu @@ -69,11 +69,43 @@ namespace{ res[gid] = left[gid] * scalar; } - __global__ void matMulKernel(ftype* res, const ftype* const left, const ftype* const right, - const tensorDim_t leftRows, const tensorDim_t leftCols, tensorDim_t rightRows, tensorDim_t rightCols) + /** + * @brief 2D matMul operation. Assumes that res is where result is written to, and res is zero-indexed. + * For higher dimensionalities the caller is responsible for proper offset computation!!! The same goes + * for left and right, which are also assumed to be 2D matrices, and offsets have to be computed by caller. + * + * @param res The result matrix + * @param left Left matrix to be multiplied + * @param right Right matrix to multiply with + * @param leftRows n rows of + * @param leftCols n columns of left matrix; leftCols == rightRows => rightRows not handed as param + * @param rightCols n columns of right matrix. + * @param resSize Size of the resulting 2D matrix. + */ + __global__ void matMul2DKernel(ftype* res, const ftype* const left, const ftype* const right, + const tensorDim_t leftRows, const tensorDim_t leftCols, + const tensorDim_t rightCols, const tensorSize_t resSize) { - int gid = blockDim.x * blockIdx.x + threadIdx.x; - //if() + const int gid = blockDim.x * blockIdx.x + threadIdx.x; + if(gid >= resSize) + return; + const int tid = threadIdx.x; + + // 48KB of shared mem -> ~12K floats of 4 bytes -> ~10 blocks per SM of shared memory is limit + extern __shared__ ftype smem[]; + smem[tid] = 0.0f; + __syncthreads(); + + const int resCol = gid % rightCols; + const int resRow = gid / rightCols; + const int leftBase = resRow * leftCols; + + // C[i, j] = sum_{k=0}^{leftCols} A[i, k] * B[k, j] + for(int k = 0; k < leftCols; k++) { + smem[tid] += left[leftBase + k] * right[k * rightCols + resCol]; + } + + res[gid] = smem[tid]; } /** @@ -87,7 +119,7 @@ namespace{ __global__ void createContiguousCopyKernel(ftype* dst, const ftype* const src, const tensorSize_t* const strides, const tensorDim_t* const dims, const int ndims, const tensorSize_t size) { - tensorSize_t flatIdx = blockIdx.x * blockDim.x + threadIdx.x; + const tensorSize_t flatIdx = blockIdx.x * blockDim.x + threadIdx.x; if(flatIdx >= size) return; @@ -95,8 +127,8 @@ namespace{ tensorSize_t srcOffset = 0; for (int i = ndims - 1; i >= 0; i--) { tensorSize_t coord = remainder % dims[i]; - remainder /= dims[i]; srcOffset += coord * strides[i]; + remainder /= dims[i]; } dst[flatIdx] = src[srcOffset]; } @@ -119,7 +151,6 @@ namespace cuda_impl { cudaErrchk(cudaDeviceSynchronize()); } - // TODO: fix this one void broadcastadd(Tensor& res, const Tensor& matrix, const Tensor& vec) { const auto size = res.getSize(); @@ -148,11 +179,28 @@ namespace cuda_impl { } void matmul(Tensor& res, const Tensor& left, const Tensor& right) { -/* constexpr int threadsPerBlock = 256; - const int blocksPerGrid = (size + threadsPerBlock - 1) / threadsPerBlock; + constexpr int threadsPerBlock = 256; + const int blocksPerGrid = (res.getSize() + threadsPerBlock - 1) / threadsPerBlock; + + // sizes of the 2D matrices respectively + const tensorSize_t leftSize = left.getDims().get(-1) * left.getDims().get(-2); + const tensorSize_t rightSize = right.getDims().get(-1) * right.getDims().get(-2); + const tensorSize_t resSize = left.getDims().get(-2) * right.getDims().get(-1); + + tensorSize_t leftOffset = 0; + tensorSize_t rightOffset = 0; + tensorSize_t resOffset = 0; + + while(leftOffset < left.getSize()){ + matMul2DKernel<<>>(res.getData() + resOffset, left.getData() + leftOffset, right.getData() + rightOffset, + left.getDims().get(-2), left.getDims().get(-1), right.getDims().get(-1), resSize); + + leftOffset += leftSize; + rightOffset += rightSize; + resOffset += resSize; + } - matMulKernel<<>>(res, left, right, size); - cudaErrchk(cudaDeviceSynchronize()); */ + cudaErrchk(cudaDeviceSynchronize()); } void scalarFill(Tensor& t, ftype value) { diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index 90934d1..6a0256e 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -382,14 +382,14 @@ Tensor Tensor::createContiguousCopy() const { switch(values->getDevice()) { case Device::CPU: { - for (tensorSize_t flatIdx = 0; flatIdx < values->getSize(); flatIdx++) { + for(tensorSize_t flatIdx = 0; flatIdx < values->getSize(); flatIdx++) { tensorSize_t remainder = flatIdx; tensorSize_t srcOffset = 0; - for (int i=dims.nDims()-1; i>=0; i--) { + for(int i = dims.nDims() - 1; i >= 0; i--) { tensorSize_t coord = remainder % dims[i]; - remainder /= dims[i]; srcOffset += coord * dims.getStride(i); + remainder /= dims[i]; } res.values->data()[flatIdx] = (*values)[srcOffset]; @@ -469,19 +469,19 @@ Tensor Tensor::matMulImpl(const Tensor& left, const Tensor& right) { // broadcasting auto resDims = left.dims.nDims() > right.dims.nDims() ? left.dims.toVector() : right.dims.toVector(); - resDims[resDims.size()-2] = left.dims.get(-2); // rows - resDims[resDims.size()-1] = right.dims.get(-1); // cols + resDims[resDims.size() - 2] = left.dims.get(-2); // rows + resDims[resDims.size() - 1] = right.dims.get(-1); // cols Tensor res(resDims, left.values->getDevice(), false); - // sizes of the 2D matrices respectively - const tensorSize_t leftSize = left.dims.get(-1) * left.dims.get(-2); - const tensorSize_t rightSize = right.dims.get(-1) * right.dims.get(-2); - const tensorSize_t resSize = left.dims.get(-2) * right.dims.get(-1); - switch(left.values->getDevice()){ case Device::CPU: { + // sizes of the 2D matrices respectively + const tensorSize_t leftSize = left.dims.get(-1) * left.dims.get(-2); + const tensorSize_t rightSize = right.dims.get(-1) * right.dims.get(-2); + const tensorSize_t resSize = left.dims.get(-2) * right.dims.get(-1); + tensorSize_t leftOffset = 0; tensorSize_t rightOffset = 0; tensorSize_t resOffset = 0; @@ -543,14 +543,10 @@ void Tensor::matMul2DCpu(Tensor& res, const Tensor& left, const Tensor& right, c * @brief Matrix multiplication. */ Tensor Tensor::matmul(const Tensor& other) const { - assert(values->getDevice()==other.values->getDevice()); - - makeContiguous(); - other.makeContiguous(); - if(values->getDevice()!=other.values->getDevice()){ __throw_runtime_error("Tensors on different devices."); } + return matMulImpl(getContiguous(), other.getContiguous()); } @@ -782,7 +778,7 @@ void Tensor::backward() { } vector sortedTensors = cgraph::TopologicalSort::reverseSort(this); - for(auto tPtr: sortedTensors){ + for(auto tPtr : sortedTensors){ auto& tensor = *tPtr; tensor.makeContiguous(); assert(tensor.grads && !tensor.grads->requiresGrad); // gradient should not require grad @@ -791,7 +787,7 @@ void Tensor::backward() { const auto& parents = tensor.cgNode->getParents(); - for(size_t i=0; irequiresGrad){ continue; @@ -818,7 +814,7 @@ shared_ptr Tensor::getGrads() const { * NumPy we map from the end to the beginning in that case. */ tensorDim_t Tensor::mapDim(const int dim, const Dimension& dims) { - if(dim>=0){ + if(dim >= 0){ return dim; } else if(dim + dims.nDims() < 0){ @@ -967,7 +963,6 @@ Tensor Tensor::getSlice(span indices) { assert(indices.size()>0); makeContiguous(); - auto resDims = dims.toVector(); resDims[0] = indices.size(); diff --git a/tests/backend/cuda/test_tensorops_cuda.cu b/tests/backend/cuda/test_tensorops_cuda.cu index ba14f54..5def02f 100644 --- a/tests/backend/cuda/test_tensorops_cuda.cu +++ b/tests/backend/cuda/test_tensorops_cuda.cu @@ -15,7 +15,7 @@ #include "data_modeling/tensor_functions.h" TEST(CudaTensorOpsTest, ScalarAddWorks) { - auto t1 = TensorFunctions::Ones({10000, 10000}, Device::CUDA); + auto t1 = TensorFunctions::Ones({500, 500}, Device::CUDA); auto res = t1 + 1.5; res.setDevice(Device::CPU); @@ -29,7 +29,7 @@ TEST(CudaTensorOpsTest, ScalarAddWorks) { } TEST(CudaTensorOpsTest, ScalarMulWorks) { - auto t1 = TensorFunctions::Ones({10000, 10000}, Device::CUDA); + auto t1 = TensorFunctions::Ones({500, 500}, Device::CUDA); constexpr ftype f = 2.5; auto res = t1 * f; @@ -43,8 +43,8 @@ TEST(CudaTensorOpsTest, ScalarMulWorks) { } TEST(CudaTensorOpsTest, TensorAddWorks) { - auto t1 = TensorFunctions::Ones({10000, 10000}, Device::CUDA); - auto t2 = TensorFunctions::Ones({10000, 10000}, Device::CUDA) * 4; + auto t1 = TensorFunctions::Ones({500, 500}, Device::CUDA); + auto t2 = TensorFunctions::Ones({500, 500}, Device::CUDA) * 4; auto res = t1 + t2; res.setDevice(Device::CPU); @@ -105,8 +105,8 @@ TEST(CudaTensorOpsTest, TensorAddThrowsOnDimMismatch) { } TEST(CudaTensorOpsTest, MatrixAddGivesCorrectResults) { - auto t1 = TensorFunctions::Ones({200, 20000}, Device::CUDA); - auto t2 = TensorFunctions::Ones({200, 20000}, Device::CUDA); + auto t1 = TensorFunctions::Ones({200, 200}, Device::CUDA); + auto t2 = TensorFunctions::Ones({200, 200}, Device::CUDA); auto res = t1 + t2; res.setDevice(Device::CPU); @@ -114,22 +114,22 @@ TEST(CudaTensorOpsTest, MatrixAddGivesCorrectResults) { constexpr ftype resSum = 2.0; for(auto i=0; i +#ifdef __CUDA +#include "computational_graph/activation_functions/cuda/activation_nodes.cuh" +#else +#include +#endif + using namespace std; using namespace cgraph; vector> LeakyReLuNode::backward(const Tensor& upstreamGrad) { assert(!upstreamGrad.getRequiresGrad()); - constexpr ftype zero = 0.0; - + auto res = make_shared(upstreamGrad.getDims(), upstreamGrad.getDevice(), false); - const auto& parent = parents[0]; - for(tensorSize_t i=0; iset((*parent)[i] > zero ? upstreamGrad[i] : upstreamGrad[i] * eps, i); + + switch(upstreamGrad.getDevice()) { + case Device::CPU: { + constexpr ftype zero = 0.0; + for(tensorSize_t i=0; iset((*parent)[i] > zero ? upstreamGrad[i] : upstreamGrad[i] * eps, i); + } + break; + } + case Device::CUDA: + #ifdef __CUDA + cuda_impl::leakyReluBackward(*res, upstreamGrad, *parent, eps); + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); + #endif + break; } return {res}; -} \ No newline at end of file +} diff --git a/src/backend/computational_graph/activation_functions/relu_node.cpp b/src/backend/computational_graph/activation_functions/relu_node.cpp index 3fcc958..76e4f08 100644 --- a/src/backend/computational_graph/activation_functions/relu_node.cpp +++ b/src/backend/computational_graph/activation_functions/relu_node.cpp @@ -1,31 +1,49 @@ /** * @file relu_node.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-02-15 - * + * * @copyright Copyright (c) 2026 - * + * */ #include "relu_node.h" #include +#ifdef __CUDA +#include "computational_graph/activation_functions/cuda/activation_nodes.cuh" +#else +#include +#endif + using namespace std; using namespace cgraph; vector> ReLuNode::backward(const Tensor& upstreamGrad) { assert(!upstreamGrad.getRequiresGrad()); - constexpr ftype zero = 0.0; - - auto res = make_shared(upstreamGrad.getDims(), upstreamGrad.getDevice(), false); + auto res = make_shared(upstreamGrad.getDims(), upstreamGrad.getDevice(), false); const auto& parent = parents[0]; - for(tensorSize_t i=0; iset((*parent)[i] > zero ? upstreamGrad[i] : zero, i); + + switch(upstreamGrad.getDevice()) { + case Device::CPU: { + constexpr ftype zero = 0.0; + for(tensorSize_t i=0; iset((*parent)[i] > zero ? upstreamGrad[i] : zero, i); + } + break; + } + case Device::CUDA: + #ifdef __CUDA + cuda_impl::reluBackward(*res, upstreamGrad, *parent); + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); + #endif + break; } return {res}; -} \ No newline at end of file +} diff --git a/src/backend/computational_graph/activation_functions/sigmoid_node.cpp b/src/backend/computational_graph/activation_functions/sigmoid_node.cpp index 5873724..7226d12 100644 --- a/src/backend/computational_graph/activation_functions/sigmoid_node.cpp +++ b/src/backend/computational_graph/activation_functions/sigmoid_node.cpp @@ -1,35 +1,51 @@ /** * @file sigmoid_node.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-14 - * + * * @copyright Copyright (c) 2026 - * + * */ #include "sigmoid_node.h" #include +#ifdef __CUDA +#include "computational_graph/activation_functions/cuda/activation_nodes.cuh" +#else +#include +#endif + using namespace std; using namespace cgraph; vector> SigmoidNode::backward(const Tensor& upstreamGrad) { assert(!upstreamGrad.getRequiresGrad()); - constexpr ftype zero = 0.0; - + auto res = make_shared(upstreamGrad.getDims(), upstreamGrad.getDevice(), false); - // s is result from forward pass sigmoid - auto derivative = [](ftype s){ - return s * (1-s); - }; + switch(upstreamGrad.getDevice()) { + case Device::CPU: { + auto derivative = [](ftype s){ + return s * (1-s); + }; - for(tensorSize_t i=0; iset(derivative((*sigmoid)[i]) * upstreamGrad[i], i); + for(tensorSize_t i=0; iset(derivative((*sigmoid)[i]) * upstreamGrad[i], i); + } + break; + } + case Device::CUDA: + #ifdef __CUDA + cuda_impl::sigmoidBackward(*res, upstreamGrad, *sigmoid); + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); + #endif + break; } return {res}; -} \ No newline at end of file +} diff --git a/src/backend/computational_graph/activation_functions/softmax_node.cpp b/src/backend/computational_graph/activation_functions/softmax_node.cpp index 8603355..debcea6 100644 --- a/src/backend/computational_graph/activation_functions/softmax_node.cpp +++ b/src/backend/computational_graph/activation_functions/softmax_node.cpp @@ -1,12 +1,12 @@ /** * @file softmax_node.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-15 - * + * * @copyright Copyright (c) 2026 - * + * */ #include "softmax_node.h" @@ -15,31 +15,49 @@ #include +#ifdef __CUDA +#include "computational_graph/activation_functions/cuda/activation_nodes.cuh" +#else +#include +#endif + using namespace std; using namespace cgraph; vector< shared_ptr > SoftmaxNode::backward(const Tensor& upstreamGrad) { assert(!upstreamGrad.getRequiresGrad()); - + const auto& yPred = parents[0]; auto res = make_shared(yPred->createEmptyCopy()); - const auto bSize = yPred->getDims()[0]; - assert(bSize>0); - - for(tensorDim_t b=0; bgetDims()[1]; i++){ - ftype grad = 0; - const ftype yi = softmax->get(b, i); - - for(tensorDim_t j=0; jgetDims()[1]; j++){ - ftype yj = softmax->get(b, j); - ftype jacobian = (i==j) ? yi*(1-yj) : -yi*yj; - grad += upstreamGrad.get(b, j) * jacobian; + switch(upstreamGrad.getDevice()) { + case Device::CPU: { + const auto bSize = yPred->getDims()[0]; + assert(bSize>0); + + for(tensorDim_t b=0; bgetDims()[1]; i++){ + ftype grad = 0; + const ftype yi = softmax->get(b, i); + + for(tensorDim_t j=0; jgetDims()[1]; j++){ + ftype yj = softmax->get(b, j); + ftype jacobian = (i==j) ? yi*(1-yj) : -yi*yj; + grad += upstreamGrad.get(b, j) * jacobian; + } + res->set(grad, b, i); + } } - res->set(grad, b, i); + break; } + case Device::CUDA: + #ifdef __CUDA + cuda_impl::softmaxBackward(*res, upstreamGrad, *softmax); + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); + #endif + break; } return {res}; -} \ No newline at end of file +} diff --git a/src/backend/module/activation_functions/leaky_relu.cpp b/src/backend/module/activation_functions/leaky_relu.cpp index 0cd46bc..3f46e5a 100644 --- a/src/backend/module/activation_functions/leaky_relu.cpp +++ b/src/backend/module/activation_functions/leaky_relu.cpp @@ -12,14 +12,31 @@ #include "leaky_relu.h" #include "computational_graph/activation_functions/leaky_relu_node.h" +#ifdef __CUDA +#include "module/activation_functions/cuda/activations.cuh" +#else +#include +#endif + using namespace std; using namespace module; Tensor LeakyReLu::operator()(const Tensor& t) const { auto res = t.createDeepCopy(); - for(tensorSize_t i=0; i +#endif + using namespace std; using namespace module; Tensor ReLu::operator()(const Tensor& t) const { auto res = t.createDeepCopy(); - for(tensorSize_t i=0; i +#ifdef __CUDA +#include "module/activation_functions/cuda/activations.cuh" +#else +#include +#endif + using namespace std; using namespace module; @@ -24,17 +29,29 @@ using namespace module; Tensor Sigmoid::operator()(const Tensor& t) const { auto res = t.createEmptyCopy(); - constexpr ftype one = 1.0; - auto compute = [](ftype x){ - if(x>=0){ - return one / (one + exp(-x)); - } - auto e = exp(x); - return e / (one + e); - }; + switch(t.getDevice()) { + case Device::CPU: { + constexpr ftype one = 1.0; + auto compute = [](ftype x){ + if(x>=0){ + return one / (one + exp(-x)); + } + auto e = exp(x); + return e / (one + e); + }; - for(tensorSize_t i=0; i +#ifdef __CUDA +#include "module/activation_functions/cuda/activations.cuh" +#else +#include +#endif + using namespace std; using namespace module; @@ -28,42 +33,55 @@ Tensor Softmax::operator()(const Tensor& t) const { __throw_invalid_argument("Softmax expects input shape of minimum two dimensions"); } - const auto nRows = t.getDims()[-2]; - const auto nCols = t.getDims()[-1]; + auto res = t.createEmptyCopy(); - // pre-compute exponents - Tensor tmp(t.getDims(), t.getDevice(), false); - for(tensorDim_t i=0; i::infinity(); - for(tensorDim_t j=0; j::infinity(); + for(tensorDim_t j=0; j #include +#ifdef __CUDA +#include "module/layers/cuda/layers.cuh" +#else +#include +#endif + using namespace std; using namespace module; using namespace utility; @@ -55,13 +61,29 @@ FfLayer::FfLayer(tensorDim_t inSize, tensorDim_t outSize, Device d, * Assumption for input: (b-size, ..., dim1, in-size) */ Tensor FfLayer::operator()(const Tensor& input) const { - auto res = input.matmul(*weights); - - if(useBias){ - res = res + *bias; + switch(input.getDevice()) { + case Device::CPU: { + auto res = input.matmul(*weights); + if(useBias) res = res + *bias; + return res; + } + case Device::CUDA: + #ifdef __CUDA + { + auto resDims = input.getDims().toVector(); + resDims.back() = static_cast(weights->getDims().get(-1)); + Tensor res(resDims, input.getDevice(), false); + if(useBias) { + cuda_impl::forwardBias(res, input, *weights, *bias); + } else { + cuda_impl::forward(res, input, *weights); + } + return res; + } + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); + #endif } - - return res; } /** diff --git a/tests/backend/cuda/main.cu b/tests/backend/cuda/main.cu index c567981..1738d17 100644 --- a/tests/backend/cuda/main.cu +++ b/tests/backend/cuda/main.cu @@ -9,7 +9,11 @@ * */ - #include +#ifndef __CUDA +static_assert(false, "File should not be compiled without CUDA enabled"); +#endif // __CUDA + +#include class CudaEnvironment : public ::testing::Environment { public: diff --git a/tests/backend/cuda/test_module_cuda.cu b/tests/backend/cuda/test_module_cuda.cu new file mode 100644 index 0000000..31c5e2d --- /dev/null +++ b/tests/backend/cuda/test_module_cuda.cu @@ -0,0 +1,204 @@ +/** + * @file test_module_cuda.cu + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-05-13 + * + * @copyright Copyright (c) 2026 + * + */ + +#ifndef __CUDA +static_assert(false, "File should not be compiled without CUDA enabled"); +#endif // __CUDA + +#include + +#include "data_modeling/tensor_functions.h" +#include "computational_graph/tensor_ops/graph_creation.h" + +#include "module/activation_functions/relu.h" +#include "module/activation_functions/leaky_relu.h" +#include "module/activation_functions/softmax.h" +#include "module/activation_functions/sigmoid.h" +#include "module/layers/ff_layer.h" + +using namespace std; + +constexpr ftype delta = 1e-3; + +TEST(CudaActivationTest, ReluForward) { + auto t1 = TensorFunctions::Ones({300, 500}, Device::CUDA); + auto f = module::ReLu(); + + auto res = f(t1); + + for(size_t i=0; ibackward(); + + ASSERT_DOUBLE_EQ(x->getGrads()->get(0), 0.0); + ASSERT_DOUBLE_EQ(x->getGrads()->get(1), 0.0); + ASSERT_NEAR(x->getGrads()->get(2), 1.0, 1e-5); +} + +TEST(CudaActivationTest, LeakyReluForward) { + auto t1 = TensorFunctions::Ones({300, 500}, Device::CUDA); + + auto f = module::LeakyReLu(0.3); + auto res = f(t1); + + for(size_t i=0; ibackward(); + + ASSERT_DOUBLE_EQ(x->getGrads()->get(0), eps); + ASSERT_DOUBLE_EQ(x->getGrads()->get(1), eps); + ASSERT_NEAR(x->getGrads()->get(2), 1.0, 1e-5); +} + +TEST(CudaActivationTest, SigmoidLargePositive) { + auto t = TensorFunctions::makeSharedTensor({1}, vector{100.0}, Device::CUDA); + + module::Sigmoid sig; + auto res = sig(*t); + + ASSERT_NEAR(res[0], 1.0, 1e-5); + ASSERT_FALSE(std::isnan(res[0])); + ASSERT_FALSE(std::isinf(res[0])); +} + +TEST(CudaActivationTest, SigmoidLargeNegative) { + auto t = TensorFunctions::makeSharedTensor({1}, vector{-100.0}, Device::CUDA); + + module::Sigmoid sig; + auto res = sig(*t); + + ASSERT_NEAR(res[0], 0.0, 1e-5); + ASSERT_FALSE(std::isnan(res[0])); + ASSERT_FALSE(std::isinf(res[0])); +} + +TEST(CudaAutogradTest, SigmoidBackward) { + auto t = TensorFunctions::makeSharedTensor({2}, {0.0, 1.0}, Device::CUDA, true); + + module::Sigmoid sig; + auto res = sig(t); + res->backward(); + + auto grads = t->getGrads(); + ASSERT_NEAR((*grads)[0], 0.25, delta); + ASSERT_NEAR((*grads)[1], 0.1966, delta); +} + +TEST(CudaActivationTest, SoftmaxForward) { + auto t = TensorFunctions::makeSharedTensor({1, 3}, {1.0, 2.0, 3.0}, Device::CUDA); + + module::Softmax sm; + auto res = sm(*t); + + ASSERT_NEAR(res[0], 0.0900, delta); + ASSERT_NEAR(res[1], 0.2447, delta); + ASSERT_NEAR(res[2], 0.6652, delta); +} + +TEST(CudaActivationTest, SoftmaxSumsToOne) { + auto t = TensorFunctions::makeSharedTensor({2, 4}, + {1.0, 2.0, 3.0, 4.0, + 2.0, 1.0, 4.0, 3.0}, Device::CUDA); + + module::Softmax sm; + auto res = sm(*t); + + ftype row0sum = res[0] + res[1] + res[2] + res[3]; + ftype row1sum = res[4] + res[5] + res[6] + res[7]; + ASSERT_NEAR(row0sum, 1.0, delta); + ASSERT_NEAR(row1sum, 1.0, delta); +} + +TEST(CudaActivationTest, SoftmaxForwardNumericalStability) { + auto t = TensorFunctions::makeSharedTensor({1, 3}, {100.0, 101.0, 102.0}, Device::CUDA); + + module::Softmax sm; + auto res = sm(*t); + + for(int i = 0; i < 3; i++) { + ASSERT_FALSE(std::isnan(res[i])); + ASSERT_FALSE(std::isinf(res[i])); + } + ftype rowsum = res[0] + res[1] + res[2]; + ASSERT_NEAR(rowsum, 1.0, delta); +} + +TEST(CudaAutogradTest, SoftmaxBackward) { + auto t = TensorFunctions::makeSharedTensor({1, 3}, {1.0, 2.0, 3.0}, Device::CUDA, true); + + module::Softmax sm; + auto resPtr = sm(t); + + auto upstream = TensorFunctions::makeSharedTensor({1, 3}, {1.0, 0.0, 0.0}, Device::CUDA); + resPtr->setGrads(upstream); + resPtr->backward(); + + auto grads = t->getGrads(); + ASSERT_NEAR((*grads)[0], 0.0819, delta); + ASSERT_NEAR((*grads)[1], -0.0220, delta); + ASSERT_NEAR((*grads)[2], -0.0599, delta); +} + +TEST(CudaLayerTest, TestFfLayer) { + auto t1 = TensorFunctions::Ones({3, 2}, Device::CUDA); + auto layer = module::FfLayer(2, 1, Device::CUDA); + + auto res = layer(t1); + + ASSERT_EQ(res.getDims(), Dimension({3, 1})); +} diff --git a/tests/backend/cuda/test_tensorops_cuda.cu b/tests/backend/cuda/test_tensorops_cuda.cu index c0dd3d6..f47ded9 100644 --- a/tests/backend/cuda/test_tensorops_cuda.cu +++ b/tests/backend/cuda/test_tensorops_cuda.cu @@ -9,6 +9,10 @@ * */ +#ifndef __CUDA +static_assert(false, "File should not be compiled without CUDA enabled"); +#endif // __CUDA + #include #include "data_modeling/tensor.h" diff --git a/tests/backend/test_module.cpp b/tests/backend/test_module.cpp index 243073d..bd7c5a7 100644 --- a/tests/backend/test_module.cpp +++ b/tests/backend/test_module.cpp @@ -23,21 +23,23 @@ #include +using namespace std; + constexpr ftype delta = 1e-3; TEST(ActivationTest, ReluForward) { - auto t1 = TensorFunctions::Ones({3, 2}, false); + auto t1 = TensorFunctions::Ones({3, 2}); auto f = module::ReLu(); auto res = f(t1); for(size_t i=0; i 0) ASSERT_DOUBLE_EQ(x->getGrads()->get(0), 0.0); ASSERT_DOUBLE_EQ(x->getGrads()->get(1), 0.0); - EXPECT_NEAR(x->getGrads()->get(2), 1.0, 1e-5); + ASSERT_NEAR(x->getGrads()->get(2), 1.0, 1e-5); } TEST(ActivationTest, LeakyReluForward) { - auto t1 = TensorFunctions::Ones({3, 2}, false); + auto t1 = TensorFunctions::Ones({3, 2}); auto f = module::LeakyReLu(0.3); auto res = f(t1); for(size_t i=0; i 0) ASSERT_DOUBLE_EQ(x->getGrads()->get(0), eps); ASSERT_DOUBLE_EQ(x->getGrads()->get(1), eps); // by convention - EXPECT_NEAR(x->getGrads()->get(2), 1.0, 1e-5); + ASSERT_NEAR(x->getGrads()->get(2), 1.0, 1e-5); } TEST(ActivationTest, SigmoidForward) { // sigmoid(0) = 0.5, sigmoid(1) = 0.7311, sigmoid(-1) = 0.2689 - auto t = Tensor({3}, {0.0, 1.0, -1.0}, true); + auto t = Tensor({3}, {0.0, 1.0, -1.0}); module::Sigmoid sig; auto res = sig(t); - EXPECT_NEAR(res[0], 0.5, delta); - EXPECT_NEAR(res[1], 0.7311, delta); - EXPECT_NEAR(res[2], 0.2689, delta); + ASSERT_NEAR(res[0], 0.5, delta); + ASSERT_NEAR(res[1], 0.7311, delta); + ASSERT_NEAR(res[2], 0.2689, delta); } TEST(ActivationTest, SigmoidLargePositive) { // sigmoid(100) should be ~1, not inf or nan - auto t = Tensor({1}, {100.0}, true); + auto t = Tensor({1}, vector{100.0}); module::Sigmoid sig; auto res = sig(t); - EXPECT_NEAR(res[0], 1.0, delta); + ASSERT_NEAR(res[0], 1.0, delta); EXPECT_FALSE(std::isnan(res[0])); EXPECT_FALSE(std::isinf(res[0])); } TEST(ActivationTest, SigmoidLargeNegative) { // sigmoid(-100) should be ~0, not nan - auto t = Tensor({1}, {-100.0}, true); + auto t = Tensor({1}, vector{-100.0}); module::Sigmoid sig; auto res = sig(t); - EXPECT_NEAR(res[0], 0.0, delta); + ASSERT_NEAR(res[0], 0.0, delta); EXPECT_FALSE(std::isnan(res[0])); EXPECT_FALSE(std::isinf(res[0])); } @@ -152,8 +154,8 @@ TEST(AutogradTest, SigmoidBackward) { res->backward(); auto grads = t->getGrads(); - EXPECT_NEAR((*grads)[0], 0.25, delta); - EXPECT_NEAR((*grads)[1], 0.1966, delta); + ASSERT_NEAR((*grads)[0], 0.25, delta); + ASSERT_NEAR((*grads)[1], 0.1966, delta); } TEST(ActivationTest, SoftmaxForward) { @@ -161,21 +163,20 @@ TEST(ActivationTest, SoftmaxForward) { // exp([1,2,3]) = [2.7183, 7.3891, 20.0855] // sum = 30.1929 // softmax = [0.0900, 0.2447, 0.6652] - auto t = Tensor({1, 3}, {1.0, 2.0, 3.0}, true); + auto t = Tensor({1, 3}, {1.0, 2.0, 3.0}); module::Softmax sm; auto res = sm(t); - EXPECT_NEAR(res[0], 0.0900, delta); - EXPECT_NEAR(res[1], 0.2447, delta); - EXPECT_NEAR(res[2], 0.6652, delta); + ASSERT_NEAR(res[0], 0.0900, delta); + ASSERT_NEAR(res[1], 0.2447, delta); + ASSERT_NEAR(res[2], 0.6652, delta); } TEST(ActivationTest, SoftmaxSumsToOne) { auto t = Tensor({2, 4}, {1.0, 2.0, 3.0, 4.0, - 2.0, 1.0, 4.0, 3.0}, - true); + 2.0, 1.0, 4.0, 3.0}); module::Softmax sm; auto res = sm(t); @@ -183,13 +184,13 @@ TEST(ActivationTest, SoftmaxSumsToOne) { // each row must sum to 1 ftype row0sum = res[0] + res[1] + res[2] + res[3]; ftype row1sum = res[4] + res[5] + res[6] + res[7]; - EXPECT_NEAR(row0sum, 1.0, delta); - EXPECT_NEAR(row1sum, 1.0, delta); + ASSERT_NEAR(row0sum, 1.0, delta); + ASSERT_NEAR(row1sum, 1.0, delta); } TEST(ActivationTest, SoftmaxForwardNumericalStability) { // large values should not produce nan or inf - auto t = Tensor({1, 3}, {100.0, 101.0, 102.0}, true); + auto t = Tensor({1, 3}, {100.0, 101.0, 102.0}); module::Softmax sm; auto res = sm(t); @@ -199,7 +200,7 @@ TEST(ActivationTest, SoftmaxForwardNumericalStability) { EXPECT_FALSE(std::isinf(res[i])); } ftype rowsum = res[0] + res[1] + res[2]; - EXPECT_NEAR(rowsum, 1.0, delta); + ASSERT_NEAR(rowsum, 1.0, delta); } TEST(AutogradTest, SoftmaxBackward) { @@ -220,21 +221,89 @@ TEST(AutogradTest, SoftmaxBackward) { // set upstream gradient to [1, 0, 0] auto upstream = TensorFunctions::makeSharedTensor( - {1, 3}, {1.0, 0.0, 0.0}, false); + {1, 3}, {1.0, 0.0, 0.0}); resPtr->setGrads(upstream); resPtr->backward(); auto grads = t->getGrads(); - EXPECT_NEAR((*grads)[0], 0.0819, delta); - EXPECT_NEAR((*grads)[1], -0.0220, delta); - EXPECT_NEAR((*grads)[2], -0.0599, delta); + ASSERT_NEAR((*grads)[0], 0.0819, delta); + ASSERT_NEAR((*grads)[1], -0.0220, delta); + ASSERT_NEAR((*grads)[2], -0.0599, delta); } TEST(LayerTest, TestFfLayer) { - auto t1 = TensorFunctions::Ones({3, 2}, false); - auto layer = module::FfLayer(2, 1, true, false); + auto t1 = TensorFunctions::Ones({3, 2}); + auto layer = module::FfLayer(2, 1); auto res = layer(t1); ASSERT_EQ(res.getDims(), Dimension({3, 1})); +} + +TEST(AutogradTest, FfLayerBackward) { + // x: (2,3) all ones, W: (3,2) all ones + // y = x @ W => (2,2) all threes + // loss = sum(y), upstream = ones(2,2) + // grad_x = upstream @ W^T = ones(2,2) @ ones(2,3) => all 2s + // grad_W = x^T @ upstream = ones(3,2) @ ones(2,2) => all 2s + auto x = TensorFunctions::makeSharedTensor({2, 3}, { + 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0 + }, true); + + auto layer = module::FfLayer(3, 2, /*useBias=*/false, /*requiresGrad=*/true); + auto w = layer.getWeights(); + for(tensorSize_t i = 0; i < w->getSize(); i++) w->set(1.0, i); + + auto res = layer(x); + auto loss = cgraph::sumTensor(res); + loss->backward(); + + auto xGrads = x->getGrads(); + ASSERT_NE(xGrads, nullptr); + ASSERT_EQ(xGrads->getDims(), x->getDims()); + for(tensorSize_t i = 0; i < xGrads->getSize(); i++) { + ASSERT_NEAR((*xGrads)[i], 2.0, delta); + } + + auto wGrads = layer.getWeights()->getGrads(); + ASSERT_NE(wGrads, nullptr); + ASSERT_EQ(wGrads->getDims(), layer.getWeights()->getDims()); + for(tensorSize_t i = 0; i < wGrads->getSize(); i++) { + ASSERT_NEAR((*wGrads)[i], 2.0, delta); + } +} + +TEST(AutogradTest, FfLayerBackwardWithBias) { + // Same as above but with bias (initialised to zero by constructor). + // grad_b = sum(upstream, axis=0) = [2, 2] + auto x = TensorFunctions::makeSharedTensor({2, 3}, { + 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0 + }, true); + + auto layer = module::FfLayer(3, 2, /*useBias=*/true, /*requiresGrad=*/true); + auto w = layer.getWeights(); + for(tensorSize_t i = 0; i < w->getSize(); i++) w->set(1.0, i); + + auto res = layer(x); + auto loss = cgraph::sumTensor(res); + loss->backward(); + + auto xGrads = x->getGrads(); + ASSERT_NE(xGrads, nullptr); + for(tensorSize_t i = 0; i < xGrads->getSize(); i++) { + ASSERT_NEAR((*xGrads)[i], 2.0, delta); + } + + auto wGrads = layer.getWeights()->getGrads(); + ASSERT_NE(wGrads, nullptr); + for(tensorSize_t i = 0; i < wGrads->getSize(); i++) { + ASSERT_NEAR((*wGrads)[i], 2.0, delta); + } + + auto bGrads = layer.getBias()->getGrads(); + ASSERT_NE(bGrads, nullptr); + ASSERT_NEAR((*bGrads)[0], 2.0, delta); + ASSERT_NEAR((*bGrads)[1], 2.0, delta); } \ No newline at end of file From 4d674553ed64b8e047c951504a658f8513efda14 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Wed, 13 May 2026 15:48:02 +0200 Subject: [PATCH 35/73] More infra prepared for CUDA --- .../training/loss_functions/bce_loss.cpp | 36 +++--- .../loss_functions/bce_sigmoid_loss.cpp | 55 +++++--- .../loss_functions/crossentropy_loss.cpp | 54 +++++--- .../crossentropy_softmax_loss.cpp | 117 ++++++++++-------- .../loss_functions/cuda/loss_functions.cu | 74 +++++++---- .../loss_functions/cuda/loss_functions.cuh | 18 +-- .../training/loss_functions/rmse_loss.cpp | 56 ++++++--- .../training/optimizers/cuda/optimizers.cu | 45 +++++++ .../training/optimizers/cuda/optimizers.cuh | 24 ++++ src/backend/training/optimizers/rmsprop.cpp | 70 +++++++---- src/backend/training/optimizers/sgd.cpp | 31 +++-- 11 files changed, 393 insertions(+), 187 deletions(-) create mode 100644 src/backend/training/optimizers/cuda/optimizers.cu create mode 100644 src/backend/training/optimizers/cuda/optimizers.cuh diff --git a/src/backend/training/loss_functions/bce_loss.cpp b/src/backend/training/loss_functions/bce_loss.cpp index 48eaf4f..43b1d87 100644 --- a/src/backend/training/loss_functions/bce_loss.cpp +++ b/src/backend/training/loss_functions/bce_loss.cpp @@ -1,23 +1,25 @@ /** * @file bce_loss.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-07 - * + * * @copyright Copyright (c) 2026 - * + * */ #include "bce_loss.h" #include "computational_graph/loss_functions/bce_node.h" -#ifdef CUDA +#include + +#ifdef __CUDA #include "training/loss_functions/cuda/loss_functions.cuh" +#else +#include #endif -#include - using namespace std; using namespace train; @@ -28,7 +30,7 @@ using namespace train; shared_ptr BceLoss::operator()(const shared_ptr y, const shared_ptr ypred) const { if(!ypred->getRequiresGrad()) { __throw_invalid_argument("ypred must have gradient enabled"); - } + } else if(y->getDevice() != ypred->getDevice()){ __throw_invalid_argument("y and ypred must be on same device"); } @@ -36,18 +38,17 @@ shared_ptr BceLoss::operator()(const shared_ptr y, const shared_ __throw_invalid_argument("Tensors must be of same shape"); } - shared_ptr res = nullptr; + shared_ptr res = nullptr; switch(y->getDevice()) { case Device::CUDA: - #ifdef CUDA - res = cuda_impl::bceLoss(*y, *ypred); - break; - #else - __throw_runtime_error("Should not reach this line"); + #ifdef __CUDA + res = make_shared(cuda_impl::bceLoss(*y, *ypred)); + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); #endif - case Device::CPU: - { + break; + case Device::CPU: { auto bce = [](ftype y, ftype ypred){ return y * log(std::max(ypred, EPS_BCE)) + (1 - y) * log(std::max(1-ypred, EPS_BCE)); }; @@ -58,14 +59,11 @@ shared_ptr BceLoss::operator()(const shared_ptr y, const shared_ loss += bce((*y)[i], (*ypred)[i]); } res = make_shared(std::vector{1}, std::vector{-loss / nBatches}, Device::CPU, true); - break; } - default: - __throw_invalid_argument("Unexpected device encountered"); } res->setCgNode(make_shared(y, ypred)); assert(res->getRequiresGrad()); return res; -} \ No newline at end of file +} diff --git a/src/backend/training/loss_functions/bce_sigmoid_loss.cpp b/src/backend/training/loss_functions/bce_sigmoid_loss.cpp index 2634bf4..21cbe29 100644 --- a/src/backend/training/loss_functions/bce_sigmoid_loss.cpp +++ b/src/backend/training/loss_functions/bce_sigmoid_loss.cpp @@ -1,19 +1,24 @@ /** * @file bce_logits_loss.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-17 - * + * * @copyright Copyright (c) 2026 - * + * */ - #include "bce_sigmoid_loss.h" +#include "bce_sigmoid_loss.h" +#include "computational_graph/loss_functions/bce_sigmoid_node.h" - #include "computational_graph/loss_functions/bce_sigmoid_node.h" +#include - #include +#ifdef __CUDA +#include "training/loss_functions/cuda/loss_functions.cuh" +#else +#include +#endif using namespace std; using namespace train; @@ -25,7 +30,7 @@ using namespace train; shared_ptr BceSigmoidLoss::operator()(const shared_ptr y, const shared_ptr logits) const { if(!logits->getRequiresGrad()) { __throw_invalid_argument("logits must have gradient enabled"); - } + } else if(y->getDevice() != logits->getDevice()){ __throw_invalid_argument("y and logits must be on same device"); } @@ -33,21 +38,33 @@ shared_ptr BceSigmoidLoss::operator()(const shared_ptr y, const __throw_invalid_argument("Tensors must be of same shape"); } - auto bceSimplified = [](ftype y, ftype logit){ - constexpr ftype zero = 0; - return std::max(logit, zero) - logit*y + log(1+exp(-std::abs(logit))); - }; + shared_ptr res = nullptr; - const auto nBatches = y->getDims()[0]; + switch(y->getDevice()) { + case Device::CPU: { + auto bceSimplified = [](ftype y, ftype logit){ + constexpr ftype zero = 0; + return std::max(logit, zero) - logit*y + log(1+exp(-std::abs(logit))); + }; - ftype loss = 0; - for(tensorSize_t i=0; igetDims()[0]; + ftype loss = 0; + for(tensorSize_t i=0; i(std::vector{1}, std::vector{loss / nBatches}, y->getDevice(), true); + break; + } + case Device::CUDA: + #ifdef __CUDA + res = make_shared(cuda_impl::bceSigmoidLoss(*y, *logits)); + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); + #endif + break; } - auto res = make_shared(std::vector{1}, std::vector{loss / nBatches}, y->getDevice(), true); res->setCgNode(make_shared(y, logits)); assert(res->getRequiresGrad()); - - return res; -} \ No newline at end of file + return res; +} diff --git a/src/backend/training/loss_functions/crossentropy_loss.cpp b/src/backend/training/loss_functions/crossentropy_loss.cpp index ee6b07c..c598f88 100644 --- a/src/backend/training/loss_functions/crossentropy_loss.cpp +++ b/src/backend/training/loss_functions/crossentropy_loss.cpp @@ -1,20 +1,25 @@ /** * @file crossentropy_loss.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-17 - * + * * @copyright Copyright (c) 2026 - * + * */ #include "crossentropy_loss.h" - #include "computational_graph/loss_functions/crossentropy_node.h" #include +#ifdef __CUDA +#include "training/loss_functions/cuda/loss_functions.cuh" +#else +#include +#endif + using namespace std; using namespace train; @@ -33,23 +38,36 @@ shared_ptr CrossEntropyLoss::operator()(const shared_ptr y, cons __throw_invalid_argument("Tensors must be of same shape"); } - auto ce = [&y, &ypred](const tensorDim_t b){ - ftype res = 0; - for(tensorDim_t i=0; igetDims()[-1]; i++){ - res += y->get(b, i) * log(std::max(ypred->get(b, i), EPS_CROSSENTROPY)); - } - return res; - }; + shared_ptr res = nullptr; - const auto nBatches = y->getDims()[0]; - ftype loss = 0; - for(tensorSize_t b=0; bgetDevice()) { + case Device::CPU: { + auto ce = [&y, &ypred](const tensorDim_t b){ + ftype r = 0; + for(tensorDim_t i=0; igetDims()[-1]; i++){ + r += y->get(b, i) * log(std::max(ypred->get(b, i), EPS_CROSSENTROPY)); + } + return r; + }; + + const auto nBatches = y->getDims()[0]; + ftype loss = 0; + for(tensorSize_t b=0; b(std::vector{1}, std::vector{-loss / nBatches}, y->getDevice(), true); + break; + } + case Device::CUDA: + #ifdef __CUDA + res = make_shared(cuda_impl::crossEntropyLoss(*y, *ypred)); + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); + #endif + break; } - auto res = make_shared(std::vector{1}, std::vector{-loss / nBatches}, y->getDevice(), true); res->setCgNode(std::make_shared(y, ypred)); assert(res->getRequiresGrad()); - return res; -} \ No newline at end of file +} diff --git a/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp b/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp index a2b7866..a82b188 100644 --- a/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp +++ b/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp @@ -1,20 +1,25 @@ /** * @file crossentropy_logits_loss.cpp - * @author Robert Baumgartner (r.baumgartner-1@tudelflogits->nl) - * @brief + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief * @version 0.1 * @date 2026-03-17 - * + * * @copyright Copyright (c) 2026 - * + * */ #include "crossentropy_softmax_loss.h" - #include "computational_graph/loss_functions/crossentropy_softmax_node.h" #include +#ifdef __CUDA +#include "training/loss_functions/cuda/loss_functions.cuh" +#else +#include +#endif + using namespace std; using namespace train; @@ -33,61 +38,73 @@ shared_ptr CrossEntropySoftmaxLoss::operator()(const shared_ptr __throw_invalid_argument("Tensors must be of same shape"); } - //////////////////////////////////////////////// + shared_ptr res = nullptr; - const auto nRows = logits->getDims()[-2]; - const auto nCols = logits->getDims()[-1]; + switch(y->getDevice()) { + case Device::CPU: { + const auto nRows = logits->getDims()[-2]; + const auto nCols = logits->getDims()[-1]; - // pre-compute exponents and max-values - vector maxValues(nRows); - Tensor tmp(logits->getDims(), logits->getDevice(), false); - for(tensorDim_t i=0; i::infinity(); - for(tensorDim_t j=0; jget(i, j)); - } + // pre-compute exponents and max-values + vector maxValues(nRows); + Tensor tmp(logits->getDims(), logits->getDevice(), false); + for(tensorDim_t i=0; i::infinity(); + for(tensorDim_t j=0; jget(i, j)); + } - maxValues[i] = maxV; + maxValues[i] = maxV; - for(tensorDim_t j=0; jget(i, j)-maxV; - tmp.set(exp(e), i, j); - } - } + for(tensorDim_t j=0; jget(i, j)-maxV; + tmp.set(exp(e), i, j); + } + } - const tensorSize_t stride = logits->getDims()[-1]; - ftype loss = 0; - - /** - * CE = -sum_i(y_i * z_i) + log(sum_j(exp(z_j))) with - * log(sum_j(exp(z_j))) = max(z) + log(sum_j(exp(z_j - max(z)))). - * for numerical stability - */ - auto compute = [&loss, &y, &logits, &tmp, &maxValues, stride](tensorSize_t start){ - ftype lsum = 0; - for(tensorSize_t i=start; igetDims()[-1]; + ftype loss = 0; + + /** + * CE = -sum_i(y_i * z_i) + log(sum_j(exp(z_j))) with + * log(sum_j(exp(z_j))) = max(z) + log(sum_j(exp(z_j - max(z)))). + * for numerical stability + */ + auto compute = [&loss, &y, &logits, &tmp, &maxValues, stride](tensorSize_t start){ + ftype lsum = 0; + for(tensorSize_t i=start; i0){ // y either zero or one + loss += -(*logits)[i] + maxValues[j] + lsum; + } + } + }; - const tensorSize_t j = start/stride; - for(tensorSize_t i=start; i0){ // y either zero or one - loss += -(*logits)[i] + maxValues[j] + lsum; + tensorSize_t offset=0; + while(offsetgetSize()) { + compute(offset); + offset += stride; } + + res = make_shared(std::vector{1}, std::vector{loss / logits->getDims()[0]}, y->getDevice(), true); + break; } - }; - - tensorSize_t offset=0; - while(offsetgetSize()) { - compute(offset); - offset += stride; + case Device::CUDA: + #ifdef __CUDA + res = make_shared(cuda_impl::crossEntropySoftmaxLoss(*y, *logits)); + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); + #endif + break; } - auto res = make_shared(std::vector{1}, std::vector{loss / logits->getDims()[0]}, y->getDevice(), true); res->setCgNode(std::make_shared(y, logits)); assert(res->getRequiresGrad()); - return res; -} \ No newline at end of file +} diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cu b/src/backend/training/loss_functions/cuda/loss_functions.cu index 68dd776..0a0114a 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cu +++ b/src/backend/training/loss_functions/cuda/loss_functions.cu @@ -1,15 +1,14 @@ /** * @file loss_functions.cu * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-05-10 - * + * * @copyright Copyright (c) 2026 - * + * */ - #ifndef __CUDA static_assert(false, "File should not be compiled without CUDA enabled"); #endif // __CUDA @@ -17,8 +16,6 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include "loss_functions.cuh" #include "utility/cuda/cuda_common.cuh" -#include - using namespace std; namespace { @@ -61,22 +58,22 @@ namespace { sdata[tid] += sdata[tid + 32]; } __syncthreads(); - + if(tid < 16 && gid + 16 < size) { sdata[tid] += sdata[tid + 16]; } __syncthreads(); - + if(tid < 8 && gid + 8 < size) { sdata[tid] += sdata[tid + 8]; } __syncthreads(); - + if(tid < 4 && gid + 4 < size) { sdata[tid] += sdata[tid + 4]; } __syncthreads(); - + if(tid < 2 && gid + 2 < size) { sdata[tid] += sdata[tid + 2]; } @@ -86,35 +83,70 @@ namespace { sdata[0] = (sdata[0] + sdata[1]) / size; } } + + // TODO: bceSigmoidLoss kernel + + // TODO: crossEntropyLoss kernel + + // TODO: crossEntropySoftmaxLoss kernel + + // TODO: rmseLoss kernel } namespace cuda_impl { - Tensor&& bceLoss(const Tensor& y, const Tensor& yPred) { - + Tensor bceLoss(const Tensor& y, const Tensor& yPred) { constexpr int threadsPerBlock = 256; const int blocks = (y.getSize() + threadsPerBlock - 1) / (threadsPerBlock * 2); - auto res = Tensor(vector{1}, Device::CUDA, true); - - bceKernel<<>>(res.getData(), y.getData(), yPred.getData(), y.getDims()[0]); + Tensor res(vector{1}, Device::CUDA, true); + bceKernel<<>>( + res.getData(), y.getData(), yPred.getData(), y.getDims()[0]); cudaErrchk(cudaDeviceSynchronize()); - return std::move(res); + return res; } - Tensor&& bceSigmoidLoss(const Tensor& y, const Tensor& yPred) { + Tensor bceSigmoidLoss(const Tensor& y, const Tensor& yPred) { + const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; + + Tensor res(vector{1}, Device::CUDA, true); + // TODO: launch kernel + cudaErrchk(cudaDeviceSynchronize()); + return res; } - Tensor&& crossEntropyLoss(const Tensor& y, const Tensor& yPred) { + Tensor crossEntropyLoss(const Tensor& y, const Tensor& yPred) { + const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; + + Tensor res(vector{1}, Device::CUDA, true); + // TODO: launch kernel + cudaErrchk(cudaDeviceSynchronize()); + return res; } - Tensor&& crossEntropySoftmaxLoss(const Tensor& y, const Tensor& yPred) { + Tensor crossEntropySoftmaxLoss(const Tensor& y, const Tensor& yPred) { + const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; + Tensor res(vector{1}, Device::CUDA, true); + // TODO: launch kernel + cudaErrchk(cudaDeviceSynchronize()); + + return res; } - Tensor&& rmseLoss(const Tensor& y, const Tensor& yPred) { + Tensor rmseLoss(const Tensor& y, const Tensor& yPred) { + const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; + Tensor res(vector{1}, Device::CUDA, true); + // TODO: launch kernel + cudaErrchk(cudaDeviceSynchronize()); + + return res; } -} \ No newline at end of file +} diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cuh b/src/backend/training/loss_functions/cuda/loss_functions.cuh index f88fdad..87eee44 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cuh +++ b/src/backend/training/loss_functions/cuda/loss_functions.cuh @@ -1,12 +1,12 @@ /** * @file loss_functions.cuh * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-05-10 - * + * * @copyright Copyright (c) 2026 - * + * */ #pragma once @@ -18,11 +18,11 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "data_modeling/tensor.h" namespace cuda_impl { - [[nodiscard]] Tensor&& bceLoss(const Tensor& y, const Tensor& yPred); - [[nodiscard]] Tensor&& bceSigmoidLoss(const Tensor& y, const Tensor& yPred); + [[nodiscard]] Tensor bceLoss(const Tensor& y, const Tensor& yPred); + [[nodiscard]] Tensor bceSigmoidLoss(const Tensor& y, const Tensor& yPred); - [[nodiscard]] Tensor&& crossEntropyLoss(const Tensor& y, const Tensor& yPred); - [[nodiscard]] Tensor&& crossEntropySoftmaxLoss(const Tensor& y, const Tensor& yPred); + [[nodiscard]] Tensor crossEntropyLoss(const Tensor& y, const Tensor& yPred); + [[nodiscard]] Tensor crossEntropySoftmaxLoss(const Tensor& y, const Tensor& yPred); - [[nodiscard]] Tensor&& rmseLoss(const Tensor& y, const Tensor& yPred); -} \ No newline at end of file + [[nodiscard]] Tensor rmseLoss(const Tensor& y, const Tensor& yPred); +} diff --git a/src/backend/training/loss_functions/rmse_loss.cpp b/src/backend/training/loss_functions/rmse_loss.cpp index 30c750a..872fddf 100644 --- a/src/backend/training/loss_functions/rmse_loss.cpp +++ b/src/backend/training/loss_functions/rmse_loss.cpp @@ -1,21 +1,26 @@ /** * @file rmse_loss.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-14 - * + * * @copyright Copyright (c) 2026 - * + * */ #include "rmse_loss.h" - #include "computational_graph/loss_functions/rmse_node.h" #include #include +#ifdef __CUDA +#include "training/loss_functions/cuda/loss_functions.cuh" +#else +#include +#endif + using namespace std; using namespace train; @@ -26,7 +31,7 @@ using namespace train; shared_ptr RmseLoss::operator()(const shared_ptr y, const shared_ptr ypred) const { if(!ypred->getRequiresGrad()) { __throw_invalid_argument("ypred must have gradient enabled"); - } + } else if(y->getDevice() != ypred->getDevice()){ __throw_invalid_argument("y and ypred must be on same device"); } @@ -34,22 +39,35 @@ shared_ptr RmseLoss::operator()(const shared_ptr y, const shared __throw_invalid_argument("Tensors must be of same shape"); } - auto diffPow = [](ftype y, ftype ypred){ - auto diff = y - ypred; - return diff * diff; - }; + shared_ptr res = nullptr; - const auto nBatches = y->getDims()[0]; + switch(y->getDevice()) { + case Device::CPU: { + auto diffPow = [](ftype y, ftype ypred){ + auto diff = y - ypred; + return diff * diff; + }; - ftype loss = 0; - for(tensorSize_t i=0; igetDims()[0]; + ftype loss = 0; + for(tensorSize_t i=0; i(std::vector{1}, std::vector{loss}, y->getDevice(), true); + break; + } + case Device::CUDA: + #ifdef __CUDA + res = make_shared(cuda_impl::rmseLoss(*y, *ypred)); + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); + #endif + break; } - loss = sqrt(loss/nBatches); - auto res = make_shared(std::vector{1}, std::vector{loss}, y->getDevice(), true); - res->setCgNode(make_shared(y, ypred, loss)); + res->setCgNode(make_shared(y, ypred, res->get(0))); assert(res->getRequiresGrad()); - - return res; -} \ No newline at end of file + return res; +} diff --git a/src/backend/training/optimizers/cuda/optimizers.cu b/src/backend/training/optimizers/cuda/optimizers.cu new file mode 100644 index 0000000..f20bc26 --- /dev/null +++ b/src/backend/training/optimizers/cuda/optimizers.cu @@ -0,0 +1,45 @@ +/** + * @file optimizers.cu + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-05-13 + * + * @copyright Copyright (c) 2026 + * + */ + +#ifndef __CUDA +static_assert(false, "File should not be compiled without CUDA enabled"); +#endif // __CUDA + +#include "optimizers.cuh" +#include "utility/cuda/cuda_common.cuh" + +using namespace std; + +namespace { + // TODO: sgdStep kernel + + // TODO: rmspropStep kernel +} + +namespace cuda_impl { + void sgdStep(Tensor& param, const Tensor& grad, ftype lr) { + const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + const int blocks = (param.getSize() + threadsPerBlock - 1) / threadsPerBlock; + + // TODO: launch kernel + + cudaErrchk(cudaDeviceSynchronize()); + } + + void rmspropStep(Tensor& param, Tensor& movingAvg, const Tensor& grad, ftype lr, ftype decay, ftype eps) { + const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + const int blocks = (param.getSize() + threadsPerBlock - 1) / threadsPerBlock; + + // TODO: launch kernel + + cudaErrchk(cudaDeviceSynchronize()); + } +} diff --git a/src/backend/training/optimizers/cuda/optimizers.cuh b/src/backend/training/optimizers/cuda/optimizers.cuh new file mode 100644 index 0000000..89e0f11 --- /dev/null +++ b/src/backend/training/optimizers/cuda/optimizers.cuh @@ -0,0 +1,24 @@ +/** + * @file optimizers.cuh + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-05-13 + * + * @copyright Copyright (c) 2026 + * + */ + +#pragma once + +#ifndef __CUDA +static_assert(false, "File should not be included without CUDA enabled"); +#endif // __CUDA + +#include "utility/global_params.h" +#include "data_modeling/tensor.h" + +namespace cuda_impl { + void sgdStep(Tensor& param, const Tensor& grad, ftype lr); + void rmspropStep(Tensor& param, Tensor& movingAvg, const Tensor& grad, ftype lr, ftype decay, ftype eps); +} diff --git a/src/backend/training/optimizers/rmsprop.cpp b/src/backend/training/optimizers/rmsprop.cpp index 6878f56..8cf84a8 100644 --- a/src/backend/training/optimizers/rmsprop.cpp +++ b/src/backend/training/optimizers/rmsprop.cpp @@ -1,16 +1,22 @@ /** * @file rmsprop.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-10 - * + * * @copyright Copyright (c) 2026 - * + * */ #include "rmsprop.h" +#ifdef __CUDA +#include "training/optimizers/cuda/optimizers.cuh" +#else +#include +#endif + using namespace std; using namespace train; @@ -19,29 +25,43 @@ void RmsPropOptimizer::step() { for(const auto& param: params){ auto tPtr = param.get(); const auto gPtr = tPtr->getGrads().get(); - auto vPtr = movingAvg[tPtr].get(); - // update moving avg - if(vPtr!=nullptr) { // hot path - for(tensorSize_t i=0; igetSize(); i++){ - auto g = (*gPtr)[i]; - auto update = decay * (*vPtr)[i] + (1-decay)*g*g; - vPtr->set(update, i); - } - } - else { // init loop - movingAvg[tPtr] = make_unique(tPtr->getDims(), tPtr->getDevice(), false); // create empty tensor - vPtr = movingAvg[tPtr].get(); - for(tensorSize_t i=0; igetSize(); i++) { - auto g = (*gPtr)[i]; - vPtr->set((1-decay)*g*g, i); - } - } + switch(tPtr->getDevice()) { + case Device::CPU: { + auto vPtr = movingAvg[tPtr].get(); + + if(vPtr != nullptr) { + for(tensorSize_t i=0; igetSize(); i++){ + auto g = (*gPtr)[i]; + auto update = decay * (*vPtr)[i] + (1-decay)*g*g; + vPtr->set(update, i); + } + } + else { + movingAvg[tPtr] = make_unique(tPtr->getDims(), tPtr->getDevice(), false); + vPtr = movingAvg[tPtr].get(); + for(tensorSize_t i=0; igetSize(); i++) { + auto g = (*gPtr)[i]; + vPtr->set((1-decay)*g*g, i); + } + } - // update gradients - for(tensorSize_t i=0; igetSize(); i++) { - auto update = (*tPtr)[i] - lr * (*gPtr)[i] / ((*vPtr)[i] + eps); - tPtr->set(update, i); + for(tensorSize_t i=0; igetSize(); i++) { + auto update = (*tPtr)[i] - lr * (*gPtr)[i] / ((*vPtr)[i] + eps); + tPtr->set(update, i); + } + break; + } + case Device::CUDA: + #ifdef __CUDA + if(movingAvg[tPtr] == nullptr) { + movingAvg[tPtr] = make_unique(tPtr->getDims(), tPtr->getDevice(), false); + } + cuda_impl::rmspropStep(*tPtr, *movingAvg[tPtr], *gPtr, lr, decay, eps); + #else + __throw_invalid_argument("Attempted to use CUDA tensor"); + #endif + break; } } -} \ No newline at end of file +} diff --git a/src/backend/training/optimizers/sgd.cpp b/src/backend/training/optimizers/sgd.cpp index 83f122d..f132dbd 100644 --- a/src/backend/training/optimizers/sgd.cpp +++ b/src/backend/training/optimizers/sgd.cpp @@ -1,25 +1,42 @@ /** * @file sgd.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-08 - * + * * @copyright Copyright (c) 2026 - * + * */ #include "sgd.h" +#ifdef __CUDA +#include "training/optimizers/cuda/optimizers.cuh" +#else +#include +#endif + using namespace std; using namespace train; void SgdOptimizer::step() { for(auto& t: params){ auto grads = t->getGrads(); - for(auto idx=0; idxgetSize(); idx++){ - auto updatedWeight = (*t)[idx] - lr*(*grads)[idx]; - t->set(updatedWeight, idx); + switch(t->getDevice()) { + case Device::CPU: + for(auto idx=0; idxgetSize(); idx++){ + auto updatedWeight = (*t)[idx] - lr*(*grads)[idx]; + t->set(updatedWeight, idx); + } + break; + case Device::CUDA: + #ifdef __CUDA + cuda_impl::sgdStep(*t, *grads, lr); + #else + __throw_invalid_argument("Attempted to use CUDA tensor"); + #endif + break; } } -} \ No newline at end of file +} From f82f8999e85460710ba0337eb4c994b5392a4c59 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Wed, 13 May 2026 15:55:39 +0200 Subject: [PATCH 36/73] Last pieces of infrastructure --- .../loss_functions/bce_node.cpp | 42 +++++++---- .../loss_functions/bce_sigmoid_node.cpp | 60 ++++++++++------ .../loss_functions/crossentropy_node.cpp | 48 ++++++++----- .../crossentropy_softmax_node.cpp | 43 ++++++++---- .../loss_functions/cuda/loss_nodes.cu | 70 +++++++++++++++++-- .../loss_functions/cuda/loss_nodes.cuh | 19 ++--- .../loss_functions/rmse_node.cpp | 44 ++++++++---- 7 files changed, 234 insertions(+), 92 deletions(-) diff --git a/src/backend/computational_graph/loss_functions/bce_node.cpp b/src/backend/computational_graph/loss_functions/bce_node.cpp index c0f30be..8c25e0a 100644 --- a/src/backend/computational_graph/loss_functions/bce_node.cpp +++ b/src/backend/computational_graph/loss_functions/bce_node.cpp @@ -1,18 +1,23 @@ /** * @file bce_node.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-14 - * + * * @copyright Copyright (c) 2026 - * + * */ #include "bce_node.h" - #include "data_modeling/tensor_functions.h" +#ifdef __CUDA +#include "computational_graph/loss_functions/cuda/loss_nodes.cuh" +#else +#include +#endif + using namespace std; using namespace cgraph; @@ -22,14 +27,25 @@ vector< shared_ptr > BceNode::backward(const Tensor& upstreamGrad) { const auto& yPred = parents[0]; auto res = make_shared(yPred->createEmptyCopy()); - ftype bSize = yPred->getDims()[0]; - for(tensorSize_t i=0; igetDims()[0]; i++){ - auto yi = (*yTrue)[i]; - auto yiHat = (*yPred)[i]; - - auto g = -yi/std::max(yiHat, EPS_BCE) + (1-yi)/std::max(1-yiHat, EPS_BCE); - res->set(g/bSize, i); + switch(upstreamGrad.getDevice()) { + case Device::CPU: { + ftype bSize = yPred->getDims()[0]; + for(tensorSize_t i=0; igetDims()[0]; i++){ + auto yi = (*yTrue)[i]; + auto yiHat = (*yPred)[i]; + auto g = -yi/std::max(yiHat, EPS_BCE) + (1-yi)/std::max(1-yiHat, EPS_BCE); + res->set(g/bSize, i); + } + break; + } + case Device::CUDA: + #ifdef __CUDA + cuda_impl::bceBackward(*res, *yPred, *yTrue); + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); + #endif + break; } - + return {res}; -} \ No newline at end of file +} diff --git a/src/backend/computational_graph/loss_functions/bce_sigmoid_node.cpp b/src/backend/computational_graph/loss_functions/bce_sigmoid_node.cpp index 998e110..16f2e6f 100644 --- a/src/backend/computational_graph/loss_functions/bce_sigmoid_node.cpp +++ b/src/backend/computational_graph/loss_functions/bce_sigmoid_node.cpp @@ -1,46 +1,62 @@ /** * @file bce_sigmoid_node.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-17 - * + * * @copyright Copyright (c) 2026 - * + * */ #include "bce_sigmoid_node.h" - #include "data_modeling/tensor_functions.h" #include +#ifdef __CUDA +#include "computational_graph/loss_functions/cuda/loss_nodes.cuh" +#else +#include +#endif + using namespace std; using namespace cgraph; vector< shared_ptr > BceSigmoidNode::backward(const Tensor& upstreamGrad) { assert(!upstreamGrad.getRequiresGrad()); - - auto sigmoid = [](ftype x){ - constexpr ftype one = 1.0; - if(x>=0){ - return one / (one + exp(-x)); - } - auto e = exp(x); - return e / (one + e); - }; const auto& logits = parents[0]; auto res = make_shared(logits->createEmptyCopy()); - ftype bSize = logits->getDims()[0]; - for(tensorSize_t i=0; igetDims()[0]; i++){ - auto y = (*yTrue)[i]; - auto s = sigmoid((*logits)[i]); - - auto g = s - y; - res->set(g/bSize, i); + switch(upstreamGrad.getDevice()) { + case Device::CPU: { + auto sigmoid = [](ftype x){ + constexpr ftype one = 1.0; + if(x>=0){ + return one / (one + exp(-x)); + } + auto e = exp(x); + return e / (one + e); + }; + + ftype bSize = logits->getDims()[0]; + for(tensorSize_t i=0; igetDims()[0]; i++){ + auto y = (*yTrue)[i]; + auto s = sigmoid((*logits)[i]); + auto g = s - y; + res->set(g/bSize, i); + } + break; + } + case Device::CUDA: + #ifdef __CUDA + cuda_impl::bceSigmoidBackward(*res, *logits, *yTrue); + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); + #endif + break; } - + return {res}; -} \ No newline at end of file +} diff --git a/src/backend/computational_graph/loss_functions/crossentropy_node.cpp b/src/backend/computational_graph/loss_functions/crossentropy_node.cpp index 8d74229..347f59d 100644 --- a/src/backend/computational_graph/loss_functions/crossentropy_node.cpp +++ b/src/backend/computational_graph/loss_functions/crossentropy_node.cpp @@ -1,37 +1,53 @@ /** - * @file add_node.cpp + * @file crossentropy_node.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-02-03 - * + * * @copyright Copyright (c) 2026 - * + * */ #include "crossentropy_node.h" - #include "data_modeling/tensor_functions.h" +#ifdef __CUDA +#include "computational_graph/loss_functions/cuda/loss_nodes.cuh" +#else +#include +#endif + using namespace std; using namespace cgraph; vector< shared_ptr > CrossEntropyNode::backward(const Tensor& upstreamGrad) { assert(!upstreamGrad.getRequiresGrad()); - + const auto& yPred = parents[0]; auto res = make_shared(yPred->createEmptyCopy()); - ftype bSize = yPred->getDims()[0]; - for(tensorDim_t i=0; igetDims()[0]; i++){ - for(tensorDim_t j=0; jgetDims()[1]; j++){ - auto yij = yTrue->get(i, j); - auto yijHat = yPred->get(i, j); - - auto g = -yij/std::max(yijHat, EPS_CROSSENTROPY); - res->set(g/bSize, i, j); + switch(upstreamGrad.getDevice()) { + case Device::CPU: { + ftype bSize = yPred->getDims()[0]; + for(tensorDim_t i=0; igetDims()[0]; i++){ + for(tensorDim_t j=0; jgetDims()[1]; j++){ + auto yij = yTrue->get(i, j); + auto yijHat = yPred->get(i, j); + auto g = -yij/std::max(yijHat, EPS_CROSSENTROPY); + res->set(g/bSize, i, j); + } + } + break; } + case Device::CUDA: + #ifdef __CUDA + cuda_impl::crossEntropyBackward(*res, *yPred, *yTrue); + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); + #endif + break; } - + return {res}; -} \ No newline at end of file +} diff --git a/src/backend/computational_graph/loss_functions/crossentropy_softmax_node.cpp b/src/backend/computational_graph/loss_functions/crossentropy_softmax_node.cpp index ca0d1c7..c820031 100644 --- a/src/backend/computational_graph/loss_functions/crossentropy_softmax_node.cpp +++ b/src/backend/computational_graph/loss_functions/crossentropy_softmax_node.cpp @@ -1,18 +1,23 @@ /** * @file crossentropy_softmax_node.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-17 - * + * * @copyright Copyright (c) 2026 - * + * */ #include "crossentropy_softmax_node.h" - #include "module/activation_functions/softmax.h" +#ifdef __CUDA +#include "computational_graph/loss_functions/cuda/loss_nodes.cuh" +#else +#include +#endif + using namespace std; using namespace cgraph; @@ -22,16 +27,28 @@ vector< shared_ptr > CrossEntropySoftmaxNode::backward(const Tensor& ups const auto& logits = parents[0]; auto res = make_shared(logits->createEmptyCopy()); - const auto softmax = module::Softmax(); - const auto s = softmax(*logits); - - ftype bSize = logits->getDims()[0]; - for(tensorSize_t b=0; bgetDims()[0]; b++){ - for(tensorSize_t i=0; igetDims()[1]; i++){ - auto g = s.get(b, i) - yTrue->get(b, i); - res->set(g / bSize, b, i); + switch(upstreamGrad.getDevice()) { + case Device::CPU: { + const auto softmax = module::Softmax(); + const auto s = softmax(*logits); + + ftype bSize = logits->getDims()[0]; + for(tensorSize_t b=0; bgetDims()[0]; b++){ + for(tensorSize_t i=0; igetDims()[1]; i++){ + auto g = s.get(b, i) - yTrue->get(b, i); + res->set(g / bSize, b, i); + } + } + break; } + case Device::CUDA: + #ifdef __CUDA + cuda_impl::crossEntropySoftmaxBackward(*res, *logits, *yTrue); + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); + #endif + break; } return {res}; -} \ No newline at end of file +} diff --git a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu index dfb251c..8c45915 100644 --- a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu +++ b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu @@ -1,18 +1,78 @@ /** * @file loss_nodes.cu * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-23 - * + * * @copyright Copyright (c) 2026 - * + * */ #ifndef __CUDA -static_assert(false, "File should not be included without CUDA enabled"); +static_assert(false, "File should not be compiled without CUDA enabled"); #endif // __CUDA #include "loss_nodes.cuh" +#include "utility/cuda/cuda_common.cuh" -using namespace cuda; \ No newline at end of file +using namespace std; + +namespace { + // TODO: bceBackward kernel + + // TODO: bceSigmoidBackward kernel + + // TODO: crossEntropyBackward kernel + + // TODO: crossEntropySoftmaxBackward kernel + + // TODO: rmseBackward kernel +} + +namespace cuda_impl { + void bceBackward(Tensor& res, const Tensor& yPred, const Tensor& yTrue) { + const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + const int blocks = (yPred.getSize() + threadsPerBlock - 1) / threadsPerBlock; + + // TODO: launch kernel + + cudaErrchk(cudaDeviceSynchronize()); + } + + void bceSigmoidBackward(Tensor& res, const Tensor& logits, const Tensor& yTrue) { + const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + const int blocks = (logits.getSize() + threadsPerBlock - 1) / threadsPerBlock; + + // TODO: launch kernel + + cudaErrchk(cudaDeviceSynchronize()); + } + + void crossEntropyBackward(Tensor& res, const Tensor& yPred, const Tensor& yTrue) { + const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + const int blocks = (yPred.getSize() + threadsPerBlock - 1) / threadsPerBlock; + + // TODO: launch kernel + + cudaErrchk(cudaDeviceSynchronize()); + } + + void crossEntropySoftmaxBackward(Tensor& res, const Tensor& logits, const Tensor& yTrue) { + const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + const int blocks = (logits.getSize() + threadsPerBlock - 1) / threadsPerBlock; + + // TODO: launch kernel + + cudaErrchk(cudaDeviceSynchronize()); + } + + void rmseBackward(Tensor& res, const Tensor& yPred, const Tensor& yTrue, ftype rmse) { + const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + const int blocks = (yPred.getSize() + threadsPerBlock - 1) / threadsPerBlock; + + // TODO: launch kernel + + cudaErrchk(cudaDeviceSynchronize()); + } +} diff --git a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cuh b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cuh index 95e313d..24486bf 100644 --- a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cuh +++ b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cuh @@ -1,12 +1,12 @@ /** * @file loss_nodes.cuh * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-23 - * + * * @copyright Copyright (c) 2026 - * + * */ #pragma once @@ -16,13 +16,14 @@ static_assert(false, "File should not be included without CUDA enabled"); #endif // __CUDA #include "utility/global_params.h" +#include "data_modeling/tensor.h" namespace cuda_impl { - __global__ void bceBackward(ftype* res, const ftype* const upstreamGrad, tensorSize_t size); - __global__ void bceWithSigmoidBackward(ftype* res, const ftype* const upstreamGrad, ftype eps, tensorSize_t size); + void bceBackward(Tensor& res, const Tensor& yPred, const Tensor& yTrue); + void bceSigmoidBackward(Tensor& res, const Tensor& logits, const Tensor& yTrue); - __global__ void crossEntropyBackward(ftype* res, const ftype* const sigmoids, const ftype* const upstreamGrad, tensorSize_t size); - __global__ void crossEntropyWithSoftmaxBackward(ftype* res, const ftype* const softmax, const ftype* const upstreamGrad, tensorSize_t size); + void crossEntropyBackward(Tensor& res, const Tensor& yPred, const Tensor& yTrue); + void crossEntropySoftmaxBackward(Tensor& res, const Tensor& logits, const Tensor& yTrue); - __global__ void rmseBackward(ftype* res, const ftype* const sigmoids, const ftype* const upstreamGrad, tensorSize_t size); -} \ No newline at end of file + void rmseBackward(Tensor& res, const Tensor& yPred, const Tensor& yTrue, ftype rmse); +} diff --git a/src/backend/computational_graph/loss_functions/rmse_node.cpp b/src/backend/computational_graph/loss_functions/rmse_node.cpp index e3eb11e..3a2d74b 100644 --- a/src/backend/computational_graph/loss_functions/rmse_node.cpp +++ b/src/backend/computational_graph/loss_functions/rmse_node.cpp @@ -1,20 +1,25 @@ /** * @file rmse_node.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-03-14 - * + * * @copyright Copyright (c) 2026 - * + * */ #include "rmse_node.h" - #include "data_modeling/tensor_functions.h" #include +#ifdef __CUDA +#include "computational_graph/loss_functions/cuda/loss_nodes.cuh" +#else +#include +#endif + using namespace std; using namespace cgraph; @@ -25,15 +30,26 @@ vector< shared_ptr > RmseNode::backward(const Tensor& upstreamGrad) { const auto& yPred = parents[0]; auto res = make_shared(yPred->createEmptyCopy()); - ftype bSize = yPred->getDims()[0]; - for(tensorSize_t i=0; igetDims()[0]; i++){ - auto yi = (*yTrue)[i]; - auto yiHat = (*yPred)[i]; - - auto denom = rmse * bSize + eps; - auto g = (yiHat-yi) / denom; - res->set(g, i); + switch(upstreamGrad.getDevice()) { + case Device::CPU: { + ftype bSize = yPred->getDims()[0]; + for(tensorSize_t i=0; igetDims()[0]; i++){ + auto yi = (*yTrue)[i]; + auto yiHat = (*yPred)[i]; + auto denom = rmse * bSize + eps; + auto g = (yiHat-yi) / denom; + res->set(g, i); + } + break; + } + case Device::CUDA: + #ifdef __CUDA + cuda_impl::rmseBackward(*res, *yPred, *yTrue, rmse); + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); + #endif + break; } - + return {res}; -} \ No newline at end of file +} From 95bce13ad9c87b1b3c26d80c83f40c988f45fab1 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Thu, 14 May 2026 12:37:34 +0200 Subject: [PATCH 37/73] Added warp reduce kernel code to compute softmax. Needs dimension that softmax computes over <= 64 == 2 * warpSize; added kernel code for backward functions of relu, leakyrelu and sigmoid; added unit tests for all new kernels --- CMakeLists.txt | 4 +- .../cuda/activation_nodes.cu | 31 ++- src/backend/data_modeling/cuda/tensorops.cu | 2 +- .../activation_functions/cuda/activations.cu | 215 ++++++++++++++++-- src/backend/utility/cuda/cuda_common.cuh | 57 ++--- tests/backend/cuda/test_module_cuda.cu | 34 +-- 6 files changed, 267 insertions(+), 76 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 08f8d3b..2936cac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ endif() add_compile_options("$<$:/utf-8>") add_compile_options("$<$:/utf-8>") -option(CUDA "Enable CUDA execution for some faster data structures" OFF) +option(CUDA "Enable CUDA execution for some faster data structures" ON) if (CUDA) include(CheckLanguage) @@ -40,7 +40,7 @@ if (CUDA) set(CMAKE_CUDA_STANDARD 20) set(CMAKE_CUDA_STANDARD_REQUIRED ON) else() - message(WARNING "Could not find CUDA on system. Exiting.") + message(WARNING "Could not find CUDA on system. Compiling without CUDA enabled") endif() endif() diff --git a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu index a5100b7..fd96968 100644 --- a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu +++ b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu @@ -19,11 +19,27 @@ static_assert(false, "File should not be compiled without CUDA enabled"); using namespace std; namespace { - // TODO: reluBackward kernel + __global__ void reluBackwardKernel(ftype* res, const ftype* const upstreamGrad, const ftype* const parent, const tensorSize_t size) { + int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) return; - // TODO: leakyReluBackward kernel + res[gid] = parent[gid] > 0 ? upstreamGrad[gid] : 0; + } + + __global__ void leakyReluBackwardKernel(ftype* res, const ftype* const upstreamGrad, const ftype* const parent, const ftype eps, const tensorSize_t size) { + int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) return; + + res[gid] = parent[gid] > 0 ? upstreamGrad[gid] : eps * upstreamGrad[gid]; + } + + __global__ void sigmoidBackwardKernel(ftype* res, const ftype* const upstreamGrad, const ftype* const sigmoid, const tensorSize_t size) { + int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) return; - // TODO: sigmoidBackward kernel + ftype si = sigmoid[gid]; + res[gid] = si * (1 - si) * upstreamGrad[gid]; + } // TODO: softmaxBackward kernel } @@ -33,8 +49,7 @@ namespace cuda_impl { constexpr int threadsPerBlock = 256; const int blocks = (upstreamGrad.getSize() + threadsPerBlock - 1) / threadsPerBlock; - // TODO: launch kernel - + reluBackwardKernel<<>>(res.getData(), upstreamGrad.getData(), parent.getData(), res.getSize()); cudaErrchk(cudaDeviceSynchronize()); } @@ -42,8 +57,7 @@ namespace cuda_impl { constexpr int threadsPerBlock = 256; const int blocks = (upstreamGrad.getSize() + threadsPerBlock - 1) / threadsPerBlock; - // TODO: launch kernel - + leakyReluBackwardKernel<<>>(res.getData(), upstreamGrad.getData(), parent.getData(), eps, res.getSize()); cudaErrchk(cudaDeviceSynchronize()); } @@ -51,8 +65,7 @@ namespace cuda_impl { constexpr int threadsPerBlock = 256; const int blocks = (upstreamGrad.getSize() + threadsPerBlock - 1) / threadsPerBlock; - // TODO: launch kernel - + sigmoidBackwardKernel<<>>(res.getData(), upstreamGrad.getData(), sigmoid.getData(), res.getSize()); cudaErrchk(cudaDeviceSynchronize()); } diff --git a/src/backend/data_modeling/cuda/tensorops.cu b/src/backend/data_modeling/cuda/tensorops.cu index fcbda95..e092020 100644 --- a/src/backend/data_modeling/cuda/tensorops.cu +++ b/src/backend/data_modeling/cuda/tensorops.cu @@ -194,7 +194,7 @@ namespace cuda_impl { tensorSize_t resOffset = 0; while(leftOffset < left.getSize()){ - //const auto smemSize = min(resSize, DeviceProperties::getThreadsPerBlock()) * sizeof(ftype); + //const auto smemSize = min(resSize, threadsPerBlock) * sizeof(ftype); //matMul2DKernel<<>>(res.getData() + resOffset, left.getData() + leftOffset, right.getData() + rightOffset, matMul2DKernel<<>>(res.getData() + resOffset, left.getData() + leftOffset, right.getData() + rightOffset, left.getDims().get(-2), left.getDims().get(-1), right.getDims().get(-1), resSize); diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index d9af1b1..1fd4fe0 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -21,7 +21,7 @@ static_assert(false, "File should not be compiled without CUDA enabled"); using namespace std; namespace { - __global__ void reluKernel(ftype* res, const ftype* const input, const tensorSize_t size) { + __global__ void reluKernel(ftype* const res, const ftype* const input, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; @@ -30,12 +30,12 @@ namespace { res[gid] = fmaxf(input[gid], zero); } - __global__ void leakyReluKernel(ftype* res, const ftype* const input, const ftype eps, const tensorSize_t size) { + __global__ void leakyReluKernel(ftype* const res, const ftype* const input, const ftype eps, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; - res[gid] = fmaxf(input[gid], eps*input[gid]); // eps < 1 + res[gid] = fmaxf(input[gid], eps * input[gid]); // eps < 1 } __device__ __forceinline__ ftype sigmoid(ftype x) { @@ -44,7 +44,7 @@ namespace { return (x >= 0.f) ? s : z * s; // x < 0 => e^x/(e^x+1) } - __global__ void sigmoidKernel(ftype* res, const ftype* const input, const tensorSize_t size) { + __global__ void sigmoidKernel(ftype* const res, const ftype* const input, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; @@ -52,8 +52,119 @@ namespace { res[gid] = sigmoid(input[gid]); } + /** + * @brief Reduction kernel that computes the maximum within the size of 2 * warpsize at maximum. + */ + template + __forceinline__ __device__ void warpMaxReduce(volatile ftype* const input, const tensorSize_t stride, const int offset) { + if(maxoffset == 32) { + if(offset + 32 < stride) input[offset] = max(input[offset], input[offset + 32]); + } + if(maxoffset >= 16) { + if(offset + 16 < stride) input[offset] = max(input[offset], input[offset + 16]); + } + if(maxoffset >= 8) { + if(offset + 8 < stride) input[offset] = max(input[offset], input[offset + 8]); + } + if(maxoffset >= 4) { + if(offset + 4 < stride) input[offset] = max(input[offset], input[offset + 4]); + } + if(maxoffset >= 2) { + if(offset + 2 < stride) input[offset] = max(input[offset], input[offset + 2]); + } + if(maxoffset >= 1) { + if(offset + 1 < stride) input[offset] = max(input[offset], input[offset + 1]); + } + } + + /** + * @brief Reduction kernel that computes the sum over an array within the size of 2 * warpsize at maximum. + */ + template + __forceinline__ __device__ void warpSumReduce(volatile ftype* const input, const tensorSize_t stride, const int offset) { + if(maxoffset == 32) { + if(offset + 32 < stride) input[offset] += input[offset + 32]; + } + if(maxoffset >= 16) { + if(offset + 16 < stride) input[offset] += input[offset + 16]; + } + if(maxoffset >= 8) { + if(offset + 8 < stride) input[offset] += input[offset + 8]; + } + if(maxoffset >= 4) { + if(offset + 4 < stride) input[offset] += input[offset + 4]; + } + if(maxoffset >= 2) { + if(offset + 2 < stride) input[offset] += input[offset + 2]; + } + if(maxoffset >= 1) { + if(offset + 1 < stride) input[offset] += input[offset + 1]; + } + } + + /** + * @brief Here we find the maximum within 'stride'. Assumption: One warp does exactly one element of stride! + * Reduction via warp reduce. res has the maximum values stored. + */ + template + __global__ void findMaxKernelOneWarp(ftype* const res, const ftype* const input, const tensorSize_t stride, const tensorSize_t size) { + int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) + return; + + int tid = threadIdx.x; + extern __shared__ ftype smem[]; + smem[tid] = input[gid]; + __syncthreads(); + + volatile ftype* const start = smem + (tid / stride) * stride; + const int offset = gid % stride; + warpMaxReduce(start, stride, offset); + + // one warp reduces one 'stride' + if(offset == 0) + res[tid / 32] = smem[tid]; + } + + /** + * @brief Numerically stable version of + */ + template + __global__ void stableSoftmaxKernelOneWarp(ftype* const res, const ftype* const input, const ftype* const maxValues, + const tensorSize_t stride, const tensorSize_t size) { + int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) + return; + + const auto strideOffset = gid / stride; + const auto maxValue = maxValues[strideOffset]; + + int tid = threadIdx.x; + extern __shared__ ftype smem[]; // can lead to bank conflicts iff std::is_same_v + __syncthreads(); + + ftype expVal = 0; + if constexpr (std::is_same_v) { + expVal = expf(input[gid] - maxValue); + } + else if constexpr (std::is_same_v) { + expVal = exp(input[gid] - maxValue); + } + else { + static_assert(always_false, "ftype encountered unexpected type"); + } + smem[tid] = expVal; + __syncthreads(); + + volatile ftype* const start = smem + (tid / stride) * stride; + const int offset = gid % stride; + warpSumReduce(start, stride, offset); + + res[gid] = expVal / start[0]; + } + // TODO: use shared memory - __global__ void findMaxKernel(ftype* res, ftype* const input, const tensorSize_t stride, const tensorSize_t size) { + __global__ void findMaxKernelOneBlock(ftype* res, ftype* const input, const tensorSize_t stride, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; @@ -98,7 +209,7 @@ namespace { // TODO: use shared memory template - __global__ void expAndSumKernel(T* res, T* const tmp, const T* const input, const T* const maxValues, + __global__ void expAndSumKernelOneBlock(T* res, T* const tmp, const T* const input, const T* const maxValues, const tensorSize_t stride, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) @@ -150,17 +261,16 @@ namespace { } } - __global__ void softmaxDivisionKernel(ftype* res, const ftype* const tmp, const tensorSize_t stride, const tensorSize_t size) { + __global__ void softmaxDivisionKernel(ftype* res, const ftype* const sums, const tensorSize_t stride, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; const auto strideOffset = gid / stride; - res[gid] = res[gid] / tmp[strideOffset]; + res[gid] = res[gid] / sums[strideOffset]; } } - namespace cuda_impl { void relu(Tensor& res, const Tensor& in) { constexpr int threadsPerBlock = 256; @@ -188,34 +298,91 @@ namespace cuda_impl { void softmax(Tensor& res, const Tensor& in) { const tensorSize_t stride = static_cast(in.getDims().get(-1)); + if(stride == 1) { + __throw_invalid_argument("Attemped softmax of one element"); + } - if (stride <= 1 << 10) { - // threads per block needs to be multiple of stride - int threadsPerBlock = 256 / stride * stride; // 256 >= threadsPerBlock >= 256 - stride + 1 - const int blocks = (in.getSize()+threadsPerBlock-1) / threadsPerBlock; + // TODO: use some static struct here to prevent those guys from keeping on re-allocating memory +/* ftype* tmp; + cudaErrchk(cudaMalloc(&tmp, in.getSize() * sizeof(ftype))); + cudaErrchk(cudaMemcpy(tmp, in.getData(), in.getSize() * sizeof(ftype), cudaMemcpyDeviceToDevice)); */ + + ftype* maxValues; + const tensorSize_t nMaxValues = in.getSize() / stride; + cudaErrchk(cudaMalloc(&maxValues, nMaxValues * sizeof(ftype))); - ftype* tmp; - cudaErrchk(cudaMalloc(&tmp, in.getSize() * sizeof(ftype))); - cudaErrchk(cudaMemcpy(tmp, in.getData(), in.getSize() * sizeof(ftype), cudaMemcpyDeviceToDevice)); + static const auto warpSizeT2 = 2 * DeviceProperties::getWarpSize(); // TODO: can this be a problem in a multi-GPU setting? + if(stride <= warpSizeT2) { + assert(DeviceProperties::getWarpSize() == 32); - ftype* maxValues; - const tensorSize_t nMaxValues = in.getSize() / stride; - cudaErrchk(cudaMalloc(&maxValues, nMaxValues * sizeof(ftype))); + const int threadsPerBlock = 256; + const int blocks = (in.getSize() + threadsPerBlock - 1) / threadsPerBlock; + + if(stride == 2) { + findMaxKernelOneWarp<1> <<>>(maxValues, in.getData(), stride, in.getSize()); + } + else if(stride <= 4) { + findMaxKernelOneWarp<2> <<>>(maxValues, in.getData(), stride, in.getSize()); + } + else if(stride <= 8) { + findMaxKernelOneWarp<4> <<>>(maxValues, in.getData(), stride, in.getSize()); + } + else if(stride <= 16) { + findMaxKernelOneWarp<8> <<>>(maxValues, in.getData(), stride, in.getSize()); + } + else if(stride <= 32) { + findMaxKernelOneWarp<16> <<>>(maxValues, in.getData(), stride, in.getSize()); + } + else if(stride <= 64) { + findMaxKernelOneWarp<32> <<>>(maxValues, in.getData(), stride, in.getSize()); + } + cudaErrchk(cudaDeviceSynchronize()); - findMaxKernel<<>>(maxValues, tmp, stride, in.getSize()); + if(stride == 2) { + stableSoftmaxKernelOneWarp <<>> + (res.getData(), in.getData(), maxValues, stride, in.getSize()); + } + else if(stride <= 4) { + stableSoftmaxKernelOneWarp <<>> + (res.getData(), in.getData(), maxValues, stride, in.getSize()); + } + else if(stride <= 8) { + stableSoftmaxKernelOneWarp <<>> + (res.getData(), in.getData(), maxValues, stride, in.getSize()); + } + else if(stride <= 16) { + stableSoftmaxKernelOneWarp <<>> + (res.getData(), in.getData(), maxValues, stride, in.getSize()); + } + else if(stride <= 32) { + stableSoftmaxKernelOneWarp <<>> + (res.getData(), in.getData(), maxValues, stride, in.getSize()); + } + else if(stride <= 64) { + stableSoftmaxKernelOneWarp <<>> + (res.getData(), in.getData(), maxValues, stride, in.getSize()); + } cudaErrchk(cudaDeviceSynchronize()); + } + else if (stride <= 256) { + // threads per block needs to be multiple of stride +/* constexpr int threadsPerBlock = 256; + const int blocks = (in.getSize()+threadsPerBlock-1) / threadsPerBlock; - expAndSumKernel<<>>(res.getData(), tmp, in.getData(), maxValues, stride, in.getSize()); + findMaxKernelOneBlock<<>>(maxValues, tmp, stride, in.getSize()); cudaErrchk(cudaDeviceSynchronize()); - softmaxDivisionKernel<<>>(res.getData(), tmp, stride, in.getSize()); + expAndSumKernelOneBlock<<>>(res.getData(), tmp, in.getData(), maxValues, stride, in.getSize()); cudaErrchk(cudaDeviceSynchronize()); - cudaErrchk(cudaFree(tmp)); - cudaErrchk(cudaFree(maxValues)); + softmaxDivisionKernel<<>>(res.getData(), tmp, stride, in.getSize()); + cudaErrchk(cudaDeviceSynchronize()); */ } else { __throw_runtime_error("Softmax kernels not yet implemented at inter-block level"); } + + //cudaErrchk(cudaFree(tmp)); + cudaErrchk(cudaFree(maxValues)); } } \ No newline at end of file diff --git a/src/backend/utility/cuda/cuda_common.cuh b/src/backend/utility/cuda/cuda_common.cuh index bcc7ce4..d1e8311 100644 --- a/src/backend/utility/cuda/cuda_common.cuh +++ b/src/backend/utility/cuda/cuda_common.cuh @@ -27,31 +27,34 @@ constexpr bool always_false = false; #define cudaErrchk(ans) { utility::gpuAssert((ans), __FILE__, __LINE__); } namespace cuda_impl { -struct DeviceProperties final { - private: - int threadsPerBlock; - - static const DeviceProperties& get() { - static DeviceProperties instance; - return instance; - } - - DeviceProperties() { - cudaDeviceProp prop; - cudaGetDeviceProperties(&prop, 0); - - threadsPerBlock = prop.maxThreadsPerBlock; - } - - public: - DeviceProperties(const DeviceProperties&) = delete; - DeviceProperties& operator=(const DeviceProperties&) = delete; - - DeviceProperties(DeviceProperties&&) = delete; - DeviceProperties& operator=(DeviceProperties&&) = delete; - - ~DeviceProperties() = default; - - static int getThreadsPerBlock() { return get().threadsPerBlock; } -}; + struct DeviceProperties final { + private: + int threadsPerBlock; + int warpSize; + + static const DeviceProperties& get() { + static DeviceProperties instance; + return instance; + } + + DeviceProperties() { + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + + threadsPerBlock = prop.maxThreadsPerBlock; + warpSize = prop.warpSize; + } + + public: + DeviceProperties(const DeviceProperties&) = delete; + DeviceProperties& operator=(const DeviceProperties&) = delete; + + DeviceProperties(DeviceProperties&&) = delete; + DeviceProperties& operator=(DeviceProperties&&) = delete; + + ~DeviceProperties() = default; + + static int getThreadsPerBlock() { return get().threadsPerBlock; } + static int getWarpSize() { return get().warpSize; } + }; } diff --git a/tests/backend/cuda/test_module_cuda.cu b/tests/backend/cuda/test_module_cuda.cu index 31c5e2d..4fb79e4 100644 --- a/tests/backend/cuda/test_module_cuda.cu +++ b/tests/backend/cuda/test_module_cuda.cu @@ -26,13 +26,13 @@ static_assert(false, "File should not be compiled without CUDA enabled"); using namespace std; -constexpr ftype delta = 1e-3; - TEST(CudaActivationTest, ReluForward) { auto t1 = TensorFunctions::Ones({300, 500}, Device::CUDA); auto f = module::ReLu(); auto res = f(t1); + res.setDevice(Device::CPU); + t1.setDevice(Device::CPU); for(size_t i=0; ibackward(); auto grads = t->getGrads(); - ASSERT_NEAR((*grads)[0], 0.25, delta); - ASSERT_NEAR((*grads)[1], 0.1966, delta); + ASSERT_NEAR((*grads)[0], 0.25, 1e-4); + ASSERT_NEAR((*grads)[1], 0.1966, 1e-4); } TEST(CudaActivationTest, SoftmaxForward) { @@ -145,9 +151,9 @@ TEST(CudaActivationTest, SoftmaxForward) { module::Softmax sm; auto res = sm(*t); - ASSERT_NEAR(res[0], 0.0900, delta); - ASSERT_NEAR(res[1], 0.2447, delta); - ASSERT_NEAR(res[2], 0.6652, delta); + ASSERT_NEAR(res[0], 0.0900, 1e-4); + ASSERT_NEAR(res[1], 0.2447, 1e-4); + ASSERT_NEAR(res[2], 0.6652, 1e-4); } TEST(CudaActivationTest, SoftmaxSumsToOne) { @@ -160,8 +166,8 @@ TEST(CudaActivationTest, SoftmaxSumsToOne) { ftype row0sum = res[0] + res[1] + res[2] + res[3]; ftype row1sum = res[4] + res[5] + res[6] + res[7]; - ASSERT_NEAR(row0sum, 1.0, delta); - ASSERT_NEAR(row1sum, 1.0, delta); + ASSERT_NEAR(row0sum, 1.0, 1e-5); + ASSERT_NEAR(row1sum, 1.0, 1e-5); } TEST(CudaActivationTest, SoftmaxForwardNumericalStability) { @@ -175,9 +181,10 @@ TEST(CudaActivationTest, SoftmaxForwardNumericalStability) { ASSERT_FALSE(std::isinf(res[i])); } ftype rowsum = res[0] + res[1] + res[2]; - ASSERT_NEAR(rowsum, 1.0, delta); + ASSERT_NEAR(rowsum, 1.0, 1e-5); } +/* TEST(CudaAutogradTest, SoftmaxBackward) { auto t = TensorFunctions::makeSharedTensor({1, 3}, {1.0, 2.0, 3.0}, Device::CUDA, true); @@ -189,9 +196,9 @@ TEST(CudaAutogradTest, SoftmaxBackward) { resPtr->backward(); auto grads = t->getGrads(); - ASSERT_NEAR((*grads)[0], 0.0819, delta); - ASSERT_NEAR((*grads)[1], -0.0220, delta); - ASSERT_NEAR((*grads)[2], -0.0599, delta); + ASSERT_NEAR((*grads)[0], 0.0819, 1e-5); + ASSERT_NEAR((*grads)[1], -0.0220, 1e-5); + ASSERT_NEAR((*grads)[2], -0.0599, 1e-5); } TEST(CudaLayerTest, TestFfLayer) { @@ -202,3 +209,4 @@ TEST(CudaLayerTest, TestFfLayer) { ASSERT_EQ(res.getDims(), Dimension({3, 1})); } + */ \ No newline at end of file From 138d83bd4666c6668a47c038007dcc4275554c5c Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Thu, 14 May 2026 16:00:24 +0200 Subject: [PATCH 38/73] Added softmax kernel for medium large strides --- .../cuda/activation_nodes.cu | 6 +- src/backend/data_modeling/cuda/tensorops.cu | 34 ++-- src/backend/data_modeling/cuda/tensorops.cuh | 4 +- .../activation_functions/cuda/activations.cu | 156 +++++++++--------- .../module/activation_functions/softmax.cpp | 16 +- .../loss_functions/cuda/loss_functions.cu | 2 +- tests/backend/test_module.cpp | 9 +- 7 files changed, 119 insertions(+), 108 deletions(-) diff --git a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu index fd96968..f6939f2 100644 --- a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu +++ b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu @@ -19,21 +19,21 @@ static_assert(false, "File should not be compiled without CUDA enabled"); using namespace std; namespace { - __global__ void reluBackwardKernel(ftype* res, const ftype* const upstreamGrad, const ftype* const parent, const tensorSize_t size) { + __global__ void reluBackwardKernel(ftype* const res, const ftype* const upstreamGrad, const ftype* const parent, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; res[gid] = parent[gid] > 0 ? upstreamGrad[gid] : 0; } - __global__ void leakyReluBackwardKernel(ftype* res, const ftype* const upstreamGrad, const ftype* const parent, const ftype eps, const tensorSize_t size) { + __global__ void leakyReluBackwardKernel(ftype* const res, const ftype* const upstreamGrad, const ftype* const parent, const ftype eps, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; res[gid] = parent[gid] > 0 ? upstreamGrad[gid] : eps * upstreamGrad[gid]; } - __global__ void sigmoidBackwardKernel(ftype* res, const ftype* const upstreamGrad, const ftype* const sigmoid, const tensorSize_t size) { + __global__ void sigmoidBackwardKernel(ftype* const res, const ftype* const upstreamGrad, const ftype* const sigmoid, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; diff --git a/src/backend/data_modeling/cuda/tensorops.cu b/src/backend/data_modeling/cuda/tensorops.cu index e092020..2b0b75e 100644 --- a/src/backend/data_modeling/cuda/tensorops.cu +++ b/src/backend/data_modeling/cuda/tensorops.cu @@ -22,8 +22,10 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include namespace{ - __global__ void elementwiseaddKernel(ftype* res, const ftype* const left, const ftype* const right, const tensorSize_t size) - { + /** + * @brief Kernel for simple elementwise addition. + */ + __global__ void elementwiseaddKernel(ftype* const res, const ftype* const left, const ftype* const right, const tensorSize_t size) { int gid = blockDim.x * blockIdx.x + threadIdx.x; if(gid>=size) return; @@ -31,9 +33,11 @@ namespace{ res[gid] = left[gid] + right[gid]; } - __global__ void broadcastaddKernel(ftype* res, const ftype* const matrix, const ftype* const vec, - const tensorSize_t vectorSize, const tensorSize_t matrixSize) - { + /** + * @brief Kernel for broadcasted addition, e.g. when adding a bias to a matrix. + */ + __global__ void broadcastaddKernel(ftype* const res, const ftype* const matrix, const ftype* const vec, + const tensorSize_t vectorSize, const tensorSize_t matrixSize) { int gid = blockDim.x * blockIdx.x + threadIdx.x; if(gid>=matrixSize) return; @@ -42,8 +46,10 @@ namespace{ res[gid] = matrix[gid] + vec[vectorIdx]; } - __global__ void elementwisemulKernel(ftype* res, const ftype* const left, const ftype* const right, const tensorSize_t size) - { + /** + * @brief Kernel for simple elementwise multiplication. + */ + __global__ void elementwisemulKernel(ftype* const res, const ftype* const left, const ftype* const right, const tensorSize_t size) { int gid = blockDim.x * blockIdx.x + threadIdx.x; if(gid>=size) return; @@ -51,8 +57,10 @@ namespace{ res[gid] = left[gid] * right[gid]; } - __global__ void scalaraddKernel(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size) - { + /** + * @brief Kernel for scalar addition. + */ + __global__ void scalaraddKernel(ftype* const res, const ftype* const left, ftype scalar, tensorSize_t size) { int gid = blockDim.x * blockIdx.x + threadIdx.x; if(gid>=size) return; @@ -60,8 +68,10 @@ namespace{ res[gid] = left[gid] + scalar; } - __global__ void scalarmulKernel(ftype* res, const ftype* const left, ftype scalar, tensorSize_t size) - { + /** + * @brief Kernel for scalar multiplication. + */ + __global__ void scalarmulKernel(ftype* const res, const ftype* const left, ftype scalar, tensorSize_t size) { int gid = blockDim.x * blockIdx.x + threadIdx.x; if(gid>=size) return; @@ -82,7 +92,7 @@ namespace{ * @param rightCols n columns of right matrix. * @param resSize Size of the resulting 2D matrix. */ - __global__ void matMul2DKernel(ftype* res, const ftype* const left, const ftype* const right, + __global__ void matMul2DKernel(ftype* const res, const ftype* const left, const ftype* const right, const tensorDim_t leftRows, const tensorDim_t leftCols, const tensorDim_t rightCols, const tensorSize_t resSize) { diff --git a/src/backend/data_modeling/cuda/tensorops.cuh b/src/backend/data_modeling/cuda/tensorops.cuh index 595f564..0dc420d 100644 --- a/src/backend/data_modeling/cuda/tensorops.cuh +++ b/src/backend/data_modeling/cuda/tensorops.cuh @@ -32,8 +32,8 @@ namespace cuda_impl { void elementwisemul(Tensor& res, const Tensor& left, const Tensor& right); void matmul(Tensor& res, const Tensor& left, const Tensor& right); - void transpose2D(ftype* res, const ftype* const src, Dimension dims, tensorDim_t dim1, tensorDim_t dim2); - void transpose(ftype* res, const ftype* const src, Dimension dims, tensorDim_t dim1, tensorDim_t dim2); + void transpose2D(ftype* const res, const ftype* const src, Dimension dims, tensorDim_t dim1, tensorDim_t dim2); + void transpose(ftype* const res, const ftype* const src, Dimension dims, tensorDim_t dim1, tensorDim_t dim2); void scalarFill(Tensor& t, ftype value); diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index 1fd4fe0..c424137 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -14,13 +14,18 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #endif // __CUDA #include "activations.cuh" + #include "utility/cuda/cuda_common.cuh" +#include "utility/macros.h" #include using namespace std; namespace { + /** + * @brief Kernel for forward ReLU function. + */ __global__ void reluKernel(ftype* const res, const ftype* const input, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) @@ -30,6 +35,9 @@ namespace { res[gid] = fmaxf(input[gid], zero); } + /** + * @brief Kernel for forward Leaky-ReLU function. + */ __global__ void leakyReluKernel(ftype* const res, const ftype* const input, const ftype eps, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) @@ -38,12 +46,18 @@ namespace { res[gid] = fmaxf(input[gid], eps * input[gid]); // eps < 1 } + /** + * @brief Single sigmoid computation. + */ __device__ __forceinline__ ftype sigmoid(ftype x) { ftype z = expf(-fabsf(x)); ftype s = 1.0f / (1.0f + z); return (x >= 0.f) ? s : z * s; // x < 0 => e^x/(e^x+1) } + /** + * @brief Kernel for forward Sigmoid function. + */ __global__ void sigmoidKernel(ftype* const res, const ftype* const input, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) @@ -108,6 +122,8 @@ namespace { */ template __global__ void findMaxKernelOneWarp(ftype* const res, const ftype* const input, const tensorSize_t stride, const tensorSize_t size) { + assert(blockDim.x % 32 == 0); + int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; @@ -122,16 +138,20 @@ namespace { warpMaxReduce(start, stride, offset); // one warp reduces one 'stride' - if(offset == 0) + if(offset == 0) { res[tid / 32] = smem[tid]; + } } /** - * @brief Numerically stable version of + * @brief Numerically stable version of softmax kernel. Just as in findMaxKernelOneWarp we assume that stride <= 2 * warpsize. + * Numerical stability comes from computing the maximum values per row, see findMaxKernelOneWarp and argument maxValues. */ template __global__ void stableSoftmaxKernelOneWarp(ftype* const res, const ftype* const input, const ftype* const maxValues, - const tensorSize_t stride, const tensorSize_t size) { + const tensorSize_t stride, const tensorSize_t size) { + assert(blockDim.x % 32 == 0); + int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; @@ -141,7 +161,6 @@ namespace { int tid = threadIdx.x; extern __shared__ ftype smem[]; // can lead to bank conflicts iff std::is_same_v - __syncthreads(); ftype expVal = 0; if constexpr (std::is_same_v) { @@ -163,105 +182,87 @@ namespace { res[gid] = expVal / start[0]; } - // TODO: use shared memory - __global__ void findMaxKernelOneBlock(ftype* res, ftype* const input, const tensorSize_t stride, const tensorSize_t size) { + /** + * @brief Like findMaxKernelOneWarp. The difference now is that the input size can be much larger. stride is + * 64 < stride <= threadsPerBlock. res has the maximum values stored. + */ + __global__ void findMaxKernelOneBlock(ftype* const res, const ftype* const input, const tensorSize_t stride, const tensorSize_t size) { + assert(blockDim.x % stride == 0); // guarantees that stride not spread across two blocks + int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; int tid = threadIdx.x; + extern __shared__ ftype smem[]; // can lead to bank conflicts iff std::is_same_v + smem[tid] = input[gid]; + __syncthreads(); + const tensorSize_t baseOffset = gid % stride; - for(tensorSize_t offset = (stride + 1) / 2; offset > 32; offset = (offset + 1) / 2) { // ceil-division - // ceil-division over two -> last thread crosses boundary iff stride % 2 == 1 + for(tensorSize_t offset = (stride + 1) / 2; offset > 32; offset = (offset + 1) / 2) { + // ceil-division over two -> at least one thread crosses boundary iff stride % 2 == 1 if(baseOffset + offset < stride) { - input[gid] = max(input[gid], input[gid + offset]); + smem[tid] = max(smem[tid], smem[tid + offset]); } __syncthreads(); } - // loop unrolling - if(baseOffset + 16 < stride) { - input[gid] = max(input[gid], input[gid + 16]); - __syncthreads(); - } - if(baseOffset + 8 < stride) { - input[gid] = max(input[gid], input[gid + 8]); - __syncthreads(); - } - if(baseOffset + 4 < stride) { - input[gid] = max(input[gid], input[gid + 4]); - __syncthreads(); - } - if(baseOffset + 2 < stride) { - input[gid] = max(input[gid], input[gid + 2]); - __syncthreads(); - } - if(baseOffset + 1 < stride) { - input[gid] = max(input[gid], input[gid + 1]); - __syncthreads(); - } + volatile ftype* const start = smem + (tid / stride) * stride; + const int offset = gid % stride; + warpMaxReduce<32>(start, stride, offset); - // write to result if(baseOffset == 0) { - res[gid / stride] = input[gid]; + res[gid / stride] = start[0]; } } - // TODO: use shared memory + /** + * @brief Just like stableSoftmaxKernelOneWarp, but this one works across a whole block, not just a warp. + */ template - __global__ void expAndSumKernelOneBlock(T* res, T* const tmp, const T* const input, const T* const maxValues, - const tensorSize_t stride, const tensorSize_t size) { + __global__ void stableSoftmaxKernelOneBlock(ftype* res, const ftype* const input, const ftype* const maxValues, + const tensorSize_t stride, const tensorSize_t size) { + assert(blockDim.x % stride == 0); // guarantees that stride not spread across two blocks + int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; const auto strideOffset = gid / stride; const auto maxValue = maxValues[strideOffset]; + + extern __shared__ ftype smem[]; + int tid = threadIdx.x; + + ftype expVal = 0; if constexpr (std::is_same_v) { - res[gid] = expf(input[gid] - maxValue); + expVal = expf(input[gid] - maxValue); } else if constexpr (std::is_same_v) { - res[gid] = exp(input[gid] - maxValue); + expVal = exp(input[gid] - maxValue); } else { static_assert(always_false, "ftype encountered unexpected type"); } - + smem[tid] = expVal; __syncthreads(); const tensorSize_t baseOffset = gid % stride; - tensorSize_t offset = (stride + 1) / 2; // ceil-division - while(offset > 32) { - if(baseOffset + offset >= stride) continue; // ceil-division -> last thread can cross boundary - - tmp[gid] = res[gid] + res[gid + offset]; - offset = (offset + 1) / 2; + for(tensorSize_t offset = (stride + 1) / 2; offset > 32; offset = (offset + 1) / 2) { + if(baseOffset + offset < stride) { + smem[tid] = smem[tid] + smem[tid + offset]; // ceil-division -> last thread can cross boundary + } + __syncthreads(); } - // loop unrolling - if(offset <= 32 && baseOffset + 16 < stride) { - tmp[gid] = res[gid] + res[gid + 16]; - } - if(offset <= 16 && baseOffset + 8 < stride) { - tmp[gid] = res[gid] + res[gid + 8]; - } - if(offset <= 8 && baseOffset + 4 < stride) { - tmp[gid] = res[gid] + res[gid + 4]; - } - if(offset <= 4 && baseOffset + 2 < stride) { - tmp[gid] = res[gid] + res[gid + 2]; - } - if(offset <= 2 && baseOffset + 1 < stride) { - tmp[gid] = res[gid] + res[gid + 1]; - } + volatile ftype* const start = smem + (tid / stride) * stride; + const int offset = gid % stride; + warpSumReduce<32>(start, stride, offset); - // write to result - if(baseOffset == 0) { - tmp[strideOffset] = tmp[gid]; - } + res[gid] = expVal / start[0]; } - __global__ void softmaxDivisionKernel(ftype* res, const ftype* const sums, const tensorSize_t stride, const tensorSize_t size) { + __global__ void softmaxDivisionKernel(ftype* const res, const ftype* const sums, const tensorSize_t stride, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; @@ -302,11 +303,7 @@ namespace cuda_impl { __throw_invalid_argument("Attemped softmax of one element"); } - // TODO: use some static struct here to prevent those guys from keeping on re-allocating memory -/* ftype* tmp; - cudaErrchk(cudaMalloc(&tmp, in.getSize() * sizeof(ftype))); - cudaErrchk(cudaMemcpy(tmp, in.getData(), in.getSize() * sizeof(ftype), cudaMemcpyDeviceToDevice)); */ - + // TODO: use some static struct here to prevent this guy from keeping on re-allocating memory ftype* maxValues; const tensorSize_t nMaxValues = in.getSize() / stride; cudaErrchk(cudaMalloc(&maxValues, nMaxValues * sizeof(ftype))); @@ -366,23 +363,26 @@ namespace cuda_impl { } else if (stride <= 256) { // threads per block needs to be multiple of stride -/* constexpr int threadsPerBlock = 256; - const int blocks = (in.getSize()+threadsPerBlock-1) / threadsPerBlock; + constexpr int maxThreadsPerBlock = 256; + assert_debug(stride <= maxThreadsPerBlock, "If you adapt maxThreadsPerBlock you also have to adapt the limit in the if-clause above"); - findMaxKernelOneBlock<<>>(maxValues, tmp, stride, in.getSize()); - cudaErrchk(cudaDeviceSynchronize()); + const int nStrides = in.getSize() / stride; + const int stridesPerBlock = max(maxThreadsPerBlock / stride, 1); + + const int threadsPerBlock = stridesPerBlock * stride; + const int blocks = (nStrides + stridesPerBlock - 1) / stridesPerBlock; - expAndSumKernelOneBlock<<>>(res.getData(), tmp, in.getData(), maxValues, stride, in.getSize()); + findMaxKernelOneBlock<<>>(maxValues, in.getData(), stride, in.getSize()); cudaErrchk(cudaDeviceSynchronize()); - softmaxDivisionKernel<<>>(res.getData(), tmp, stride, in.getSize()); - cudaErrchk(cudaDeviceSynchronize()); */ + stableSoftmaxKernelOneBlock<<>>(res.getData(), in.getData(), + maxValues, stride, in.getSize()); + cudaErrchk(cudaDeviceSynchronize()); } else { __throw_runtime_error("Softmax kernels not yet implemented at inter-block level"); } - //cudaErrchk(cudaFree(tmp)); cudaErrchk(cudaFree(maxValues)); } } \ No newline at end of file diff --git a/src/backend/module/activation_functions/softmax.cpp b/src/backend/module/activation_functions/softmax.cpp index c5171a4..1cd2f07 100644 --- a/src/backend/module/activation_functions/softmax.cpp +++ b/src/backend/module/activation_functions/softmax.cpp @@ -42,16 +42,16 @@ Tensor Softmax::operator()(const Tensor& t) const { // pre-compute exponents Tensor tmp(t.getDims(), t.getDevice(), false); - for(tensorDim_t i=0; i::infinity(); - for(tensorDim_t j=0; j= size) return; diff --git a/tests/backend/test_module.cpp b/tests/backend/test_module.cpp index bd7c5a7..6c07237 100644 --- a/tests/backend/test_module.cpp +++ b/tests/backend/test_module.cpp @@ -147,7 +147,7 @@ TEST(AutogradTest, SigmoidBackward) { // for x=0: grad = 0.5 * 0.5 = 0.25 // for x=1: grad = 0.7311 * 0.2689 = 0.1966 auto t = TensorFunctions::makeSharedTensor( - {2}, {0.0, 1.0}, true); + {2}, {0.0, 1.0}, true); module::Sigmoid sig; auto res = sig(t); @@ -196,9 +196,10 @@ TEST(ActivationTest, SoftmaxForwardNumericalStability) { auto res = sm(t); for(int i = 0; i < 3; i++) { - EXPECT_FALSE(std::isnan(res[i])); - EXPECT_FALSE(std::isinf(res[i])); + EXPECT_FALSE(std::isnan(res[i])); + EXPECT_FALSE(std::isinf(res[i])); } + ftype rowsum = res[0] + res[1] + res[2]; ASSERT_NEAR(rowsum, 1.0, delta); } @@ -221,7 +222,7 @@ TEST(AutogradTest, SoftmaxBackward) { // set upstream gradient to [1, 0, 0] auto upstream = TensorFunctions::makeSharedTensor( - {1, 3}, {1.0, 0.0, 0.0}); + {1, 3}, {1.0, 0.0, 0.0}); resPtr->setGrads(upstream); resPtr->backward(); From 17d65c8dc128f31c284028645bf7961b75f2a5ab Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Thu, 14 May 2026 16:34:27 +0200 Subject: [PATCH 39/73] Fixed indexing bug for crossentropy- and softmax that arises in dimensions >= 3; added unit test for larger softmax kernel; removed unused function stubs --- .../loss_functions/crossentropy_node.cpp | 17 ++++---- src/backend/data_modeling/cuda/tensorops.cuh | 3 -- .../activation_functions/cuda/activations.cu | 9 ----- .../module/activation_functions/softmax.cpp | 24 ++++++------ .../crossentropy_softmax_loss.cpp | 39 +++++++++---------- tests/backend/cuda/test_module_cuda.cu | 29 ++++++++++++++ 6 files changed, 68 insertions(+), 53 deletions(-) diff --git a/src/backend/computational_graph/loss_functions/crossentropy_node.cpp b/src/backend/computational_graph/loss_functions/crossentropy_node.cpp index 347f59d..be15b0f 100644 --- a/src/backend/computational_graph/loss_functions/crossentropy_node.cpp +++ b/src/backend/computational_graph/loss_functions/crossentropy_node.cpp @@ -21,6 +21,10 @@ using namespace std; using namespace cgraph; +/** + * @brief Backward function on crossentropy-node. Uses cached values of forward pass + * for higher efficiency. + */ vector< shared_ptr > CrossEntropyNode::backward(const Tensor& upstreamGrad) { assert(!upstreamGrad.getRequiresGrad()); @@ -29,14 +33,11 @@ vector< shared_ptr > CrossEntropyNode::backward(const Tensor& upstreamGr switch(upstreamGrad.getDevice()) { case Device::CPU: { - ftype bSize = yPred->getDims()[0]; - for(tensorDim_t i=0; igetDims()[0]; i++){ - for(tensorDim_t j=0; jgetDims()[1]; j++){ - auto yij = yTrue->get(i, j); - auto yijHat = yPred->get(i, j); - auto g = -yij/std::max(yijHat, EPS_CROSSENTROPY); - res->set(g/bSize, i, j); - } + const tensorSize_t stride = yPred->getDims()[-1]; + const ftype bSize = static_cast(yPred->getSize() / stride); + for(tensorSize_t i = 0; i < yPred->getSize(); i++) { + auto g = -(*yTrue)[i] / std::max((*yPred)[i], EPS_CROSSENTROPY); + res->set(g / bSize, i); } break; } diff --git a/src/backend/data_modeling/cuda/tensorops.cuh b/src/backend/data_modeling/cuda/tensorops.cuh index 0dc420d..ef0e6ae 100644 --- a/src/backend/data_modeling/cuda/tensorops.cuh +++ b/src/backend/data_modeling/cuda/tensorops.cuh @@ -32,9 +32,6 @@ namespace cuda_impl { void elementwisemul(Tensor& res, const Tensor& left, const Tensor& right); void matmul(Tensor& res, const Tensor& left, const Tensor& right); - void transpose2D(ftype* const res, const ftype* const src, Dimension dims, tensorDim_t dim1, tensorDim_t dim2); - void transpose(ftype* const res, const ftype* const src, Dimension dims, tensorDim_t dim1, tensorDim_t dim2); - void scalarFill(Tensor& t, ftype value); void createContiguousCopy(Tensor& res, const Tensor& src); diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index c424137..99a3ddd 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -261,15 +261,6 @@ namespace { res[gid] = expVal / start[0]; } - - __global__ void softmaxDivisionKernel(ftype* const res, const ftype* const sums, const tensorSize_t stride, const tensorSize_t size) { - int gid = blockIdx.x * blockDim.x + threadIdx.x; - if(gid >= size) - return; - - const auto strideOffset = gid / stride; - res[gid] = res[gid] / sums[strideOffset]; - } } namespace cuda_impl { diff --git a/src/backend/module/activation_functions/softmax.cpp b/src/backend/module/activation_functions/softmax.cpp index 1cd2f07..df01f7a 100644 --- a/src/backend/module/activation_functions/softmax.cpp +++ b/src/backend/module/activation_functions/softmax.cpp @@ -37,26 +37,24 @@ Tensor Softmax::operator()(const Tensor& t) const { switch(t.getDevice()) { case Device::CPU: { - const auto nRows = t.getDims()[-2]; - const auto nCols = t.getDims()[-1]; + const tensorSize_t stride = t.getDims()[-1]; - // pre-compute exponents + // pre-compute exponents, centering each slice around its max for numerical stability Tensor tmp(t.getDims(), t.getDevice(), false); - for(tensorDim_t i = 0; i < nRows; i++){ - // for numerical stability, avoid large values - // by centering around maxValue for each sample + tensorSize_t offset = 0; + while(offset < t.getSize()) { ftype maxValue = -std::numeric_limits::infinity(); - for(tensorDim_t j = 0; j < nCols; j++){ - maxValue = std::max(maxValue, t.get(i, j)); + for(tensorSize_t i = offset; i < offset + stride; i++) { + maxValue = std::max(maxValue, t[i]); } - for(tensorDim_t j = 0; j < nCols; j++){ - ftype e = t.get(i, j) - maxValue; - tmp.set(exp(e), i, j); + for(tensorSize_t i = offset; i < offset + stride; i++) { + tmp.set(exp(t[i] - maxValue), i); } + + offset += stride; } - const tensorSize_t stride = t.getDims()[-1]; auto compute = [&res, &tmp, stride](tensorSize_t start){ ftype sum = 0; for(tensorSize_t i = start; i < start+stride; i++){ @@ -68,7 +66,7 @@ Tensor Softmax::operator()(const Tensor& t) const { } }; - tensorSize_t offset = 0; + offset = 0; while(offset < res.getSize()) { compute(offset); offset += stride; diff --git a/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp b/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp index a82b188..028b5ea 100644 --- a/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp +++ b/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp @@ -42,28 +42,27 @@ shared_ptr CrossEntropySoftmaxLoss::operator()(const shared_ptr switch(y->getDevice()) { case Device::CPU: { - const auto nRows = logits->getDims()[-2]; - const auto nCols = logits->getDims()[-1]; + const tensorSize_t stride = logits->getDims()[-1]; + const tensorSize_t nSamples = logits->getSize() / stride; - // pre-compute exponents and max-values - vector maxValues(nRows); + // pre-compute exponents and max-values, centering each slice for numerical stability + vector maxValues(nSamples); Tensor tmp(logits->getDims(), logits->getDevice(), false); - for(tensorDim_t i=0; igetSize()) { ftype maxV = -std::numeric_limits::infinity(); - for(tensorDim_t j=0; jget(i, j)); + for(tensorSize_t i = offset; i < offset + stride; i++) { + maxV = std::max(maxV, (*logits)[i]); } - maxValues[i] = maxV; + maxValues[offset / stride] = maxV; - for(tensorDim_t j=0; jget(i, j)-maxV; - tmp.set(exp(e), i, j); + for(tensorSize_t i = offset; i < offset + stride; i++) { + tmp.set(exp((*logits)[i] - maxV), i); } - } - const tensorSize_t stride = logits->getDims()[-1]; + offset += stride; + } ftype loss = 0; /** @@ -73,26 +72,26 @@ shared_ptr CrossEntropySoftmaxLoss::operator()(const shared_ptr */ auto compute = [&loss, &y, &logits, &tmp, &maxValues, stride](tensorSize_t start){ ftype lsum = 0; - for(tensorSize_t i=start; i0){ // y either zero or one loss += -(*logits)[i] + maxValues[j] + lsum; } } }; - tensorSize_t offset=0; - while(offsetgetSize()) { + offset = 0; + while(offset < logits->getSize()) { compute(offset); offset += stride; } - res = make_shared(std::vector{1}, std::vector{loss / logits->getDims()[0]}, y->getDevice(), true); + res = make_shared(std::vector{1}, std::vector{loss / static_cast(nSamples)}, y->getDevice(), true); break; } case Device::CUDA: diff --git a/tests/backend/cuda/test_module_cuda.cu b/tests/backend/cuda/test_module_cuda.cu index 4fb79e4..7fc49e4 100644 --- a/tests/backend/cuda/test_module_cuda.cu +++ b/tests/backend/cuda/test_module_cuda.cu @@ -180,10 +180,39 @@ TEST(CudaActivationTest, SoftmaxForwardNumericalStability) { ASSERT_FALSE(std::isnan(res[i])); ASSERT_FALSE(std::isinf(res[i])); } + ftype rowsum = res[0] + res[1] + res[2]; ASSERT_NEAR(rowsum, 1.0, 1e-5); } +TEST(CudaActivationTest, SoftmaxMediumLargeInput) { + constexpr tensorDim_t testDim = 190; + assert(testDim <= 256 && testDim > 64); // see the kernel call + + auto t = TensorFunctions::Gaussian({5, 10, testDim}, 2.0f, Device::CUDA); + auto tCopy = t.createDeepCopy(); + tCopy.setDevice(Device::CPU); + + module::Softmax sm; + auto resGpu = sm(t); + auto resCpu = sm(tCopy); + + cout << resCpu << endl; + cout << resGpu << endl; + + resGpu.setDevice(Device::CUDA); + for(int i = 0; i < resGpu.getDims().get(0); i++) { + for(int j = 0; j < resGpu.getDims().get(1); j++) { + for(int k = 0; k < resGpu.getDims().get(2); k++) { + ASSERT_NEAR(resCpu.get(i, j, k), resGpu.get(i, j, k), 1e-4) + << "Mismatch at (" << i << ", " << j << ", " << k << ")" + << " cpu=" << resCpu.get(i, j, k) + << " gpu=" << resGpu.get(i, j, k); + } + } + } +} + /* TEST(CudaAutogradTest, SoftmaxBackward) { auto t = TensorFunctions::makeSharedTensor({1, 3}, {1.0, 2.0, 3.0}, Device::CUDA, true); From 3e9601805a1eab93a2c6672a521e209347828de4 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Thu, 14 May 2026 16:38:27 +0200 Subject: [PATCH 40/73] Update readme --- readme.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index a2a846c..5575cbd 100644 --- a/readme.md +++ b/readme.md @@ -54,10 +54,11 @@ make ### Building with CUDA -To build with CUDA, a flag has to be supplied to CMake. +Project automatically detects whether CUDA is installed, and compiles with it. +If CUDA compilation not desired you can switch it off via ```bash -cmake --DCUDA=On .. +cmake --DCUDA=Off .. ``` From 39a9086318a763bf489ac47d61e8647f78e28ad2 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Fri, 15 May 2026 09:06:33 +0200 Subject: [PATCH 41/73] Adjusting spacing --- .../activation_functions/leaky_relu_node.cpp | 2 +- .../activation_functions/relu_node.cpp | 2 +- .../activation_functions/sigmoid_node.cpp | 4 ++-- .../activation_functions/softmax_node.cpp | 8 ++++---- .../computational_graph/loss_functions/bce_node.cpp | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/backend/computational_graph/activation_functions/leaky_relu_node.cpp b/src/backend/computational_graph/activation_functions/leaky_relu_node.cpp index bdffd6f..24607e8 100644 --- a/src/backend/computational_graph/activation_functions/leaky_relu_node.cpp +++ b/src/backend/computational_graph/activation_functions/leaky_relu_node.cpp @@ -31,7 +31,7 @@ vector> LeakyReLuNode::backward(const Tensor& upstreamGrad) { switch(upstreamGrad.getDevice()) { case Device::CPU: { constexpr ftype zero = 0.0; - for(tensorSize_t i=0; iset((*parent)[i] > zero ? upstreamGrad[i] : upstreamGrad[i] * eps, i); } break; diff --git a/src/backend/computational_graph/activation_functions/relu_node.cpp b/src/backend/computational_graph/activation_functions/relu_node.cpp index 76e4f08..78c6ec3 100644 --- a/src/backend/computational_graph/activation_functions/relu_node.cpp +++ b/src/backend/computational_graph/activation_functions/relu_node.cpp @@ -31,7 +31,7 @@ vector> ReLuNode::backward(const Tensor& upstreamGrad) { switch(upstreamGrad.getDevice()) { case Device::CPU: { constexpr ftype zero = 0.0; - for(tensorSize_t i=0; iset((*parent)[i] > zero ? upstreamGrad[i] : zero, i); } break; diff --git a/src/backend/computational_graph/activation_functions/sigmoid_node.cpp b/src/backend/computational_graph/activation_functions/sigmoid_node.cpp index 7226d12..7f6f85b 100644 --- a/src/backend/computational_graph/activation_functions/sigmoid_node.cpp +++ b/src/backend/computational_graph/activation_functions/sigmoid_node.cpp @@ -30,10 +30,10 @@ vector> SigmoidNode::backward(const Tensor& upstreamGrad) { switch(upstreamGrad.getDevice()) { case Device::CPU: { auto derivative = [](ftype s){ - return s * (1-s); + return s * (1 - s); }; - for(tensorSize_t i=0; iset(derivative((*sigmoid)[i]) * upstreamGrad[i], i); } break; diff --git a/src/backend/computational_graph/activation_functions/softmax_node.cpp b/src/backend/computational_graph/activation_functions/softmax_node.cpp index debcea6..e385bb3 100644 --- a/src/backend/computational_graph/activation_functions/softmax_node.cpp +++ b/src/backend/computational_graph/activation_functions/softmax_node.cpp @@ -35,14 +35,14 @@ vector< shared_ptr > SoftmaxNode::backward(const Tensor& upstreamGrad) { const auto bSize = yPred->getDims()[0]; assert(bSize>0); - for(tensorDim_t b=0; bgetDims()[1]; i++){ + for(tensorDim_t b = 0; b < bSize; b++){ + for(tensorDim_t i = 0; i < yPred->getDims()[1]; i++){ ftype grad = 0; const ftype yi = softmax->get(b, i); - for(tensorDim_t j=0; jgetDims()[1]; j++){ + for(tensorDim_t j = 0; j < yPred->getDims()[1]; j++){ ftype yj = softmax->get(b, j); - ftype jacobian = (i==j) ? yi*(1-yj) : -yi*yj; + ftype jacobian = (i == j) ? yi * (1 - yj) : -yi * yj; grad += upstreamGrad.get(b, j) * jacobian; } res->set(grad, b, i); diff --git a/src/backend/computational_graph/loss_functions/bce_node.cpp b/src/backend/computational_graph/loss_functions/bce_node.cpp index 8c25e0a..b49af11 100644 --- a/src/backend/computational_graph/loss_functions/bce_node.cpp +++ b/src/backend/computational_graph/loss_functions/bce_node.cpp @@ -30,11 +30,11 @@ vector< shared_ptr > BceNode::backward(const Tensor& upstreamGrad) { switch(upstreamGrad.getDevice()) { case Device::CPU: { ftype bSize = yPred->getDims()[0]; - for(tensorSize_t i=0; igetDims()[0]; i++){ + for(tensorSize_t i = 0; i < yPred->getDims()[0]; i++){ auto yi = (*yTrue)[i]; auto yiHat = (*yPred)[i]; - auto g = -yi/std::max(yiHat, EPS_BCE) + (1-yi)/std::max(1-yiHat, EPS_BCE); - res->set(g/bSize, i); + auto g = -yi / std::max(yiHat, EPS_BCE) + (1 - yi) / std::max(1 - yiHat, EPS_BCE); + res->set(g / bSize, i); } break; } From 2e13f164e83ae504b13b43f6f893e60be4dcfb4c Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Fri, 15 May 2026 12:09:18 +0200 Subject: [PATCH 42/73] Fix GPU softmax --- .../activation_functions/cuda/activations.cu | 148 +++++++++++------- 1 file changed, 88 insertions(+), 60 deletions(-) diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index 99a3ddd..f1cb526 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -71,6 +71,7 @@ namespace { */ template __forceinline__ __device__ void warpMaxReduce(volatile ftype* const input, const tensorSize_t stride, const int offset) { + // TODO: warp shuffle for newer architectures if(maxoffset == 32) { if(offset + 32 < stride) input[offset] = max(input[offset], input[offset + 32]); } @@ -96,6 +97,7 @@ namespace { */ template __forceinline__ __device__ void warpSumReduce(volatile ftype* const input, const tensorSize_t stride, const int offset) { + // TODO: warp shuffle for newer architectures if(maxoffset == 32) { if(offset + 32 < stride) input[offset] += input[offset + 32]; } @@ -115,6 +117,22 @@ namespace { if(offset + 1 < stride) input[offset] += input[offset + 1]; } } + + /** + * @brief For the softmax implementations. + */ + template + __forceinline__ __device__ ftype stableExp(const ftype x, const ftype maxValue) { + if constexpr (std::is_same_v) { + return expf(x - maxValue); + } + else if constexpr (std::is_same_v) { + return exp(x - maxValue); + } + else { + static_assert(always_false, "ftype encountered unexpected type"); + } + } /** * @brief Here we find the maximum within 'stride'. Assumption: One warp does exactly one element of stride! @@ -158,20 +176,10 @@ namespace { const auto strideOffset = gid / stride; const auto maxValue = maxValues[strideOffset]; + ftype expVal = stableExp(input[gid], maxValue); int tid = threadIdx.x; extern __shared__ ftype smem[]; // can lead to bank conflicts iff std::is_same_v - - ftype expVal = 0; - if constexpr (std::is_same_v) { - expVal = expf(input[gid] - maxValue); - } - else if constexpr (std::is_same_v) { - expVal = exp(input[gid] - maxValue); - } - else { - static_assert(always_false, "ftype encountered unexpected type"); - } smem[tid] = expVal; __syncthreads(); @@ -185,81 +193,100 @@ namespace { /** * @brief Like findMaxKernelOneWarp. The difference now is that the input size can be much larger. stride is * 64 < stride <= threadsPerBlock. res has the maximum values stored. + * + * In this initial version we assume one kernel per stride, to make matters simple to understand. */ __global__ void findMaxKernelOneBlock(ftype* const res, const ftype* const input, const tensorSize_t stride, const tensorSize_t size) { - assert(blockDim.x % stride == 0); // guarantees that stride not spread across two blocks + assert_debug(blockDim.x / stride == 0, "Kernel built for one stride per block, blockDim.x is < stride"); - int gid = blockIdx.x * blockDim.x + threadIdx.x; - if(gid >= size) - return; + const int tid = threadIdx.x; + const int gid = blockIdx.x * stride + tid; - int tid = threadIdx.x; extern __shared__ ftype smem[]; // can lead to bank conflicts iff std::is_same_v - smem[tid] = input[gid]; + + const tensorSize_t maxIdx = tid + blockDim.x; + const bool doPadding = maxIdx >= stride; + if(doPadding) { + smem[tid] = input[gid]; + } + else { + smem[tid] = max(input[gid], input[gid + blockDim.x]); + } __syncthreads(); - const tensorSize_t baseOffset = gid % stride; - for(tensorSize_t offset = (stride + 1) / 2; offset > 32; offset = (offset + 1) / 2) { - // ceil-division over two -> at least one thread crosses boundary iff stride % 2 == 1 - if(baseOffset + offset < stride) { + for(tensorSize_t offset = blockDim.x >> 1; offset > 32; offset >>= 1) { + if(tid < offset) { smem[tid] = max(smem[tid], smem[tid + offset]); } __syncthreads(); } - volatile ftype* const start = smem + (tid / stride) * stride; - const int offset = gid % stride; - warpMaxReduce<32>(start, stride, offset); - - if(baseOffset == 0) { - res[gid / stride] = start[0]; + // TODO: warp shuffle for newer architectures + volatile ftype* const start = smem; + if(tid < 32) start[tid] = max(start[tid], start[tid + 32]); + if(tid < 16) start[tid] = max(start[tid], start[tid + 16]); + if(tid < 8) start[tid] = max(start[tid], start[tid + 8]); + if(tid < 4) start[tid] = max(start[tid], start[tid + 4]); + if(tid < 2) start[tid] = max(start[tid], start[tid + 2]); + if(tid < 1) start[tid] = max(start[tid], start[tid + 1]); + + if(tid == 0) { // one block per stride + res[blockIdx.x] = start[0]; } } /** * @brief Just like stableSoftmaxKernelOneWarp, but this one works across a whole block, not just a warp. + * + * In this initial version we assume one kernel per stride, to make matters simple to understand. */ template __global__ void stableSoftmaxKernelOneBlock(ftype* res, const ftype* const input, const ftype* const maxValues, - const tensorSize_t stride, const tensorSize_t size) { - assert(blockDim.x % stride == 0); // guarantees that stride not spread across two blocks + const tensorSize_t stride, const tensorSize_t size) { + assert_debug(blockDim.x / stride == 0, "Kernel built for one stride per block, blockDim.x is < stride"); - int gid = blockIdx.x * blockDim.x + threadIdx.x; - if(gid >= size) - return; + const int tid = threadIdx.x; + const int gid = blockIdx.x * stride + tid; - const auto strideOffset = gid / stride; - const auto maxValue = maxValues[strideOffset]; + const auto maxValue = maxValues[blockIdx.x]; // TODO: i can share this one within a warp? - extern __shared__ ftype smem[]; - int tid = threadIdx.x; + extern __shared__ ftype smem[]; // can lead to bank conflicts iff std::is_same_v - ftype expVal = 0; - if constexpr (std::is_same_v) { - expVal = expf(input[gid] - maxValue); - } - else if constexpr (std::is_same_v) { - expVal = exp(input[gid] - maxValue); - } - else { - static_assert(always_false, "ftype encountered unexpected type"); - } + ftype expVal = stableExp(input[gid], maxValue); smem[tid] = expVal; __syncthreads(); - const tensorSize_t baseOffset = gid % stride; - for(tensorSize_t offset = (stride + 1) / 2; offset > 32; offset = (offset + 1) / 2) { - if(baseOffset + offset < stride) { - smem[tid] = smem[tid] + smem[tid + offset]; // ceil-division -> last thread can cross boundary + const tensorSize_t maxIdx = tid + blockDim.x; + const bool doPadding = maxIdx >= stride; + + ftype expValOffset = 0; + if(!doPadding) { // some threads will be idle here + expValOffset = stableExp(input[gid + blockDim.x], maxValue); + } + + smem[maxIdx] = expValOffset; + __syncthreads(); + + for(tensorSize_t offset = blockDim.x; offset > 32; offset >>= 1) { + if(tid < offset) { + smem[tid] = smem[tid] + smem[tid + offset]; } __syncthreads(); } - volatile ftype* const start = smem + (tid / stride) * stride; - const int offset = gid % stride; - warpSumReduce<32>(start, stride, offset); + // TODO: warp shuffle for newer architectures + volatile ftype* const start = smem; + if(tid < 32) start[tid] = start[tid] += start[tid + 32]; + if(tid < 16) start[tid] = start[tid] += start[tid + 16]; + if(tid < 8) start[tid] = start[tid] += start[tid + 8]; + if(tid < 4) start[tid] = start[tid] += start[tid + 4]; + if(tid < 2) start[tid] = start[tid] += start[tid + 2]; + if(tid < 1) start[tid] = start[tid] += start[tid + 1]; res[gid] = expVal / start[0]; + if(!doPadding) { + res[gid + blockDim.x] = expValOffset / start[0]; + } } } @@ -353,21 +380,22 @@ namespace cuda_impl { cudaErrchk(cudaDeviceSynchronize()); } else if (stride <= 256) { - // threads per block needs to be multiple of stride constexpr int maxThreadsPerBlock = 256; assert_debug(stride <= maxThreadsPerBlock, "If you adapt maxThreadsPerBlock you also have to adapt the limit in the if-clause above"); const int nStrides = in.getSize() / stride; - const int stridesPerBlock = max(maxThreadsPerBlock / stride, 1); + constexpr int stridesPerBlock = 1; // if multiple strides per block allowed (adapt kernels!): max(maxThreadsPerBlock / stride, 1); - const int threadsPerBlock = stridesPerBlock * stride; - const int blocks = (nStrides + stridesPerBlock - 1) / stridesPerBlock; + // threads per block needs to be multiple of stride, rounded toward the next multiple of 32 + const int paddedStride = ((stride + 31) / 32) * 32; + const int threadsPerBlock = paddedStride * stridesPerBlock / 2; // over 2 for efficiency in reduction + const int blocks = (nStrides + stridesPerBlock - 1) / stridesPerBlock; // gerneralized version iff multiple strides per block allowed - findMaxKernelOneBlock<<>>(maxValues, in.getData(), stride, in.getSize()); + findMaxKernelOneBlock<<>>(maxValues, in.getData(), stride, in.getSize()); cudaErrchk(cudaDeviceSynchronize()); - stableSoftmaxKernelOneBlock<<>>(res.getData(), in.getData(), - maxValues, stride, in.getSize()); + stableSoftmaxKernelOneBlock<<>>(res.getData(), in.getData(), maxValues, + stride, in.getSize()); cudaErrchk(cudaDeviceSynchronize()); } else { From 7091379cae44322e11180754b7955929223c506d Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Fri, 15 May 2026 16:23:58 +0200 Subject: [PATCH 43/73] Fixed softmax for good --- .../activation_functions/cuda/activations.cu | 48 ++++++++++--------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index f1cb526..042d858 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -223,12 +223,14 @@ namespace { // TODO: warp shuffle for newer architectures volatile ftype* const start = smem; - if(tid < 32) start[tid] = max(start[tid], start[tid + 32]); - if(tid < 16) start[tid] = max(start[tid], start[tid + 16]); - if(tid < 8) start[tid] = max(start[tid], start[tid + 8]); - if(tid < 4) start[tid] = max(start[tid], start[tid + 4]); - if(tid < 2) start[tid] = max(start[tid], start[tid + 2]); - if(tid < 1) start[tid] = max(start[tid], start[tid + 1]); + if(tid < 32) { + start[tid] = max(start[tid], start[tid + 32]); + start[tid] = max(start[tid], start[tid + 16]); + start[tid] = max(start[tid], start[tid + 8]); + start[tid] = max(start[tid], start[tid + 4]); + start[tid] = max(start[tid], start[tid + 2]); + start[tid] = max(start[tid], start[tid + 1]); + } if(tid == 0) { // one block per stride res[blockIdx.x] = start[0]; @@ -248,13 +250,10 @@ namespace { const int tid = threadIdx.x; const int gid = blockIdx.x * stride + tid; - const auto maxValue = maxValues[blockIdx.x]; // TODO: i can share this one within a warp? - extern __shared__ ftype smem[]; // can lead to bank conflicts iff std::is_same_v - + const auto maxValue = maxValues[blockIdx.x]; ftype expVal = stableExp(input[gid], maxValue); smem[tid] = expVal; - __syncthreads(); const tensorSize_t maxIdx = tid + blockDim.x; const bool doPadding = maxIdx >= stride; @@ -269,19 +268,22 @@ namespace { for(tensorSize_t offset = blockDim.x; offset > 32; offset >>= 1) { if(tid < offset) { - smem[tid] = smem[tid] + smem[tid + offset]; + smem[tid] += smem[tid + offset]; } __syncthreads(); } // TODO: warp shuffle for newer architectures volatile ftype* const start = smem; - if(tid < 32) start[tid] = start[tid] += start[tid + 32]; - if(tid < 16) start[tid] = start[tid] += start[tid + 16]; - if(tid < 8) start[tid] = start[tid] += start[tid + 8]; - if(tid < 4) start[tid] = start[tid] += start[tid + 4]; - if(tid < 2) start[tid] = start[tid] += start[tid + 2]; - if(tid < 1) start[tid] = start[tid] += start[tid + 1]; + if(tid < 32) { + start[tid] += start[tid + 32]; + start[tid] += start[tid + 16]; + start[tid] += start[tid + 8]; + start[tid] += start[tid + 4]; + start[tid] += start[tid + 2]; + start[tid] += start[tid + 1]; + } + __syncthreads(); res[gid] = expVal / start[0]; if(!doPadding) { @@ -379,16 +381,18 @@ namespace cuda_impl { } cudaErrchk(cudaDeviceSynchronize()); } - else if (stride <= 256) { + else if (stride <= 512) { constexpr int maxThreadsPerBlock = 256; - assert_debug(stride <= maxThreadsPerBlock, "If you adapt maxThreadsPerBlock you also have to adapt the limit in the if-clause above"); + assert_debug(stride <= 2 * maxThreadsPerBlock, "If you adapt maxThreadsPerBlock you also have to adapt the limit in the if-clause above"); const int nStrides = in.getSize() / stride; constexpr int stridesPerBlock = 1; // if multiple strides per block allowed (adapt kernels!): max(maxThreadsPerBlock / stride, 1); - // threads per block needs to be multiple of stride, rounded toward the next multiple of 32 - const int paddedStride = ((stride + 31) / 32) * 32; - const int threadsPerBlock = paddedStride * stridesPerBlock / 2; // over 2 for efficiency in reduction + // threads per block needs to be power of 2 for reduction to resolve cleanly + int paddedStride = 1; + while(paddedStride < stride) paddedStride <<= 1; + int threadsPerBlock = paddedStride / 2; + const int blocks = (nStrides + stridesPerBlock - 1) / stridesPerBlock; // gerneralized version iff multiple strides per block allowed findMaxKernelOneBlock<<>>(maxValues, in.getData(), stride, in.getSize()); From bc5ae6dfca39548367442d71a9dc8d6e842340fc Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Mon, 18 May 2026 13:03:35 +0200 Subject: [PATCH 44/73] Softmax kernels for large case; need debugging --- .../activation_functions/cuda/activations.cu | 285 +++++++++++++++++- src/backend/utility/cuda/cuda_common.cuh | 13 + src/python/py_core/py_core.cpp | 8 + tests/backend/cuda/test_module_cuda.cu | 28 ++ 4 files changed, 324 insertions(+), 10 deletions(-) diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index 042d858..e2afb96 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -196,7 +196,7 @@ namespace { * * In this initial version we assume one kernel per stride, to make matters simple to understand. */ - __global__ void findMaxKernelOneBlock(ftype* const res, const ftype* const input, const tensorSize_t stride, const tensorSize_t size) { + __global__ void findMaxKernelOneBlock(ftype* const res, const ftype* const input, const tensorSize_t stride) { assert_debug(blockDim.x / stride == 0, "Kernel built for one stride per block, blockDim.x is < stride"); const int tid = threadIdx.x; @@ -243,8 +243,7 @@ namespace { * In this initial version we assume one kernel per stride, to make matters simple to understand. */ template - __global__ void stableSoftmaxKernelOneBlock(ftype* res, const ftype* const input, const ftype* const maxValues, - const tensorSize_t stride, const tensorSize_t size) { + __global__ void stableSoftmaxKernelOneBlock(ftype* const res, const ftype* const input, const ftype* const maxValues, const tensorSize_t stride) { assert_debug(blockDim.x / stride == 0, "Kernel built for one stride per block, blockDim.x is < stride"); const int tid = threadIdx.x; @@ -290,6 +289,214 @@ namespace { res[gid + blockDim.x] = expValOffset / start[0]; } } + + /** + * @brief Like findMaxKernelOneBlock, but finding partial maximum. This is the case when the stride is too + * large to fit in one block. + */ + __global__ void findMaxKernelLargePass1(ftype* const partialMaxValues, const ftype* const input, + const tensorSize_t stride, const int blocksPerStride) { + const int tid = threadIdx.x; + const int strideIdx = blockIdx.x / blocksPerStride; + const int blockWithinStride = blockIdx.x % blocksPerStride; + + // block 0 within stride handles elements [0, 2*blockDim.x), block 1 within stride handles elements [2*blockDim.x, 4*blockDim.x), ... + const int inputBase = strideIdx * stride + blockWithinStride * 2 * blockDim.x; + + extern __shared__ ftype smem[]; + const tensorSize_t localIdx0 = inputBase + tid; + const tensorSize_t localIdx1 = inputBase + tid + blockDim.x; + + // localIdx0 < (strideIdx + 1) * stride <- checks whether thread idx exceeds bounds of this stride; one stride per block at max + smem[tid] = (localIdx0 < (strideIdx + 1) * stride) ? input[localIdx0] : -INFINITY; + smem[tid + blockDim.x] = (localIdx1 < (strideIdx + 1) * stride) ? input[localIdx1] : -INFINITY; + __syncthreads(); + + // same reduction as findMaxKernelOneBlock from here + for(tensorSize_t offset = blockDim.x >> 1; offset > 32; offset >>= 1) { + if(tid < offset){ + smem[tid] = max(smem[tid], smem[tid + offset]); + } + __syncthreads(); + } + + volatile ftype* start = smem; + if(tid < 32) { + start[tid] = max(start[tid], start[tid + 32]); + start[tid] = max(start[tid], start[tid + 16]); + start[tid] = max(start[tid], start[tid + 8]); + start[tid] = max(start[tid], start[tid + 4]); + start[tid] = max(start[tid], start[tid + 2]); + start[tid] = max(start[tid], start[tid + 1]); + } + + if(tid == 0) { + partialMaxValues[blockIdx.x] = start[0]; + } + } + + /** + * @brief Self explanatory following findMaxKernelLargePass1. Assumption: All remaining max values do fit into + * one single block now -> we launch one block per stride this time. + */ + __global__ void findMaxKernelLargePass2(ftype* const maxValues, const ftype* const partialMaxValues, const tensorSize_t blocksPerStride) { + assert_debug(blockDim.x / blocksPerStride == 0, "Kernel built for one stride per block, blockDim.x is < stride"); + + const int tid = threadIdx.x; + const int gid = blockIdx.x * blocksPerStride + tid; + + extern __shared__ ftype smem[]; // can lead to bank conflicts iff std::is_same_v + + const tensorSize_t maxIdx = tid + blockDim.x; + const bool doPadding = maxIdx >= blocksPerStride; + if(doPadding) { + smem[tid] = partialMaxValues[gid]; + } + else { + smem[tid] = max(partialMaxValues[gid], partialMaxValues[gid + blockDim.x]); + } + __syncthreads(); + + for(tensorSize_t offset = blockDim.x >> 1; offset > 32; offset >>= 1) { + if(tid < offset) { + smem[tid] = max(smem[tid], smem[tid + offset]); + } + __syncthreads(); + } + + // TODO: warp shuffle for newer architectures + volatile ftype* const start = smem; + if(tid < 32) { + if(32 < blockDim.x * 2) start[tid] = max(start[tid], start[tid + 32]); + if(16 < blockDim.x * 2) start[tid] = max(start[tid], start[tid + 16]); + if(8 < blockDim.x * 2) start[tid] = max(start[tid], start[tid + 8]); + if(4 < blockDim.x * 2) start[tid] = max(start[tid], start[tid + 4]); + if(2 < blockDim.x * 2) start[tid] = max(start[tid], start[tid + 2]); + if(1 < blockDim.x * 2) start[tid] = max(start[tid], start[tid + 1]); + } + + if(tid == 0) { // one block per stride + maxValues[blockIdx.x] = start[0]; + } + } + + /** + * @brief Does the first part of stableSoftmaxKernelOneBlock, namely the sums. Because here again we have a partial sum + * and assume the stride did not fit into the block entirely, we do a partial sum only. Additionally, write the max-adjusted + * exp values back to res to prepare for the division. + */ + template + __global__ void stableSoftmaxLargePass1(ftype* const res, ftype* const partialSums, const ftype* const input, const ftype* const maxValues, + const tensorSize_t stride, const int blocksPerStride) { + const int tid = threadIdx.x; + const int strideIdx = blockIdx.x / blocksPerStride; + const int blockWithinStride = blockIdx.x % blocksPerStride; + + // same logic as in findMaxKernelLargePass1 + const int inputBase = strideIdx * stride + blockWithinStride * 2 * blockDim.x; + const ftype maxValue = maxValues[strideIdx]; + + extern __shared__ ftype smem[]; + const tensorSize_t localIdx0 = inputBase + tid; + const tensorSize_t localIdx1 = inputBase + tid + blockDim.x; + + // same logic as in findMaxKernelLargePass1 + ftype expVal0 = (localIdx0 < (strideIdx + 1) * stride) ? stableExp(input[localIdx0], maxValue) : 0.0f; + ftype expVal1 = (localIdx1 < (strideIdx + 1) * stride) ? stableExp(input[localIdx1], maxValue) : 0.0f; + + smem[tid] = expVal0; + smem[tid + blockDim.x] = expVal1; + __syncthreads(); + + // write exp values to output + if(localIdx0 < (strideIdx + 1) * stride) { + res[localIdx0] = expVal0; + } + if(localIdx1 < (strideIdx + 1) * stride) { + res[localIdx1] = expVal1; + } + + // reduce sum + for(tensorSize_t offset = blockDim.x; offset > 32; offset >>= 1) { + if(tid < offset) { + smem[tid] += smem[tid + offset]; + } + __syncthreads(); + } + + volatile ftype* start = smem; + if(tid < 32) { + start[tid] += start[tid + 32]; + start[tid] += start[tid + 16]; + start[tid] += start[tid + 8]; + start[tid] += start[tid + 4]; + start[tid] += start[tid + 2]; + start[tid] += start[tid + 1]; + } + //__syncthreads(); + + if(tid == 0) { + partialSums[blockIdx.x] = start[0]; + } + } + + /** + * @brief Self explanatory after stableSoftmaxLargePass1. Continues the sum reduce, does not need to write further to res, since + * pass 1 already did that for us. + */ + __global__ void stableSoftmaxLargePass2(ftype* const sums, const ftype* const partialSums, const tensorSize_t blocksPerStride) { + assert_debug(blockDim.x / blocksPerStride == 0, "Kernel built for one stride per block, blockDim.x is < stride"); + + const int tid = threadIdx.x; + const int gid = blockIdx.x * blocksPerStride + tid; + + extern __shared__ ftype smem[]; // can lead to bank conflicts iff std::is_same_v + + const tensorSize_t maxIdx = tid + blockDim.x; + const bool doPadding = maxIdx >= blocksPerStride; + if(doPadding) { + smem[tid] = partialSums[gid]; + } + else { + smem[tid] = partialSums[gid] + partialSums[gid + blockDim.x]; + } + __syncthreads(); + + for(tensorSize_t offset = blockDim.x >> 1; offset > 32; offset >>= 1) { + if(tid < offset) { + smem[tid] += smem[tid + offset]; + } + __syncthreads(); + } + + // TODO: warp shuffle for newer architectures + volatile ftype* const start = smem; + if(tid < 32) { + if(32 < blockDim.x * 2) start[tid] += start[tid + 32]; + if(16 < blockDim.x * 2) start[tid] += start[tid + 16]; + if(8 < blockDim.x * 2) start[tid] += start[tid + 8]; + if(4 < blockDim.x * 2) start[tid] += start[tid + 4]; + if(2 < blockDim.x * 2) start[tid] += start[tid + 2]; + if(1 < blockDim.x * 2) start[tid] += start[tid + 1]; + } + //__syncthreads(); + + if(tid == 0) { // one block per stride + sums[blockIdx.x] = start[0]; + } + } + + /** + * @brief Simple kernel doing the division of softmax in the case of a large stride. + */ + __global__ void divideKernel(ftype* const res, const ftype* const sums, const tensorSize_t stride, const tensorSize_t size) { + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) { + return; + } + + res[gid] /= sums[gid / stride]; + } } namespace cuda_impl { @@ -315,17 +522,28 @@ namespace cuda_impl { sigmoidKernel<<>>(res.getData(), in.getData(), in.getSize()); cudaErrchk(cudaDeviceSynchronize()); -} + } + /** + * @brief Does the softmax computation. Warning: Current implementation can only handle a stride of + * at max 512 * 512 = 262144 floating point numbers. If number exceeds this an exception is throws. + * + * For simplicity and for reasons of CUDA efficiency this function is split into 3 segments. + * 1. stride <= 64 -> we use warp level. + * 2. stride > 64 && stride < 512 -> we can fit one stride into one block. + * 3. stride > 512 -> we have to use two-stage kernel cascading for parallel reduction. + */ void softmax(Tensor& res, const Tensor& in) { const tensorSize_t stride = static_cast(in.getDims().get(-1)); if(stride == 1) { __throw_invalid_argument("Attemped softmax of one element"); } + const int nStrides = in.getSize() / stride; + // TODO: use some static struct here to prevent this guy from keeping on re-allocating memory ftype* maxValues; - const tensorSize_t nMaxValues = in.getSize() / stride; + const tensorSize_t nMaxValues = nStrides; cudaErrchk(cudaMalloc(&maxValues, nMaxValues * sizeof(ftype))); static const auto warpSizeT2 = 2 * DeviceProperties::getWarpSize(); // TODO: can this be a problem in a multi-GPU setting? @@ -385,7 +603,6 @@ namespace cuda_impl { constexpr int maxThreadsPerBlock = 256; assert_debug(stride <= 2 * maxThreadsPerBlock, "If you adapt maxThreadsPerBlock you also have to adapt the limit in the if-clause above"); - const int nStrides = in.getSize() / stride; constexpr int stridesPerBlock = 1; // if multiple strides per block allowed (adapt kernels!): max(maxThreadsPerBlock / stride, 1); // threads per block needs to be power of 2 for reduction to resolve cleanly @@ -395,17 +612,65 @@ namespace cuda_impl { const int blocks = (nStrides + stridesPerBlock - 1) / stridesPerBlock; // gerneralized version iff multiple strides per block allowed - findMaxKernelOneBlock<<>>(maxValues, in.getData(), stride, in.getSize()); + findMaxKernelOneBlock<<>>(maxValues, in.getData(), stride); cudaErrchk(cudaDeviceSynchronize()); stableSoftmaxKernelOneBlock<<>>(res.getData(), in.getData(), maxValues, - stride, in.getSize()); + stride); cudaErrchk(cudaDeviceSynchronize()); } else { - __throw_runtime_error("Softmax kernels not yet implemented at inter-block level"); - } + // stride does not fit into one block. We employ a 2 pass system, where pass one does a partial + // reduction, and pass two does a reduction over the partial reductions. + + // each block handles up to 512 elements (2 * 256 threads) + constexpr int maxThreadsPerBlock = 256; + constexpr int elemsPerBlock = 2 * maxThreadsPerBlock; // constant folding + const int blocksPerStride = (stride + elemsPerBlock - 1) / elemsPerBlock; + assert_debug(blocksPerStride <= 512, "Stride too large for two-pass reduction"); + + const int totalBlocks = nStrides * blocksPerStride; + + // intermediate max values: one per block per stride + ftype* partialMaxValues; + const tensorSize_t nPartialMax = totalBlocks; + cudaErrchk(cudaMalloc(&maxValues, nStrides * sizeof(ftype))); + cudaErrchk(cudaMalloc(&partialMaxValues, nPartialMax * sizeof(ftype))); + + // pass 1: reduce each chunk of 512 elements to one partial max + // launch blocksPerStride blocks per stride + findMaxKernelLargePass1<<>>( + partialMaxValues, in.getData(), stride, blocksPerStride); + cudaErrchk(cudaDeviceSynchronize()); + + // pass 2: reduce partial maxes to one max per stride + int threadsPass2 = 1; + while(threadsPass2 < blocksPerStride) threadsPass2 <<= 1; // threadsPass2 needs to be power of 2 + threadsPass2 /= 2; + + findMaxKernelLargePass2<<>>(maxValues, partialMaxValues, blocksPerStride); + cudaErrchk(cudaDeviceSynchronize()); + // softmax: same two-pass structure for the sum + ftype* partialSums; + cudaErrchk(cudaMalloc(&partialSums, nPartialMax * sizeof(ftype))); + + stableSoftmaxLargePass1<<>>( + res.getData(), partialSums, in.getData(), maxValues, stride, blocksPerStride); + cudaErrchk(cudaDeviceSynchronize()); + + // pass 2: reduce partial sums + stableSoftmaxLargePass2<<>>(partialSums, partialSums, blocksPerStride); + cudaErrchk(cudaDeviceSynchronize()); + + // final division pass: divide each exp value by its stride's sum + const int nBlocksDivision = (in.getSize() + maxThreadsPerBlock - 1) / maxThreadsPerBlock; + divideKernel<<>>(res.getData(), partialSums, stride, in.getSize()); + cudaErrchk(cudaDeviceSynchronize()); + + cudaErrchk(cudaFree(partialMaxValues)); + cudaErrchk(cudaFree(partialSums)); + } cudaErrchk(cudaFree(maxValues)); } } \ No newline at end of file diff --git a/src/backend/utility/cuda/cuda_common.cuh b/src/backend/utility/cuda/cuda_common.cuh index d1e8311..40f67d1 100644 --- a/src/backend/utility/cuda/cuda_common.cuh +++ b/src/backend/utility/cuda/cuda_common.cuh @@ -17,6 +17,19 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "cuda_runtime.h" +#include + +template +struct FtypeWarning { + static constexpr void check() {} +}; + +template<> +struct FtypeWarning { + [[deprecated("ftype=double has serious CUDA performance implications")]] + static constexpr void check() {} +}; + namespace utility { void gpuAssert(cudaError_t code, const char *file, int line, bool abort=true); } diff --git a/src/python/py_core/py_core.cpp b/src/python/py_core/py_core.cpp index 545ccc4..d5f44ca 100644 --- a/src/python/py_core/py_core.cpp +++ b/src/python/py_core/py_core.cpp @@ -19,6 +19,10 @@ #include "data_modeling/tensor_functions.h" #include "computational_graph/tensor_ops/graph_creation.h" +#ifdef __CUDA +#include "utility/cuda/cuda_common.cuh" +#endif + #include #include #include @@ -28,6 +32,10 @@ BOOST_PYTHON_MODULE(_core) { +#ifdef __CUDA + FtypeWarning::check(); +#endif + using namespace boost::python; // some macros to make code below easier to read diff --git a/tests/backend/cuda/test_module_cuda.cu b/tests/backend/cuda/test_module_cuda.cu index 7fc49e4..2af85ab 100644 --- a/tests/backend/cuda/test_module_cuda.cu +++ b/tests/backend/cuda/test_module_cuda.cu @@ -213,6 +213,34 @@ TEST(CudaActivationTest, SoftmaxMediumLargeInput) { } } +TEST(CudaActivationTest, SoftmaxLargeInput) { + constexpr tensorDim_t testDim = 1500; + assert(testDim > 256); // see the kernel call + + auto t = TensorFunctions::Gaussian({2, 2, testDim}, 2.0f, Device::CUDA); + auto tCopy = t.createDeepCopy(); + tCopy.setDevice(Device::CPU); + + module::Softmax sm; + auto resGpu = sm(t); + auto resCpu = sm(tCopy); + + cout << resCpu << endl; + cout << resGpu << endl; + + resGpu.setDevice(Device::CUDA); + for(int i = 0; i < resGpu.getDims().get(0); i++) { + for(int j = 0; j < resGpu.getDims().get(1); j++) { + for(int k = 0; k < resGpu.getDims().get(2); k++) { + ASSERT_NEAR(resCpu.get(i, j, k), resGpu.get(i, j, k), 1e-4) + << "Mismatch at (" << i << ", " << j << ", " << k << ")" + << " cpu=" << resCpu.get(i, j, k) + << " gpu=" << resGpu.get(i, j, k); + } + } + } +} + /* TEST(CudaAutogradTest, SoftmaxBackward) { auto t = TensorFunctions::makeSharedTensor({1, 3}, {1.0, 2.0, 3.0}, Device::CUDA, true); From af8b5dfb10caee24cc91f1da7104884050640035 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Mon, 18 May 2026 17:37:38 +0200 Subject: [PATCH 45/73] Fixed large softmax kernel --- .../activation_functions/cuda/activations.cu | 48 +++++++++---------- tests/backend/cuda/test_module_cuda.cu | 3 -- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index e2afb96..ce330b0 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -282,7 +282,7 @@ namespace { start[tid] += start[tid + 2]; start[tid] += start[tid + 1]; } - __syncthreads(); + __syncthreads(); // needed because threads > 32 will also use start[0] res[gid] = expVal / start[0]; if(!doPadding) { @@ -311,9 +311,9 @@ namespace { smem[tid] = (localIdx0 < (strideIdx + 1) * stride) ? input[localIdx0] : -INFINITY; smem[tid + blockDim.x] = (localIdx1 < (strideIdx + 1) * stride) ? input[localIdx1] : -INFINITY; __syncthreads(); - + // same reduction as findMaxKernelOneBlock from here - for(tensorSize_t offset = blockDim.x >> 1; offset > 32; offset >>= 1) { + for(tensorSize_t offset = blockDim.x; offset > 32; offset >>= 1) { if(tid < offset){ smem[tid] = max(smem[tid], smem[tid + offset]); } @@ -367,12 +367,12 @@ namespace { // TODO: warp shuffle for newer architectures volatile ftype* const start = smem; if(tid < 32) { - if(32 < blockDim.x * 2) start[tid] = max(start[tid], start[tid + 32]); - if(16 < blockDim.x * 2) start[tid] = max(start[tid], start[tid + 16]); - if(8 < blockDim.x * 2) start[tid] = max(start[tid], start[tid + 8]); - if(4 < blockDim.x * 2) start[tid] = max(start[tid], start[tid + 4]); - if(2 < blockDim.x * 2) start[tid] = max(start[tid], start[tid + 2]); - if(1 < blockDim.x * 2) start[tid] = max(start[tid], start[tid + 1]); + if(tid + 32 < blockDim.x) start[tid] = max(start[tid], start[tid + 32]); + if(tid + 16 < blockDim.x) start[tid] = max(start[tid], start[tid + 16]); + if(tid + 8 < blockDim.x) start[tid] = max(start[tid], start[tid + 8]); + if(tid + 4 < blockDim.x) start[tid] = max(start[tid], start[tid + 4]); + if(tid + 2 < blockDim.x) start[tid] = max(start[tid], start[tid + 2]); + if(tid + 1 < blockDim.x) start[tid] = max(start[tid], start[tid + 1]); } if(tid == 0) { // one block per stride @@ -408,7 +408,7 @@ namespace { smem[tid + blockDim.x] = expVal1; __syncthreads(); - // write exp values to output + // write values to output -> will be nominator in division if(localIdx0 < (strideIdx + 1) * stride) { res[localIdx0] = expVal0; } @@ -433,7 +433,6 @@ namespace { start[tid] += start[tid + 2]; start[tid] += start[tid + 1]; } - //__syncthreads(); if(tid == 0) { partialSums[blockIdx.x] = start[0]; @@ -472,14 +471,13 @@ namespace { // TODO: warp shuffle for newer architectures volatile ftype* const start = smem; if(tid < 32) { - if(32 < blockDim.x * 2) start[tid] += start[tid + 32]; - if(16 < blockDim.x * 2) start[tid] += start[tid + 16]; - if(8 < blockDim.x * 2) start[tid] += start[tid + 8]; - if(4 < blockDim.x * 2) start[tid] += start[tid + 4]; - if(2 < blockDim.x * 2) start[tid] += start[tid + 2]; - if(1 < blockDim.x * 2) start[tid] += start[tid + 1]; + if(tid + 32 < blockDim.x) start[tid] += start[tid + 32]; + if(tid + 16 < blockDim.x) start[tid] += start[tid + 16]; + if(tid + 8 < blockDim.x) start[tid] += start[tid + 8]; + if(tid + 4 < blockDim.x) start[tid] += start[tid + 4]; + if(tid + 2 < blockDim.x) start[tid] += start[tid + 2]; + if(tid + 1 < blockDim.x) start[tid] += start[tid + 1]; } - //__syncthreads(); if(tid == 0) { // one block per stride sums[blockIdx.x] = start[0]; @@ -606,9 +604,9 @@ namespace cuda_impl { constexpr int stridesPerBlock = 1; // if multiple strides per block allowed (adapt kernels!): max(maxThreadsPerBlock / stride, 1); // threads per block needs to be power of 2 for reduction to resolve cleanly - int paddedStride = 1; - while(paddedStride < stride) paddedStride <<= 1; - int threadsPerBlock = paddedStride / 2; + int threadsPerBlock = 1; + while(threadsPerBlock < stride) threadsPerBlock <<= 1; + threadsPerBlock /= 2; const int blocks = (nStrides + stridesPerBlock - 1) / stridesPerBlock; // gerneralized version iff multiple strides per block allowed @@ -646,12 +644,12 @@ namespace cuda_impl { // pass 2: reduce partial maxes to one max per stride int threadsPass2 = 1; while(threadsPass2 < blocksPerStride) threadsPass2 <<= 1; // threadsPass2 needs to be power of 2 - threadsPass2 /= 2; + threadsPass2 = max(1, threadsPass2 / 2); - findMaxKernelLargePass2<<>>(maxValues, partialMaxValues, blocksPerStride); + findMaxKernelLargePass2<<>>(maxValues, partialMaxValues, blocksPerStride); cudaErrchk(cudaDeviceSynchronize()); - // softmax: same two-pass structure for the sum + // same two-pass structure for the sum ftype* partialSums; cudaErrchk(cudaMalloc(&partialSums, nPartialMax * sizeof(ftype))); @@ -660,7 +658,7 @@ namespace cuda_impl { cudaErrchk(cudaDeviceSynchronize()); // pass 2: reduce partial sums - stableSoftmaxLargePass2<<>>(partialSums, partialSums, blocksPerStride); + stableSoftmaxLargePass2<<>>(partialSums, partialSums, blocksPerStride); cudaErrchk(cudaDeviceSynchronize()); // final division pass: divide each exp value by its stride's sum diff --git a/tests/backend/cuda/test_module_cuda.cu b/tests/backend/cuda/test_module_cuda.cu index 2af85ab..0be8e48 100644 --- a/tests/backend/cuda/test_module_cuda.cu +++ b/tests/backend/cuda/test_module_cuda.cu @@ -225,9 +225,6 @@ TEST(CudaActivationTest, SoftmaxLargeInput) { auto resGpu = sm(t); auto resCpu = sm(tCopy); - cout << resCpu << endl; - cout << resGpu << endl; - resGpu.setDevice(Device::CUDA); for(int i = 0; i < resGpu.getDims().get(0); i++) { for(int j = 0; j < resGpu.getDims().get(1); j++) { From e262f0921f230900fe1c6830f144bed1affd68a1 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Tue, 19 May 2026 10:31:51 +0200 Subject: [PATCH 46/73] Fixed softmax backward indexing to process multidimensional softmax --- .../activation_functions/softmax_node.cpp | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/backend/computational_graph/activation_functions/softmax_node.cpp b/src/backend/computational_graph/activation_functions/softmax_node.cpp index e385bb3..57e2dbe 100644 --- a/src/backend/computational_graph/activation_functions/softmax_node.cpp +++ b/src/backend/computational_graph/activation_functions/softmax_node.cpp @@ -24,6 +24,9 @@ using namespace std; using namespace cgraph; +/** + * @brief Shape assumptions of softmax forward: [bsize, dim1, dim2, ..., stride] + */ vector< shared_ptr > SoftmaxNode::backward(const Tensor& upstreamGrad) { assert(!upstreamGrad.getRequiresGrad()); @@ -32,21 +35,21 @@ vector< shared_ptr > SoftmaxNode::backward(const Tensor& upstreamGrad) { switch(upstreamGrad.getDevice()) { case Device::CPU: { - const auto bSize = yPred->getDims()[0]; - assert(bSize>0); - - for(tensorDim_t b = 0; b < bSize; b++){ - for(tensorDim_t i = 0; i < yPred->getDims()[1]; i++){ + const tensorSize_t stride = yPred->getDims()[-1]; + tensorSize_t offset = 0; + while(offset < yPred->getSize()) { + for(tensorSize_t i = 0; i < stride; i++) { ftype grad = 0; - const ftype yi = softmax->get(b, i); + const ftype yi = softmax->get(offset + i); - for(tensorDim_t j = 0; j < yPred->getDims()[1]; j++){ - ftype yj = softmax->get(b, j); - ftype jacobian = (i == j) ? yi * (1 - yj) : -yi * yj; - grad += upstreamGrad.get(b, j) * jacobian; + for(tensorSize_t j = 0; j < stride; j++) { + const ftype yj = softmax->get(offset + j); + const ftype jacobian = (i == j) ? yi * (1 - yj) : -yi * yj; + grad += upstreamGrad.get(offset + j) * jacobian; } - res->set(grad, b, i); + res->set(grad, offset + i); } + offset += stride; } break; } From 083dc34457898714a170bf8ae5d9321fcc06ee95 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Tue, 19 May 2026 10:32:31 +0200 Subject: [PATCH 47/73] Softmax backward kernel for small stride in CUDA; fix some indentation --- .../cuda/activation_nodes.cu | 98 +++++++++++++++++-- .../activation_functions/cuda/activations.cu | 7 +- 2 files changed, 95 insertions(+), 10 deletions(-) diff --git a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu index f6939f2..1c4d15a 100644 --- a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu +++ b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu @@ -19,6 +19,9 @@ static_assert(false, "File should not be compiled without CUDA enabled"); using namespace std; namespace { + /** + * @brief Relu backward kernel. + */ __global__ void reluBackwardKernel(ftype* const res, const ftype* const upstreamGrad, const ftype* const parent, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; @@ -26,6 +29,9 @@ namespace { res[gid] = parent[gid] > 0 ? upstreamGrad[gid] : 0; } + /** + * @brief Leaky relu backward kernel. + */ __global__ void leakyReluBackwardKernel(ftype* const res, const ftype* const upstreamGrad, const ftype* const parent, const ftype eps, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; @@ -33,6 +39,9 @@ namespace { res[gid] = parent[gid] > 0 ? upstreamGrad[gid] : eps * upstreamGrad[gid]; } + /** + * @brief Sigmoid backward kernel, optimized by using the forward sigmoid. + */ __global__ void sigmoidBackwardKernel(ftype* const res, const ftype* const upstreamGrad, const ftype* const sigmoid, const tensorSize_t size) { int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; @@ -41,7 +50,61 @@ namespace { res[gid] = si * (1 - si) * upstreamGrad[gid]; } - // TODO: softmaxBackward kernel + __global__ void softmaxBackwardKernelOneWarp(ftype* const res, const ftype* const softmax, const ftype* const upstreamGrad, + const tensorSize_t stride, tensorSize_t size) { + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) + return; + + const int tid = threadIdx.x; + } + + /** + * @brief Softmax backward kernel. This kernel is different than others since it is warp aligned. The inner loop avoids shared memory bank + * conflicts by broadcasting. + * + * stridesWidthPerBlock is an awkward name. It is the product of number of strides per block (times) stride. We pre-compute it on host. + */ + __global__ void softmaxBackwardKernelOneBlock(ftype* const res, const ftype* const softmax, const ftype* const upstreamGrad, + const tensorSize_t stride, const int stridesWidthPerBlock, const int threadsPerStride, tensorSize_t size) { + const int tid = threadIdx.x; + + const int strideNumber = (tid / stride); + if(tid - (strideNumber * threadsPerStride) > stride) { + // this warp is padded, i.e. it only exists to align warps with strides + return; + } + + const int gid = blockIdx.x * stridesWidthPerBlock + tid; + const ftype yi = softmax[gid]; + + const int strideOffset = strideNumber * stride; + const int withinStrideOffset = tid % threadsPerStride; + const int smemOffset = strideOffset + withinStrideOffset; + + extern __shared__ ftype smem[]; + smem[smemOffset] = yi; + smem[smemOffset + stride] = upstreamGrad[gid]; + __syncthreads(); + + + ftype grad = 0; + for(int j = 0; j < stride; j++) { + // warp alignment -> smem-reads are broadcasted per warp -> no bank conflicts + ftype yj = smem[strideOffset + j]; + ftype gj = smem[strideOffset + j + stride]; + + auto jacobian = (withinStrideOffset == j) ? yi * (1 - yj) : -yi * yj; + grad += gj * jacobian; + } + + res[gid] = grad; + } + + __global__ void softmaxBackwardKernelLargePass1(ftype* const res, const ftype* const softmax, const ftype* const upstreamGrad, + const tensorSize_t stride, tensorSize_t size) { + // TODO: code here + } } namespace cuda_impl { @@ -69,12 +132,33 @@ namespace cuda_impl { cudaErrchk(cudaDeviceSynchronize()); } + /** + * @brief The backward of the softmax. Due to optimization this function distinguishes three cases of stride size, where stride + * is the size of the dimension the softmax operation is applied to. The two cases are a stride either fitting into one block or not. + */ void softmaxBackward(Tensor& res, const Tensor& upstreamGrad, const Tensor& softmax) { - constexpr int threadsPerBlock = 256; - const int blocks = (upstreamGrad.getSize() + threadsPerBlock - 1) / threadsPerBlock; - - // TODO: launch kernel - - cudaErrchk(cudaDeviceSynchronize()); + assert(upstreamGrad.getSize() == softmax.getSize()); + + constexpr int maxThreadsPerBlock = 256; + const int stride = softmax.getDims()[-1]; + + if(stride < maxThreadsPerBlock) { + const int threadsPerStride = max(1, ((stride + 31) / 32)) * 32; // == warps per stride * 32 + const int stridesPerBlock = maxThreadsPerBlock / threadsPerStride; + const int strideWidthPerBlock = stridesPerBlock * stride; // for smem idx computation + + int threadsPerBlock = 1; + while(threadsPerBlock < threadsPerStride * stridesPerBlock) threadsPerBlock <<= 1; + // threadsPerBlock now larger than threadsPerStride * stridesPerBlock + + const int blocks = (upstreamGrad.getSize() + threadsPerBlock - 1) / threadsPerBlock; + softmaxBackwardKernelOneBlock<<>>( + res.getData(), upstreamGrad.getData(), softmax.getData(), stride, strideWidthPerBlock, threadsPerStride, softmax.getSize()); + cudaErrchk(cudaDeviceSynchronize()); + } + else { + __throw_runtime_error("Not implemented yet"); + // TODO: do multi pass kernel + } } } diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index ce330b0..e36f9c3 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -500,7 +500,7 @@ namespace { namespace cuda_impl { void relu(Tensor& res, const Tensor& in) { constexpr int threadsPerBlock = 256; - const int blocks = (in.getSize()+threadsPerBlock-1) / threadsPerBlock; + const int blocks = (in.getSize() + threadsPerBlock - 1) / threadsPerBlock; reluKernel<<>>(res.getData(), in.getData(), in.getSize()); cudaErrchk(cudaDeviceSynchronize()); @@ -508,7 +508,7 @@ namespace cuda_impl { void leakyRelu(Tensor& res, const Tensor& in, ftype eps) { constexpr int threadsPerBlock = 256; - const int blocks = (in.getSize()+threadsPerBlock-1) / threadsPerBlock; + const int blocks = (in.getSize() + threadsPerBlock - 1) / threadsPerBlock; leakyReluKernel<<>>(res.getData(), in.getData(), eps, in.getSize()); cudaErrchk(cudaDeviceSynchronize()); @@ -516,7 +516,7 @@ namespace cuda_impl { void sigmoid(Tensor& res, const Tensor& in) { constexpr int threadsPerBlock = 256; - const int blocks = (in.getSize()+threadsPerBlock-1) / threadsPerBlock; + const int blocks = (in.getSize() + threadsPerBlock - 1) / threadsPerBlock; sigmoidKernel<<>>(res.getData(), in.getData(), in.getSize()); cudaErrchk(cudaDeviceSynchronize()); @@ -669,6 +669,7 @@ namespace cuda_impl { cudaErrchk(cudaFree(partialMaxValues)); cudaErrchk(cudaFree(partialSums)); } + cudaErrchk(cudaFree(maxValues)); } } \ No newline at end of file From b0c6020ef4731b7a6ed34d65ed7427de2a5e9307 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Tue, 19 May 2026 13:39:32 +0200 Subject: [PATCH 48/73] Minor fix in kernel, add unit test, update readme. Kernel still needs fixing --- readme.md | 2 +- .../cuda/activation_nodes.cu | 25 ++++++++++--------- tests/backend/cuda/test_module_cuda.cu | 5 +--- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/readme.md b/readme.md index 5575cbd..3c8d55a 100644 --- a/readme.md +++ b/readme.md @@ -40,7 +40,7 @@ Roadmap: - [x] Python Binding Unit Tests - [x] Optimizers and training framework - [x] MNIST example -- [ ] CUDA mode for operations +- [x] CUDA mode for operations - [ ] Additional layer types (Conv2D, Dropout, etc.) - [ ] AlexNet reference implementation - [ ] Docker deployment example diff --git a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu index 1c4d15a..9fab7b7 100644 --- a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu +++ b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu @@ -69,24 +69,25 @@ namespace { const tensorSize_t stride, const int stridesWidthPerBlock, const int threadsPerStride, tensorSize_t size) { const int tid = threadIdx.x; - const int strideNumber = (tid / stride); - if(tid - (strideNumber * threadsPerStride) > stride) { - // this warp is padded, i.e. it only exists to align warps with strides - return; - } + const int withinStrideOffset = tid % threadsPerStride; + const bool isPadded = withinStrideOffset >= stride; // padded threads only exists to align warps with strides - const int gid = blockIdx.x * stridesWidthPerBlock + tid; - const ftype yi = softmax[gid]; + const int strideOffset = (tid / stride) * stride; + const int gid = blockIdx.x * stridesWidthPerBlock + strideOffset + withinStrideOffset; - const int strideOffset = strideNumber * stride; - const int withinStrideOffset = tid % threadsPerStride; + const ftype yi = softmax[gid]; const int smemOffset = strideOffset + withinStrideOffset; extern __shared__ ftype smem[]; - smem[smemOffset] = yi; - smem[smemOffset + stride] = upstreamGrad[gid]; + if(!isPadded) { + smem[smemOffset] = yi; + smem[smemOffset + stride] = upstreamGrad[gid]; + } __syncthreads(); - + + if(isPadded) { + return; + } ftype grad = 0; for(int j = 0; j < stride; j++) { diff --git a/tests/backend/cuda/test_module_cuda.cu b/tests/backend/cuda/test_module_cuda.cu index 0be8e48..0f4860d 100644 --- a/tests/backend/cuda/test_module_cuda.cu +++ b/tests/backend/cuda/test_module_cuda.cu @@ -197,9 +197,6 @@ TEST(CudaActivationTest, SoftmaxMediumLargeInput) { auto resGpu = sm(t); auto resCpu = sm(tCopy); - cout << resCpu << endl; - cout << resGpu << endl; - resGpu.setDevice(Device::CUDA); for(int i = 0; i < resGpu.getDims().get(0); i++) { for(int j = 0; j < resGpu.getDims().get(1); j++) { @@ -238,7 +235,6 @@ TEST(CudaActivationTest, SoftmaxLargeInput) { } } -/* TEST(CudaAutogradTest, SoftmaxBackward) { auto t = TensorFunctions::makeSharedTensor({1, 3}, {1.0, 2.0, 3.0}, Device::CUDA, true); @@ -255,6 +251,7 @@ TEST(CudaAutogradTest, SoftmaxBackward) { ASSERT_NEAR((*grads)[2], -0.0599, 1e-5); } +/* TEST(CudaLayerTest, TestFfLayer) { auto t1 = TensorFunctions::Ones({3, 2}, Device::CUDA); auto layer = module::FfLayer(2, 1, Device::CUDA); From 6bb7030a4ecb7999127ff067181ec47f80b38013 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Tue, 19 May 2026 14:03:06 +0200 Subject: [PATCH 49/73] Update unit tests with sharper delta --- tests/backend/test_module.cpp | 46 +++++++++++++++++------------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/tests/backend/test_module.cpp b/tests/backend/test_module.cpp index 6c07237..5f82c91 100644 --- a/tests/backend/test_module.cpp +++ b/tests/backend/test_module.cpp @@ -25,8 +25,6 @@ using namespace std; -constexpr ftype delta = 1e-3; - TEST(ActivationTest, ReluForward) { auto t1 = TensorFunctions::Ones({3, 2}); auto f = module::ReLu(); @@ -113,9 +111,9 @@ TEST(ActivationTest, SigmoidForward) { module::Sigmoid sig; auto res = sig(t); - ASSERT_NEAR(res[0], 0.5, delta); - ASSERT_NEAR(res[1], 0.7311, delta); - ASSERT_NEAR(res[2], 0.2689, delta); + ASSERT_NEAR(res[0], 0.5, 1e-4); + ASSERT_NEAR(res[1], 0.7311, 1e-4); + ASSERT_NEAR(res[2], 0.2689, 1e-4); } TEST(ActivationTest, SigmoidLargePositive) { @@ -125,7 +123,7 @@ TEST(ActivationTest, SigmoidLargePositive) { module::Sigmoid sig; auto res = sig(t); - ASSERT_NEAR(res[0], 1.0, delta); + ASSERT_NEAR(res[0], 1.0, 1e-5); EXPECT_FALSE(std::isnan(res[0])); EXPECT_FALSE(std::isinf(res[0])); } @@ -137,7 +135,7 @@ TEST(ActivationTest, SigmoidLargeNegative) { module::Sigmoid sig; auto res = sig(t); - ASSERT_NEAR(res[0], 0.0, delta); + ASSERT_NEAR(res[0], 0.0, 1e-5); EXPECT_FALSE(std::isnan(res[0])); EXPECT_FALSE(std::isinf(res[0])); } @@ -154,8 +152,8 @@ TEST(AutogradTest, SigmoidBackward) { res->backward(); auto grads = t->getGrads(); - ASSERT_NEAR((*grads)[0], 0.25, delta); - ASSERT_NEAR((*grads)[1], 0.1966, delta); + ASSERT_NEAR((*grads)[0], 0.25, 1e-4); + ASSERT_NEAR((*grads)[1], 0.1966, 1e-4); } TEST(ActivationTest, SoftmaxForward) { @@ -168,9 +166,9 @@ TEST(ActivationTest, SoftmaxForward) { module::Softmax sm; auto res = sm(t); - ASSERT_NEAR(res[0], 0.0900, delta); - ASSERT_NEAR(res[1], 0.2447, delta); - ASSERT_NEAR(res[2], 0.6652, delta); + ASSERT_NEAR(res[0], 0.0900, 1e-4); + ASSERT_NEAR(res[1], 0.2447, 1e-4); + ASSERT_NEAR(res[2], 0.6652, 1e-4); } TEST(ActivationTest, SoftmaxSumsToOne) { @@ -184,8 +182,8 @@ TEST(ActivationTest, SoftmaxSumsToOne) { // each row must sum to 1 ftype row0sum = res[0] + res[1] + res[2] + res[3]; ftype row1sum = res[4] + res[5] + res[6] + res[7]; - ASSERT_NEAR(row0sum, 1.0, delta); - ASSERT_NEAR(row1sum, 1.0, delta); + ASSERT_NEAR(row0sum, 1.0, 1e-5); + ASSERT_NEAR(row1sum, 1.0, 1e-5); } TEST(ActivationTest, SoftmaxForwardNumericalStability) { @@ -201,7 +199,7 @@ TEST(ActivationTest, SoftmaxForwardNumericalStability) { } ftype rowsum = res[0] + res[1] + res[2]; - ASSERT_NEAR(rowsum, 1.0, delta); + ASSERT_NEAR(rowsum, 1.0, 1e-5); } TEST(AutogradTest, SoftmaxBackward) { @@ -227,9 +225,9 @@ TEST(AutogradTest, SoftmaxBackward) { resPtr->backward(); auto grads = t->getGrads(); - ASSERT_NEAR((*grads)[0], 0.0819, delta); - ASSERT_NEAR((*grads)[1], -0.0220, delta); - ASSERT_NEAR((*grads)[2], -0.0599, delta); + ASSERT_NEAR((*grads)[0], 0.0819, 1e-4); + ASSERT_NEAR((*grads)[1], -0.0220, 1e-4); + ASSERT_NEAR((*grads)[2], -0.0599, 1e-4); } TEST(LayerTest, TestFfLayer) { @@ -264,14 +262,14 @@ TEST(AutogradTest, FfLayerBackward) { ASSERT_NE(xGrads, nullptr); ASSERT_EQ(xGrads->getDims(), x->getDims()); for(tensorSize_t i = 0; i < xGrads->getSize(); i++) { - ASSERT_NEAR((*xGrads)[i], 2.0, delta); + ASSERT_NEAR((*xGrads)[i], 2.0, 1e-5); } auto wGrads = layer.getWeights()->getGrads(); ASSERT_NE(wGrads, nullptr); ASSERT_EQ(wGrads->getDims(), layer.getWeights()->getDims()); for(tensorSize_t i = 0; i < wGrads->getSize(); i++) { - ASSERT_NEAR((*wGrads)[i], 2.0, delta); + ASSERT_NEAR((*wGrads)[i], 2.0, 1e-5); } } @@ -294,17 +292,17 @@ TEST(AutogradTest, FfLayerBackwardWithBias) { auto xGrads = x->getGrads(); ASSERT_NE(xGrads, nullptr); for(tensorSize_t i = 0; i < xGrads->getSize(); i++) { - ASSERT_NEAR((*xGrads)[i], 2.0, delta); + ASSERT_NEAR((*xGrads)[i], 2.0, 1e-5); } auto wGrads = layer.getWeights()->getGrads(); ASSERT_NE(wGrads, nullptr); for(tensorSize_t i = 0; i < wGrads->getSize(); i++) { - ASSERT_NEAR((*wGrads)[i], 2.0, delta); + ASSERT_NEAR((*wGrads)[i], 2.0, 1e-5); } auto bGrads = layer.getBias()->getGrads(); ASSERT_NE(bGrads, nullptr); - ASSERT_NEAR((*bGrads)[0], 2.0, delta); - ASSERT_NEAR((*bGrads)[1], 2.0, delta); + ASSERT_NEAR((*bGrads)[0], 2.0, 1e-5); + ASSERT_NEAR((*bGrads)[1], 2.0, 1e-5); } \ No newline at end of file From b029b52fd96e4486e9499f30b9276af098c29914 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Wed, 20 May 2026 10:21:08 +0200 Subject: [PATCH 50/73] Fix error in small softmax backward kernel --- .../cuda/activation_nodes.cu | 19 +++++++++++-------- tests/backend/cuda/test_module_cuda.cu | 6 +++--- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu index 9fab7b7..21fc71b 100644 --- a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu +++ b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu @@ -65,7 +65,7 @@ namespace { * * stridesWidthPerBlock is an awkward name. It is the product of number of strides per block (times) stride. We pre-compute it on host. */ - __global__ void softmaxBackwardKernelOneBlock(ftype* const res, const ftype* const softmax, const ftype* const upstreamGrad, + __global__ void softmaxBackwardKernelOneBlock(ftype* const res, const ftype* const upstreamGrad, const ftype* const softmax, const tensorSize_t stride, const int stridesWidthPerBlock, const int threadsPerStride, tensorSize_t size) { const int tid = threadIdx.x; @@ -75,17 +75,18 @@ namespace { const int strideOffset = (tid / stride) * stride; const int gid = blockIdx.x * stridesWidthPerBlock + strideOffset + withinStrideOffset; - const ftype yi = softmax[gid]; + ftype yi = 0; const int smemOffset = strideOffset + withinStrideOffset; extern __shared__ ftype smem[]; if(!isPadded) { + yi = softmax[gid]; smem[smemOffset] = yi; smem[smemOffset + stride] = upstreamGrad[gid]; } __syncthreads(); - - if(isPadded) { + + if(isPadded || gid > size) { return; } @@ -145,15 +146,17 @@ namespace cuda_impl { if(stride < maxThreadsPerBlock) { const int threadsPerStride = max(1, ((stride + 31) / 32)) * 32; // == warps per stride * 32 - const int stridesPerBlock = maxThreadsPerBlock / threadsPerStride; - const int strideWidthPerBlock = stridesPerBlock * stride; // for smem idx computation + // min over maximum possible strides per block and actual number of strides + const int stridesPerBlock = min(maxThreadsPerBlock / threadsPerStride, softmax.getSize() / stride); + const int strideWidthPerBlock = stridesPerBlock * stride; // for smem idx computation + int threadsPerBlock = 1; while(threadsPerBlock < threadsPerStride * stridesPerBlock) threadsPerBlock <<= 1; // threadsPerBlock now larger than threadsPerStride * stridesPerBlock - const int blocks = (upstreamGrad.getSize() + threadsPerBlock - 1) / threadsPerBlock; - softmaxBackwardKernelOneBlock<<>>( + + softmaxBackwardKernelOneBlock<<>>( res.getData(), upstreamGrad.getData(), softmax.getData(), stride, strideWidthPerBlock, threadsPerStride, softmax.getSize()); cudaErrchk(cudaDeviceSynchronize()); } diff --git a/tests/backend/cuda/test_module_cuda.cu b/tests/backend/cuda/test_module_cuda.cu index 0f4860d..42b1df6 100644 --- a/tests/backend/cuda/test_module_cuda.cu +++ b/tests/backend/cuda/test_module_cuda.cu @@ -246,9 +246,9 @@ TEST(CudaAutogradTest, SoftmaxBackward) { resPtr->backward(); auto grads = t->getGrads(); - ASSERT_NEAR((*grads)[0], 0.0819, 1e-5); - ASSERT_NEAR((*grads)[1], -0.0220, 1e-5); - ASSERT_NEAR((*grads)[2], -0.0599, 1e-5); + ASSERT_NEAR((*grads)[0], 0.0819, 1e-4); + ASSERT_NEAR((*grads)[1], -0.0220, 1e-4); + ASSERT_NEAR((*grads)[2], -0.0599, 1e-4); } /* From 2716fc600439350e88829f3acde4b2b06a60ac35 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Wed, 20 May 2026 12:08:24 +0200 Subject: [PATCH 51/73] Kernel for large backward softmax --- .../cuda/activation_nodes.cu | 63 ++++++++++++++++--- tests/backend/cuda/test_module_cuda.cu | 27 ++++++++ tests/backend/test_losses.cpp | 4 -- 3 files changed, 83 insertions(+), 11 deletions(-) diff --git a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu index 21fc71b..966f173 100644 --- a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu +++ b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu @@ -103,9 +103,52 @@ namespace { res[gid] = grad; } - __global__ void softmaxBackwardKernelLargePass1(ftype* const res, const ftype* const softmax, const ftype* const upstreamGrad, - const tensorSize_t stride, tensorSize_t size) { - // TODO: code here + /** + * @brief Large softmax pass. Because the stride now does not fit into one block anymore we do a grid-stride loop. + */ + __global__ void softmaxBackwardKernelLargePass(ftype* const res, const ftype* const upstreamGrad, const ftype* const softmax, const int blocksPerStride, const tensorSize_t stride) { + const int strideNumber = blockIdx.x / blocksPerStride; + const int strideOffset = strideNumber * stride; + const int i = (blockIdx.x % blocksPerStride) * blockDim.x + threadIdx.x; + // blockIdx.x % blocksPerStride = block number within this stride + + const int tid = threadIdx.x; + const int gid = strideOffset + i; + + extern __shared__ ftype smem[]; + + const bool isNotPadded = i < stride; + const ftype yi = isNotPadded ? softmax[gid] : 0; + + ftype grad = 0; + for(int offset = 0; offset < stride; offset += blockDim.x) { + // load int smem + { + const int j = offset + tid; + if(j < stride) { + smem[tid] = softmax[strideOffset + j]; + smem[tid + blockDim.x] = upstreamGrad[strideOffset + j]; + } + __syncthreads(); + } + + + for(int k = 0; k < blockDim.x; k++) { + const int j = offset + k; + if(j < stride) { + ftype yj = smem[k]; + ftype gj = smem[k + blockDim.x]; + + auto jacobian = (i == j) ? yi * (1 - yj) : -yi * yj; + grad += gj * jacobian; + } + } + __syncthreads(); + } + + if(isNotPadded) { + res[gid] = grad; + } } } @@ -158,11 +201,17 @@ namespace cuda_impl { softmaxBackwardKernelOneBlock<<>>( res.getData(), upstreamGrad.getData(), softmax.getData(), stride, strideWidthPerBlock, threadsPerStride, softmax.getSize()); - cudaErrchk(cudaDeviceSynchronize()); } else { - __throw_runtime_error("Not implemented yet"); - // TODO: do multi pass kernel - } + constexpr int maxThreadsPerBlock = 256; + + const int nStrides = softmax.getSize() / stride; + const int threadsPerBlock = maxThreadsPerBlock; // TODO: do that one better, this can result in gross imbalance; also for normal softmax + const int blocksPerStride = (stride + threadsPerBlock - 1) / threadsPerBlock; + + softmaxBackwardKernelLargePass<<>>( + res.getData(), upstreamGrad.getData(), softmax.getData(), blocksPerStride, stride); + } + cudaErrchk(cudaDeviceSynchronize()); } } diff --git a/tests/backend/cuda/test_module_cuda.cu b/tests/backend/cuda/test_module_cuda.cu index 42b1df6..39002cb 100644 --- a/tests/backend/cuda/test_module_cuda.cu +++ b/tests/backend/cuda/test_module_cuda.cu @@ -251,6 +251,33 @@ TEST(CudaAutogradTest, SoftmaxBackward) { ASSERT_NEAR((*grads)[2], -0.0599, 1e-4); } +TEST(CudaAutogradTest, SoftmaxBackwardLarge) { + constexpr tensorDim_t testDim = 1300; + auto tCpu = make_shared(TensorFunctions::Gaussian({2, 2, testDim}, 2.0f, true)); + auto tGpu = make_shared(tCpu->createDeepCopy()); + tGpu->setDevice(Device::CUDA); + + module::Softmax sm; + auto resPtrCpu = sm(tCpu); + auto resPtrGpu = sm(tGpu); + + auto upstreamGradCpu = make_shared(TensorFunctions::Ones(tCpu->getDims().toVector())); + auto upstreamGradGpu = make_shared(TensorFunctions::Ones(tCpu->getDims().toVector(), Device::CUDA)); + + resPtrCpu->setGrads(upstreamGradCpu); + resPtrGpu->setGrads(upstreamGradGpu); + + resPtrCpu->backward(); + resPtrGpu->backward(); + + auto gradsCpu = tCpu->getGrads(); + auto gradsGpu = tGpu->getGrads(); + + for(int i = 0; i < tCpu->getSize(); i++) { + EXPECT_NEAR((*gradsCpu)[i], (*gradsGpu)[i], 1e-4); + } +} + /* TEST(CudaLayerTest, TestFfLayer) { auto t1 = TensorFunctions::Ones({3, 2}, Device::CUDA); diff --git a/tests/backend/test_losses.cpp b/tests/backend/test_losses.cpp index 180d6ae..cfcc007 100644 --- a/tests/backend/test_losses.cpp +++ b/tests/backend/test_losses.cpp @@ -23,8 +23,6 @@ using namespace train; static constexpr ftype kTol = 1e-4f; -// ─── CrossEntropy ──────────────────────────────────────────────────────────── - TEST(LossTest, CrossEntropyFoward) { auto y = TensorFunctions::makeSharedTensor( {2, 3}, {1.0, 0.0, 0.0, @@ -112,8 +110,6 @@ TEST(LossTest, CrossEntropyBackward) { EXPECT_NEAR((*grads)[5], 0.0f, kTol); } -// ─── BCE ───────────────────────────────────────────────────────────────────── - TEST(LossTest, BceForward) { auto y = TensorFunctions::makeSharedTensor( {4, 1}, {0.0, 1.0, 1.0, 0.0}, false); From b8e8622aaa7189c2d6d9b258cee6959a4fa331b7 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Wed, 20 May 2026 12:30:48 +0200 Subject: [PATCH 52/73] CUDA version of FfLayer with unit tests --- src/backend/module/layers/cuda/layers.cu | 15 +---- src/backend/module/layers/cuda/layers.cuh | 3 +- src/backend/module/layers/ff_layer.cpp | 27 +++++--- src/backend/module/layers/ff_layer.h | 8 ++- src/backend/module/module_base.h | 2 + tests/backend/cuda/test_module_cuda.cu | 82 ++++++++++++++++++++++- 6 files changed, 109 insertions(+), 28 deletions(-) diff --git a/src/backend/module/layers/cuda/layers.cu b/src/backend/module/layers/cuda/layers.cu index ba73ef2..1ba6c7b 100644 --- a/src/backend/module/layers/cuda/layers.cu +++ b/src/backend/module/layers/cuda/layers.cu @@ -19,22 +19,11 @@ static_assert(false, "File should not be compiled without CUDA enabled"); using namespace std; namespace { - // TODO: forward kernel - - // TODO: forwardBias kernel + // TODO: matMulPlusBias kernel } namespace cuda_impl { - void forward(Tensor& res, const Tensor& input, const Tensor& weights) { - constexpr int threadsPerBlock = 256; - const int blocks = (res.getSize() + threadsPerBlock - 1) / threadsPerBlock; - - // TODO: launch kernel - - cudaErrchk(cudaDeviceSynchronize()); - } - - void forwardBias(Tensor& res, const Tensor& input, const Tensor& weights, const Tensor& bias) { + void matMulPlusBias(Tensor& res, const Tensor& input, const Tensor& weights, const Tensor& bias) { constexpr int threadsPerBlock = 256; const int blocks = (res.getSize() + threadsPerBlock - 1) / threadsPerBlock; diff --git a/src/backend/module/layers/cuda/layers.cuh b/src/backend/module/layers/cuda/layers.cuh index 15725a7..1f74d78 100644 --- a/src/backend/module/layers/cuda/layers.cuh +++ b/src/backend/module/layers/cuda/layers.cuh @@ -19,6 +19,5 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "data_modeling/tensor.h" namespace cuda_impl { - void forward(Tensor& res, const Tensor& input, const Tensor& weights); - void forwardBias(Tensor& res, const Tensor& input, const Tensor& weights, const Tensor& bias); + void matMulPlusBias(Tensor& res, const Tensor& input, const Tensor& weights, const Tensor& bias); } diff --git a/src/backend/module/layers/ff_layer.cpp b/src/backend/module/layers/ff_layer.cpp index 646d487..e254a42 100644 --- a/src/backend/module/layers/ff_layer.cpp +++ b/src/backend/module/layers/ff_layer.cpp @@ -40,7 +40,7 @@ FfLayer::FfLayer(tensorDim_t inSize, tensorDim_t outSize, bool useBias, bool req */ FfLayer::FfLayer(tensorDim_t inSize, tensorDim_t outSize, Device d, bool useBias, bool requiresGrad, shared_ptr init) - : useBias{useBias}, requiresGrad{requiresGrad} + : requiresGrad{requiresGrad} { if(!init){ init = make_shared(inSize, outSize); @@ -64,24 +64,30 @@ Tensor FfLayer::operator()(const Tensor& input) const { switch(input.getDevice()) { case Device::CPU: { auto res = input.matmul(*weights); - if(useBias) res = res + *bias; + if(bias) res = res + *bias; return res; } case Device::CUDA: #ifdef __CUDA { - auto resDims = input.getDims().toVector(); - resDims.back() = static_cast(weights->getDims().get(-1)); - Tensor res(resDims, input.getDevice(), false); - if(useBias) { - cuda_impl::forwardBias(res, input, *weights, *bias); + if(bias) { + // TODO: the following should be an optimized fusion kernel + /* auto resDims = input.getDims().toVector(); + resDims.back() = static_cast(weights->getDims().get(-1)); + Tensor res(resDims, input.getDevice(), false); + + cuda_impl::matMulPlusBias(res, input, *weights, *bias); */ + + auto res = input.matmul(*weights); + res = res + *bias; + return res; } else { - cuda_impl::forward(res, input, *weights); + return input.matmul(*weights); } - return res; } #else __throw_invalid_argument("Attempted to give CUDA tensor"); + return input.createShallowCopy(); // line should not be reached #endif } } @@ -90,8 +96,9 @@ Tensor FfLayer::operator()(const Tensor& input) const { * @brief Like overload, but creates computational graph. */ std::shared_ptr FfLayer::operator()(const std::shared_ptr& input) const { + // TODO: if you fuse kernel you'll also have to do it here. Perhaps give tensor a matmulplusbias method? auto res = cgraph::matmul(input, weights); - if(useBias){ + if(bias){ res = cgraph::add(res, bias); } diff --git a/src/backend/module/layers/ff_layer.h b/src/backend/module/layers/ff_layer.h index 2c99057..7ee5f50 100644 --- a/src/backend/module/layers/ff_layer.h +++ b/src/backend/module/layers/ff_layer.h @@ -19,7 +19,6 @@ namespace module { class FfLayer : public ModuleBase { bool requiresGrad = false; - bool useBias = false; std::shared_ptr weights = nullptr; std::shared_ptr bias = nullptr; @@ -50,6 +49,13 @@ namespace module { return {weights, bias}; } + void setDevice(Device d) noexcept override { + weights->setDevice(d); + if(bias) { + bias->setDevice(d); + } + } + void print(std::ostream& os) const noexcept override; }; } diff --git a/src/backend/module/module_base.h b/src/backend/module/module_base.h index 28247ab..6b05860 100644 --- a/src/backend/module/module_base.h +++ b/src/backend/module/module_base.h @@ -61,6 +61,8 @@ namespace module { #endif }; + virtual void setDevice(Device d) noexcept {} + friend std::ostream& operator<<(std::ostream& os, const ModuleBase& t) noexcept; }; } \ No newline at end of file diff --git a/tests/backend/cuda/test_module_cuda.cu b/tests/backend/cuda/test_module_cuda.cu index 39002cb..c956c51 100644 --- a/tests/backend/cuda/test_module_cuda.cu +++ b/tests/backend/cuda/test_module_cuda.cu @@ -278,7 +278,6 @@ TEST(CudaAutogradTest, SoftmaxBackwardLarge) { } } -/* TEST(CudaLayerTest, TestFfLayer) { auto t1 = TensorFunctions::Ones({3, 2}, Device::CUDA); auto layer = module::FfLayer(2, 1, Device::CUDA); @@ -287,4 +286,83 @@ TEST(CudaLayerTest, TestFfLayer) { ASSERT_EQ(res.getDims(), Dimension({3, 1})); } - */ \ No newline at end of file + +TEST(CudaLayerTest, TestFfLayerLarge) { + constexpr tensorDim_t largeDim = 200; + auto tCpu = TensorFunctions::Ones({30, largeDim}); + + auto tGpu = tCpu.createDeepCopy(); + tGpu.setDevice(Device::CUDA); + + auto layer = module::FfLayer(largeDim, 10); + auto resCpu = layer(tCpu); + + layer.setDevice(Device::CUDA); + auto resGpu = layer(tGpu); + + for(int i = 0; i < resCpu.getSize(); i++) { + EXPECT_NEAR(resCpu[i], resGpu[i], 1e-4); + } +} + + +TEST(CudaAutogradTest, FfLayerBackward) { + auto x = TensorFunctions::makeSharedTensor({2, 3}, { + 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0 + }, Device::CUDA, true); + + auto layer = module::FfLayer(3, 2, Device::CUDA, false, true); + auto w = layer.getWeights(); + for(tensorSize_t i = 0; i < w->getSize(); i++) w->set(1.0, i); + + auto res = layer(x); + auto loss = cgraph::sumTensor(res); + loss->backward(); + + auto xGrads = x->getGrads(); + ASSERT_NE(xGrads, nullptr); + ASSERT_EQ(xGrads->getDims(), x->getDims()); + for(tensorSize_t i = 0; i < xGrads->getSize(); i++) { + ASSERT_NEAR((*xGrads)[i], 2.0, 1e-5); + } + + auto wGrads = layer.getWeights()->getGrads(); + ASSERT_NE(wGrads, nullptr); + ASSERT_EQ(wGrads->getDims(), layer.getWeights()->getDims()); + for(tensorSize_t i = 0; i < wGrads->getSize(); i++) { + ASSERT_NEAR((*wGrads)[i], 2.0, 1e-5); + } +} + +TEST(CudaAutogradTest, FfLayerBackwardWithBias) { + auto x = TensorFunctions::makeSharedTensor({2, 3}, { + 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0 + }, Device::CUDA, true); + + auto layer = module::FfLayer(3, 2, Device::CUDA, true, true); + auto w = layer.getWeights(); + for(tensorSize_t i = 0; i < w->getSize(); i++) w->set(1.0, i); + + auto res = layer(x); + auto loss = cgraph::sumTensor(res); + loss->backward(); + + auto xGrads = x->getGrads(); + ASSERT_NE(xGrads, nullptr); + for(tensorSize_t i = 0; i < xGrads->getSize(); i++) { + ASSERT_NEAR((*xGrads)[i], 2.0, 1e-5); + } + + auto wGrads = layer.getWeights()->getGrads(); + ASSERT_NE(wGrads, nullptr); + for(tensorSize_t i = 0; i < wGrads->getSize(); i++) { + ASSERT_NEAR((*wGrads)[i], 2.0, 1e-5); + } + + auto bGrads = layer.getBias()->getGrads(); + ASSERT_NE(bGrads, nullptr); + ASSERT_NEAR((*bGrads)[0], 2.0, 1e-5); + ASSERT_NEAR((*bGrads)[1], 2.0, 1e-5); +} \ No newline at end of file From a644134398561236bff99cee0845995779ca018c Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Wed, 20 May 2026 14:52:29 +0200 Subject: [PATCH 53/73] Updated unit tests --- tests/CMakeLists.txt | 72 ++-- tests/backend/cuda/test_module_cuda.cu | 9 +- tests/backend/cuda/test_tensorops_cuda.cu | 66 ++-- tests/backend/main.cpp | 20 ++ tests/backend/net_factories.h | 64 ++++ tests/backend/test_computational_graph.cpp | 307 ++++++++++++------ tests/backend/test_losses.cpp | 44 +-- tests/backend/test_module.cpp | 53 ++- ...t_data_modeling.cpp => test_tensorops.cpp} | 164 +++++++--- tests/backend/test_train_loop.cpp | 278 ---------------- 10 files changed, 551 insertions(+), 526 deletions(-) create mode 100644 tests/backend/main.cpp create mode 100644 tests/backend/net_factories.h rename tests/backend/{test_data_modeling.cpp => test_tensorops.cpp} (55%) delete mode 100644 tests/backend/test_train_loop.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7d880d8..5d71801 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -8,57 +8,57 @@ FetchContent_Declare( FetchContent_MakeAvailable(googletest) add_executable(unit_tests_backend - backend/test_data_modeling.cpp - backend/test_computational_graph.cpp - backend/test_module.cpp - backend/test_losses.cpp - backend/test_train_loop.cpp - ) + backend/main.cpp + backend/test_tensorops.cpp + backend/test_module.cpp + backend/test_losses.cpp + backend/test_computational_graph.cpp + ) target_link_libraries(unit_tests_backend PRIVATE - gtest_main # pre-built main, avoids boilerplate if no custom initialization needed - BackendCore + gtest + BackendCore ) include(GoogleTest) gtest_discover_tests(unit_tests_backend) if(CMAKE_CUDA_COMPILER) - file(GLOB_RECURSE CUDA_TEST_SOURCES backend/cuda/*.cu) - add_executable(unit_tests_cuda ${CUDA_TEST_SOURCES}) + file(GLOB_RECURSE CUDA_TEST_SOURCES backend/cuda/*.cu) + add_executable(unit_tests_cuda ${CUDA_TEST_SOURCES}) - find_package(Threads REQUIRED) - target_link_libraries(unit_tests_cuda PRIVATE - gtest - Threads::Threads # platform agnostic - BackendCore - CUDA::cudart - ) + find_package(Threads REQUIRED) + target_link_libraries(unit_tests_cuda PRIVATE + gtest + Threads::Threads # platform agnostic + BackendCore + CUDA::cudart + ) - set_target_properties(unit_tests_cuda PROPERTIES - CUDA_SEPARABLE_COMPILATION ON - CMAKE_CUDA_ARCHITECTURES native - ) + set_target_properties(unit_tests_cuda PROPERTIES + CUDA_SEPARABLE_COMPILATION ON + CMAKE_CUDA_ARCHITECTURES native + ) - find_package(CUDAToolkit REQUIRED) - target_include_directories(unit_tests_cuda PRIVATE - ${CUDAToolkit_INCLUDE_DIRS} - ) + find_package(CUDAToolkit REQUIRED) + target_include_directories(unit_tests_cuda PRIVATE + ${CUDAToolkit_INCLUDE_DIRS} + ) - gtest_discover_tests(unit_tests_cuda) + gtest_discover_tests(unit_tests_cuda) endif() find_package(Python3 COMPONENTS Interpreter) if(Python3_FOUND) - # replace the placeholder variables and copy resulting file in .py file - add_test( - NAME python_tests - COMMAND ${Python3_EXECUTABLE} -m pytest - ${CMAKE_CURRENT_SOURCE_DIR}/python - ) + # replace the placeholder variables and copy resulting file in .py file + add_test( + NAME python_tests + COMMAND ${Python3_EXECUTABLE} -m pytest + ${CMAKE_CURRENT_SOURCE_DIR}/python + ) - # Set environment for Python to find the module - set_tests_properties(python_tests PROPERTIES - ENVIRONMENT "PYTHONPATH=${PYTHON_MODULE_DIR}:$ENV{PYTHONPATH}" - ) + # Set environment for Python to find the module + set_tests_properties(python_tests PROPERTIES + ENVIRONMENT "PYTHONPATH=${PYTHON_MODULE_DIR}:$ENV{PYTHONPATH}" + ) endif() \ No newline at end of file diff --git a/tests/backend/cuda/test_module_cuda.cu b/tests/backend/cuda/test_module_cuda.cu index c956c51..fb78d18 100644 --- a/tests/backend/cuda/test_module_cuda.cu +++ b/tests/backend/cuda/test_module_cuda.cu @@ -61,8 +61,8 @@ TEST(CudaAutogradTest, ReLUBackward) { loss->backward(); - ASSERT_DOUBLE_EQ(x->getGrads()->get(0), 0.0); - ASSERT_DOUBLE_EQ(x->getGrads()->get(1), 0.0); + ASSERT_NEAR(x->getGrads()->get(0), 0.0, 1e-5); + ASSERT_NEAR(x->getGrads()->get(1), 0.0, 1e-5); ASSERT_NEAR(x->getGrads()->get(2), 1.0, 1e-5); } @@ -106,8 +106,8 @@ TEST(CudaAutogradTest, LeakyReLUBackward) { loss->backward(); - ASSERT_DOUBLE_EQ(x->getGrads()->get(0), eps); - ASSERT_DOUBLE_EQ(x->getGrads()->get(1), eps); + ASSERT_NEAR(x->getGrads()->get(0), eps, 1e-5); + ASSERT_NEAR(x->getGrads()->get(1), eps, 1e-5); ASSERT_NEAR(x->getGrads()->get(2), 1.0, 1e-5); } @@ -305,7 +305,6 @@ TEST(CudaLayerTest, TestFfLayerLarge) { } } - TEST(CudaAutogradTest, FfLayerBackward) { auto x = TensorFunctions::makeSharedTensor({2, 3}, { 1.0, 1.0, 1.0, diff --git a/tests/backend/cuda/test_tensorops_cuda.cu b/tests/backend/cuda/test_tensorops_cuda.cu index f47ded9..b8e9f47 100644 --- a/tests/backend/cuda/test_tensorops_cuda.cu +++ b/tests/backend/cuda/test_tensorops_cuda.cu @@ -18,7 +18,20 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include "data_modeling/tensor.h" #include "data_modeling/tensor_functions.h" -TEST(CudaTensorOpsTest, ScalarAddWorks) { +TEST(CudaTensorOpsTest, TestCtor) { + auto t = Tensor({2, 2}, {2.0, 3.0, 4.0, 5.0}, Device::CUDA); + + ASSERT_EQ(t.getDims(), Dimension({2, 2})); + ASSERT_EQ(t.getDevice(), Device::CPU); + ASSERT_TRUE(!t.getRequiresGrad()); + + ASSERT_NEAR(t.get(0, 0), 2.0, 1e-5); + ASSERT_NEAR(t.get(0, 1), 3.0, 1e-5); + ASSERT_NEAR(t.get(1, 0), 4.0, 1e-5); + ASSERT_NEAR(t.get(1, 1), 5.0, 1e-5); +} + +TEST(CudaTensorOpsTest, ScalarAdd) { auto t1 = TensorFunctions::Ones({500, 500}, Device::CUDA); auto res = t1 + 1.5; @@ -32,7 +45,7 @@ TEST(CudaTensorOpsTest, ScalarAddWorks) { } } -TEST(CudaTensorOpsTest, ScalarMulWorks) { +TEST(CudaTensorOpsTest, ScalarMul) { auto t1 = TensorFunctions::Ones({500, 500}, Device::CUDA); constexpr ftype f = 2.5; @@ -46,7 +59,7 @@ TEST(CudaTensorOpsTest, ScalarMulWorks) { } } -TEST(CudaTensorOpsTest, TensorAddWorks) { +TEST(CudaTensorOpsTest, TensorAdd) { auto t1 = TensorFunctions::Ones({500, 500}, Device::CUDA); auto t2 = TensorFunctions::Ones({500, 500}, Device::CUDA) * 4; @@ -61,7 +74,20 @@ TEST(CudaTensorOpsTest, TensorAddWorks) { } } -TEST(CudaTensorOpsTest, TensorAddCanBroadCast) { +/* TEST(CudaAutogradTest, TensorAdd) { + auto t1 = TensorFunctions::makeSharedTensor({1}, {3.0}, Device::CUDA, true); + auto t2 = TensorFunctions::makeSharedTensor({1}, {2.0}, Device::CUDA, true); + + auto t3 = cgraph::add(t1, t2); + auto loss = cgraph::mul(t3, t3); + + loss->backward(); + + ASSERT_NEAR(t1->getGrads()->get(0), 10.0, 1e-5); + ASSERT_NEAR(t2->getGrads()->get(0), 10.0, 1e-5); +} */ + +TEST(CudaTensorOpsTest, BroadcastAdd) { auto t1 = TensorFunctions::Ones({3, 2, 2}, Device::CUDA); auto t2 = Tensor({2}, {2, 3}, Device::CUDA); @@ -71,8 +97,8 @@ TEST(CudaTensorOpsTest, TensorAddCanBroadCast) { for(auto i=0; i + +#include "system/sys_functions.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + sys::setRandomSeed(42); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/tests/backend/net_factories.h b/tests/backend/net_factories.h new file mode 100644 index 0000000..45ecbc6 --- /dev/null +++ b/tests/backend/net_factories.h @@ -0,0 +1,64 @@ +/** + * @file net_factories.h + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-05-20 + * + * @copyright Copyright (c) 2026 + * + */ + +#pragma once + +#include "module/networks/sequential.h" +#include "module/layers/ff_layer.h" + +#include "module/activation_functions/sigmoid.h" +#include "module/activation_functions/relu.h" +#include "module/activation_functions/leaky_relu.h" +#include "module/activation_functions/softmax.h" + +#include + +static std::shared_ptr makeBinaryNet() { + auto net = std::make_shared(); + + net->append(std::make_shared(2, 4, true, true)); + net->append(std::make_shared(0.01)); + net->append(std::make_shared(4, 1, true, true)); + net->append(std::make_shared()); + + return net; +} + +static std::shared_ptr makeBinaryNet2() { + auto net = std::make_shared(); + + net->append(std::make_shared(2, 4, true, true)); + net->append(std::make_shared(0.01)); + net->append(std::make_shared(4, 1, true, true)); + + return net; +} + +static std::shared_ptr makeMulticlassNet() { + auto net = std::make_shared(); + + net->append(std::make_shared(2, 8, true, true)); + net->append(std::make_shared(0.01)); + net->append(std::make_shared(8, 3, true, true)); + net->append(std::make_shared()); + + return net; +} + +static std::shared_ptr makeMulticlassNet2() { + auto net = std::make_shared(); + + net->append(std::make_shared(2, 8, true, true)); + net->append(std::make_shared(0.01)); + net->append(std::make_shared(8, 3, true, true)); + + return net; +} \ No newline at end of file diff --git a/tests/backend/test_computational_graph.cpp b/tests/backend/test_computational_graph.cpp index ef07e65..b77091e 100644 --- a/tests/backend/test_computational_graph.cpp +++ b/tests/backend/test_computational_graph.cpp @@ -1,9 +1,9 @@ /** - * @file test_computational_graph.cpp + * @file test_training.cpp * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) * @brief * @version 0.1 - * @date 2026-02-16 + * @date 2026-03-14 * * @copyright Copyright (c) 2026 * @@ -11,109 +11,30 @@ #include -#include "data_modeling/tensor.h" #include "data_modeling/tensor_functions.h" - #include "computational_graph/tensor_ops/graph_creation.h" -#include - -TEST(AutogradTest, ThrowsIfNoGradientSet) { - auto t1 = TensorFunctions::makeSharedTensor({1}, {3.0}, false); - auto t2 = TensorFunctions::makeSharedTensor({1}, {2.0}, false); - - auto loss = cgraph::add(t1, t2); - - EXPECT_THROW(loss->backward(), std::runtime_error); -} +#include "net_factories.h" -TEST(AutogradTest, SimpleAddition) { - auto t1 = TensorFunctions::makeSharedTensor({1}, {3.0}, true); - auto t2 = TensorFunctions::makeSharedTensor({1}, {2.0}, true); +#include "training/optimizers/sgd.h" +#include "training/optimizers/rmsprop.h" - auto t3 = cgraph::add(t1, t2); - auto loss = cgraph::mul(t3, t3); - - loss->backward(); - - EXPECT_NEAR(t1->getGrads()->get(0), 10.0, 1e-5); - EXPECT_NEAR(t2->getGrads()->get(0), 10.0, 1e-5); -} +#include "training/loss_functions/bce_loss.h" +#include "training/loss_functions/crossentropy_loss.h" +#include "training/loss_functions/bce_sigmoid_loss.h" +#include "training/loss_functions/crossentropy_softmax_loss.h" -TEST(AutogradTest, BroadcastAdd) { - // gradient of broadcast add w.r.t. bias should be sum over batch dimension - // upstream grad: (2,3) of ones → bias grad should be (3) of twos - auto t1 = TensorFunctions::makeSharedTensor({2, 3}, - {1.0, 2.0, 3.0, - 4.0, 5.0, 6.0}, true); - auto bias = TensorFunctions::makeSharedTensor({3}, - {0.0, 0.0, 0.0}, true); - - auto res = cgraph::add(t1, bias); - - // set upstream grad to ones and backprop - auto upstreamGrad = TensorFunctions::makeSharedTensor({2, 3}, - {1.0, 1.0, 1.0, - 1.0, 1.0, 1.0}, false); - res->backward(); - - // bias grad should be sum over batch: [2, 2, 2] - auto biasGrad = bias->getGrads(); - ASSERT_DOUBLE_EQ((*biasGrad)[0], 2.0); - ASSERT_DOUBLE_EQ((*biasGrad)[1], 2.0); - ASSERT_DOUBLE_EQ((*biasGrad)[2], 2.0); - - // t1 grad should be ones (add is identity for non-broadcast operand) - auto t1Grad = t1->getGrads(); - for(int i = 0; i < 6; i++) { - ASSERT_DOUBLE_EQ((*t1Grad)[i], 1.0); - } -} +#include "training/trainers/base_train_loop.h" -TEST(AutogradTest, ScalarMultiplication) { - auto t1 = TensorFunctions::makeSharedTensor({1}, {2.0}, true); - auto t2 = TensorFunctions::makeSharedTensor({1}, {3.0}, true); +using namespace std; - auto t3 = cgraph::mul(t1, t2); - auto loss = cgraph::mul(t3, t3); - - loss->backward(); - - ASSERT_DOUBLE_EQ(t1->getGrads()->get(0), 36.0); - ASSERT_DOUBLE_EQ(t2->getGrads()->get(0), 24.0); -} - -TEST(AutogradTest, MatMul) { - auto t1 = TensorFunctions::makeSharedTensor({2, 3}, {1, 2, 3, 4, 5, 6}, true); - auto t2 = TensorFunctions::makeSharedTensor({3, 2}, {1, 2, 3, 4, 5, 6}, true); - - auto t3 = cgraph::matmul(t1, t2); +TEST(AutogradTest, ThrowsIfNoGradientSet) { + auto t1 = TensorFunctions::makeSharedTensor({1}, {3.0}, false); + auto t2 = TensorFunctions::makeSharedTensor({1}, {2.0}, false); - auto loss = TensorFunctions::makeSharedTensor({1}, {0.0}, true); - for (size_t i = 0; i < t3->getSize(); ++i) { - loss = cgraph::add(loss, cgraph::get(t3, i)); - } - - loss->backward(); + auto loss = cgraph::add(t1, t2); - EXPECT_TRUE(t1->hasGrads()); - EXPECT_TRUE(t2->hasGrads()); - - // dL/dt1 = dloss/dt3 @ t2^t = Ones({2, 2}) @ t2^t - ASSERT_DOUBLE_EQ(t1->getGrads()->get({0, 0}), 3.0); - ASSERT_DOUBLE_EQ(t1->getGrads()->get({0, 1}), 7.0); - ASSERT_DOUBLE_EQ(t1->getGrads()->get({0, 2}), 11.0); - ASSERT_DOUBLE_EQ(t1->getGrads()->get({1, 0}), 3.0); - ASSERT_DOUBLE_EQ(t1->getGrads()->get({1, 1}), 7.0); - ASSERT_DOUBLE_EQ(t1->getGrads()->get({1, 2}), 11.0); - - // dL/dt2 = t1^t @ dloss/dt3 = t1^t @ Ones({2, 2}) - ASSERT_DOUBLE_EQ(t2->getGrads()->get({0, 0}), 5.0); - ASSERT_DOUBLE_EQ(t2->getGrads()->get({0, 1}), 5.0); - ASSERT_DOUBLE_EQ(t2->getGrads()->get({1, 0}), 7.0); - ASSERT_DOUBLE_EQ(t2->getGrads()->get({1, 1}), 7.0); - ASSERT_DOUBLE_EQ(t2->getGrads()->get({2, 0}), 9.0); - ASSERT_DOUBLE_EQ(t2->getGrads()->get({2, 1}), 9.0); + ASSERT_THROW(loss->backward(), std::runtime_error); } TEST(AutogradTest, ChainRule) { @@ -127,7 +48,7 @@ TEST(AutogradTest, ChainRule) { // dloss/dx = 2(x^2 + x) * (2x + 1) // At x=2: 2(4 + 2) * (4 + 1) = 2 * 6 * 5 = 60 - ASSERT_DOUBLE_EQ(x->getGrads()->get(0), 60.0); + ASSERT_NEAR(x->getGrads()->get(0), 60.0, 1e-5); } TEST(AutogradTest, MultiVariateChainRule) { @@ -142,9 +63,195 @@ TEST(AutogradTest, MultiVariateChainRule) { loss->backward(); // dloss/dx = scalar = 3 - ASSERT_DOUBLE_EQ(x->getGrads()->get(0), 3.0); - ASSERT_DOUBLE_EQ(x->getGrads()->get(1), 3.0); + ASSERT_NEAR(x->getGrads()->get(0), 3.0, 1e-5); + ASSERT_NEAR(x->getGrads()->get(1), 3.0, 1e-5); + + ASSERT_NEAR(y->getGrads()->get(0), 1.0, 1e-5); + ASSERT_NEAR(y->getGrads()->get(1), 1.0, 1e-5); +} + +TEST(OverfitTest, BceSgdOverfitsSmallDataset) { + // XOR-like: 4 samples, 2 features, binary labels + auto x = TensorFunctions::makeSharedTensor( + {4, 2}, {0.0, 0.0, + 0.0, 1.0, + 1.0, 0.0, + 1.0, 1.0}, false); + + auto y = TensorFunctions::makeSharedTensor( + {4, 1}, {0.0, + 1.0, + 1.0, + 0.0}, false); + + auto net = makeBinaryNet(); + auto loss = make_shared(); + auto optim = make_shared( + net->parameters(), /*lr=*/0.05); + + auto trainLoop = train::BaseTrainLoop( + net, loss, optim, /*epochs=*/2000, /*bsize=*/static_cast(4)); - ASSERT_DOUBLE_EQ(y->getGrads()->get(0), 1.0); - ASSERT_DOUBLE_EQ(y->getGrads()->get(1), 1.0); + trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); + + // forward one more time to get final loss + auto pred = (*net)(x); + auto finalLoss = (*loss)(y, pred); + + EXPECT_LT((*finalLoss)[0], 0.05f) + << "Network failed to overfit binary dataset\n" + << "Final prediction: " << *pred << "\nFinal loss: " << *finalLoss; +} + +TEST(OverfitTest, BceSgdOverfitsSmallDataset_OptimizedLoss) { + // XOR-like: 4 samples, 2 features, binary labels + auto x = TensorFunctions::makeSharedTensor( + {4, 2}, {0.0, 0.0, + 0.0, 1.0, + 1.0, 0.0, + 1.0, 1.0}, false); + + auto y = TensorFunctions::makeSharedTensor( + {4, 1}, {0.0, + 1.0, + 1.0, + 0.0}, false); + + auto net = makeBinaryNet2(); + auto loss = make_shared(); + auto optim = make_shared( + net->parameters(), /*lr=*/0.05); + + auto trainLoop = train::BaseTrainLoop( + net, loss, optim, /*epochs=*/2000, /*bsize=*/static_cast(4)); + + trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); + + // forward one more time to get final loss + auto pred = (*net)(x); + auto finalLoss = (*loss)(y, pred); + + auto sigmoid = module::Sigmoid(); + EXPECT_LT((*finalLoss)[0], 0.05f) + << "Network failed to overfit binary dataset\n" + << "Final prediction: " << sigmoid(*pred) << "\nFinal loss: " << *finalLoss; +} + +TEST(OverfitTest, CrossEntropyRMSPropOverfitsSmallDataset) { + // 6 samples, 2 features, 3 classes + auto x = TensorFunctions::makeSharedTensor( + {6, 2}, {1.0, 0.0, + 1.0, 0.1, + 0.0, 1.0, + 0.1, 1.0, + 0.5, 0.5, + 0.4, 0.6}, false); + + // one-hot encoded labels + auto y = TensorFunctions::makeSharedTensor( + {6, 3}, {1.0, 0.0, 0.0, + 1.0, 0.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 0.0, 1.0, + 0.0, 0.0, 1.0}, false); + + auto net = makeMulticlassNet(); + auto loss = make_shared(); + auto optim = make_shared( + net->parameters(), /*lr=*/0.0001, /*decay=*/0.95); + + auto trainLoop = train::BaseTrainLoop( + net, loss, optim, /*epochs=*/2000, /*bsize=*/6); + + trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); + + auto pred = (*net)(x); + auto finalLoss = (*loss)(y, pred); + + EXPECT_LT((*finalLoss)[0], 0.05f) + << "Network failed to overfit multiclass dataset" + << "Final prediction: " << *pred << "\nFinal loss: " << *finalLoss; +} + +TEST(OverfitTest, CrossEntropyRMSPropOverfitsSmallDataset_OptimizedLoss) { + // 6 samples, 2 features, 3 classes + auto x = TensorFunctions::makeSharedTensor( + {6, 2}, {1.0, 0.0, + 1.0, 0.1, + 0.0, 1.0, + 0.1, 1.0, + 0.5, 0.5, + 0.4, 0.6}, false); + + // one-hot encoded labels + auto y = TensorFunctions::makeSharedTensor( + {6, 3}, {1.0, 0.0, 0.0, + 1.0, 0.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 0.0, 1.0, + 0.0, 0.0, 1.0}, false); + + auto net = makeMulticlassNet2(); + auto loss = make_shared(); + auto optim = make_shared( + net->parameters(), /*lr=*/0.0001, /*decay=*/0.95); + + auto trainLoop = train::BaseTrainLoop( + net, loss, optim, /*epochs=*/2000, /*bsize=*/6); + + trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); + + auto pred = (*net)(x); + auto finalLoss = (*loss)(y, pred); + + auto softmax = module::Softmax(); + EXPECT_LT((*finalLoss)[0], 0.05f) + << "Network failed to overfit multiclass dataset" + << "Final prediction: " << softmax(*pred) << "\nFinal loss: " << *finalLoss; +} + +TEST(OptimizerTest, ZeroGrad_ClearsAllGradients) { + auto x = TensorFunctions::makeSharedTensor( + {4, 2}, {0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0}, false); + auto y = TensorFunctions::makeSharedTensor( + {4, 1}, {0.0, 1.0, 1.0, 0.0}, false); + + auto net = makeBinaryNet(); + auto loss = std::make_shared(); + auto optim = std::make_shared( + net->parameters(), 0.01f); + + // one forward/backward pass to populate gradients + auto pred = (*net)(x); + auto l = (*loss)(y, pred); + l->backward(); + + // verify gradients are non-zero before zeroing + bool anyNonZero = false; + for(auto& p : net->parameters()) { + if(p->getGrads()) { + for(tensorSize_t i = 0; i < p->getGrads()->getSize(); i++) { + if((*p->getGrads())[i] != 0.0f) { + anyNonZero = true; + break; + } + } + } + } + EXPECT_TRUE(anyNonZero) << "Expected some non-zero gradients before zeroGrad"; + + // zero gradients + optim->zeroGrad(); + + // verify all gradients are zero after zeroing + for(auto& p : net->parameters()) { + if(p->getGrads()) { + for(tensorSize_t i = 0; i < p->getGrads()->getSize(); i++) { + ASSERT_NEAR((*p->getGrads())[i], 0.0f, 1e-5) + << "Gradient not zeroed at index " << i; + } + } + } } \ No newline at end of file diff --git a/tests/backend/test_losses.cpp b/tests/backend/test_losses.cpp index cfcc007..3b8b8ac 100644 --- a/tests/backend/test_losses.cpp +++ b/tests/backend/test_losses.cpp @@ -21,7 +21,7 @@ using namespace train; -static constexpr ftype kTol = 1e-4f; +static constexpr ftype delta = 1e-4f; TEST(LossTest, CrossEntropyFoward) { auto y = TensorFunctions::makeSharedTensor( @@ -37,7 +37,7 @@ TEST(LossTest, CrossEntropyFoward) { // expected: -( log(0.7) + log(0.8) ) / 2 = 0.2899 const ftype expected = -(std::log(0.7f) + std::log(0.8f)) / 2.0f; - EXPECT_NEAR((*result)[0], expected, kTol); + ASSERT_NEAR((*result)[0], expected, delta); } TEST(LossTest, CrossEntropyPerfectPrediction) { @@ -54,7 +54,7 @@ TEST(LossTest, CrossEntropyPerfectPrediction) { auto result = loss(y, ypred); // loss should be very small - EXPECT_LT((*result)[0], 0.01f); + ASSERT_LT((*result)[0], 0.01f); } TEST(LossTest, CrossEntropyUniformPrediction) { @@ -68,7 +68,7 @@ TEST(LossTest, CrossEntropyUniformPrediction) { CrossEntropyLoss loss; auto result = loss(y, ypred); - EXPECT_NEAR((*result)[0], std::log(3.0f), kTol); + ASSERT_NEAR((*result)[0], std::log(3.0f), delta); } TEST(LossTest, CrossEntropyThrowsOnDimMismatch) { @@ -78,7 +78,7 @@ TEST(LossTest, CrossEntropyThrowsOnDimMismatch) { {2, 2}, {0.5, 0.5, 0.5, 0.5}, true); CrossEntropyLoss loss; - EXPECT_THROW(loss(y, ypred), std::invalid_argument); + ASSERT_THROW(loss(y, ypred), std::invalid_argument); } TEST(LossTest, CrossEntropyBackward) { @@ -102,12 +102,12 @@ TEST(LossTest, CrossEntropyBackward) { result->backward(); auto grads = ypred->getGrads(); - EXPECT_NEAR((*grads)[0], -0.7143f, kTol); - EXPECT_NEAR((*grads)[1], 0.0f, kTol); - EXPECT_NEAR((*grads)[2], 0.0f, kTol); - EXPECT_NEAR((*grads)[3], 0.0f, kTol); - EXPECT_NEAR((*grads)[4], -0.625f, kTol); - EXPECT_NEAR((*grads)[5], 0.0f, kTol); + ASSERT_NEAR((*grads)[0], -0.7143f, delta); + ASSERT_NEAR((*grads)[1], 0.0f, delta); + ASSERT_NEAR((*grads)[2], 0.0f, delta); + ASSERT_NEAR((*grads)[3], 0.0f, delta); + ASSERT_NEAR((*grads)[4], -0.625f, delta); + ASSERT_NEAR((*grads)[5], 0.0f, delta); } TEST(LossTest, BceForward) { @@ -123,7 +123,7 @@ TEST(LossTest, BceForward) { // expected: -( log(0.9) + log(0.9) + log(0.8) + log(0.8) ) / 4 = 0.1643 const ftype expected = -(std::log(0.9f) + std::log(0.9f) + std::log(0.8f) + std::log(0.8f)) / 4.0f; - EXPECT_NEAR((*result)[0], expected, kTol); + ASSERT_NEAR((*result)[0], expected, delta); } TEST(LossTest, BcePerfectPrediction) { @@ -136,7 +136,7 @@ TEST(LossTest, BcePerfectPrediction) { BceLoss loss; auto result = loss(y, ypred); - EXPECT_LT((*result)[0], 0.01f); + ASSERT_LT((*result)[0], 0.01f); } TEST(LossTest, BceRandomPrediction) { @@ -150,7 +150,7 @@ TEST(LossTest, BceRandomPrediction) { BceLoss loss; auto result = loss(y, ypred); - EXPECT_NEAR((*result)[0], std::log(2.0f), kTol); + ASSERT_NEAR((*result)[0], std::log(2.0f), delta); } TEST(LossTest, BceThrowsOnDimMismatch) { @@ -160,7 +160,7 @@ TEST(LossTest, BceThrowsOnDimMismatch) { {3, 1}, {0.5, 0.5, 0.5}, true); BceLoss loss; - EXPECT_THROW(loss(y, ypred), std::invalid_argument); + ASSERT_THROW(loss(y, ypred), std::invalid_argument); } TEST(LossTest, BceNoInfOrNanOnNearZeroPred) { @@ -173,7 +173,7 @@ TEST(LossTest, BceNoInfOrNanOnNearZeroPred) { auto result = loss(y, ypred); // clipping prevents log(0) - EXPECT_FALSE(std::isinf((*result)[0])); + ASSERT_FALSE(std::isinf((*result)[0])); } TEST(LossTest, BceBackward) { @@ -191,8 +191,8 @@ TEST(LossTest, BceBackward) { result->backward(); auto grads = ypred->getGrads(); - EXPECT_NEAR((*grads)[0], -0.625f, kTol); - EXPECT_NEAR((*grads)[1], 0.7143f, kTol); + ASSERT_NEAR((*grads)[0], -0.625f, delta); + ASSERT_NEAR((*grads)[1], 0.7143f, delta); } TEST(LossTest, RmseForward) { @@ -208,7 +208,7 @@ TEST(LossTest, RmseForward) { auto loss = RmseLoss{}; auto result = loss(y, ypred); - EXPECT_NEAR((*result)[0], 0.5f, kTol); + ASSERT_NEAR((*result)[0], 0.5f, delta); } TEST(LossTest, RmsePerfectPrediction) { @@ -220,7 +220,7 @@ TEST(LossTest, RmsePerfectPrediction) { RmseLoss loss; auto result = loss(y, ypred); - EXPECT_NEAR((*result)[0], 0.0f, kTol); + ASSERT_NEAR((*result)[0], 0.0f, delta); } TEST(LossTest, RmseBackward) { @@ -239,6 +239,6 @@ TEST(LossTest, RmseBackward) { result->backward(); auto grads = ypred->getGrads(); - EXPECT_NEAR((*grads)[0], -0.5f, kTol); - EXPECT_NEAR((*grads)[1], 0.5f, kTol); + ASSERT_NEAR((*grads)[0], -0.5f, delta); + ASSERT_NEAR((*grads)[1], 0.5f, delta); } \ No newline at end of file diff --git a/tests/backend/test_module.cpp b/tests/backend/test_module.cpp index 5f82c91..24e12aa 100644 --- a/tests/backend/test_module.cpp +++ b/tests/backend/test_module.cpp @@ -49,11 +49,11 @@ TEST(ActivationTest, ReluInputNegative) { } TEST(AutogradTest, ReLUBackward) { - auto x = TensorFunctions::makeSharedTensor({3}, {-1.0, 0.0, 2.0}, true); - auto relu = module::ReLu(); + auto x = TensorFunctions::makeSharedTensor({3}, {-1.0, 0.0, 2.0}, true); + auto relu = module::ReLu(); - auto y = relu(x); // [0, 0, 2] - auto loss = cgraph::sumTensor(y); // loss = 2 + auto y = relu(x); // [0, 0, 2] + auto loss = cgraph::sumTensor(y); // loss = 2 loss->backward(); @@ -203,31 +203,30 @@ TEST(ActivationTest, SoftmaxForwardNumericalStability) { } TEST(AutogradTest, SoftmaxBackward) { - // for softmax with upstream grad of ones, the gradient is zero - // because d/dx_i sum(softmax(x)) = 0 (softmax sums to 1 always) - // more useful: upstream = [1, 0, 0] - // grad[i] = softmax[i] * (upstream[i] - dot(upstream, softmax)) - // for x=[1,2,3], softmax=[0.09, 0.2447, 0.6652] - // dot([1,0,0], softmax) = 0.09 - // grad[0] = 0.09 * (1 - 0.09) = 0.0819 - // grad[1] = 0.2447 * (0 - 0.09) = -0.0220 - // grad[2] = 0.6652 * (0 - 0.09) = -0.0599 - auto t = TensorFunctions::makeSharedTensor( - {1, 3}, {1.0, 2.0, 3.0}, true); + // for softmax with upstream grad of ones, the gradient is zero + // because d/dx_i sum(softmax(x)) = 0 (softmax sums to 1 always) + // more useful: upstream = [1, 0, 0] + // grad[i] = softmax[i] * (upstream[i] - dot(upstream, softmax)) + // for x=[1,2,3], softmax=[0.09, 0.2447, 0.6652] + // dot([1,0,0], softmax) = 0.09 + // grad[0] = 0.09 * (1 - 0.09) = 0.0819 + // grad[1] = 0.2447 * (0 - 0.09) = -0.0220 + // grad[2] = 0.6652 * (0 - 0.09) = -0.0599 + auto t = TensorFunctions::makeSharedTensor({1, 3}, {1.0, 2.0, 3.0}, true); - module::Softmax sm; - auto resPtr = sm(t); + module::Softmax sm; + auto resPtr = sm(t); - // set upstream gradient to [1, 0, 0] - auto upstream = TensorFunctions::makeSharedTensor( - {1, 3}, {1.0, 0.0, 0.0}); - resPtr->setGrads(upstream); - resPtr->backward(); - - auto grads = t->getGrads(); - ASSERT_NEAR((*grads)[0], 0.0819, 1e-4); - ASSERT_NEAR((*grads)[1], -0.0220, 1e-4); - ASSERT_NEAR((*grads)[2], -0.0599, 1e-4); + // set upstream gradient to [1, 0, 0] + auto upstream = TensorFunctions::makeSharedTensor( + {1, 3}, {1.0, 0.0, 0.0}); + resPtr->setGrads(upstream); + resPtr->backward(); + + auto grads = t->getGrads(); + ASSERT_NEAR((*grads)[0], 0.0819, 1e-4); + ASSERT_NEAR((*grads)[1], -0.0220, 1e-4); + ASSERT_NEAR((*grads)[2], -0.0599, 1e-4); } TEST(LayerTest, TestFfLayer) { diff --git a/tests/backend/test_data_modeling.cpp b/tests/backend/test_tensorops.cpp similarity index 55% rename from tests/backend/test_data_modeling.cpp rename to tests/backend/test_tensorops.cpp index 367a81f..70780fc 100644 --- a/tests/backend/test_data_modeling.cpp +++ b/tests/backend/test_tensorops.cpp @@ -14,9 +14,7 @@ #include "data_modeling/tensor.h" #include "data_modeling/tensor_functions.h" -#include - -#include +#include "computational_graph/tensor_ops/graph_creation.h" using namespace std; @@ -27,13 +25,13 @@ TEST(TensorOpsTest, TestCtor) { ASSERT_EQ(t.getDevice(), Device::CPU); ASSERT_TRUE(!t.getRequiresGrad()); - EXPECT_NEAR(t.get(0, 0), 2.0, 1e-5); - EXPECT_NEAR(t.get(0, 1), 3.0, 1e-5); - EXPECT_NEAR(t.get(1, 0), 4.0, 1e-5); - EXPECT_NEAR(t.get(1, 1), 5.0, 1e-5); + ASSERT_NEAR(t.get(0, 0), 2.0, 1e-5); + ASSERT_NEAR(t.get(0, 1), 3.0, 1e-5); + ASSERT_NEAR(t.get(1, 0), 4.0, 1e-5); + ASSERT_NEAR(t.get(1, 1), 5.0, 1e-5); } -TEST(TensorOpsTest, ScalarAddWorks) { +TEST(TensorOpsTest, ScalarAdd) { auto t1 = TensorFunctions::Ones({2, 2}); auto res = t1 + 1.5; @@ -41,12 +39,12 @@ TEST(TensorOpsTest, ScalarAddWorks) { constexpr ftype sum = 2.5; for(auto i=0; ibackward(); + + ASSERT_DOUBLE_EQ(t1->getGrads()->get(0), 36.0); + ASSERT_DOUBLE_EQ(t2->getGrads()->get(0), 24.0); +} + +TEST(TensorOpsTest, TensorAdd) { auto t1 = TensorFunctions::Ones({2, 2}); auto t2 = TensorFunctions::Ones({2, 2}) * 4; @@ -68,12 +79,25 @@ TEST(TensorOpsTest, TensorAddWorks) { constexpr ftype sum = 5.0; for(auto i=0; ibackward(); + + ASSERT_NEAR(t1->getGrads()->get(0), 10.0, 1e-5); + ASSERT_NEAR(t2->getGrads()->get(0), 10.0, 1e-5); +} + +TEST(TensorOpsTest, BroadcastAdd) { auto t1 = TensorFunctions::Ones({3, 2, 2}); auto t2 = Tensor({2}, {2, 3}); @@ -83,13 +107,13 @@ TEST(TensorOpsTest, TensorAddCanBroadCast) { for(auto i=0; ibackward(); + + // bias grad should be sum over batch: [2, 2, 2] + auto biasGrad = bias->getGrads(); + ASSERT_DOUBLE_EQ((*biasGrad)[0], 2.0); + ASSERT_DOUBLE_EQ((*biasGrad)[1], 2.0); + ASSERT_DOUBLE_EQ((*biasGrad)[2], 2.0); + + // t1 grad should be ones (add is identity for non-broadcast operand) + auto t1Grad = t1->getGrads(); + for(int i = 0; i < 6; i++) { + ASSERT_DOUBLE_EQ((*t1Grad)[i], 1.0); + } } TEST(TensorOpsTest, MatrixAddGivesCorrectResults) { @@ -129,7 +183,7 @@ TEST(TensorOpsTest, MatrixAddGivesCorrectResults) { for(auto i=0; igetSize(); ++i) { + loss = cgraph::add(loss, cgraph::get(t3, i)); + } + + loss->backward(); + + EXPECT_TRUE(t1->hasGrads()); + EXPECT_TRUE(t2->hasGrads()); + + // dL/dt1 = dloss/dt3 @ t2^t = Ones({2, 2}) @ t2^t + ASSERT_DOUBLE_EQ(t1->getGrads()->get({0, 0}), 3.0); + ASSERT_DOUBLE_EQ(t1->getGrads()->get({0, 1}), 7.0); + ASSERT_DOUBLE_EQ(t1->getGrads()->get({0, 2}), 11.0); + ASSERT_DOUBLE_EQ(t1->getGrads()->get({1, 0}), 3.0); + ASSERT_DOUBLE_EQ(t1->getGrads()->get({1, 1}), 7.0); + ASSERT_DOUBLE_EQ(t1->getGrads()->get({1, 2}), 11.0); + + // dL/dt2 = t1^t @ dloss/dt3 = t1^t @ Ones({2, 2}) + ASSERT_DOUBLE_EQ(t2->getGrads()->get({0, 0}), 5.0); + ASSERT_DOUBLE_EQ(t2->getGrads()->get({0, 1}), 5.0); + ASSERT_DOUBLE_EQ(t2->getGrads()->get({1, 0}), 7.0); + ASSERT_DOUBLE_EQ(t2->getGrads()->get({1, 1}), 7.0); + ASSERT_DOUBLE_EQ(t2->getGrads()->get({2, 0}), 9.0); + ASSERT_DOUBLE_EQ(t2->getGrads()->get({2, 1}), 9.0); +} + + +TEST(TensorOpsTest, MatrixTranspose) { auto t = TensorFunctions::Gaussian({3, 2}, 1.0); auto transposed = t.createDeepCopy(); @@ -221,7 +309,7 @@ TEST(TensorOpsTest, TransposeWorksAsIntended1) { for(auto row=0; row - -#include "module/networks/sequential.h" -#include "module/layers/ff_layer.h" - -#include "module/activation_functions/sigmoid.h" -#include "module/activation_functions/relu.h" -#include "module/activation_functions/leaky_relu.h" -#include "module/activation_functions/softmax.h" - -#include "training/optimizers/sgd.h" -#include "training/optimizers/rmsprop.h" - -#include "training/loss_functions/bce_loss.h" -#include "training/loss_functions/crossentropy_loss.h" -#include "training/loss_functions/bce_sigmoid_loss.h" -#include "training/loss_functions/crossentropy_softmax_loss.h" - -#include "training/trainers/base_train_loop.h" - -#include "data_modeling/tensor_functions.h" - -#include "system/sys_functions.h" - -using namespace std; - -static shared_ptr makeBinaryNet() { - auto net = make_shared(); - - net->append(make_shared(2, 4, true, true)); - - net->append(make_shared(0.01)); - - net->append(make_shared(4, 1, true, true)); - - net->append(make_shared()); - return net; -} - -static shared_ptr makeBinaryNet2() { - auto net = make_shared(); - - net->append(make_shared(2, 4, true, true)); - - net->append(make_shared(0.01)); - - net->append(make_shared(4, 1, true, true)); - - return net; -} - -static shared_ptr makeMulticlassNet() { - auto net = make_shared(); - - net->append(make_shared(2, 8, true, true)); - - net->append(make_shared(0.01)); - - net->append(make_shared(8, 3, true, true)); - - net->append(make_shared()); - return net; -} - -static shared_ptr makeMulticlassNet2() { - auto net = make_shared(); - - net->append(make_shared(2, 8, true, true)); - - net->append(make_shared(0.01)); - - net->append(make_shared(8, 3, true, true)); - - return net; -} - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - sys::setRandomSeed(42); - return RUN_ALL_TESTS(); -} - -TEST(OverfitTest, BceSgdOverfitsSmallDataset) { - // XOR-like: 4 samples, 2 features, binary labels - auto x = TensorFunctions::makeSharedTensor( - {4, 2}, {0.0, 0.0, - 0.0, 1.0, - 1.0, 0.0, - 1.0, 1.0}, false); - - auto y = TensorFunctions::makeSharedTensor( - {4, 1}, {0.0, - 1.0, - 1.0, - 0.0}, false); - - auto net = makeBinaryNet(); - auto loss = make_shared(); - auto optim = make_shared( - net->parameters(), /*lr=*/0.05); - - auto trainLoop = train::BaseTrainLoop( - net, loss, optim, /*epochs=*/2000, /*bsize=*/static_cast(4)); - - trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); - - // forward one more time to get final loss - auto pred = (*net)(x); - auto finalLoss = (*loss)(y, pred); - - EXPECT_LT((*finalLoss)[0], 0.05f) - << "Network failed to overfit binary dataset\n" - << "Final prediction: " << *pred << "\nFinal loss: " << *finalLoss; -} - -TEST(OverfitTest, BceSgdOverfitsSmallDataset_OptimizedLoss) { - // XOR-like: 4 samples, 2 features, binary labels - auto x = TensorFunctions::makeSharedTensor( - {4, 2}, {0.0, 0.0, - 0.0, 1.0, - 1.0, 0.0, - 1.0, 1.0}, false); - - auto y = TensorFunctions::makeSharedTensor( - {4, 1}, {0.0, - 1.0, - 1.0, - 0.0}, false); - - auto net = makeBinaryNet2(); - auto loss = make_shared(); - auto optim = make_shared( - net->parameters(), /*lr=*/0.05); - - auto trainLoop = train::BaseTrainLoop( - net, loss, optim, /*epochs=*/2000, /*bsize=*/static_cast(4)); - - trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); - - // forward one more time to get final loss - auto pred = (*net)(x); - auto finalLoss = (*loss)(y, pred); - - auto sigmoid = module::Sigmoid(); - EXPECT_LT((*finalLoss)[0], 0.05f) - << "Network failed to overfit binary dataset\n" - << "Final prediction: " << sigmoid(*pred) << "\nFinal loss: " << *finalLoss; -} - -TEST(OverfitTest, CrossEntropyRMSPropOverfitsSmallDataset) { - // 6 samples, 2 features, 3 classes - auto x = TensorFunctions::makeSharedTensor( - {6, 2}, {1.0, 0.0, - 1.0, 0.1, - 0.0, 1.0, - 0.1, 1.0, - 0.5, 0.5, - 0.4, 0.6}, false); - - // one-hot encoded labels - auto y = TensorFunctions::makeSharedTensor( - {6, 3}, {1.0, 0.0, 0.0, - 1.0, 0.0, 0.0, - 0.0, 1.0, 0.0, - 0.0, 1.0, 0.0, - 0.0, 0.0, 1.0, - 0.0, 0.0, 1.0}, false); - - auto net = makeMulticlassNet(); - auto loss = make_shared(); - auto optim = make_shared( - net->parameters(), /*lr=*/0.0001, /*decay=*/0.95); - - auto trainLoop = train::BaseTrainLoop( - net, loss, optim, /*epochs=*/2000, /*bsize=*/6); - - trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); - - auto pred = (*net)(x); - auto finalLoss = (*loss)(y, pred); - - EXPECT_LT((*finalLoss)[0], 0.05f) - << "Network failed to overfit multiclass dataset" - << "Final prediction: " << *pred << "\nFinal loss: " << *finalLoss; -} - -TEST(OverfitTest, CrossEntropyRMSPropOverfitsSmallDataset_OptimizedLoss) { - // 6 samples, 2 features, 3 classes - auto x = TensorFunctions::makeSharedTensor( - {6, 2}, {1.0, 0.0, - 1.0, 0.1, - 0.0, 1.0, - 0.1, 1.0, - 0.5, 0.5, - 0.4, 0.6}, false); - - // one-hot encoded labels - auto y = TensorFunctions::makeSharedTensor( - {6, 3}, {1.0, 0.0, 0.0, - 1.0, 0.0, 0.0, - 0.0, 1.0, 0.0, - 0.0, 1.0, 0.0, - 0.0, 0.0, 1.0, - 0.0, 0.0, 1.0}, false); - - auto net = makeMulticlassNet2(); - auto loss = make_shared(); - auto optim = make_shared( - net->parameters(), /*lr=*/0.0001, /*decay=*/0.95); - - auto trainLoop = train::BaseTrainLoop( - net, loss, optim, /*epochs=*/2000, /*bsize=*/6); - - trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); - - auto pred = (*net)(x); - auto finalLoss = (*loss)(y, pred); - - auto softmax = module::Softmax(); - EXPECT_LT((*finalLoss)[0], 0.05f) - << "Network failed to overfit multiclass dataset" - << "Final prediction: " << softmax(*pred) << "\nFinal loss: " << *finalLoss; -} - -TEST(OptimizerTest, ZeroGrad_ClearsAllGradients) { - auto x = TensorFunctions::makeSharedTensor( - {4, 2}, {0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0}, false); - auto y = TensorFunctions::makeSharedTensor( - {4, 1}, {0.0, 1.0, 1.0, 0.0}, false); - - auto net = makeBinaryNet(); - auto loss = std::make_shared(); - auto optim = std::make_shared( - net->parameters(), 0.01f); - - // one forward/backward pass to populate gradients - auto pred = (*net)(x); - auto l = (*loss)(y, pred); - l->backward(); - - // verify gradients are non-zero before zeroing - bool anyNonZero = false; - for(auto& p : net->parameters()) { - if(p->getGrads()) { - for(tensorSize_t i = 0; i < p->getGrads()->getSize(); i++) { - if((*p->getGrads())[i] != 0.0f) { - anyNonZero = true; - break; - } - } - } - } - EXPECT_TRUE(anyNonZero) << "Expected some non-zero gradients before zeroGrad"; - - // zero gradients - optim->zeroGrad(); - - // verify all gradients are zero after zeroing - for(auto& p : net->parameters()) { - if(p->getGrads()) { - for(tensorSize_t i = 0; i < p->getGrads()->getSize(); i++) { - EXPECT_FLOAT_EQ((*p->getGrads())[i], 0.0f) - << "Gradient not zeroed at index " << i; - } - } - } -} \ No newline at end of file From cde3a65cef527721b65bd649498ab67905bee94e Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Wed, 20 May 2026 14:55:39 +0200 Subject: [PATCH 54/73] Fixed unit tests --- tests/backend/cuda/test_tensorops_cuda.cu | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/backend/cuda/test_tensorops_cuda.cu b/tests/backend/cuda/test_tensorops_cuda.cu index b8e9f47..d34da1c 100644 --- a/tests/backend/cuda/test_tensorops_cuda.cu +++ b/tests/backend/cuda/test_tensorops_cuda.cu @@ -22,7 +22,7 @@ TEST(CudaTensorOpsTest, TestCtor) { auto t = Tensor({2, 2}, {2.0, 3.0, 4.0, 5.0}, Device::CUDA); ASSERT_EQ(t.getDims(), Dimension({2, 2})); - ASSERT_EQ(t.getDevice(), Device::CPU); + ASSERT_EQ(t.getDevice(), Device::CUDA); ASSERT_TRUE(!t.getRequiresGrad()); ASSERT_NEAR(t.get(0, 0), 2.0, 1e-5); @@ -165,7 +165,6 @@ TEST(CudaTensorOpsTest, ElementwiseMulGivesCorrectResults) { } TEST(CudaTensorOpsTest, ElementwiseMulThrowsOnDimensionMismatch) { - constexpr ftype factor = 0.5; auto t1 = TensorFunctions::Ones({2, 2}, Device::CUDA); auto t2 = TensorFunctions::Ones({2, 3}, Device::CUDA) * 0.5; @@ -222,7 +221,6 @@ TEST(CudaTensorOpsTest, MatMulGivesCorrectValues2) { auto expectedDims = std::vector{2, 2}; ASSERT_EQ(res.getDims().toVector(), expectedDims); - constexpr ftype resSum = 3.0; for(auto i=0; i Date: Wed, 20 May 2026 15:15:27 +0200 Subject: [PATCH 55/73] Clean up unit tests further, align indentation --- tests/CMakeLists.txt | 6 +- tests/backend/cuda/main.cu | 20 +- tests/backend/cuda/test_tensorops_cuda.cu | 120 +++---- tests/backend/test_computational_graph.cpp | 400 ++++++++++----------- tests/backend/test_losses.cpp | 308 ++++++++-------- tests/backend/test_module.cpp | 178 ++++----- tests/backend/test_tensorops.cpp | 196 +++++----- 7 files changed, 614 insertions(+), 614 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5d71801..c661c6a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,9 +1,9 @@ include(FetchContent) FetchContent_Declare( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG v1.14.0 + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG v1.14.0 ) FetchContent_MakeAvailable(googletest) diff --git a/tests/backend/cuda/main.cu b/tests/backend/cuda/main.cu index 1738d17..b8f92e9 100644 --- a/tests/backend/cuda/main.cu +++ b/tests/backend/cuda/main.cu @@ -17,17 +17,17 @@ static_assert(false, "File should not be compiled without CUDA enabled"); class CudaEnvironment : public ::testing::Environment { public: - void SetUp() override { - // cuda warmup to avoid context initialization costs - void* tmp; - cudaMalloc(&tmp, 1); - cudaFree(tmp); - cudaDeviceSynchronize(); - } + void SetUp() override { + // cuda warmup to avoid context initialization costs + void* tmp; + cudaMalloc(&tmp, 1); + cudaFree(tmp); + cudaDeviceSynchronize(); + } }; int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - ::testing::AddGlobalTestEnvironment(new CudaEnvironment()); - return RUN_ALL_TESTS(); + ::testing::InitGoogleTest(&argc, argv); + ::testing::AddGlobalTestEnvironment(new CudaEnvironment()); + return RUN_ALL_TESTS(); } \ No newline at end of file diff --git a/tests/backend/cuda/test_tensorops_cuda.cu b/tests/backend/cuda/test_tensorops_cuda.cu index d34da1c..5af4b24 100644 --- a/tests/backend/cuda/test_tensorops_cuda.cu +++ b/tests/backend/cuda/test_tensorops_cuda.cu @@ -74,17 +74,24 @@ TEST(CudaTensorOpsTest, TensorAdd) { } } +TEST(CudaTensorOpsTest, TensorAddThrowsOnDimMismatch) { + auto t1 = TensorFunctions::Ones({2, 2}, Device::CUDA); + auto t2 = TensorFunctions::Ones({2, 3}, Device::CUDA) * 4; + + ASSERT_THROW(t1 + t2, std::invalid_argument); +} + /* TEST(CudaAutogradTest, TensorAdd) { - auto t1 = TensorFunctions::makeSharedTensor({1}, {3.0}, Device::CUDA, true); - auto t2 = TensorFunctions::makeSharedTensor({1}, {2.0}, Device::CUDA, true); + auto t1 = TensorFunctions::makeSharedTensor({1}, {3.0}, Device::CUDA, true); + auto t2 = TensorFunctions::makeSharedTensor({1}, {2.0}, Device::CUDA, true); - auto t3 = cgraph::add(t1, t2); - auto loss = cgraph::mul(t3, t3); - - loss->backward(); - - ASSERT_NEAR(t1->getGrads()->get(0), 10.0, 1e-5); - ASSERT_NEAR(t2->getGrads()->get(0), 10.0, 1e-5); + auto t3 = cgraph::add(t1, t2); + auto loss = cgraph::mul(t3, t3); + + loss->backward(); + + ASSERT_NEAR(t1->getGrads()->get(0), 10.0, 1e-5); + ASSERT_NEAR(t2->getGrads()->get(0), 10.0, 1e-5); } */ TEST(CudaTensorOpsTest, BroadcastAdd) { @@ -104,20 +111,20 @@ TEST(CudaTensorOpsTest, BroadcastAdd) { } TEST(CudaTensorOpsTest, BroadcastAdd_2D) { - // (2,3) + (3) - auto t1 = Tensor({2, 3}, {1.0, 2.0, 3.0, - 4.0, 5.0, 6.0}, Device::CUDA); - auto t2 = Tensor({3}, {10.0, 20.0, 30.0}, Device::CUDA); - - auto res = t1 + t2; - - // expected: each row of t1 gets t2 added elementwise - ASSERT_NEAR(res.get(0, 0), 11.0, 1e-5); - ASSERT_NEAR(res.get(0, 1), 22.0, 1e-5); - ASSERT_NEAR(res.get(0, 2), 33.0, 1e-5); - ASSERT_NEAR(res.get(1, 0), 14.0, 1e-5); - ASSERT_NEAR(res.get(1, 1), 25.0, 1e-5); - ASSERT_NEAR(res.get(1, 2), 36.0, 1e-5); + // (2,3) + (3) + auto t1 = Tensor({2, 3}, {1.0, 2.0, 3.0, + 4.0, 5.0, 6.0}, Device::CUDA); + auto t2 = Tensor({3}, {10.0, 20.0, 30.0}, Device::CUDA); + + auto res = t1 + t2; + + // expected: each row of t1 gets t2 added elementwise + ASSERT_NEAR(res.get(0, 0), 11.0, 1e-5); + ASSERT_NEAR(res.get(0, 1), 22.0, 1e-5); + ASSERT_NEAR(res.get(0, 2), 33.0, 1e-5); + ASSERT_NEAR(res.get(1, 0), 14.0, 1e-5); + ASSERT_NEAR(res.get(1, 1), 25.0, 1e-5); + ASSERT_NEAR(res.get(1, 2), 36.0, 1e-5); } TEST(CudaTensorOpsTest, BroadcastAddNotCommutative) { @@ -127,14 +134,7 @@ TEST(CudaTensorOpsTest, BroadcastAddNotCommutative) { ASSERT_THROW(t2 + t1, std::invalid_argument); } -TEST(CudaTensorOpsTest, TensorAddThrowsOnDimMismatch) { - auto t1 = TensorFunctions::Ones({2, 2}, Device::CUDA); - auto t2 = TensorFunctions::Ones({2, 3}, Device::CUDA) * 4; - - ASSERT_THROW(t1 + t2, std::invalid_argument); -} - -TEST(CudaTensorOpsTest, MatrixAddGivesCorrectResults) { +TEST(CudaTensorOpsTest, MatrixAdd) { auto t1 = TensorFunctions::Ones({200, 200}, Device::CUDA); auto t2 = TensorFunctions::Ones({200, 200}, Device::CUDA); @@ -149,7 +149,7 @@ TEST(CudaTensorOpsTest, MatrixAddGivesCorrectResults) { } } -TEST(CudaTensorOpsTest, ElementwiseMulGivesCorrectResults) { +TEST(CudaTensorOpsTest, ElementwiseMul) { constexpr ftype factor = 0.5; auto t1 = TensorFunctions::Ones({200, 200}, Device::CUDA); auto t2 = TensorFunctions::Ones({200, 200}, Device::CUDA) * 0.5; @@ -171,31 +171,7 @@ TEST(CudaTensorOpsTest, ElementwiseMulThrowsOnDimensionMismatch) { ASSERT_THROW(t1 * t2, std::invalid_argument); } -TEST(CudaTensorOpsTest, MatMulGivesCorrectValues) { - auto t1 = TensorFunctions::Gaussian({1000, 10}, 2.0); - auto t2 = TensorFunctions::Gaussian({10, 1000}, 2.0); - auto resCpu = t1.matmul(t2); - - t1.setDevice(Device::CUDA); - t2.setDevice(Device::CUDA); - - auto resGpu = t1.matmul(t2); - resGpu.setDevice(Device::CPU); - - const auto expectedDims = resCpu.getDims().toVector(); - ASSERT_EQ(resGpu.getDims().toVector(), expectedDims); - - for(auto i = 0; i< resCpu.getDims().get(0); i++) { - for(auto j = 0; j < resCpu.getDims().get(1); j++) { - ASSERT_NEAR(resCpu.get(i, j), resGpu.get(i, j), 1e-4) - << "Mismatch at (" << i << ", " << j << ")" - << " cpu=" << resCpu.get(i, j) - << " gpu=" << resGpu.get(i, j); - } - } -} - -TEST(CudaTensorOpsTest, MatMulGivesCorrectValues2) { +TEST(CudaTensorOpsTest, MatMul) { auto t1 = Tensor({2, 2}); auto t2 = Tensor({2, 2}); @@ -228,6 +204,30 @@ TEST(CudaTensorOpsTest, MatMulGivesCorrectValues2) { } } +TEST(CudaTensorOpsTest, MatMulLarge) { + auto t1 = TensorFunctions::Gaussian({1000, 10}, 2.0); + auto t2 = TensorFunctions::Gaussian({10, 1000}, 2.0); + auto resCpu = t1.matmul(t2); + + t1.setDevice(Device::CUDA); + t2.setDevice(Device::CUDA); + + auto resGpu = t1.matmul(t2); + resGpu.setDevice(Device::CPU); + + const auto expectedDims = resCpu.getDims().toVector(); + ASSERT_EQ(resGpu.getDims().toVector(), expectedDims); + + for(auto i = 0; i< resCpu.getDims().get(0); i++) { + for(auto j = 0; j < resCpu.getDims().get(1); j++) { + ASSERT_NEAR(resCpu.get(i, j), resGpu.get(i, j), 1e-4) + << "Mismatch at (" << i << ", " << j << ")" + << " cpu=" << resCpu.get(i, j) + << " gpu=" << resGpu.get(i, j); + } + } +} + TEST(CudaTensorOpsTest, MatMulThrowsWhenDimensionsNotMatched) { auto t1 = TensorFunctions::Ones({2, 2}); auto t2 = TensorFunctions::Ones({3, 2}); @@ -235,7 +235,7 @@ TEST(CudaTensorOpsTest, MatMulThrowsWhenDimensionsNotMatched) { ASSERT_THROW(t1.matmul(t2), std::runtime_error); } -TEST(CudaTensorOpsTest, TransposeAsIntended1) { +TEST(CudaTensorOpsTest, MatrixTranspose1) { auto t = TensorFunctions::Gaussian({200, 200}, 1.0, Device::CUDA); auto transposed = t.createDeepCopy(); @@ -257,7 +257,7 @@ TEST(CudaTensorOpsTest, TransposeAsIntended1) { } // Swap first two dimensions. -TEST(CudaTensorOpsTest, TransposeAsIntended2) { +TEST(CudaTensorOpsTest, MatrixTranspose2) { auto t = TensorFunctions::Gaussian({10, 20, 200}, 1.0, Device::CUDA); auto transposed = t.createDeepCopy().transpose(0, 1); transposed.setDevice(Device::CPU); @@ -281,7 +281,7 @@ TEST(CudaTensorOpsTest, TransposeAsIntended2) { } // Swap first and last dimension. -TEST(CudaTensorOpsTest, TransposeAsIntended3) { +TEST(CudaTensorOpsTest, MatrixTranspose3) { auto t = TensorFunctions::Gaussian({10, 20, 200}, 1.0, Device::CUDA); auto transposed = t.createDeepCopy().transpose(0, -1); transposed.setDevice(Device::CPU); diff --git a/tests/backend/test_computational_graph.cpp b/tests/backend/test_computational_graph.cpp index b77091e..a3d4694 100644 --- a/tests/backend/test_computational_graph.cpp +++ b/tests/backend/test_computational_graph.cpp @@ -29,229 +29,229 @@ using namespace std; TEST(AutogradTest, ThrowsIfNoGradientSet) { - auto t1 = TensorFunctions::makeSharedTensor({1}, {3.0}, false); - auto t2 = TensorFunctions::makeSharedTensor({1}, {2.0}, false); + auto t1 = TensorFunctions::makeSharedTensor({1}, {3.0}, false); + auto t2 = TensorFunctions::makeSharedTensor({1}, {2.0}, false); - auto loss = cgraph::add(t1, t2); - - ASSERT_THROW(loss->backward(), std::runtime_error); + auto loss = cgraph::add(t1, t2); + + ASSERT_THROW(loss->backward(), std::runtime_error); } TEST(AutogradTest, ChainRule) { - auto x = TensorFunctions::makeSharedTensor({1}, {2.0}, true); - - auto y = cgraph::mul(x, x); // y = x^2 - auto z = cgraph::add(x, y); // z = x^2 + x - auto loss = cgraph::mul(z, z); // loss = (x^2 + x)^2 - - loss->backward(); - - // dloss/dx = 2(x^2 + x) * (2x + 1) - // At x=2: 2(4 + 2) * (4 + 1) = 2 * 6 * 5 = 60 - ASSERT_NEAR(x->getGrads()->get(0), 60.0, 1e-5); + auto x = TensorFunctions::makeSharedTensor({1}, {2.0}, true); + + auto y = cgraph::mul(x, x); // y = x^2 + auto z = cgraph::add(x, y); // z = x^2 + x + auto loss = cgraph::mul(z, z); // loss = (x^2 + x)^2 + + loss->backward(); + + // dloss/dx = 2(x^2 + x) * (2x + 1) + // At x=2: 2(4 + 2) * (4 + 1) = 2 * 6 * 5 = 60 + ASSERT_NEAR(x->getGrads()->get(0), 60.0, 1e-5); } TEST(AutogradTest, MultiVariateChainRule) { - auto x = TensorFunctions::makeSharedTensor({2}, {1.0, 2.0}, true); - - auto y = cgraph::mul(x, 3.0); // y = [3, 6] - auto loss = TensorFunctions::makeSharedTensor({1}, {0.0}, true); - for(int i=0; igetSize(); i++){ - loss = cgraph::add(loss, cgraph::get(y, i)); - } // loss = 9 - - loss->backward(); - - // dloss/dx = scalar = 3 - ASSERT_NEAR(x->getGrads()->get(0), 3.0, 1e-5); - ASSERT_NEAR(x->getGrads()->get(1), 3.0, 1e-5); - - ASSERT_NEAR(y->getGrads()->get(0), 1.0, 1e-5); - ASSERT_NEAR(y->getGrads()->get(1), 1.0, 1e-5); + auto x = TensorFunctions::makeSharedTensor({2}, {1.0, 2.0}, true); + + auto y = cgraph::mul(x, 3.0); // y = [3, 6] + auto loss = TensorFunctions::makeSharedTensor({1}, {0.0}, true); + for(int i=0; igetSize(); i++){ + loss = cgraph::add(loss, cgraph::get(y, i)); + } // loss = 9 + + loss->backward(); + + // dloss/dx = scalar = 3 + ASSERT_NEAR(x->getGrads()->get(0), 3.0, 1e-5); + ASSERT_NEAR(x->getGrads()->get(1), 3.0, 1e-5); + + ASSERT_NEAR(y->getGrads()->get(0), 1.0, 1e-5); + ASSERT_NEAR(y->getGrads()->get(1), 1.0, 1e-5); } TEST(OverfitTest, BceSgdOverfitsSmallDataset) { - // XOR-like: 4 samples, 2 features, binary labels - auto x = TensorFunctions::makeSharedTensor( - {4, 2}, {0.0, 0.0, - 0.0, 1.0, - 1.0, 0.0, - 1.0, 1.0}, false); - - auto y = TensorFunctions::makeSharedTensor( - {4, 1}, {0.0, - 1.0, - 1.0, - 0.0}, false); - - auto net = makeBinaryNet(); - auto loss = make_shared(); - auto optim = make_shared( - net->parameters(), /*lr=*/0.05); - - auto trainLoop = train::BaseTrainLoop( - net, loss, optim, /*epochs=*/2000, /*bsize=*/static_cast(4)); - - trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); - - // forward one more time to get final loss - auto pred = (*net)(x); - auto finalLoss = (*loss)(y, pred); - - EXPECT_LT((*finalLoss)[0], 0.05f) - << "Network failed to overfit binary dataset\n" - << "Final prediction: " << *pred << "\nFinal loss: " << *finalLoss; + // XOR-like: 4 samples, 2 features, binary labels + auto x = TensorFunctions::makeSharedTensor( + {4, 2}, {0.0, 0.0, + 0.0, 1.0, + 1.0, 0.0, + 1.0, 1.0}, false); + + auto y = TensorFunctions::makeSharedTensor( + {4, 1}, {0.0, + 1.0, + 1.0, + 0.0}, false); + + auto net = makeBinaryNet(); + auto loss = make_shared(); + auto optim = make_shared( + net->parameters(), /*lr=*/0.05); + + auto trainLoop = train::BaseTrainLoop( + net, loss, optim, /*epochs=*/2000, /*bsize=*/static_cast(4)); + + trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); + + // forward one more time to get final loss + auto pred = (*net)(x); + auto finalLoss = (*loss)(y, pred); + + EXPECT_LT((*finalLoss)[0], 0.05f) + << "Network failed to overfit binary dataset\n" + << "Final prediction: " << *pred << "\nFinal loss: " << *finalLoss; } TEST(OverfitTest, BceSgdOverfitsSmallDataset_OptimizedLoss) { - // XOR-like: 4 samples, 2 features, binary labels - auto x = TensorFunctions::makeSharedTensor( - {4, 2}, {0.0, 0.0, - 0.0, 1.0, - 1.0, 0.0, - 1.0, 1.0}, false); - - auto y = TensorFunctions::makeSharedTensor( - {4, 1}, {0.0, - 1.0, - 1.0, - 0.0}, false); - - auto net = makeBinaryNet2(); - auto loss = make_shared(); - auto optim = make_shared( - net->parameters(), /*lr=*/0.05); - - auto trainLoop = train::BaseTrainLoop( - net, loss, optim, /*epochs=*/2000, /*bsize=*/static_cast(4)); - - trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); - - // forward one more time to get final loss - auto pred = (*net)(x); - auto finalLoss = (*loss)(y, pred); - - auto sigmoid = module::Sigmoid(); - EXPECT_LT((*finalLoss)[0], 0.05f) - << "Network failed to overfit binary dataset\n" - << "Final prediction: " << sigmoid(*pred) << "\nFinal loss: " << *finalLoss; + // XOR-like: 4 samples, 2 features, binary labels + auto x = TensorFunctions::makeSharedTensor( + {4, 2}, {0.0, 0.0, + 0.0, 1.0, + 1.0, 0.0, + 1.0, 1.0}, false); + + auto y = TensorFunctions::makeSharedTensor( + {4, 1}, {0.0, + 1.0, + 1.0, + 0.0}, false); + + auto net = makeBinaryNet2(); + auto loss = make_shared(); + auto optim = make_shared( + net->parameters(), /*lr=*/0.05); + + auto trainLoop = train::BaseTrainLoop( + net, loss, optim, /*epochs=*/2000, /*bsize=*/static_cast(4)); + + trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); + + // forward one more time to get final loss + auto pred = (*net)(x); + auto finalLoss = (*loss)(y, pred); + + auto sigmoid = module::Sigmoid(); + EXPECT_LT((*finalLoss)[0], 0.05f) + << "Network failed to overfit binary dataset\n" + << "Final prediction: " << sigmoid(*pred) << "\nFinal loss: " << *finalLoss; } TEST(OverfitTest, CrossEntropyRMSPropOverfitsSmallDataset) { - // 6 samples, 2 features, 3 classes - auto x = TensorFunctions::makeSharedTensor( - {6, 2}, {1.0, 0.0, - 1.0, 0.1, - 0.0, 1.0, - 0.1, 1.0, - 0.5, 0.5, - 0.4, 0.6}, false); - - // one-hot encoded labels - auto y = TensorFunctions::makeSharedTensor( - {6, 3}, {1.0, 0.0, 0.0, - 1.0, 0.0, 0.0, - 0.0, 1.0, 0.0, - 0.0, 1.0, 0.0, - 0.0, 0.0, 1.0, - 0.0, 0.0, 1.0}, false); - - auto net = makeMulticlassNet(); - auto loss = make_shared(); - auto optim = make_shared( - net->parameters(), /*lr=*/0.0001, /*decay=*/0.95); - - auto trainLoop = train::BaseTrainLoop( - net, loss, optim, /*epochs=*/2000, /*bsize=*/6); - - trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); - - auto pred = (*net)(x); - auto finalLoss = (*loss)(y, pred); - - EXPECT_LT((*finalLoss)[0], 0.05f) - << "Network failed to overfit multiclass dataset" - << "Final prediction: " << *pred << "\nFinal loss: " << *finalLoss; + // 6 samples, 2 features, 3 classes + auto x = TensorFunctions::makeSharedTensor( + {6, 2}, {1.0, 0.0, + 1.0, 0.1, + 0.0, 1.0, + 0.1, 1.0, + 0.5, 0.5, + 0.4, 0.6}, false); + + // one-hot encoded labels + auto y = TensorFunctions::makeSharedTensor( + {6, 3}, {1.0, 0.0, 0.0, + 1.0, 0.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 0.0, 1.0, + 0.0, 0.0, 1.0}, false); + + auto net = makeMulticlassNet(); + auto loss = make_shared(); + auto optim = make_shared( + net->parameters(), /*lr=*/0.0001, /*decay=*/0.95); + + auto trainLoop = train::BaseTrainLoop( + net, loss, optim, /*epochs=*/2000, /*bsize=*/6); + + trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); + + auto pred = (*net)(x); + auto finalLoss = (*loss)(y, pred); + + EXPECT_LT((*finalLoss)[0], 0.05f) + << "Network failed to overfit multiclass dataset" + << "Final prediction: " << *pred << "\nFinal loss: " << *finalLoss; } TEST(OverfitTest, CrossEntropyRMSPropOverfitsSmallDataset_OptimizedLoss) { - // 6 samples, 2 features, 3 classes - auto x = TensorFunctions::makeSharedTensor( - {6, 2}, {1.0, 0.0, - 1.0, 0.1, - 0.0, 1.0, - 0.1, 1.0, - 0.5, 0.5, - 0.4, 0.6}, false); - - // one-hot encoded labels - auto y = TensorFunctions::makeSharedTensor( - {6, 3}, {1.0, 0.0, 0.0, - 1.0, 0.0, 0.0, - 0.0, 1.0, 0.0, - 0.0, 1.0, 0.0, - 0.0, 0.0, 1.0, - 0.0, 0.0, 1.0}, false); - - auto net = makeMulticlassNet2(); - auto loss = make_shared(); - auto optim = make_shared( - net->parameters(), /*lr=*/0.0001, /*decay=*/0.95); - - auto trainLoop = train::BaseTrainLoop( - net, loss, optim, /*epochs=*/2000, /*bsize=*/6); - - trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); - - auto pred = (*net)(x); - auto finalLoss = (*loss)(y, pred); - - auto softmax = module::Softmax(); - EXPECT_LT((*finalLoss)[0], 0.05f) - << "Network failed to overfit multiclass dataset" - << "Final prediction: " << softmax(*pred) << "\nFinal loss: " << *finalLoss; + // 6 samples, 2 features, 3 classes + auto x = TensorFunctions::makeSharedTensor( + {6, 2}, {1.0, 0.0, + 1.0, 0.1, + 0.0, 1.0, + 0.1, 1.0, + 0.5, 0.5, + 0.4, 0.6}, false); + + // one-hot encoded labels + auto y = TensorFunctions::makeSharedTensor( + {6, 3}, {1.0, 0.0, 0.0, + 1.0, 0.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 0.0, 1.0, + 0.0, 0.0, 1.0}, false); + + auto net = makeMulticlassNet2(); + auto loss = make_shared(); + auto optim = make_shared( + net->parameters(), /*lr=*/0.0001, /*decay=*/0.95); + + auto trainLoop = train::BaseTrainLoop( + net, loss, optim, /*epochs=*/2000, /*bsize=*/6); + + trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); + + auto pred = (*net)(x); + auto finalLoss = (*loss)(y, pred); + + auto softmax = module::Softmax(); + EXPECT_LT((*finalLoss)[0], 0.05f) + << "Network failed to overfit multiclass dataset" + << "Final prediction: " << softmax(*pred) << "\nFinal loss: " << *finalLoss; } TEST(OptimizerTest, ZeroGrad_ClearsAllGradients) { - auto x = TensorFunctions::makeSharedTensor( - {4, 2}, {0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0}, false); - auto y = TensorFunctions::makeSharedTensor( - {4, 1}, {0.0, 1.0, 1.0, 0.0}, false); - - auto net = makeBinaryNet(); - auto loss = std::make_shared(); - auto optim = std::make_shared( - net->parameters(), 0.01f); - - // one forward/backward pass to populate gradients - auto pred = (*net)(x); - auto l = (*loss)(y, pred); - l->backward(); - - // verify gradients are non-zero before zeroing - bool anyNonZero = false; - for(auto& p : net->parameters()) { - if(p->getGrads()) { - for(tensorSize_t i = 0; i < p->getGrads()->getSize(); i++) { - if((*p->getGrads())[i] != 0.0f) { - anyNonZero = true; - break; - } - } + auto x = TensorFunctions::makeSharedTensor( + {4, 2}, {0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0}, false); + auto y = TensorFunctions::makeSharedTensor( + {4, 1}, {0.0, 1.0, 1.0, 0.0}, false); + + auto net = makeBinaryNet(); + auto loss = std::make_shared(); + auto optim = std::make_shared( + net->parameters(), 0.01f); + + // one forward/backward pass to populate gradients + auto pred = (*net)(x); + auto l = (*loss)(y, pred); + l->backward(); + + // verify gradients are non-zero before zeroing + bool anyNonZero = false; + for(auto& p : net->parameters()) { + if(p->getGrads()) { + for(tensorSize_t i = 0; i < p->getGrads()->getSize(); i++) { + if((*p->getGrads())[i] != 0.0f) { + anyNonZero = true; + break; } + } } - EXPECT_TRUE(anyNonZero) << "Expected some non-zero gradients before zeroGrad"; - - // zero gradients - optim->zeroGrad(); - - // verify all gradients are zero after zeroing - for(auto& p : net->parameters()) { - if(p->getGrads()) { - for(tensorSize_t i = 0; i < p->getGrads()->getSize(); i++) { - ASSERT_NEAR((*p->getGrads())[i], 0.0f, 1e-5) - << "Gradient not zeroed at index " << i; - } - } + } + EXPECT_TRUE(anyNonZero) << "Expected some non-zero gradients before zeroGrad"; + + // zero gradients + optim->zeroGrad(); + + // verify all gradients are zero after zeroing + for(auto& p : net->parameters()) { + if(p->getGrads()) { + for(tensorSize_t i = 0; i < p->getGrads()->getSize(); i++) { + ASSERT_NEAR((*p->getGrads())[i], 0.0f, 1e-5) + << "Gradient not zeroed at index " << i; + } } + } } \ No newline at end of file diff --git a/tests/backend/test_losses.cpp b/tests/backend/test_losses.cpp index 3b8b8ac..efde0ef 100644 --- a/tests/backend/test_losses.cpp +++ b/tests/backend/test_losses.cpp @@ -24,221 +24,221 @@ using namespace train; static constexpr ftype delta = 1e-4f; TEST(LossTest, CrossEntropyFoward) { - auto y = TensorFunctions::makeSharedTensor( - {2, 3}, {1.0, 0.0, 0.0, - 0.0, 1.0, 0.0}, false); + auto y = TensorFunctions::makeSharedTensor( + {2, 3}, {1.0, 0.0, 0.0, + 0.0, 1.0, 0.0}, false); - auto ypred = TensorFunctions::makeSharedTensor( - {2, 3}, {0.7, 0.2, 0.1, - 0.1, 0.8, 0.1}, true); + auto ypred = TensorFunctions::makeSharedTensor( + {2, 3}, {0.7, 0.2, 0.1, + 0.1, 0.8, 0.1}, true); - CrossEntropyLoss loss; - auto result = loss(y, ypred); + CrossEntropyLoss loss; + auto result = loss(y, ypred); - // expected: -( log(0.7) + log(0.8) ) / 2 = 0.2899 - const ftype expected = -(std::log(0.7f) + std::log(0.8f)) / 2.0f; - ASSERT_NEAR((*result)[0], expected, delta); + // expected: -( log(0.7) + log(0.8) ) / 2 = 0.2899 + const ftype expected = -(std::log(0.7f) + std::log(0.8f)) / 2.0f; + ASSERT_NEAR((*result)[0], expected, delta); } TEST(LossTest, CrossEntropyPerfectPrediction) { - auto y = TensorFunctions::makeSharedTensor( - {2, 3}, {1.0, 0.0, 0.0, - 0.0, 1.0, 0.0}, false); + auto y = TensorFunctions::makeSharedTensor( + {2, 3}, {1.0, 0.0, 0.0, + 0.0, 1.0, 0.0}, false); - // near-perfect predictions — can't use exactly 1.0 due to log(0) - auto ypred = TensorFunctions::makeSharedTensor( - {2, 3}, {0.999, 0.0005, 0.0005, - 0.0005, 0.999, 0.0005}, true); + // near-perfect predictions — can't use exactly 1.0 due to log(0) + auto ypred = TensorFunctions::makeSharedTensor( + {2, 3}, {0.999, 0.0005, 0.0005, + 0.0005, 0.999, 0.0005}, true); - CrossEntropyLoss loss; - auto result = loss(y, ypred); + CrossEntropyLoss loss; + auto result = loss(y, ypred); - // loss should be very small - ASSERT_LT((*result)[0], 0.01f); + // loss should be very small + ASSERT_LT((*result)[0], 0.01f); } TEST(LossTest, CrossEntropyUniformPrediction) { - // uniform prediction should give log(3) ~ 1.0986 - auto y = TensorFunctions::makeSharedTensor( - {1, 3}, {1.0, 0.0, 0.0}, false); + // uniform prediction should give log(3) ~ 1.0986 + auto y = TensorFunctions::makeSharedTensor( + {1, 3}, {1.0, 0.0, 0.0}, false); - auto ypred = TensorFunctions::makeSharedTensor( - {1, 3}, {1.0f/3, 1.0f/3, 1.0f/3}, true); + auto ypred = TensorFunctions::makeSharedTensor( + {1, 3}, {1.0f/3, 1.0f/3, 1.0f/3}, true); - CrossEntropyLoss loss; - auto result = loss(y, ypred); + CrossEntropyLoss loss; + auto result = loss(y, ypred); - ASSERT_NEAR((*result)[0], std::log(3.0f), delta); + ASSERT_NEAR((*result)[0], std::log(3.0f), delta); } TEST(LossTest, CrossEntropyThrowsOnDimMismatch) { - auto y = TensorFunctions::makeSharedTensor( - {2, 3}, {1.0, 0.0, 0.0, 0.0, 1.0, 0.0}, false); - auto ypred = TensorFunctions::makeSharedTensor( - {2, 2}, {0.5, 0.5, 0.5, 0.5}, true); + auto y = TensorFunctions::makeSharedTensor( + {2, 3}, {1.0, 0.0, 0.0, 0.0, 1.0, 0.0}, false); + auto ypred = TensorFunctions::makeSharedTensor( + {2, 2}, {0.5, 0.5, 0.5, 0.5}, true); - CrossEntropyLoss loss; - ASSERT_THROW(loss(y, ypred), std::invalid_argument); + CrossEntropyLoss loss; + ASSERT_THROW(loss(y, ypred), std::invalid_argument); } TEST(LossTest, CrossEntropyBackward) { - // y = [[1,0,0],[0,1,0]], ypred = [[0.7,0.2,0.1],[0.1,0.8,0.1]] - // grad CE w.r.t. ypred[b,i] = -y[b,i] / (ypred[b,i] * n) - // grad[0,0] = -1/(0.7*2) = -0.7143 - // grad[0,1] = 0 - // grad[0,2] = 0 - // grad[1,0] = 0 - // grad[1,1] = -1/(0.8*2) = -0.625 - // grad[1,2] = 0 - auto y = TensorFunctions::makeSharedTensor( - {2, 3}, {1.0, 0.0, 0.0, - 0.0, 1.0, 0.0}, false); - auto ypred = TensorFunctions::makeSharedTensor( - {2, 3}, {0.7, 0.2, 0.1, - 0.1, 0.8, 0.1}, true); - - CrossEntropyLoss loss; - auto result = loss(y, ypred); - result->backward(); - - auto grads = ypred->getGrads(); - ASSERT_NEAR((*grads)[0], -0.7143f, delta); - ASSERT_NEAR((*grads)[1], 0.0f, delta); - ASSERT_NEAR((*grads)[2], 0.0f, delta); - ASSERT_NEAR((*grads)[3], 0.0f, delta); - ASSERT_NEAR((*grads)[4], -0.625f, delta); - ASSERT_NEAR((*grads)[5], 0.0f, delta); + // y = [[1,0,0],[0,1,0]], ypred = [[0.7,0.2,0.1],[0.1,0.8,0.1]] + // grad CE w.r.t. ypred[b,i] = -y[b,i] / (ypred[b,i] * n) + // grad[0,0] = -1/(0.7*2) = -0.7143 + // grad[0,1] = 0 + // grad[0,2] = 0 + // grad[1,0] = 0 + // grad[1,1] = -1/(0.8*2) = -0.625 + // grad[1,2] = 0 + auto y = TensorFunctions::makeSharedTensor( + {2, 3}, {1.0, 0.0, 0.0, + 0.0, 1.0, 0.0}, false); + auto ypred = TensorFunctions::makeSharedTensor( + {2, 3}, {0.7, 0.2, 0.1, + 0.1, 0.8, 0.1}, true); + + CrossEntropyLoss loss; + auto result = loss(y, ypred); + result->backward(); + + auto grads = ypred->getGrads(); + ASSERT_NEAR((*grads)[0], -0.7143f, delta); + ASSERT_NEAR((*grads)[1], 0.0f, delta); + ASSERT_NEAR((*grads)[2], 0.0f, delta); + ASSERT_NEAR((*grads)[3], 0.0f, delta); + ASSERT_NEAR((*grads)[4], -0.625f, delta); + ASSERT_NEAR((*grads)[5], 0.0f, delta); } TEST(LossTest, BceForward) { - auto y = TensorFunctions::makeSharedTensor( - {4, 1}, {0.0, 1.0, 1.0, 0.0}, false); + auto y = TensorFunctions::makeSharedTensor( + {4, 1}, {0.0, 1.0, 1.0, 0.0}, false); - auto ypred = TensorFunctions::makeSharedTensor( - {4, 1}, {0.1, 0.9, 0.8, 0.2}, true); + auto ypred = TensorFunctions::makeSharedTensor( + {4, 1}, {0.1, 0.9, 0.8, 0.2}, true); - BceLoss loss; - auto result = loss(y, ypred); + BceLoss loss; + auto result = loss(y, ypred); - // expected: -( log(0.9) + log(0.9) + log(0.8) + log(0.8) ) / 4 = 0.1643 - const ftype expected = -(std::log(0.9f) + std::log(0.9f) + - std::log(0.8f) + std::log(0.8f)) / 4.0f; - ASSERT_NEAR((*result)[0], expected, delta); + // expected: -( log(0.9) + log(0.9) + log(0.8) + log(0.8) ) / 4 = 0.1643 + const ftype expected = -(std::log(0.9f) + std::log(0.9f) + + std::log(0.8f) + std::log(0.8f)) / 4.0f; + ASSERT_NEAR((*result)[0], expected, delta); } TEST(LossTest, BcePerfectPrediction) { - auto y = TensorFunctions::makeSharedTensor( - {2, 1}, {1.0, 0.0}, false); + auto y = TensorFunctions::makeSharedTensor( + {2, 1}, {1.0, 0.0}, false); - auto ypred = TensorFunctions::makeSharedTensor( - {2, 1}, {0.999, 0.001}, true); + auto ypred = TensorFunctions::makeSharedTensor( + {2, 1}, {0.999, 0.001}, true); - BceLoss loss; - auto result = loss(y, ypred); + BceLoss loss; + auto result = loss(y, ypred); - ASSERT_LT((*result)[0], 0.01f); + ASSERT_LT((*result)[0], 0.01f); } TEST(LossTest, BceRandomPrediction) { - // ypred = 0.5 for all -> loss = log(2) ~ 0.6931 - auto y = TensorFunctions::makeSharedTensor( - {2, 1}, {1.0, 0.0}, false); + // ypred = 0.5 for all -> loss = log(2) ~ 0.6931 + auto y = TensorFunctions::makeSharedTensor( + {2, 1}, {1.0, 0.0}, false); - auto ypred = TensorFunctions::makeSharedTensor( - {2, 1}, {0.5, 0.5}, true); + auto ypred = TensorFunctions::makeSharedTensor( + {2, 1}, {0.5, 0.5}, true); - BceLoss loss; - auto result = loss(y, ypred); + BceLoss loss; + auto result = loss(y, ypred); - ASSERT_NEAR((*result)[0], std::log(2.0f), delta); + ASSERT_NEAR((*result)[0], std::log(2.0f), delta); } TEST(LossTest, BceThrowsOnDimMismatch) { - auto y = TensorFunctions::makeSharedTensor( - {2, 1}, {1.0, 0.0}, false); - auto ypred = TensorFunctions::makeSharedTensor( - {3, 1}, {0.5, 0.5, 0.5}, true); + auto y = TensorFunctions::makeSharedTensor( + {2, 1}, {1.0, 0.0}, false); + auto ypred = TensorFunctions::makeSharedTensor( + {3, 1}, {0.5, 0.5, 0.5}, true); - BceLoss loss; - ASSERT_THROW(loss(y, ypred), std::invalid_argument); + BceLoss loss; + ASSERT_THROW(loss(y, ypred), std::invalid_argument); } TEST(LossTest, BceNoInfOrNanOnNearZeroPred) { - auto y = TensorFunctions::makeSharedTensor( - {1, 1}, {1.0}, false); - auto ypred = TensorFunctions::makeSharedTensor( - {1, 1}, {0.0}, true); + auto y = TensorFunctions::makeSharedTensor( + {1, 1}, {1.0}, false); + auto ypred = TensorFunctions::makeSharedTensor( + {1, 1}, {0.0}, true); - BceLoss loss; - auto result = loss(y, ypred); + BceLoss loss; + auto result = loss(y, ypred); - // clipping prevents log(0) - ASSERT_FALSE(std::isinf((*result)[0])); + // clipping prevents log(0) + ASSERT_FALSE(std::isinf((*result)[0])); } TEST(LossTest, BceBackward) { - // y = [1, 0], ypred = [0.8, 0.3] - // grad BCE w.r.t. ypred_i = (-y/ypred + (1-y)/(1-ypred)) / n - // grad[0] = (-1/0.8 + 0) / 2 = -0.625 - // grad[1] = (0 + 1/0.7) / 2 = 0.7143 - auto y = TensorFunctions::makeSharedTensor( - {2, 1}, {1.0, 0.0}, false); - auto ypred = TensorFunctions::makeSharedTensor( - {2, 1}, {0.8, 0.3}, true); - - BceLoss loss; - auto result = loss(y, ypred); - result->backward(); - - auto grads = ypred->getGrads(); - ASSERT_NEAR((*grads)[0], -0.625f, delta); - ASSERT_NEAR((*grads)[1], 0.7143f, delta); + // y = [1, 0], ypred = [0.8, 0.3] + // grad BCE w.r.t. ypred_i = (-y/ypred + (1-y)/(1-ypred)) / n + // grad[0] = (-1/0.8 + 0) / 2 = -0.625 + // grad[1] = (0 + 1/0.7) / 2 = 0.7143 + auto y = TensorFunctions::makeSharedTensor( + {2, 1}, {1.0, 0.0}, false); + auto ypred = TensorFunctions::makeSharedTensor( + {2, 1}, {0.8, 0.3}, true); + + BceLoss loss; + auto result = loss(y, ypred); + result->backward(); + + auto grads = ypred->getGrads(); + ASSERT_NEAR((*grads)[0], -0.625f, delta); + ASSERT_NEAR((*grads)[1], 0.7143f, delta); } TEST(LossTest, RmseForward) { - // y = [1, 2, 3], ypred = [1.5, 2.5, 2.5] - // diffs = [-0.5, -0.5, 0.5] - // MSE = (0.25 + 0.25 + 0.25) / 3 = 0.25 - // RMSE = 0.5 - auto y = TensorFunctions::makeSharedTensor( - {3}, {1.0, 2.0, 3.0}, false); - auto ypred = TensorFunctions::makeSharedTensor( - {3}, {1.5, 2.5, 2.5}, true); - - auto loss = RmseLoss{}; - auto result = loss(y, ypred); - - ASSERT_NEAR((*result)[0], 0.5f, delta); + // y = [1, 2, 3], ypred = [1.5, 2.5, 2.5] + // diffs = [-0.5, -0.5, 0.5] + // MSE = (0.25 + 0.25 + 0.25) / 3 = 0.25 + // RMSE = 0.5 + auto y = TensorFunctions::makeSharedTensor( + {3}, {1.0, 2.0, 3.0}, false); + auto ypred = TensorFunctions::makeSharedTensor( + {3}, {1.5, 2.5, 2.5}, true); + + auto loss = RmseLoss{}; + auto result = loss(y, ypred); + + ASSERT_NEAR((*result)[0], 0.5f, delta); } TEST(LossTest, RmsePerfectPrediction) { - auto y = TensorFunctions::makeSharedTensor( - {3}, {1.0, 2.0, 3.0}, false); - auto ypred = TensorFunctions::makeSharedTensor( - {3}, {1.0, 2.0, 3.0}, true); + auto y = TensorFunctions::makeSharedTensor( + {3}, {1.0, 2.0, 3.0}, false); + auto ypred = TensorFunctions::makeSharedTensor( + {3}, {1.0, 2.0, 3.0}, true); - RmseLoss loss; - auto result = loss(y, ypred); + RmseLoss loss; + auto result = loss(y, ypred); - ASSERT_NEAR((*result)[0], 0.0f, delta); + ASSERT_NEAR((*result)[0], 0.0f, delta); } TEST(LossTest, RmseBackward) { - // y = [1, 0], ypred = [0.5, 0.5] - // diffs = [0.5, -0.5], MSE = 0.25, RMSE = 0.5 - // grad_i = -(y_i - ypred_i) / (n * RMSE) - // grad[0] = -(1 - 0.5) / (2 * 0.5) = -0.5 - // grad[1] = -(0 - 0.5) / (2 * 0.5) = 0.5 - auto y = TensorFunctions::makeSharedTensor( - {2}, {1.0, 0.0}, false); - auto ypred = TensorFunctions::makeSharedTensor( - {2}, {0.5, 0.5}, true); - - RmseLoss loss; - auto result = loss(y, ypred); - result->backward(); - - auto grads = ypred->getGrads(); - ASSERT_NEAR((*grads)[0], -0.5f, delta); - ASSERT_NEAR((*grads)[1], 0.5f, delta); + // y = [1, 0], ypred = [0.5, 0.5] + // diffs = [0.5, -0.5], MSE = 0.25, RMSE = 0.5 + // grad_i = -(y_i - ypred_i) / (n * RMSE) + // grad[0] = -(1 - 0.5) / (2 * 0.5) = -0.5 + // grad[1] = -(0 - 0.5) / (2 * 0.5) = 0.5 + auto y = TensorFunctions::makeSharedTensor( + {2}, {1.0, 0.0}, false); + auto ypred = TensorFunctions::makeSharedTensor( + {2}, {0.5, 0.5}, true); + + RmseLoss loss; + auto result = loss(y, ypred); + result->backward(); + + auto grads = ypred->getGrads(); + ASSERT_NEAR((*grads)[0], -0.5f, delta); + ASSERT_NEAR((*grads)[1], 0.5f, delta); } \ No newline at end of file diff --git a/tests/backend/test_module.cpp b/tests/backend/test_module.cpp index 24e12aa..ca0b1ac 100644 --- a/tests/backend/test_module.cpp +++ b/tests/backend/test_module.cpp @@ -54,13 +54,13 @@ TEST(AutogradTest, ReLUBackward) { auto y = relu(x); // [0, 0, 2] auto loss = cgraph::sumTensor(y); // loss = 2 - - loss->backward(); - - // Gradient: [0, 0, 1] (only where input > 0) - ASSERT_DOUBLE_EQ(x->getGrads()->get(0), 0.0); - ASSERT_DOUBLE_EQ(x->getGrads()->get(1), 0.0); - ASSERT_NEAR(x->getGrads()->get(2), 1.0, 1e-5); + + loss->backward(); + + // Gradient: [0, 0, 1] (only where input > 0) + ASSERT_DOUBLE_EQ(x->getGrads()->get(0), 0.0); + ASSERT_DOUBLE_EQ(x->getGrads()->get(1), 0.0); + ASSERT_NEAR(x->getGrads()->get(2), 1.0, 1e-5); } TEST(ActivationTest, LeakyReluForward) { @@ -88,118 +88,118 @@ TEST(ActivationTest, LeakyReluInputNegative) { } TEST(AutogradTest, LeakyReLUBackward) { - auto x = TensorFunctions::makeSharedTensor({3}, {-1.0, 0.0, 2.0}, true); + auto x = TensorFunctions::makeSharedTensor({3}, {-1.0, 0.0, 2.0}, true); - constexpr ftype eps = 0.3; - auto relu = module::LeakyReLu(eps); + constexpr ftype eps = 0.3; + auto relu = module::LeakyReLu(eps); - auto y = relu(x); // [0, 0, 2] - auto loss = cgraph::sumTensor(y); // loss = 2 - - loss->backward(); - - // Gradient: [0, 0, 1] (only where input > 0) - ASSERT_DOUBLE_EQ(x->getGrads()->get(0), eps); - ASSERT_DOUBLE_EQ(x->getGrads()->get(1), eps); // by convention - ASSERT_NEAR(x->getGrads()->get(2), 1.0, 1e-5); + auto y = relu(x); // [0, 0, 2] + auto loss = cgraph::sumTensor(y); // loss = 2 + + loss->backward(); + + // Gradient: [0, 0, 1] (only where input > 0) + ASSERT_DOUBLE_EQ(x->getGrads()->get(0), eps); + ASSERT_DOUBLE_EQ(x->getGrads()->get(1), eps); // by convention + ASSERT_NEAR(x->getGrads()->get(2), 1.0, 1e-5); } TEST(ActivationTest, SigmoidForward) { - // sigmoid(0) = 0.5, sigmoid(1) = 0.7311, sigmoid(-1) = 0.2689 - auto t = Tensor({3}, {0.0, 1.0, -1.0}); - - module::Sigmoid sig; - auto res = sig(t); + // sigmoid(0) = 0.5, sigmoid(1) = 0.7311, sigmoid(-1) = 0.2689 + auto t = Tensor({3}, {0.0, 1.0, -1.0}); - ASSERT_NEAR(res[0], 0.5, 1e-4); - ASSERT_NEAR(res[1], 0.7311, 1e-4); - ASSERT_NEAR(res[2], 0.2689, 1e-4); + module::Sigmoid sig; + auto res = sig(t); + + ASSERT_NEAR(res[0], 0.5, 1e-4); + ASSERT_NEAR(res[1], 0.7311, 1e-4); + ASSERT_NEAR(res[2], 0.2689, 1e-4); } TEST(ActivationTest, SigmoidLargePositive) { - // sigmoid(100) should be ~1, not inf or nan - auto t = Tensor({1}, vector{100.0}); - - module::Sigmoid sig; - auto res = sig(t); + // sigmoid(100) should be ~1, not inf or nan + auto t = Tensor({1}, vector{100.0}); - ASSERT_NEAR(res[0], 1.0, 1e-5); - EXPECT_FALSE(std::isnan(res[0])); - EXPECT_FALSE(std::isinf(res[0])); + module::Sigmoid sig; + auto res = sig(t); + + ASSERT_NEAR(res[0], 1.0, 1e-5); + EXPECT_FALSE(std::isnan(res[0])); + EXPECT_FALSE(std::isinf(res[0])); } TEST(ActivationTest, SigmoidLargeNegative) { - // sigmoid(-100) should be ~0, not nan - auto t = Tensor({1}, vector{-100.0}); - - module::Sigmoid sig; - auto res = sig(t); + // sigmoid(-100) should be ~0, not nan + auto t = Tensor({1}, vector{-100.0}); + + module::Sigmoid sig; + auto res = sig(t); - ASSERT_NEAR(res[0], 0.0, 1e-5); - EXPECT_FALSE(std::isnan(res[0])); - EXPECT_FALSE(std::isinf(res[0])); + ASSERT_NEAR(res[0], 0.0, 1e-5); + EXPECT_FALSE(std::isnan(res[0])); + EXPECT_FALSE(std::isinf(res[0])); } TEST(AutogradTest, SigmoidBackward) { - // grad of sigmoid = sigmoid(x) * (1 - sigmoid(x)) - // for x=0: grad = 0.5 * 0.5 = 0.25 - // for x=1: grad = 0.7311 * 0.2689 = 0.1966 - auto t = TensorFunctions::makeSharedTensor( - {2}, {0.0, 1.0}, true); - - module::Sigmoid sig; - auto res = sig(t); - res->backward(); + // grad of sigmoid = sigmoid(x) * (1 - sigmoid(x)) + // for x=0: grad = 0.5 * 0.5 = 0.25 + // for x=1: grad = 0.7311 * 0.2689 = 0.1966 + auto t = TensorFunctions::makeSharedTensor( + {2}, {0.0, 1.0}, true); + + module::Sigmoid sig; + auto res = sig(t); + res->backward(); - auto grads = t->getGrads(); - ASSERT_NEAR((*grads)[0], 0.25, 1e-4); - ASSERT_NEAR((*grads)[1], 0.1966, 1e-4); + auto grads = t->getGrads(); + ASSERT_NEAR((*grads)[0], 0.25, 1e-4); + ASSERT_NEAR((*grads)[1], 0.1966, 1e-4); } TEST(ActivationTest, SoftmaxForward) { - // softmax([1, 2, 3]) - // exp([1,2,3]) = [2.7183, 7.3891, 20.0855] - // sum = 30.1929 - // softmax = [0.0900, 0.2447, 0.6652] - auto t = Tensor({1, 3}, {1.0, 2.0, 3.0}); - - module::Softmax sm; - auto res = sm(t); + // softmax([1, 2, 3]) + // exp([1,2,3]) = [2.7183, 7.3891, 20.0855] + // sum = 30.1929 + // softmax = [0.0900, 0.2447, 0.6652] + auto t = Tensor({1, 3}, {1.0, 2.0, 3.0}); + + module::Softmax sm; + auto res = sm(t); - ASSERT_NEAR(res[0], 0.0900, 1e-4); - ASSERT_NEAR(res[1], 0.2447, 1e-4); - ASSERT_NEAR(res[2], 0.6652, 1e-4); + ASSERT_NEAR(res[0], 0.0900, 1e-4); + ASSERT_NEAR(res[1], 0.2447, 1e-4); + ASSERT_NEAR(res[2], 0.6652, 1e-4); } TEST(ActivationTest, SoftmaxSumsToOne) { - auto t = Tensor({2, 4}, - {1.0, 2.0, 3.0, 4.0, - 2.0, 1.0, 4.0, 3.0}); - - module::Softmax sm; - auto res = sm(t); - - // each row must sum to 1 - ftype row0sum = res[0] + res[1] + res[2] + res[3]; - ftype row1sum = res[4] + res[5] + res[6] + res[7]; - ASSERT_NEAR(row0sum, 1.0, 1e-5); - ASSERT_NEAR(row1sum, 1.0, 1e-5); + auto t = Tensor({2, 4}, + {1.0, 2.0, 3.0, 4.0, + 2.0, 1.0, 4.0, 3.0}); + + module::Softmax sm; + auto res = sm(t); + + // each row must sum to 1 + ftype row0sum = res[0] + res[1] + res[2] + res[3]; + ftype row1sum = res[4] + res[5] + res[6] + res[7]; + ASSERT_NEAR(row0sum, 1.0, 1e-5); + ASSERT_NEAR(row1sum, 1.0, 1e-5); } TEST(ActivationTest, SoftmaxForwardNumericalStability) { - // large values should not produce nan or inf - auto t = Tensor({1, 3}, {100.0, 101.0, 102.0}); - - module::Softmax sm; - auto res = sm(t); + // large values should not produce nan or inf + auto t = Tensor({1, 3}, {100.0, 101.0, 102.0}); + + module::Softmax sm; + auto res = sm(t); - for(int i = 0; i < 3; i++) { - EXPECT_FALSE(std::isnan(res[i])); - EXPECT_FALSE(std::isinf(res[i])); - } + for(int i = 0; i < 3; i++) { + EXPECT_FALSE(std::isnan(res[i])); + EXPECT_FALSE(std::isinf(res[i])); + } - ftype rowsum = res[0] + res[1] + res[2]; - ASSERT_NEAR(rowsum, 1.0, 1e-5); + ftype rowsum = res[0] + res[1] + res[2]; + ASSERT_NEAR(rowsum, 1.0, 1e-5); } TEST(AutogradTest, SoftmaxBackward) { diff --git a/tests/backend/test_tensorops.cpp b/tests/backend/test_tensorops.cpp index 70780fc..5af6c29 100644 --- a/tests/backend/test_tensorops.cpp +++ b/tests/backend/test_tensorops.cpp @@ -58,16 +58,16 @@ TEST(TensorOpsTest, ScalarMul) { } TEST(AutogradTest, ScalarMultiplication) { - auto t1 = TensorFunctions::makeSharedTensor({1}, {2.0}, true); - auto t2 = TensorFunctions::makeSharedTensor({1}, {3.0}, true); + auto t1 = TensorFunctions::makeSharedTensor({1}, {2.0}, true); + auto t2 = TensorFunctions::makeSharedTensor({1}, {3.0}, true); - auto t3 = cgraph::mul(t1, t2); - auto loss = cgraph::mul(t3, t3); - - loss->backward(); - - ASSERT_DOUBLE_EQ(t1->getGrads()->get(0), 36.0); - ASSERT_DOUBLE_EQ(t2->getGrads()->get(0), 24.0); + auto t3 = cgraph::mul(t1, t2); + auto loss = cgraph::mul(t3, t3); + + loss->backward(); + + ASSERT_DOUBLE_EQ(t1->getGrads()->get(0), 36.0); + ASSERT_DOUBLE_EQ(t2->getGrads()->get(0), 24.0); } TEST(TensorOpsTest, TensorAdd) { @@ -84,17 +84,24 @@ TEST(TensorOpsTest, TensorAdd) { } } +TEST(TensorOpsTest, TensorAddThrowsOnDimMismatch) { + auto t1 = TensorFunctions::Ones({2, 2}); + auto t2 = TensorFunctions::Ones({2, 3}) * 4; + + ASSERT_THROW(t1 + t2, std::invalid_argument); +} + TEST(AutogradTest, TensorAdd) { - auto t1 = TensorFunctions::makeSharedTensor({1}, {3.0}, true); - auto t2 = TensorFunctions::makeSharedTensor({1}, {2.0}, true); + auto t1 = TensorFunctions::makeSharedTensor({1}, {3.0}, true); + auto t2 = TensorFunctions::makeSharedTensor({1}, {2.0}, true); - auto t3 = cgraph::add(t1, t2); - auto loss = cgraph::mul(t3, t3); - - loss->backward(); - - ASSERT_NEAR(t1->getGrads()->get(0), 10.0, 1e-5); - ASSERT_NEAR(t2->getGrads()->get(0), 10.0, 1e-5); + auto t3 = cgraph::add(t1, t2); + auto loss = cgraph::mul(t3, t3); + + loss->backward(); + + ASSERT_NEAR(t1->getGrads()->get(0), 10.0, 1e-5); + ASSERT_NEAR(t2->getGrads()->get(0), 10.0, 1e-5); } TEST(TensorOpsTest, BroadcastAdd) { @@ -114,19 +121,19 @@ TEST(TensorOpsTest, BroadcastAdd) { } TEST(TensorOpsTest, BroadcastAdd2D) { - // (2,3) + (3) - auto t1 = Tensor({2, 3}, {1.0, 2.0, 3.0, - 4.0, 5.0, 6.0}); - auto t2 = Tensor({3}, {10.0, 20.0, 30.0}); - auto res = t1 + t2; - - // expected: each row of t1 gets t2 added elementwise - ASSERT_NEAR(res.get(0, 0), 11.0, 1e-5); - ASSERT_NEAR(res.get(0, 1), 22.0, 1e-5); - ASSERT_NEAR(res.get(0, 2), 33.0, 1e-5); - ASSERT_NEAR(res.get(1, 0), 14.0, 1e-5); - ASSERT_NEAR(res.get(1, 1), 25.0, 1e-5); - ASSERT_NEAR(res.get(1, 2), 36.0, 1e-5); + // (2,3) + (3) + auto t1 = Tensor({2, 3}, {1.0, 2.0, 3.0, + 4.0, 5.0, 6.0}); + auto t2 = Tensor({3}, {10.0, 20.0, 30.0}); + auto res = t1 + t2; + + // expected: each row of t1 gets t2 added elementwise + ASSERT_NEAR(res.get(0, 0), 11.0, 1e-5); + ASSERT_NEAR(res.get(0, 1), 22.0, 1e-5); + ASSERT_NEAR(res.get(0, 2), 33.0, 1e-5); + ASSERT_NEAR(res.get(1, 0), 14.0, 1e-5); + ASSERT_NEAR(res.get(1, 1), 25.0, 1e-5); + ASSERT_NEAR(res.get(1, 2), 36.0, 1e-5); } TEST(TensorOpsTest, BroadcastAddNotCommutative) { @@ -136,44 +143,37 @@ TEST(TensorOpsTest, BroadcastAddNotCommutative) { ASSERT_THROW(t2 + t1, std::invalid_argument); } -TEST(TensorOpsTest, TensorAddThrowsOnDimMismatch) { - auto t1 = TensorFunctions::Ones({2, 2}); - auto t2 = TensorFunctions::Ones({2, 3}) * 4; - - ASSERT_THROW(t1 + t2, std::invalid_argument); -} - TEST(AutogradTest, BroadcastAdd) { - // gradient of broadcast add w.r.t. bias should be sum over batch dimension - // upstream grad: (2,3) of ones → bias grad should be (3) of twos - auto t1 = TensorFunctions::makeSharedTensor({2, 3}, - {1.0, 2.0, 3.0, - 4.0, 5.0, 6.0}, true); - auto bias = TensorFunctions::makeSharedTensor({3}, - {0.0, 0.0, 0.0}, true); - - auto res = cgraph::add(t1, bias); - - // set upstream grad to ones and backprop - auto upstreamGrad = TensorFunctions::makeSharedTensor({2, 3}, - {1.0, 1.0, 1.0, - 1.0, 1.0, 1.0}, false); - res->backward(); - - // bias grad should be sum over batch: [2, 2, 2] - auto biasGrad = bias->getGrads(); - ASSERT_DOUBLE_EQ((*biasGrad)[0], 2.0); - ASSERT_DOUBLE_EQ((*biasGrad)[1], 2.0); - ASSERT_DOUBLE_EQ((*biasGrad)[2], 2.0); - - // t1 grad should be ones (add is identity for non-broadcast operand) - auto t1Grad = t1->getGrads(); - for(int i = 0; i < 6; i++) { - ASSERT_DOUBLE_EQ((*t1Grad)[i], 1.0); - } + // gradient of broadcast add w.r.t. bias should be sum over batch dimension + // upstream grad: (2,3) of ones → bias grad should be (3) of twos + auto t1 = TensorFunctions::makeSharedTensor({2, 3}, + {1.0, 2.0, 3.0, + 4.0, 5.0, 6.0}, true); + auto bias = TensorFunctions::makeSharedTensor({3}, + {0.0, 0.0, 0.0}, true); + + auto res = cgraph::add(t1, bias); + + // set upstream grad to ones and backprop + auto upstreamGrad = TensorFunctions::makeSharedTensor({2, 3}, + {1.0, 1.0, 1.0, + 1.0, 1.0, 1.0}, false); + res->backward(); + + // bias grad should be sum over batch: [2, 2, 2] + auto biasGrad = bias->getGrads(); + ASSERT_DOUBLE_EQ((*biasGrad)[0], 2.0); + ASSERT_DOUBLE_EQ((*biasGrad)[1], 2.0); + ASSERT_DOUBLE_EQ((*biasGrad)[2], 2.0); + + // t1 grad should be ones (add is identity for non-broadcast operand) + auto t1Grad = t1->getGrads(); + for(int i = 0; i < 6; i++) { + ASSERT_DOUBLE_EQ((*t1Grad)[i], 1.0); + } } -TEST(TensorOpsTest, MatrixAddGivesCorrectResults) { +TEST(TensorOpsTest, MatrixAdd) { auto t1 = TensorFunctions::Ones({2, 2}); auto t2 = TensorFunctions::Ones({2, 2}); @@ -188,7 +188,7 @@ TEST(TensorOpsTest, MatrixAddGivesCorrectResults) { } } -TEST(TensorOpsTest, ElementwiseMulGivesCorrectResults) { +TEST(TensorOpsTest, ElementwiseMul) { constexpr ftype factor = 0.5; auto t1 = TensorFunctions::Ones({2, 2}); auto t2 = TensorFunctions::Ones({2, 2}) * 0.5; @@ -210,7 +210,7 @@ TEST(TensorOpsTest, ElementwiseMulThrowsOnDimensionMismatch) { ASSERT_THROW(t1 * t2, std::invalid_argument); } -TEST(TensorOpsTest, MatMulGivesCorrectValues1) { +TEST(TensorOpsTest, MatMul) { auto t1 = TensorFunctions::Ones({3, 2}); auto t2 = TensorFunctions::Ones({2, 6}) * 1.5; @@ -226,7 +226,7 @@ TEST(TensorOpsTest, MatMulGivesCorrectValues1) { } } -TEST(TensorOpsTest, MatMulGivesCorrectValues2) { +TEST(TensorOpsTest, MatMul2) { auto t1 = Tensor({2, 2}); auto t2 = Tensor({2, 2}); @@ -264,36 +264,36 @@ TEST(TensorOpsTest, MatMulThrowsWhenDimensionsNotMatched) { } TEST(AutogradTest, MatMul) { - auto t1 = TensorFunctions::makeSharedTensor({2, 3}, {1, 2, 3, 4, 5, 6}, true); - auto t2 = TensorFunctions::makeSharedTensor({3, 2}, {1, 2, 3, 4, 5, 6}, true); - - auto t3 = cgraph::matmul(t1, t2); + auto t1 = TensorFunctions::makeSharedTensor({2, 3}, {1, 2, 3, 4, 5, 6}, true); + auto t2 = TensorFunctions::makeSharedTensor({3, 2}, {1, 2, 3, 4, 5, 6}, true); - auto loss = TensorFunctions::makeSharedTensor({1}, {0.0}, true); - for (size_t i = 0; i < t3->getSize(); ++i) { - loss = cgraph::add(loss, cgraph::get(t3, i)); - } - - loss->backward(); - - EXPECT_TRUE(t1->hasGrads()); - EXPECT_TRUE(t2->hasGrads()); - - // dL/dt1 = dloss/dt3 @ t2^t = Ones({2, 2}) @ t2^t - ASSERT_DOUBLE_EQ(t1->getGrads()->get({0, 0}), 3.0); - ASSERT_DOUBLE_EQ(t1->getGrads()->get({0, 1}), 7.0); - ASSERT_DOUBLE_EQ(t1->getGrads()->get({0, 2}), 11.0); - ASSERT_DOUBLE_EQ(t1->getGrads()->get({1, 0}), 3.0); - ASSERT_DOUBLE_EQ(t1->getGrads()->get({1, 1}), 7.0); - ASSERT_DOUBLE_EQ(t1->getGrads()->get({1, 2}), 11.0); - - // dL/dt2 = t1^t @ dloss/dt3 = t1^t @ Ones({2, 2}) - ASSERT_DOUBLE_EQ(t2->getGrads()->get({0, 0}), 5.0); - ASSERT_DOUBLE_EQ(t2->getGrads()->get({0, 1}), 5.0); - ASSERT_DOUBLE_EQ(t2->getGrads()->get({1, 0}), 7.0); - ASSERT_DOUBLE_EQ(t2->getGrads()->get({1, 1}), 7.0); - ASSERT_DOUBLE_EQ(t2->getGrads()->get({2, 0}), 9.0); - ASSERT_DOUBLE_EQ(t2->getGrads()->get({2, 1}), 9.0); + auto t3 = cgraph::matmul(t1, t2); + + auto loss = TensorFunctions::makeSharedTensor({1}, {0.0}, true); + for(size_t i = 0; i < t3->getSize(); ++i) { + loss = cgraph::add(loss, cgraph::get(t3, i)); + } + + loss->backward(); + + EXPECT_TRUE(t1->hasGrads()); + EXPECT_TRUE(t2->hasGrads()); + + // dL/dt1 = dloss/dt3 @ t2^t = Ones({2, 2}) @ t2^t + ASSERT_DOUBLE_EQ(t1->getGrads()->get({0, 0}), 3.0); + ASSERT_DOUBLE_EQ(t1->getGrads()->get({0, 1}), 7.0); + ASSERT_DOUBLE_EQ(t1->getGrads()->get({0, 2}), 11.0); + ASSERT_DOUBLE_EQ(t1->getGrads()->get({1, 0}), 3.0); + ASSERT_DOUBLE_EQ(t1->getGrads()->get({1, 1}), 7.0); + ASSERT_DOUBLE_EQ(t1->getGrads()->get({1, 2}), 11.0); + + // dL/dt2 = t1^t @ dloss/dt3 = t1^t @ Ones({2, 2}) + ASSERT_DOUBLE_EQ(t2->getGrads()->get({0, 0}), 5.0); + ASSERT_DOUBLE_EQ(t2->getGrads()->get({0, 1}), 5.0); + ASSERT_DOUBLE_EQ(t2->getGrads()->get({1, 0}), 7.0); + ASSERT_DOUBLE_EQ(t2->getGrads()->get({1, 1}), 7.0); + ASSERT_DOUBLE_EQ(t2->getGrads()->get({2, 0}), 9.0); + ASSERT_DOUBLE_EQ(t2->getGrads()->get({2, 1}), 9.0); } From d7bde0177e04a97feb3ef743696dbb1e90ae19d1 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Thu, 21 May 2026 18:31:28 +0200 Subject: [PATCH 56/73] backward on broadcast add --- .../tensor_ops/cuda/tensor_ops_nodes.cu | 45 +++++++++++ .../tensor_ops/cuda/tensor_ops_nodes.cuh | 24 ++++++ .../cuda/{tensorops.cu => tensor_ops.cu} | 79 ++++++++++++++++++- .../cuda/{tensorops.cuh => tensor_ops.cuh} | 3 +- src/backend/data_modeling/tensor.cpp | 2 +- .../data_modeling/tensor_functions.cpp | 38 ++++++--- tests/backend/cuda/test_tensorops_cuda.cu | 36 ++++++++- 7 files changed, 211 insertions(+), 16 deletions(-) create mode 100644 src/backend/computational_graph/tensor_ops/cuda/tensor_ops_nodes.cu create mode 100644 src/backend/computational_graph/tensor_ops/cuda/tensor_ops_nodes.cuh rename src/backend/data_modeling/cuda/{tensorops.cu => tensor_ops.cu} (77%) rename src/backend/data_modeling/cuda/{tensorops.cuh => tensor_ops.cuh} (90%) diff --git a/src/backend/computational_graph/tensor_ops/cuda/tensor_ops_nodes.cu b/src/backend/computational_graph/tensor_ops/cuda/tensor_ops_nodes.cu new file mode 100644 index 0000000..d473ed4 --- /dev/null +++ b/src/backend/computational_graph/tensor_ops/cuda/tensor_ops_nodes.cu @@ -0,0 +1,45 @@ +/** + * @file tensor_ops_nodes.cu + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-05-20 + * + * @copyright Copyright (c) 2026 + * + */ + +#ifndef __CUDA +static_assert(false, "File should not be compiled without CUDA enabled"); +#endif // __CUDA + +#include "tensor_ops_nodes.cuh" +#include "utility/cuda/cuda_common.cuh" + +using namespace std; + +namespace { + __global__ void scalarMulKernel(ftype* const res, const ftype* const upstreamGrad, const ftype factor, const tensorSize_t size) { + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) return; + res[gid] = upstreamGrad[gid] * factor; + } +} + +namespace cuda_impl { + void scalarMulBackward(Tensor& res, const Tensor& upstreamGrad, ftype factor) { + constexpr int threadsPerBlock = 256; + const int blocks = (upstreamGrad.getSize() + threadsPerBlock - 1) / threadsPerBlock; + + scalarMulKernel<<>>(res.getData(), upstreamGrad.getData(), factor, upstreamGrad.getSize()); + cudaErrchk(cudaDeviceSynchronize()); + } + + void getterBackward(Tensor& res, ftype val, tensorSize_t linearIdx) { + cudaErrchk(cudaMemset(res.getData(), 0, res.getSize() * sizeof(ftype))); + cudaErrchk(cudaDeviceSynchronize()); + + cudaErrchk(cudaMemcpy(res.getData() + linearIdx, &val, sizeof(ftype), cudaMemcpyHostToDevice)); + cudaErrchk(cudaDeviceSynchronize()); + } +} diff --git a/src/backend/computational_graph/tensor_ops/cuda/tensor_ops_nodes.cuh b/src/backend/computational_graph/tensor_ops/cuda/tensor_ops_nodes.cuh new file mode 100644 index 0000000..5568d2b --- /dev/null +++ b/src/backend/computational_graph/tensor_ops/cuda/tensor_ops_nodes.cuh @@ -0,0 +1,24 @@ +/** + * @file tensor_ops_nodes.cuh + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-05-20 + * + * @copyright Copyright (c) 2026 + * + */ + +#pragma once + +#ifndef __CUDA +static_assert(false, "File should not be included without CUDA enabled"); +#endif // __CUDA + +#include "utility/global_params.h" +#include "data_modeling/tensor.h" + +namespace cuda_impl { + void scalarMulBackward(Tensor& res, const Tensor& upstreamGrad, ftype factor); + void getterBackward(Tensor& res, ftype val, tensorSize_t linearIdx); +} diff --git a/src/backend/data_modeling/cuda/tensorops.cu b/src/backend/data_modeling/cuda/tensor_ops.cu similarity index 77% rename from src/backend/data_modeling/cuda/tensorops.cu rename to src/backend/data_modeling/cuda/tensor_ops.cu index 2b0b75e..a6954ea 100644 --- a/src/backend/data_modeling/cuda/tensorops.cu +++ b/src/backend/data_modeling/cuda/tensor_ops.cu @@ -1,5 +1,5 @@ /** - * @file tensorops.cu + * @file tensor_ops.cu * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) * @brief * @version 0.1 @@ -15,7 +15,7 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include "data_modeling/tensor.h" -#include "tensorops.cuh" +#include "tensor_ops.cuh" #include "utility/cuda/cuda_common.cuh" #include @@ -120,6 +120,54 @@ namespace{ res[gid] = cij; // smem[tid]; } + /** + * @brief Does what you think it does. Assumption for kernel call: Each block contains one or more strides completely. + */ + __global__ void sumOverDimsKernelOneBlock(ftype* const res, const ftype* const input, const tensorSize_t stride, + const int srcDimSize, const int stridesPerBlock, const int threadsPerStride) { + const int withinStrideIdx = threadIdx.x % threadsPerStride; + const bool isNotPadded = withinStrideIdx < stride; + + const int strideInBlockNumber = threadIdx.x / threadsPerStride; + const int localIdx = strideInBlockNumber * stride + withinStrideIdx; + const int globalIdx = blockIdx.x * stridesPerBlock * stride + localIdx; + + extern __shared__ ftype smem[]; + if(isNotPadded) { + smem[localIdx] = 0; + } + __syncthreads(); + + ftype sum = 0; + for(int k = 0; k < srcDimSize; k++) { + if(isNotPadded) { + smem[localIdx] += input[k * stride + withinStrideIdx]; + } + __syncthreads(); + } + + if(isNotPadded) { + res[withinStrideIdx] += smem[localIdx]; + } + } + +/* __global__ void sumOverDimsKernel(ftype* const res, const ftype* const input, const tensorSize_t stride, + const int srcDimSize, const tensorSize_t size) { + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid > size) { + return; + } + + const int tid = threadIdx.x; + + ftype sum = 0; + for(int k = 0; k < srcDimSize; k++) { + sum += input[k * stride + tid]; + } + + res[gid] = sum; + } */ + /** * @brief Create a contiguous copy of src and copy it into dst. Used for reshaping, transposing, etc. * @@ -217,6 +265,33 @@ namespace cuda_impl { cudaErrchk(cudaDeviceSynchronize()); } + void sumOverDims(Tensor& res, const Tensor& input, tensorDim_t dim) { + + tensorSize_t stride = 1; + for(tensorDim_t i = dim+1; i < input.getDims().nDims(); i++){ + stride *= input.getDims()[i]; + } + + constexpr int maxThreadsPerBlock = 256; + if(stride <= maxThreadsPerBlock) { + const int stridesPerBlock = (stride + maxThreadsPerBlock - 1) / maxThreadsPerBlock; + + int threadsPerStride = 1; + while(threadsPerStride < stride) threadsPerStride <<= 1; + + const int threadsPerBlock = threadsPerStride * stridesPerBlock; + const int blocks = (input.getSize() + stridesPerBlock - 1) / stridesPerBlock; + + sumOverDimsKernelOneBlock<<>>( + res.getData(), input.getData(), stride, input.getDims()[dim], stridesPerBlock, threadsPerStride); + } + else { + std::__throw_runtime_error("Not implemented yet"); + } + + cudaErrchk(cudaDeviceSynchronize()); + } + void scalarFill(Tensor& t, ftype value) { ftype* ptr = t.getData(); thrust::fill(thrust::device_pointer_cast(ptr), diff --git a/src/backend/data_modeling/cuda/tensorops.cuh b/src/backend/data_modeling/cuda/tensor_ops.cuh similarity index 90% rename from src/backend/data_modeling/cuda/tensorops.cuh rename to src/backend/data_modeling/cuda/tensor_ops.cuh index ef0e6ae..7634f46 100644 --- a/src/backend/data_modeling/cuda/tensorops.cuh +++ b/src/backend/data_modeling/cuda/tensor_ops.cuh @@ -1,5 +1,5 @@ /** - * @file tensorops.cuh + * @file tensor_ops.cuh * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) * @brief * @version 0.1 @@ -31,6 +31,7 @@ namespace cuda_impl { void broadcastadd(Tensor& res, const Tensor& matrix, const Tensor& vec); void elementwisemul(Tensor& res, const Tensor& left, const Tensor& right); void matmul(Tensor& res, const Tensor& left, const Tensor& right); + void sumOverDims(Tensor& res, const Tensor& input, tensorDim_t dim); void scalarFill(Tensor& t, ftype value); diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index 6a0256e..012e088 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -24,7 +24,7 @@ #ifdef __CUDA #include "utility/cuda/cuda_common.cuh" -#include "data_modeling/cuda/tensorops.cuh" +#include "data_modeling/cuda/tensor_ops.cuh" #endif using namespace std; diff --git a/src/backend/data_modeling/tensor_functions.cpp b/src/backend/data_modeling/tensor_functions.cpp index 66defff..077d478 100644 --- a/src/backend/data_modeling/tensor_functions.cpp +++ b/src/backend/data_modeling/tensor_functions.cpp @@ -11,6 +11,11 @@ #include "tensor_functions.h" +#ifdef __CUDA +#include "data_modeling/cuda/tensor_ops.cuh" +#include "computational_graph/tensor_ops/cuda/tensor_ops_nodes.cuh" +#endif + using namespace std; Tensor TensorFunctions::Zeros(vector dims, Device d, const bool requiresGrad) { @@ -97,17 +102,30 @@ Tensor TensorFunctions::SumOverDims(const Tensor& t, tensorDim_t dim) { auto resDims = t.getDims().collapseDimension(dim); Tensor res = Zeros(resDims.toVector(), t.getDevice(), t.getRequiresGrad()); // inefficiency toVector - tensorSize_t stride = 1; - for(tensorDim_t i=dim+1; igetGrads()->get(0), 10.0, 1e-5); ASSERT_NEAR(t2->getGrads()->get(0), 10.0, 1e-5); -} */ +} TEST(CudaTensorOpsTest, BroadcastAdd) { auto t1 = TensorFunctions::Ones({3, 2, 2}, Device::CUDA); @@ -134,6 +136,36 @@ TEST(CudaTensorOpsTest, BroadcastAddNotCommutative) { ASSERT_THROW(t2 + t1, std::invalid_argument); } +TEST(CudaAutogradTest, BroadcastAdd) { + // gradient of broadcast add w.r.t. bias should be sum over batch dimension + // upstream grad: (2,3) of ones → bias grad should be (3) of twos + auto t1 = TensorFunctions::makeSharedTensor({2, 3}, + {1.0, 2.0, 3.0, + 4.0, 5.0, 6.0}, Device::CUDA, true); + auto bias = TensorFunctions::makeSharedTensor({3}, + {0.0, 0.0, 0.0}, Device::CUDA, true); + + auto res = cgraph::add(t1, bias); + + // set upstream grad to ones and backprop + auto upstreamGrad = TensorFunctions::makeSharedTensor({2, 3}, + {1.0, 1.0, 1.0, + 1.0, 1.0, 1.0}, false); + res->backward(); + + // bias grad should be sum over batch: [2, 2, 2] + auto biasGrad = bias->getGrads(); + ASSERT_DOUBLE_EQ((*biasGrad)[0], 2.0); + ASSERT_DOUBLE_EQ((*biasGrad)[1], 2.0); + ASSERT_DOUBLE_EQ((*biasGrad)[2], 2.0); + + // t1 grad should be ones (add is identity for non-broadcast operand) + auto t1Grad = t1->getGrads(); + for(int i = 0; i < 6; i++) { + ASSERT_DOUBLE_EQ((*t1Grad)[i], 1.0); + } +} + TEST(CudaTensorOpsTest, MatrixAdd) { auto t1 = TensorFunctions::Ones({200, 200}, Device::CUDA); auto t2 = TensorFunctions::Ones({200, 200}, Device::CUDA); From 51ab71fafe2c0fecccaae77627c4f4112b4fa1bb Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Fri, 22 May 2026 17:04:45 +0200 Subject: [PATCH 57/73] Fixed sum over dims on CPU for general case and implemented general case CUDA kernel --- src/backend/data_modeling/cuda/tensor_ops.cu | 74 +++++-------------- .../data_modeling/tensor_functions.cpp | 21 ++++-- 2 files changed, 33 insertions(+), 62 deletions(-) diff --git a/src/backend/data_modeling/cuda/tensor_ops.cu b/src/backend/data_modeling/cuda/tensor_ops.cu index a6954ea..49701b3 100644 --- a/src/backend/data_modeling/cuda/tensor_ops.cu +++ b/src/backend/data_modeling/cuda/tensor_ops.cu @@ -17,6 +17,7 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include "tensor_ops.cuh" #include "utility/cuda/cuda_common.cuh" +#include "utility/macros.h" #include #include @@ -99,7 +100,7 @@ namespace{ const int gid = blockDim.x * blockIdx.x + threadIdx.x; if(gid >= resSize) return; - const int tid = threadIdx.x; + //const int tid = threadIdx.x; // 48KB of shared mem -> ~12K floats of 4 bytes -> ~10 blocks per SM of shared memory is limit //extern __shared__ ftype smem[]; @@ -121,52 +122,27 @@ namespace{ } /** - * @brief Does what you think it does. Assumption for kernel call: Each block contains one or more strides completely. + * @brief Strides in an outer size larger than one block. We use one thread per stride. */ - __global__ void sumOverDimsKernelOneBlock(ftype* const res, const ftype* const input, const tensorSize_t stride, - const int srcDimSize, const int stridesPerBlock, const int threadsPerStride) { - const int withinStrideIdx = threadIdx.x % threadsPerStride; - const bool isNotPadded = withinStrideIdx < stride; - - const int strideInBlockNumber = threadIdx.x / threadsPerStride; - const int localIdx = strideInBlockNumber * stride + withinStrideIdx; - const int globalIdx = blockIdx.x * stridesPerBlock * stride + localIdx; - - extern __shared__ ftype smem[]; - if(isNotPadded) { - smem[localIdx] = 0; - } - __syncthreads(); - - ftype sum = 0; - for(int k = 0; k < srcDimSize; k++) { - if(isNotPadded) { - smem[localIdx] += input[k * stride + withinStrideIdx]; - } - __syncthreads(); - } - - if(isNotPadded) { - res[withinStrideIdx] += smem[localIdx]; - } - } - -/* __global__ void sumOverDimsKernel(ftype* const res, const ftype* const input, const tensorSize_t stride, - const int srcDimSize, const tensorSize_t size) { - const int gid = blockIdx.x * blockDim.x + threadIdx.x; - if(gid > size) { + __global__ void sumOverDimsKernel(ftype* const res, const ftype* const input, tensorSize_t stride, + const tensorDim_t srcDimSize, const tensorSize_t size) { + const tensorSize_t gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) { return; - } + } - const int tid = threadIdx.x; + const tensorSize_t outerIdx = gid / stride; + const tensorSize_t batchOffset = outerIdx * stride * srcDimSize; + + const int withinStrideIdx = gid % stride; ftype sum = 0; for(int k = 0; k < srcDimSize; k++) { - sum += input[k * stride + tid]; + sum += input[batchOffset + k * stride + withinStrideIdx]; } res[gid] = sum; - } */ + } /** * @brief Create a contiguous copy of src and copy it into dst. Used for reshaping, transposing, etc. @@ -266,29 +242,15 @@ namespace cuda_impl { } void sumOverDims(Tensor& res, const Tensor& input, tensorDim_t dim) { - tensorSize_t stride = 1; - for(tensorDim_t i = dim+1; i < input.getDims().nDims(); i++){ + for(tensorDim_t i = dim + 1; i < input.getDims().nDims(); i++){ stride *= input.getDims()[i]; } - constexpr int maxThreadsPerBlock = 256; - if(stride <= maxThreadsPerBlock) { - const int stridesPerBlock = (stride + maxThreadsPerBlock - 1) / maxThreadsPerBlock; - - int threadsPerStride = 1; - while(threadsPerStride < stride) threadsPerStride <<= 1; - - const int threadsPerBlock = threadsPerStride * stridesPerBlock; - const int blocks = (input.getSize() + stridesPerBlock - 1) / stridesPerBlock; - - sumOverDimsKernelOneBlock<<>>( - res.getData(), input.getData(), stride, input.getDims()[dim], stridesPerBlock, threadsPerStride); - } - else { - std::__throw_runtime_error("Not implemented yet"); - } + constexpr int threadsPerBlock = 256; + const int blocks = (res.getSize() + threadsPerBlock - 1) / threadsPerBlock; + sumOverDimsKernel<<>>(res.getData(), input.getData(), stride, input.getDims()[dim], res.getSize()); cudaErrchk(cudaDeviceSynchronize()); } diff --git a/src/backend/data_modeling/tensor_functions.cpp b/src/backend/data_modeling/tensor_functions.cpp index 077d478..2744f92 100644 --- a/src/backend/data_modeling/tensor_functions.cpp +++ b/src/backend/data_modeling/tensor_functions.cpp @@ -95,7 +95,7 @@ shared_ptr TensorFunctions::makeSharedTensor(const vector& * Input dim must be smaller then t.dims.nDims()-1 */ Tensor TensorFunctions::SumOverDims(const Tensor& t, tensorDim_t dim) { - if(dim>=t.getDims().nDims()-1){ + if(dim > t.getDims().nDims() - 1){ __throw_invalid_argument("Dim parameter must be smaller than number of dims, but was " + dim); } @@ -109,12 +109,21 @@ Tensor TensorFunctions::SumOverDims(const Tensor& t, tensorDim_t dim) { for(tensorDim_t i = dim + 1; i < t.getDims().nDims(); i++){ stride *= t.getDims()[i]; } + + // size of dimensions before dim + tensorSize_t outerSize = 1; + for(tensorDim_t i = 0; i < dim; i++) { + outerSize *= t.getDims()[i]; + } - tensorSize_t targetOffset = 0; - for(tensorDim_t loop = 0; loop < t.getDims()[dim]; loop++){ - for(tensorSize_t i = 0; i < stride; i++){ - res.set(res.get(i) + t.get(targetOffset), i); - targetOffset++; + for(tensorSize_t outer = 0; outer < outerSize; outer++){ + for(tensorDim_t k = 0; k < t.getDims()[dim]; k++){ + for(tensorSize_t i = 0; i < stride; i++){ + tensorSize_t srcIdx = outer * t.getDims()[dim] * stride + k * stride + i; + tensorSize_t dstIdx = outer * stride + i; + + res.set(res.get(dstIdx) + t.get(srcIdx), dstIdx); + } } } break; From f5eef20aa2e74775aa465707672a58d38af585dc Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sat, 23 May 2026 14:59:28 +0200 Subject: [PATCH 58/73] Implemented backward loss functions, generalized forward CE loss functions for general case --- .../loss_functions/bce_node.cpp | 4 +- .../loss_functions/bce_sigmoid_node.cpp | 6 +- .../crossentropy_softmax_node.cpp | 20 +-- .../loss_functions/cuda/loss_nodes.cu | 117 +++++++++++++++--- .../loss_functions/rmse_node.cpp | 14 ++- .../tensor_ops/matmul_node.cpp | 1 + .../tensor_ops/scalar_op_nodes.cpp | 15 +++ .../activation_functions/cuda/activations.cu | 11 +- .../module/activation_functions/softmax.cpp | 2 +- .../loss_functions/crossentropy_loss.cpp | 19 +-- .../crossentropy_softmax_loss.cpp | 6 +- .../loss_functions/cuda/loss_functions.cu | 8 +- .../training/optimizers/cuda/optimizers.cu | 4 +- src/backend/utility/cuda/cuda_common.cuh | 21 ++++ src/backend/utility/global_params.h | 1 + 15 files changed, 186 insertions(+), 63 deletions(-) diff --git a/src/backend/computational_graph/loss_functions/bce_node.cpp b/src/backend/computational_graph/loss_functions/bce_node.cpp index b49af11..9419e9d 100644 --- a/src/backend/computational_graph/loss_functions/bce_node.cpp +++ b/src/backend/computational_graph/loss_functions/bce_node.cpp @@ -29,10 +29,12 @@ vector< shared_ptr > BceNode::backward(const Tensor& upstreamGrad) { switch(upstreamGrad.getDevice()) { case Device::CPU: { - ftype bSize = yPred->getDims()[0]; + const ftype bSize = yPred->getDims()[0]; + for(tensorSize_t i = 0; i < yPred->getDims()[0]; i++){ auto yi = (*yTrue)[i]; auto yiHat = (*yPred)[i]; + auto g = -yi / std::max(yiHat, EPS_BCE) + (1 - yi) / std::max(1 - yiHat, EPS_BCE); res->set(g / bSize, i); } diff --git a/src/backend/computational_graph/loss_functions/bce_sigmoid_node.cpp b/src/backend/computational_graph/loss_functions/bce_sigmoid_node.cpp index 16f2e6f..ed4aa25 100644 --- a/src/backend/computational_graph/loss_functions/bce_sigmoid_node.cpp +++ b/src/backend/computational_graph/loss_functions/bce_sigmoid_node.cpp @@ -40,12 +40,12 @@ vector< shared_ptr > BceSigmoidNode::backward(const Tensor& upstreamGrad return e / (one + e); }; - ftype bSize = logits->getDims()[0]; - for(tensorSize_t i=0; igetDims()[0]; i++){ + const ftype bSize = logits->getDims()[0]; + for(tensorSize_t i = 0; i < logits->getDims()[0]; i++){ auto y = (*yTrue)[i]; auto s = sigmoid((*logits)[i]); auto g = s - y; - res->set(g/bSize, i); + res->set(g / bSize, i); } break; } diff --git a/src/backend/computational_graph/loss_functions/crossentropy_softmax_node.cpp b/src/backend/computational_graph/loss_functions/crossentropy_softmax_node.cpp index c820031..0ab4356 100644 --- a/src/backend/computational_graph/loss_functions/crossentropy_softmax_node.cpp +++ b/src/backend/computational_graph/loss_functions/crossentropy_softmax_node.cpp @@ -22,21 +22,25 @@ using namespace std; using namespace cgraph; vector< shared_ptr > CrossEntropySoftmaxNode::backward(const Tensor& upstreamGrad) { - assert(!upstreamGrad.getRequiresGrad()); + assert(!upstreamGrad.getRequiresGrad() && logits.nDims() == 2); const auto& logits = parents[0]; auto res = make_shared(logits->createEmptyCopy()); switch(upstreamGrad.getDevice()) { - case Device::CPU: { - const auto softmax = module::Softmax(); + case Device::CPU: + { + static const auto softmax = module::Softmax(); const auto s = softmax(*logits); - ftype bSize = logits->getDims()[0]; - for(tensorSize_t b=0; bgetDims()[0]; b++){ - for(tensorSize_t i=0; igetDims()[1]; i++){ - auto g = s.get(b, i) - yTrue->get(b, i); - res->set(g / bSize, b, i); + const tensorSize_t stride = logits->getDims().get(-1); + const tensorSize_t bSize = logits->getSize() / stride; + + for(tensorSize_t b = 0; b < bSize; b++){ + for(tensorSize_t i = 0; i < stride; i++){ + const tensorSize_t flatIdx = b * stride + i; + auto g = s[flatIdx] - (*yTrue)[flatIdx]; + res->set(g / bSize, flatIdx); } } break; diff --git a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu index 8c45915..1087352 100644 --- a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu +++ b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu @@ -14,65 +14,146 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #endif // __CUDA #include "loss_nodes.cuh" +#include "module/activation_functions/softmax.h" + #include "utility/cuda/cuda_common.cuh" +#include "utility/global_params.h" + +#include using namespace std; namespace { - // TODO: bceBackward kernel + /** + * @brief Does what you think it does. + */ + __global__ void bceBackwardKernel(ftype* const res, const ftype* const yPred, const ftype* const yTrue, const tensorSize_t size) { + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) { + return; + } + + auto yi = yTrue[gid]; + auto yiHat = Pred[gid]; + + auto g = -yi / cudaMax(yiHat, EPS_BCE) + (1 - yi) / cudaMax(1 - yiHat, EPS_BCE); + res[gid] = g; + } - // TODO: bceSigmoidBackward kernel + /** + * @brief BCE backward kernel appplied with sigmoid. + * + * @param bSize Batch-size. + * @param sigmoids Sigmoids from forward pass. + */ + __global__ void bceSigmoidBackwardKernel(ftype* const res, const ftype* const sigmoids, const ftype* const yTrue, const ftype bSize, const tensorSize_t size) { + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) { + return; + } + + auto y = yTrue[gid]; + auto s = cudaSigmoid(logits[gid]); + + auto g = s - y; + res[gid] = g / bSize; + } - // TODO: crossEntropyBackward kernel + /** + * @brief The simple cross-entropy backward kernel. + * + * @param bSize The batch-size. + */ + __global__ void crossEntropyBackwardKernel(ftype* const res, const ftype* const yPred, const ftype* const yTrue, const ftype nSamples, const tensorSize_t size) { + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) { + return; + } + + auto g = -yTrue[gid] / cudaMax(yPred[gid], EPS_CROSSENTROPY); + res[gid] = g / nSamples; + } - // TODO: crossEntropySoftmaxBackward kernel + /** + * @brief Does what you think it does. + */ + __global__ void crossEntropySoftmaxBackwardKernel(ftype* const res, const ftype* const softmaxedLogits, const ftype* const yTrue, + const ftype nSamples, const tensorSize_t size) { + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) { + return; + } + + res[gid] = (softmaxedLogits[gid] - yTrue[gid]) / nSamples; + } - // TODO: rmseBackward kernel + /** + * @brief RMSE backward kernel. bSize = batch-size, rmse = the rmse from the forward pass. + */ + __global__ void rmseBackwardKernel(ftype* const res, const ftype* const yPred, const ftype* const yTrue, + const ftype rmse, const ftype bSize, const tensorSize_t size) { + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) { + return; + } + + const ftype yi = yTrue[gid]; + const ftype yiHat = yPred[gid]; + + const ftype denom = rmse * bSize + EPS_RMSE; + const ftype g = (yiHat-yi) / denom; + + res[gid] = g; + } } namespace cuda_impl { void bceBackward(Tensor& res, const Tensor& yPred, const Tensor& yTrue) { - const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + constexpr int threadsPerBlock = 256; const int blocks = (yPred.getSize() + threadsPerBlock - 1) / threadsPerBlock; - // TODO: launch kernel - + bceBackwardKernel<<>>(res.getData(), yPred.getData(), yTrue.getData(), yTrue.getSize()); cudaErrchk(cudaDeviceSynchronize()); } void bceSigmoidBackward(Tensor& res, const Tensor& logits, const Tensor& yTrue) { - const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + constexpr int threadsPerBlock = 256; const int blocks = (logits.getSize() + threadsPerBlock - 1) / threadsPerBlock; - // TODO: launch kernel - + bceSigmoidBackwardKernel<<>>(res.getData(), logits.getData(), yTrue.getData(), logits->getDims()[0], res.getSize()); cudaErrchk(cudaDeviceSynchronize()); } void crossEntropyBackward(Tensor& res, const Tensor& yPred, const Tensor& yTrue) { - const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + constexpr int threadsPerBlock = 256; const int blocks = (yPred.getSize() + threadsPerBlock - 1) / threadsPerBlock; - // TODO: launch kernel + const tensorSize_t stride = yPred->getDims()[-1]; + const ftype nSamples = static_cast(yPred->getSize() / stride); + crossEntropyBackwardKernel<<>>(res.getData(), yPred.getData(), yTrue.getData(), nSamples, yTrue.getSize()); cudaErrchk(cudaDeviceSynchronize()); } void crossEntropySoftmaxBackward(Tensor& res, const Tensor& logits, const Tensor& yTrue) { - const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + constexpr int threadsPerBlock = 256; const int blocks = (logits.getSize() + threadsPerBlock - 1) / threadsPerBlock; - // TODO: launch kernel + static const auto softmax = module::Softmax(); + const auto softmaxedLogits = softmax(*logits); + const tensorSize_t stride = logits->getDims().get(-1); + const auto nSamples = static_cast(->getSize() / stride); + + crossEntropySoftmaxBackwardKernel<<>>(res.getData(), softmaxedLogits.getData(), yTrue.getData(), bSize, logits.getSize()); cudaErrchk(cudaDeviceSynchronize()); } void rmseBackward(Tensor& res, const Tensor& yPred, const Tensor& yTrue, ftype rmse) { - const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + constexpr int threadsPerBlock = 256; const int blocks = (yPred.getSize() + threadsPerBlock - 1) / threadsPerBlock; - // TODO: launch kernel - + rmseBackwardKernel<<>>(res.getData(), yPred.getData(), yTrue.getData(), rmse, static_cast(yPred->getDims()[0]), yPred.getSize()); cudaErrchk(cudaDeviceSynchronize()); } } diff --git a/src/backend/computational_graph/loss_functions/rmse_node.cpp b/src/backend/computational_graph/loss_functions/rmse_node.cpp index 3a2d74b..9deeefc 100644 --- a/src/backend/computational_graph/loss_functions/rmse_node.cpp +++ b/src/backend/computational_graph/loss_functions/rmse_node.cpp @@ -25,19 +25,21 @@ using namespace cgraph; vector< shared_ptr > RmseNode::backward(const Tensor& upstreamGrad) { assert(!upstreamGrad.getRequiresGrad()); - constexpr ftype eps = 1e-9; const auto& yPred = parents[0]; auto res = make_shared(yPred->createEmptyCopy()); switch(upstreamGrad.getDevice()) { - case Device::CPU: { - ftype bSize = yPred->getDims()[0]; - for(tensorSize_t i=0; igetDims()[0]; i++){ + case Device::CPU: + { + const ftype bSize = yPred->getDims()[0]; + for(tensorSize_t i = 0; i < yPred->getDims()[0]; i++){ auto yi = (*yTrue)[i]; auto yiHat = (*yPred)[i]; - auto denom = rmse * bSize + eps; - auto g = (yiHat-yi) / denom; + + auto denom = rmse * bSize + EPS_RMSE; + auto g = (yiHat - yi) / denom; + res->set(g, i); } break; diff --git a/src/backend/computational_graph/tensor_ops/matmul_node.cpp b/src/backend/computational_graph/tensor_ops/matmul_node.cpp index 2237026..7edd0e7 100644 --- a/src/backend/computational_graph/tensor_ops/matmul_node.cpp +++ b/src/backend/computational_graph/tensor_ops/matmul_node.cpp @@ -16,6 +16,7 @@ using namespace cgraph; vector> MatMulNode::backward(const Tensor& upstreamGrad) { assert(!upstreamGrad.getRequiresGrad()); + // TODO: optimize operators return { make_shared(upstreamGrad.matmul(parents[1]->transpose(-2, -1))), make_shared(parents[0]->transpose(-2, -1).matmul(upstreamGrad)) diff --git a/src/backend/computational_graph/tensor_ops/scalar_op_nodes.cpp b/src/backend/computational_graph/tensor_ops/scalar_op_nodes.cpp index e0f52f0..2d65358 100644 --- a/src/backend/computational_graph/tensor_ops/scalar_op_nodes.cpp +++ b/src/backend/computational_graph/tensor_ops/scalar_op_nodes.cpp @@ -12,6 +12,11 @@ #include "scalar_op_nodes.h" #include +#include + +#ifdef __CUDA +#include "computational_graph/tensor_ops/cuda/tensor_ops_nodes.cuh" +#endif using namespace std; using namespace cgraph; @@ -25,6 +30,16 @@ vector> cgraph::ScalarMulNode::backward(const Tensor& upstrea assert(!upstreamGrad.getRequiresGrad()); auto res = make_shared(upstreamGrad.createDeepCopy()); + switch(res->getDevice()) { + case Device::CPU: + + case Device::CUDA: + #ifdef __CUDA + scalarMulBackward(res, upstreamGrad, factor); + #else + __throw_invalid_argument("Not compiled with CUDA"); + #endif + } for(tensorSize_t i=0; igetSize(); i++){ res->set(res->get(i) * factor, i); } diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index e36f9c3..a221b09 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -46,15 +46,6 @@ namespace { res[gid] = fmaxf(input[gid], eps * input[gid]); // eps < 1 } - /** - * @brief Single sigmoid computation. - */ - __device__ __forceinline__ ftype sigmoid(ftype x) { - ftype z = expf(-fabsf(x)); - ftype s = 1.0f / (1.0f + z); - return (x >= 0.f) ? s : z * s; // x < 0 => e^x/(e^x+1) - } - /** * @brief Kernel for forward Sigmoid function. */ @@ -63,7 +54,7 @@ namespace { if(gid >= size) return; - res[gid] = sigmoid(input[gid]); + res[gid] = cudaSigmoid(input[gid]); } /** diff --git a/src/backend/module/activation_functions/softmax.cpp b/src/backend/module/activation_functions/softmax.cpp index df01f7a..9833cc2 100644 --- a/src/backend/module/activation_functions/softmax.cpp +++ b/src/backend/module/activation_functions/softmax.cpp @@ -57,7 +57,7 @@ Tensor Softmax::operator()(const Tensor& t) const { auto compute = [&res, &tmp, stride](tensorSize_t start){ ftype sum = 0; - for(tensorSize_t i = start; i < start+stride; i++){ + for(tensorSize_t i = start; i < start + stride; i++){ sum += tmp[i]; } diff --git a/src/backend/training/loss_functions/crossentropy_loss.cpp b/src/backend/training/loss_functions/crossentropy_loss.cpp index c598f88..a294b5d 100644 --- a/src/backend/training/loss_functions/crossentropy_loss.cpp +++ b/src/backend/training/loss_functions/crossentropy_loss.cpp @@ -42,20 +42,25 @@ shared_ptr CrossEntropyLoss::operator()(const shared_ptr y, cons switch(y->getDevice()) { case Device::CPU: { - auto ce = [&y, &ypred](const tensorDim_t b){ + const tensorSize_t stride = y->getDims()[-1]; + const tensorSize_t nSamples = y->getSize() / stride; + + auto ce = [&y, &ypred, stride](const tensorSize_t offset){ ftype r = 0; - for(tensorDim_t i=0; igetDims()[-1]; i++){ - r += y->get(b, i) * log(std::max(ypred->get(b, i), EPS_CROSSENTROPY)); + for(tensorSize_t i = offset; i < offset + stride; i++){ + r += (*y)[i] * log(std::max((*ypred)[i], EPS_CROSSENTROPY)); } return r; }; - const auto nBatches = y->getDims()[0]; ftype loss = 0; - for(tensorSize_t b=0; bgetSize()){ + loss += ce(offset); + offset += stride; } - res = make_shared(std::vector{1}, std::vector{-loss / nBatches}, y->getDevice(), true); + + res = make_shared(std::vector{1}, std::vector{-loss / nSamples}, y->getDevice(), true); break; } case Device::CUDA: diff --git a/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp b/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp index 028b5ea..7078552 100644 --- a/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp +++ b/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp @@ -70,15 +70,15 @@ shared_ptr CrossEntropySoftmaxLoss::operator()(const shared_ptr * log(sum_j(exp(z_j))) = max(z) + log(sum_j(exp(z_j - max(z)))). * for numerical stability */ - auto compute = [&loss, &y, &logits, &tmp, &maxValues, stride](tensorSize_t start){ + auto compute = [&loss, &y, &logits, &tmp, &maxValues, stride](tensorSize_t start) { ftype lsum = 0; - for(tensorSize_t i=start; i < start+stride; i++){ + for(tensorSize_t i = start; i < start + stride; i++){ lsum += tmp[i]; } lsum = log(lsum); const tensorSize_t j = start / stride; - for(tensorSize_t i = start; i < start + stride; i++){ + for(tensorSize_t i = start; i < start + stride; i++) { if((*y)[i]>0){ // y either zero or one loss += -(*logits)[i] + maxValues[j] + lsum; } diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cu b/src/backend/training/loss_functions/cuda/loss_functions.cu index 9fce431..ca05b90 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cu +++ b/src/backend/training/loss_functions/cuda/loss_functions.cu @@ -107,7 +107,7 @@ namespace cuda_impl { } Tensor bceSigmoidLoss(const Tensor& y, const Tensor& yPred) { - const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + constexpr int threadsPerBlock = 256; const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; Tensor res(vector{1}, Device::CUDA, true); @@ -118,7 +118,7 @@ namespace cuda_impl { } Tensor crossEntropyLoss(const Tensor& y, const Tensor& yPred) { - const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + constexpr int threadsPerBlock = 256; const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; Tensor res(vector{1}, Device::CUDA, true); @@ -129,7 +129,7 @@ namespace cuda_impl { } Tensor crossEntropySoftmaxLoss(const Tensor& y, const Tensor& yPred) { - const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + constexpr int threadsPerBlock = 256; const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; Tensor res(vector{1}, Device::CUDA, true); @@ -140,7 +140,7 @@ namespace cuda_impl { } Tensor rmseLoss(const Tensor& y, const Tensor& yPred) { - const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + constexpr int threadsPerBlock = 256; const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; Tensor res(vector{1}, Device::CUDA, true); diff --git a/src/backend/training/optimizers/cuda/optimizers.cu b/src/backend/training/optimizers/cuda/optimizers.cu index f20bc26..ee050d7 100644 --- a/src/backend/training/optimizers/cuda/optimizers.cu +++ b/src/backend/training/optimizers/cuda/optimizers.cu @@ -26,7 +26,7 @@ namespace { namespace cuda_impl { void sgdStep(Tensor& param, const Tensor& grad, ftype lr) { - const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + constexpr int threadsPerBlock = 256; const int blocks = (param.getSize() + threadsPerBlock - 1) / threadsPerBlock; // TODO: launch kernel @@ -35,7 +35,7 @@ namespace cuda_impl { } void rmspropStep(Tensor& param, Tensor& movingAvg, const Tensor& grad, ftype lr, ftype decay, ftype eps) { - const int threadsPerBlock = DeviceProperties::getThreadsPerBlock(); + constexpr int threadsPerBlock = 256; const int blocks = (param.getSize() + threadsPerBlock - 1) / threadsPerBlock; // TODO: launch kernel diff --git a/src/backend/utility/cuda/cuda_common.cuh b/src/backend/utility/cuda/cuda_common.cuh index 40f67d1..c2d6ee8 100644 --- a/src/backend/utility/cuda/cuda_common.cuh +++ b/src/backend/utility/cuda/cuda_common.cuh @@ -37,6 +37,27 @@ namespace utility { template constexpr bool always_false = false; +template +__device__ __forceinline__ ftype cudaMax(const ftype left, const ftype right) { + if constexpr (std::is_same_v) { + return fmaxf(a, b); + } else if(std::is_same_v) { + return fmax(a, b); + } + else { + always_false; + } +} + +/** + * @brief Single sigmoid computation. + */ +__device__ __forceinline__ ftype cudaSigmoid(ftype x) { + ftype z = expf(-fabsf(x)); + ftype s = 1.0f / (1.0f + z); + return (x >= 0.f) ? s : z * s; // x < 0 => e^x/(e^x+1) +} + #define cudaErrchk(ans) { utility::gpuAssert((ans), __FILE__, __LINE__); } namespace cuda_impl { diff --git a/src/backend/utility/global_params.h b/src/backend/utility/global_params.h index f017553..ebd4025 100644 --- a/src/backend/utility/global_params.h +++ b/src/backend/utility/global_params.h @@ -55,6 +55,7 @@ static_assert(sizeof(tensorDim_t)<=sizeof(tensorSize_t)); constexpr ftype EPS_CROSSENTROPY = 1e-5; constexpr ftype EPS_BCE = 1e-5; +constexpr ftype EPS_RMSE = 1e-9; // ----------------- Default values ------------------------ From ae003ee34cf1aee88a67c020e0be365e93051209 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 24 May 2026 13:40:05 +0200 Subject: [PATCH 59/73] Fixed compile time issues, added unit tests for backward of basic tensorops --- .../loss_functions/cuda/loss_nodes.cu | 24 ++-- .../tensor_ops/scalar_op_nodes.cpp | 14 +- src/backend/data_modeling/tensor.cpp | 4 + src/backend/module/layers/ff_layer.cpp | 9 +- src/backend/utility/cuda/cuda_common.cuh | 5 +- tests/backend/cuda/test_tensorops_cuda.cu | 130 ++++++++++++++++-- tests/backend/test_computational_graph.cpp | 2 +- tests/backend/test_tensorops.cpp | 15 +- 8 files changed, 158 insertions(+), 45 deletions(-) diff --git a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu index 1087352..3ef7b58 100644 --- a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu +++ b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu @@ -34,9 +34,9 @@ namespace { } auto yi = yTrue[gid]; - auto yiHat = Pred[gid]; + auto yiHat = yPred[gid]; - auto g = -yi / cudaMax(yiHat, EPS_BCE) + (1 - yi) / cudaMax(1 - yiHat, EPS_BCE); + auto g = -yi / cudaMax(yiHat, EPS_BCE) + (1 - yi) / cudaMax(1 - yiHat, EPS_BCE); res[gid] = g; } @@ -46,7 +46,7 @@ namespace { * @param bSize Batch-size. * @param sigmoids Sigmoids from forward pass. */ - __global__ void bceSigmoidBackwardKernel(ftype* const res, const ftype* const sigmoids, const ftype* const yTrue, const ftype bSize, const tensorSize_t size) { + __global__ void bceSigmoidBackwardKernel(ftype* const res, const ftype* const logits, const ftype* const yTrue, const ftype bSize, const tensorSize_t size) { const int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) { return; @@ -70,7 +70,7 @@ namespace { return; } - auto g = -yTrue[gid] / cudaMax(yPred[gid], EPS_CROSSENTROPY); + auto g = -yTrue[gid] / cudaMax(yPred[gid], EPS_CROSSENTROPY); res[gid] = g / nSamples; } @@ -120,7 +120,7 @@ namespace cuda_impl { constexpr int threadsPerBlock = 256; const int blocks = (logits.getSize() + threadsPerBlock - 1) / threadsPerBlock; - bceSigmoidBackwardKernel<<>>(res.getData(), logits.getData(), yTrue.getData(), logits->getDims()[0], res.getSize()); + bceSigmoidBackwardKernel<<>>(res.getData(), logits.getData(), yTrue.getData(), logits.getDims()[0], res.getSize()); cudaErrchk(cudaDeviceSynchronize()); } @@ -128,8 +128,8 @@ namespace cuda_impl { constexpr int threadsPerBlock = 256; const int blocks = (yPred.getSize() + threadsPerBlock - 1) / threadsPerBlock; - const tensorSize_t stride = yPred->getDims()[-1]; - const ftype nSamples = static_cast(yPred->getSize() / stride); + const tensorSize_t stride = yPred.getDims()[-1]; + const ftype nSamples = static_cast(yPred.getSize() / stride); crossEntropyBackwardKernel<<>>(res.getData(), yPred.getData(), yTrue.getData(), nSamples, yTrue.getSize()); cudaErrchk(cudaDeviceSynchronize()); @@ -140,12 +140,12 @@ namespace cuda_impl { const int blocks = (logits.getSize() + threadsPerBlock - 1) / threadsPerBlock; static const auto softmax = module::Softmax(); - const auto softmaxedLogits = softmax(*logits); + const auto softmaxedLogits = softmax(logits); - const tensorSize_t stride = logits->getDims().get(-1); - const auto nSamples = static_cast(->getSize() / stride); + const tensorSize_t stride = logits.getDims().get(-1); + const auto nSamples = static_cast(logits.getSize() / stride); - crossEntropySoftmaxBackwardKernel<<>>(res.getData(), softmaxedLogits.getData(), yTrue.getData(), bSize, logits.getSize()); + crossEntropySoftmaxBackwardKernel<<>>(res.getData(), softmaxedLogits.getData(), yTrue.getData(), nSamples, logits.getSize()); cudaErrchk(cudaDeviceSynchronize()); } @@ -153,7 +153,7 @@ namespace cuda_impl { constexpr int threadsPerBlock = 256; const int blocks = (yPred.getSize() + threadsPerBlock - 1) / threadsPerBlock; - rmseBackwardKernel<<>>(res.getData(), yPred.getData(), yTrue.getData(), rmse, static_cast(yPred->getDims()[0]), yPred.getSize()); + rmseBackwardKernel<<>>(res.getData(), yPred.getData(), yTrue.getData(), rmse, static_cast(yPred.getDims()[0]), yPred.getSize()); cudaErrchk(cudaDeviceSynchronize()); } } diff --git a/src/backend/computational_graph/tensor_ops/scalar_op_nodes.cpp b/src/backend/computational_graph/tensor_ops/scalar_op_nodes.cpp index 2d65358..fc70bfe 100644 --- a/src/backend/computational_graph/tensor_ops/scalar_op_nodes.cpp +++ b/src/backend/computational_graph/tensor_ops/scalar_op_nodes.cpp @@ -32,16 +32,20 @@ vector> cgraph::ScalarMulNode::backward(const Tensor& upstrea auto res = make_shared(upstreamGrad.createDeepCopy()); switch(res->getDevice()) { case Device::CPU: - + { + for(tensorSize_t i=0; igetSize(); i++){ + res->set(res->get(i) * factor, i); + } + break; + } case Device::CUDA: #ifdef __CUDA - scalarMulBackward(res, upstreamGrad, factor); + cuda_impl::scalarMulBackward(*res, upstreamGrad, factor); #else __throw_invalid_argument("Not compiled with CUDA"); #endif + break; } - for(tensorSize_t i=0; igetSize(); i++){ - res->set(res->get(i) * factor, i); - } + return {std::move(res)}; } \ No newline at end of file diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index 012e088..3961b98 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -922,6 +922,10 @@ Device Tensor::getDefaultDevice() noexcept { void Tensor::setDevice(const Device d) noexcept { makeContiguous(); values->setDevice(d); + + if(grads) { + grads->setDevice(d); + } } Device Tensor::getDevice() const noexcept { diff --git a/src/backend/module/layers/ff_layer.cpp b/src/backend/module/layers/ff_layer.cpp index e254a42..2053ba5 100644 --- a/src/backend/module/layers/ff_layer.cpp +++ b/src/backend/module/layers/ff_layer.cpp @@ -81,15 +81,18 @@ Tensor FfLayer::operator()(const Tensor& input) const { auto res = input.matmul(*weights); res = res + *bias; return res; - } else { - return input.matmul(*weights); - } + } + + return input.matmul(*weights); } #else __throw_invalid_argument("Attempted to give CUDA tensor"); return input.createShallowCopy(); // line should not be reached #endif } + + __throw_runtime_error("This line should never be reached."); + return input.createShallowCopy(); // suppress warnings } /** diff --git a/src/backend/utility/cuda/cuda_common.cuh b/src/backend/utility/cuda/cuda_common.cuh index c2d6ee8..c78add8 100644 --- a/src/backend/utility/cuda/cuda_common.cuh +++ b/src/backend/utility/cuda/cuda_common.cuh @@ -16,6 +16,7 @@ static_assert(false, "File should not be included without CUDA enabled"); #endif // __CUDA #include "cuda_runtime.h" +#include "utility/global_params.h" #include @@ -38,14 +39,14 @@ template constexpr bool always_false = false; template -__device__ __forceinline__ ftype cudaMax(const ftype left, const ftype right) { +__device__ __forceinline__ ftype cudaMax(const ftype a, const ftype b) { if constexpr (std::is_same_v) { return fmaxf(a, b); } else if(std::is_same_v) { return fmax(a, b); } else { - always_false; + static_assert(always_false, "Unexpected value for ftype encountered"); } } diff --git a/tests/backend/cuda/test_tensorops_cuda.cu b/tests/backend/cuda/test_tensorops_cuda.cu index 439fdb6..8178d72 100644 --- a/tests/backend/cuda/test_tensorops_cuda.cu +++ b/tests/backend/cuda/test_tensorops_cuda.cu @@ -61,6 +61,19 @@ TEST(CudaTensorOpsTest, ScalarMul) { } } +TEST(CudaAutogradTest, ScalarMul) { + auto t1 = TensorFunctions::makeSharedTensor({1}, {2.0}, Device::CUDA, true); + auto t2 = TensorFunctions::makeSharedTensor({1}, {3.0}, Device::CUDA, true); + + auto t3 = cgraph::mul(t1, t2); + auto loss = cgraph::mul(t3, t3); + + loss->backward(); + + ASSERT_DOUBLE_EQ(t1->getGrads()->get(0), 36.0); + ASSERT_DOUBLE_EQ(t2->getGrads()->get(0), 24.0); +} + TEST(CudaTensorOpsTest, TensorAdd) { auto t1 = TensorFunctions::Ones({500, 500}, Device::CUDA); auto t2 = TensorFunctions::Ones({500, 500}, Device::CUDA) * 4; @@ -137,8 +150,6 @@ TEST(CudaTensorOpsTest, BroadcastAddNotCommutative) { } TEST(CudaAutogradTest, BroadcastAdd) { - // gradient of broadcast add w.r.t. bias should be sum over batch dimension - // upstream grad: (2,3) of ones → bias grad should be (3) of twos auto t1 = TensorFunctions::makeSharedTensor({2, 3}, {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}, Device::CUDA, true); @@ -146,23 +157,54 @@ TEST(CudaAutogradTest, BroadcastAdd) { {0.0, 0.0, 0.0}, Device::CUDA, true); auto res = cgraph::add(t1, bias); - - // set upstream grad to ones and backprop - auto upstreamGrad = TensorFunctions::makeSharedTensor({2, 3}, - {1.0, 1.0, 1.0, - 1.0, 1.0, 1.0}, false); res->backward(); // bias grad should be sum over batch: [2, 2, 2] auto biasGrad = bias->getGrads(); - ASSERT_DOUBLE_EQ((*biasGrad)[0], 2.0); - ASSERT_DOUBLE_EQ((*biasGrad)[1], 2.0); - ASSERT_DOUBLE_EQ((*biasGrad)[2], 2.0); + ASSERT_NEAR((*biasGrad)[0], 2.0, 1e-5); + ASSERT_NEAR((*biasGrad)[1], 2.0, 1e-5); + ASSERT_NEAR((*biasGrad)[2], 2.0, 1e-5); // t1 grad should be ones (add is identity for non-broadcast operand) auto t1Grad = t1->getGrads(); for(int i = 0; i < 6; i++) { - ASSERT_DOUBLE_EQ((*t1Grad)[i], 1.0); + ASSERT_NEAR((*t1Grad)[i], 1.0, 1e-5); + } +} + +TEST(CudaAutogradTest, BroadcastAddLarge) { + constexpr int dimsize = 1500; + + auto t1 = std::make_shared( + TensorFunctions::Gaussian({2, dimsize}, + 5.0, Device::CPU, true)); + + auto bias = std::make_shared( + TensorFunctions::Gaussian({dimsize}, + 2.0, Device::CPU, true)); + + auto t1Gpu = std::make_shared(t1->createDeepCopy()); + t1Gpu->setDevice(Device::CUDA); + auto biasGpu = std::make_shared(bias->createDeepCopy()); + biasGpu->setDevice(Device::CUDA); + + auto resCpu = cgraph::add(t1, bias); + resCpu->backward(); + + auto resGpu = cgraph::add(t1Gpu, biasGpu); + resGpu->backward(); + resGpu->setDevice(Device::CPU); + + auto biasGrad = bias->getGrads(); + auto biasGradGpu = biasGpu->getGrads(); + for(int i = 0; i < biasGrad->getSize(); i++) { + ASSERT_NEAR((*biasGrad)[i], (*biasGradGpu)[i], 1e-5); + } + + auto t1Grad = t1->getGrads(); + auto t1GpuGrads = t1Gpu->getGrads(); + for(int i = 0; i < t1Grad->getSize(); i++) { + ASSERT_NEAR((*t1Grad)[i], (*t1GpuGrads)[i], 1e-5); } } @@ -250,7 +292,7 @@ TEST(CudaTensorOpsTest, MatMulLarge) { const auto expectedDims = resCpu.getDims().toVector(); ASSERT_EQ(resGpu.getDims().toVector(), expectedDims); - for(auto i = 0; i< resCpu.getDims().get(0); i++) { + for(auto i = 0; i < resCpu.getDims().get(0); i++) { for(auto j = 0; j < resCpu.getDims().get(1); j++) { ASSERT_NEAR(resCpu.get(i, j), resGpu.get(i, j), 1e-4) << "Mismatch at (" << i << ", " << j << ")" @@ -267,6 +309,70 @@ TEST(CudaTensorOpsTest, MatMulThrowsWhenDimensionsNotMatched) { ASSERT_THROW(t1.matmul(t2), std::runtime_error); } +TEST(CudaAutogradTest, MatMul) { + constexpr int dimsize = 30; + + // init tensors + auto t1 = std::make_shared( + TensorFunctions::Gaussian( + {10, dimsize}, 2.0, Device::CPU, true)); + + auto t2 = std::make_shared( + TensorFunctions::Gaussian( + {dimsize, 10}, 2.0, Device::CPU, true)); + + auto t1Gpu = std::make_shared(t1->createDeepCopy()); + auto t2Gpu = std::make_shared(t2->createDeepCopy()); + t1Gpu->setDevice(Device::CUDA); + t2Gpu->setDevice(Device::CUDA); + + { + // compute and take loss + auto resCpu = cgraph::matmul(t1, t2); + auto lossCpu = TensorFunctions::makeSharedTensor( + {1}, {0.0}, Device::CPU, true); + for(size_t i = 0; i < resCpu->getSize(); ++i) { + lossCpu = cgraph::add(lossCpu, cgraph::get(resCpu, i)); + } + lossCpu->backward(); + } + + { + auto resGpu = cgraph::matmul(t1Gpu, t2Gpu); + auto lossGpu = TensorFunctions::makeSharedTensor( + {1}, {0.0}, Device::CUDA, true); + for(size_t i = 0; i < resGpu->getSize(); ++i) { + lossGpu = cgraph::add(lossGpu, cgraph::get(resGpu, i)); + } + lossGpu->backward(); + } + + // get grads and compare + auto t1Grads = t1->getGrads(); + auto t2Grads = t2->getGrads(); + + auto t1GpuGrads = t1Gpu->getGrads(); + auto t2GpuGrads = t2Gpu->getGrads(); + + for(auto i = 0; i < t1Grads->getDims().get(0); i++) { + for(auto j = 0; j < t1Grads->getDims().get(1); j++) { + EXPECT_NEAR(t1Grads->get(i, j), t1GpuGrads->get(i, j), 1e-5) + << "Mismatch at (" << i << ", " << j << ")" + << " cpu=" << t1Grads->get(i, j) + << " gpu=" << t1GpuGrads->get(i, j); + } + } + + for(auto i = 0; i < t2Grads->getDims().get(0); i++) { + for(auto j = 0; j < t2Grads->getDims().get(1); j++) { + EXPECT_NEAR(t2Grads->get(i, j), t2GpuGrads->get(i, j), 1e-5) + << "Mismatch at (" << i << ", " << j << ")" + << " cpu=" << t2Grads->get(i, j) + << " gpu=" << t2GpuGrads->get(i, j); + } + } +} + TEST(CudaTensorOpsTest, MatrixTranspose1) { auto t = TensorFunctions::Gaussian({200, 200}, 1.0, Device::CUDA); diff --git a/tests/backend/test_computational_graph.cpp b/tests/backend/test_computational_graph.cpp index a3d4694..d3993d9 100644 --- a/tests/backend/test_computational_graph.cpp +++ b/tests/backend/test_computational_graph.cpp @@ -56,7 +56,7 @@ TEST(AutogradTest, MultiVariateChainRule) { auto y = cgraph::mul(x, 3.0); // y = [3, 6] auto loss = TensorFunctions::makeSharedTensor({1}, {0.0}, true); - for(int i=0; igetSize(); i++){ + for(int i = 0; i < y->getSize(); i++){ loss = cgraph::add(loss, cgraph::get(y, i)); } // loss = 9 diff --git a/tests/backend/test_tensorops.cpp b/tests/backend/test_tensorops.cpp index 5af6c29..0a197e3 100644 --- a/tests/backend/test_tensorops.cpp +++ b/tests/backend/test_tensorops.cpp @@ -57,7 +57,7 @@ TEST(TensorOpsTest, ScalarMul) { } } -TEST(AutogradTest, ScalarMultiplication) { +TEST(AutogradTest, ScalarMul) { auto t1 = TensorFunctions::makeSharedTensor({1}, {2.0}, true); auto t2 = TensorFunctions::makeSharedTensor({1}, {3.0}, true); @@ -153,23 +153,18 @@ TEST(AutogradTest, BroadcastAdd) { {0.0, 0.0, 0.0}, true); auto res = cgraph::add(t1, bias); - - // set upstream grad to ones and backprop - auto upstreamGrad = TensorFunctions::makeSharedTensor({2, 3}, - {1.0, 1.0, 1.0, - 1.0, 1.0, 1.0}, false); res->backward(); // bias grad should be sum over batch: [2, 2, 2] auto biasGrad = bias->getGrads(); - ASSERT_DOUBLE_EQ((*biasGrad)[0], 2.0); - ASSERT_DOUBLE_EQ((*biasGrad)[1], 2.0); - ASSERT_DOUBLE_EQ((*biasGrad)[2], 2.0); + ASSERT_NEAR((*biasGrad)[0], 2.0, 1e-5); + ASSERT_NEAR((*biasGrad)[1], 2.0, 1e-5); + ASSERT_NEAR((*biasGrad)[2], 2.0, 1e-5); // t1 grad should be ones (add is identity for non-broadcast operand) auto t1Grad = t1->getGrads(); for(int i = 0; i < 6; i++) { - ASSERT_DOUBLE_EQ((*t1Grad)[i], 1.0); + ASSERT_NEAR((*t1Grad)[i], 1.0, 1e-5); } } From 5cde0c6577cd745536188416b3622aadb9305804 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 24 May 2026 15:14:31 +0200 Subject: [PATCH 60/73] More forward loss kernels, prepared unit tests for losses --- .../training/loss_functions/bce_loss.cpp | 19 +- .../loss_functions/bce_sigmoid_loss.cpp | 7 +- .../loss_functions/crossentropy_loss.cpp | 3 +- .../crossentropy_softmax_loss.cpp | 3 +- .../loss_functions/cuda/loss_functions.cu | 189 +++++++++++---- .../loss_functions/cuda/loss_functions.cuh | 10 +- .../training/loss_functions/rmse_loss.cpp | 3 +- tests/backend/cuda/test_losses.cu | 220 ++++++++++++++++++ 8 files changed, 386 insertions(+), 68 deletions(-) create mode 100644 tests/backend/cuda/test_losses.cu diff --git a/src/backend/training/loss_functions/bce_loss.cpp b/src/backend/training/loss_functions/bce_loss.cpp index 43b1d87..ebd481d 100644 --- a/src/backend/training/loss_functions/bce_loss.cpp +++ b/src/backend/training/loss_functions/bce_loss.cpp @@ -41,14 +41,8 @@ shared_ptr BceLoss::operator()(const shared_ptr y, const shared_ shared_ptr res = nullptr; switch(y->getDevice()) { - case Device::CUDA: - #ifdef __CUDA - res = make_shared(cuda_impl::bceLoss(*y, *ypred)); - #else - __throw_invalid_argument("Attempted to give CUDA tensor"); - #endif - break; - case Device::CPU: { + case Device::CPU: + { auto bce = [](ftype y, ftype ypred){ return y * log(std::max(ypred, EPS_BCE)) + (1 - y) * log(std::max(1-ypred, EPS_BCE)); }; @@ -58,9 +52,18 @@ shared_ptr BceLoss::operator()(const shared_ptr y, const shared_ for(tensorSize_t i = 0; i < nBatches; i++){ loss += bce((*y)[i], (*ypred)[i]); } + res = make_shared(std::vector{1}, std::vector{-loss / nBatches}, Device::CPU, true); break; } + case Device::CUDA: + #ifdef __CUDA + res = make_shared(vector{1}, Device::CUDA, true); + cuda_impl::bceLoss(*res, *y, *ypred); + #else + __throw_invalid_argument("Attempted to give CUDA tensor"); + #endif + break; } res->setCgNode(make_shared(y, ypred)); diff --git a/src/backend/training/loss_functions/bce_sigmoid_loss.cpp b/src/backend/training/loss_functions/bce_sigmoid_loss.cpp index 21cbe29..eafbc05 100644 --- a/src/backend/training/loss_functions/bce_sigmoid_loss.cpp +++ b/src/backend/training/loss_functions/bce_sigmoid_loss.cpp @@ -44,12 +44,12 @@ shared_ptr BceSigmoidLoss::operator()(const shared_ptr y, const case Device::CPU: { auto bceSimplified = [](ftype y, ftype logit){ constexpr ftype zero = 0; - return std::max(logit, zero) - logit*y + log(1+exp(-std::abs(logit))); + return std::max(logit, zero) - logit * y + log(1 + exp(-std::abs(logit))); }; const auto nBatches = y->getDims()[0]; ftype loss = 0; - for(tensorSize_t i=0; i(std::vector{1}, std::vector{loss / nBatches}, y->getDevice(), true); @@ -57,7 +57,8 @@ shared_ptr BceSigmoidLoss::operator()(const shared_ptr y, const } case Device::CUDA: #ifdef __CUDA - res = make_shared(cuda_impl::bceSigmoidLoss(*y, *logits)); + res = make_shared(vector{1}, Device::CUDA, true); + cuda_impl::bceSigmoidLoss(*res, *y, *logits); #else __throw_invalid_argument("Attempted to give CUDA tensor"); #endif diff --git a/src/backend/training/loss_functions/crossentropy_loss.cpp b/src/backend/training/loss_functions/crossentropy_loss.cpp index a294b5d..67e25c5 100644 --- a/src/backend/training/loss_functions/crossentropy_loss.cpp +++ b/src/backend/training/loss_functions/crossentropy_loss.cpp @@ -65,7 +65,8 @@ shared_ptr CrossEntropyLoss::operator()(const shared_ptr y, cons } case Device::CUDA: #ifdef __CUDA - res = make_shared(cuda_impl::crossEntropyLoss(*y, *ypred)); + res = make_shared(vector{1}, Device::CUDA, true); + cuda_impl::crossEntropyLoss(*res, *y, *ypred); #else __throw_invalid_argument("Attempted to give CUDA tensor"); #endif diff --git a/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp b/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp index 7078552..feedf8d 100644 --- a/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp +++ b/src/backend/training/loss_functions/crossentropy_softmax_loss.cpp @@ -96,7 +96,8 @@ shared_ptr CrossEntropySoftmaxLoss::operator()(const shared_ptr } case Device::CUDA: #ifdef __CUDA - res = make_shared(cuda_impl::crossEntropySoftmaxLoss(*y, *logits)); + res = make_shared(vector{1}, Device::CUDA, true); + cuda_impl::crossEntropySoftmaxLoss(*res, *y, *logits); #else __throw_invalid_argument("Attempted to give CUDA tensor"); #endif diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cu b/src/backend/training/loss_functions/cuda/loss_functions.cu index ca05b90..dd1203a 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cu +++ b/src/backend/training/loss_functions/cuda/loss_functions.cu @@ -16,76 +16,145 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include "loss_functions.cuh" #include "utility/cuda/cuda_common.cuh" +#include +#include + using namespace std; namespace { template __forceinline__ __device__ T bce(T y, T ypred) { if constexpr (std::is_same_v) { - return y * __logf(max(ypred, EPS_BCE)) + (1 - y) * __logf(max(1-ypred, EPS_BCE)); + return y * __logf(cudaMax(ypred, EPS_BCE)) + (1 - y) * __logf(cudaMax(1 - ypred, EPS_BCE)); } else if constexpr (std::is_same_v) { - return y * log(max(ypred, EPS_BCE)) + (1 - y) * log(max(1-ypred, EPS_BCE)); + return y * log(cudaMax(ypred, EPS_BCE)) + (1 - y) * log(cudaMax(1 - ypred, EPS_BCE)); } else { static_assert(always_false, "Unexpected value for ftype"); } } - __global__ void bceKernel(ftype* const res, const ftype* const y, const ftype* const ypred, tensorSize_t size) { + /** + * @brief Forward BCE loss. + */ + __global__ void bceLossKernel(ftype* const res, const ftype* const y, const ftype* const ypred, tensorSize_t size) { int gid = blockDim.x * blockIdx.x + threadIdx.x; if(gid >= size) return; int tid = threadIdx.x; - extern __shared__ ftype sdata[]; + extern __shared__ ftype smem[]; // pre-load first round { int i = blockIdx.x * (blockDim.x * 2) + threadIdx.x; - sdata[tid] = bce(y[i], ypred[i]) + bce(y[i + blockDim.x], ypred[i + blockDim.x]); + smem[tid] = bce(y[i], ypred[i]) + bce(y[i + blockDim.x], ypred[i + blockDim.x]); __syncthreads(); } for(tensorSize_t i = blockDim.x / 2; i >= 64; i >>= 1){ if(tid < i) { - sdata[tid] += sdata[tid + i]; + smem[tid] += smem[tid + i]; } __syncthreads(); } - if(tid < 32 && gid + 32 < size) { - sdata[tid] += sdata[tid + 32]; + volatile ftype* sdata = smem; + if(tid < 32) { + if(gid + 32 < size) { + sdata[tid] += sdata[tid + 32]; + } + if(gid + 16 < size) { + sdata[tid] += sdata[tid + 16]; + } + if(gid + 8 < size) { + sdata[tid] += sdata[tid + 8]; + } + if(gid + 4 < size) { + sdata[tid] += sdata[tid + 4]; + } + if(gid + 2 < size) { + sdata[tid] += sdata[tid + 2]; + } + if(gid + 1 < size) { + sdata[0] = (sdata[0] + sdata[1]) / size; + } + } + + if(tid == 0) { + res[blockIdx.x] = sdata[0]; } - __syncthreads(); + } - if(tid < 16 && gid + 16 < size) { - sdata[tid] += sdata[tid + 16]; + template + __forceinline__ __device__ T bceSimplified(T y, T logit) { + constexpr T zero = 0; + if constexpr (std::is_same_v) { + return cudaMax(logit, zero) - logit * y + __logf(1 + __expf(abs(logit))); + } + else if constexpr (std::is_same_v) { + return cudaMax(logit, zero) - logit * y + log(1 + exp(abs(logit))); } - __syncthreads(); + else { + static_assert(always_false, "Unexpected value for ftype"); + } + } + + /** + * @brief BCE kernel with integrated sigmoid. + * + * @param logits Forward logits. + */ + __global__ void bceSigmoidLossKernel(ftype* const res, const ftype* const y, const ftype* const logits, tensorSize_t size) { + int gid = blockDim.x * blockIdx.x + threadIdx.x; + if(gid >= size) + return; + + int tid = threadIdx.x; + extern __shared__ ftype smem[]; - if(tid < 8 && gid + 8 < size) { - sdata[tid] += sdata[tid + 8]; + // pre-load first round + { + int i = blockIdx.x * (blockDim.x * 2) + threadIdx.x; + smem[tid] = bceSimplified(y[i], logits[i]) + bceSimplified(y[i + blockDim.x], logits[i + blockDim.x]); + __syncthreads(); } - __syncthreads(); - if(tid < 4 && gid + 4 < size) { - sdata[tid] += sdata[tid + 4]; + for(tensorSize_t i = blockDim.x / 2; i >= 64; i >>= 1){ + if(tid < i) { + smem[tid] += smem[tid + i]; + } + __syncthreads(); } - __syncthreads(); - if(tid < 2 && gid + 2 < size) { - sdata[tid] += sdata[tid + 2]; + volatile ftype* sdata = smem; + if(tid < 32) { + if(gid + 32 < size) { + sdata[tid] += sdata[tid + 32]; + } + if(gid + 16 < size) { + sdata[tid] += sdata[tid + 16]; + } + if(gid + 8 < size) { + sdata[tid] += sdata[tid + 8]; + } + if(gid + 4 < size) { + sdata[tid] += sdata[tid + 4]; + } + if(gid + 2 < size) { + sdata[tid] += sdata[tid + 2]; + } + if(gid + 1 < size) { + sdata[0] = (sdata[0] + sdata[1]) / size; + } } - __syncthreads(); - if(tid == 0 && gid + 1 < size) { - sdata[0] = (sdata[0] + sdata[1]) / size; + if(tid == 0) { + res[blockIdx.x] = sdata[0]; } } - // TODO: bceSigmoidLoss kernel - // TODO: crossEntropyLoss kernel // TODO: crossEntropySoftmaxLoss kernel @@ -94,59 +163,81 @@ namespace { } namespace cuda_impl { - Tensor bceLoss(const Tensor& y, const Tensor& yPred) { + void bceLoss(Tensor& res, const Tensor& y, const Tensor& yPred) { constexpr int threadsPerBlock = 256; const int blocks = (y.getSize() + threadsPerBlock - 1) / (threadsPerBlock * 2); - Tensor res(vector{1}, Device::CUDA, true); - bceKernel<<>>( - res.getData(), y.getData(), yPred.getData(), y.getDims()[0]); - cudaErrchk(cudaDeviceSynchronize()); + if(blocks > 1) { + // we do two passes at max + ftype* tmp; + cudaErrchk(cudaMalloc(&tmp, blocks * sizeof(ftype))); - return res; + bceLossKernel<<>>( + tmp, y.getData(), yPred.getData(), y.getDims()[0]); + cudaErrchk(cudaDeviceSynchronize()); + + // do a sum over the residual array + thrust::device_ptr tmpPtr(tmp); + thrust::device_ptr resPtr(res.getData()); + resPtr[0] = thrust::reduce(tmpPtr, tmpPtr + blocks, static_cast(0.0f), thrust::plus()); + + cudaErrchk(cudaFree(tmp)); + } + else { + bceLossKernel<<>>( + res.getData(), y.getData(), yPred.getData(), y.getDims()[0]); + cudaErrchk(cudaDeviceSynchronize()); + } } - Tensor bceSigmoidLoss(const Tensor& y, const Tensor& yPred) { + void bceSigmoidLoss(Tensor& res, const Tensor& y, const Tensor& logits) { constexpr int threadsPerBlock = 256; - const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; + const int blocks = (y.getSize() + threadsPerBlock - 1) / (threadsPerBlock * 2); - Tensor res(vector{1}, Device::CUDA, true); - // TODO: launch kernel - cudaErrchk(cudaDeviceSynchronize()); + if(blocks > 1) { + // we do two passes at max + ftype* tmp; + cudaErrchk(cudaMalloc(&tmp, blocks * sizeof(ftype))); + + bceSigmoidLossKernel<<>>( + tmp, y.getData(), logits.getData(), y.getDims()[0]); + cudaErrchk(cudaDeviceSynchronize()); - return res; + // do a sum over the residual array + thrust::device_ptr tmpPtr(tmp); + thrust::device_ptr resPtr(res.getData()); + resPtr[0] = thrust::reduce(tmpPtr, tmpPtr + blocks, static_cast(0.0f), thrust::plus()); + + cudaErrchk(cudaFree(tmp)); + } + else { + bceSigmoidLossKernel<<>>( + res.getData(), y.getData(), logits.getData(), y.getDims()[0]); + cudaErrchk(cudaDeviceSynchronize()); + } } - Tensor crossEntropyLoss(const Tensor& y, const Tensor& yPred) { + void crossEntropyLoss(Tensor& res, const Tensor& y, const Tensor& yPred) { constexpr int threadsPerBlock = 256; const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; - Tensor res(vector{1}, Device::CUDA, true); // TODO: launch kernel cudaErrchk(cudaDeviceSynchronize()); - - return res; } - Tensor crossEntropySoftmaxLoss(const Tensor& y, const Tensor& yPred) { + void crossEntropySoftmaxLoss(Tensor& res, const Tensor& y, const Tensor& yPred) { constexpr int threadsPerBlock = 256; const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; - Tensor res(vector{1}, Device::CUDA, true); // TODO: launch kernel cudaErrchk(cudaDeviceSynchronize()); - - return res; } - Tensor rmseLoss(const Tensor& y, const Tensor& yPred) { + void rmseLoss(Tensor& res, const Tensor& y, const Tensor& yPred) { constexpr int threadsPerBlock = 256; const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; - Tensor res(vector{1}, Device::CUDA, true); // TODO: launch kernel cudaErrchk(cudaDeviceSynchronize()); - - return res; } } diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cuh b/src/backend/training/loss_functions/cuda/loss_functions.cuh index 87eee44..fa41596 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cuh +++ b/src/backend/training/loss_functions/cuda/loss_functions.cuh @@ -18,11 +18,11 @@ static_assert(false, "File should not be included without CUDA enabled"); #include "data_modeling/tensor.h" namespace cuda_impl { - [[nodiscard]] Tensor bceLoss(const Tensor& y, const Tensor& yPred); - [[nodiscard]] Tensor bceSigmoidLoss(const Tensor& y, const Tensor& yPred); + void bceLoss(Tensor& res, const Tensor& y, const Tensor& yPred); + void bceSigmoidLoss(Tensor& res, const Tensor& y, const Tensor& logits); - [[nodiscard]] Tensor crossEntropyLoss(const Tensor& y, const Tensor& yPred); - [[nodiscard]] Tensor crossEntropySoftmaxLoss(const Tensor& y, const Tensor& yPred); + void crossEntropyLoss(Tensor& res, const Tensor& y, const Tensor& yPred); + void crossEntropySoftmaxLoss(Tensor& res, const Tensor& y, const Tensor& yPred); - [[nodiscard]] Tensor rmseLoss(const Tensor& y, const Tensor& yPred); + void rmseLoss(Tensor& res, const Tensor& y, const Tensor& yPred); } diff --git a/src/backend/training/loss_functions/rmse_loss.cpp b/src/backend/training/loss_functions/rmse_loss.cpp index 872fddf..08513cf 100644 --- a/src/backend/training/loss_functions/rmse_loss.cpp +++ b/src/backend/training/loss_functions/rmse_loss.cpp @@ -60,7 +60,8 @@ shared_ptr RmseLoss::operator()(const shared_ptr y, const shared } case Device::CUDA: #ifdef __CUDA - res = make_shared(cuda_impl::rmseLoss(*y, *ypred)); + res = make_shared(vector{1}, Device::CUDA, true); + cuda_impl::rmseLoss(*res, *y, *ypred); #else __throw_invalid_argument("Attempted to give CUDA tensor"); #endif diff --git a/tests/backend/cuda/test_losses.cu b/tests/backend/cuda/test_losses.cu new file mode 100644 index 0000000..c65148a --- /dev/null +++ b/tests/backend/cuda/test_losses.cu @@ -0,0 +1,220 @@ +/** + * @file test_losses.cu + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-05-24 + * + * @copyright Copyright (c) 2026 + * + */ + +#ifndef __CUDA +static_assert(false, "File should not be compiled without CUDA enabled"); +#endif // __CUDA + +#include + +#include "data_modeling/tensor_functions.h" + +#include "training/loss_functions/rmse_loss.h" +#include "training/loss_functions/bce_loss.h" +#include "training/loss_functions/crossentropy_loss.h" + +#include + +using namespace train; + +static constexpr ftype delta = 1e-4f; + +TEST(CudaLossTest, CrossEntropyForward) { + auto y = TensorFunctions::makeSharedTensor( + {2, 3}, {1.0, 0.0, 0.0, + 0.0, 1.0, 0.0}, Device::CUDA, false); + + auto ypred = TensorFunctions::makeSharedTensor( + {2, 3}, {0.7, 0.2, 0.1, + 0.1, 0.8, 0.1}, Device::CUDA, true); + + CrossEntropyLoss loss; + auto result = loss(y, ypred); + + const ftype expected = -(std::log(0.7f) + std::log(0.8f)) / 2.0f; + ASSERT_NEAR((*result)[0], expected, delta); +} + +TEST(CudaLossTest, CrossEntropyPerfectPrediction) { + auto y = TensorFunctions::makeSharedTensor( + {2, 3}, {1.0, 0.0, 0.0, + 0.0, 1.0, 0.0}, Device::CUDA, false); + + auto ypred = TensorFunctions::makeSharedTensor( + {2, 3}, {0.999, 0.0005, 0.0005, + 0.0005, 0.999, 0.0005}, Device::CUDA, true); + + CrossEntropyLoss loss; + auto result = loss(y, ypred); + + ASSERT_LT((*result)[0], 0.01f); +} + +TEST(CudaLossTest, CrossEntropyUniformPrediction) { + auto y = TensorFunctions::makeSharedTensor( + {1, 3}, {1.0, 0.0, 0.0}, Device::CUDA, false); + + auto ypred = TensorFunctions::makeSharedTensor( + {1, 3}, {1.0f/3, 1.0f/3, 1.0f/3}, Device::CUDA, true); + + CrossEntropyLoss loss; + auto result = loss(y, ypred); + + ASSERT_NEAR((*result)[0], std::log(3.0f), delta); +} + +TEST(CudaLossTest, CrossEntropyThrowsOnDimMismatch) { + auto y = TensorFunctions::makeSharedTensor( + {2, 3}, {1.0, 0.0, 0.0, 0.0, 1.0, 0.0}, Device::CUDA, false); + auto ypred = TensorFunctions::makeSharedTensor( + {2, 2}, {0.5, 0.5, 0.5, 0.5}, Device::CUDA, true); + + CrossEntropyLoss loss; + ASSERT_THROW(loss(y, ypred), std::invalid_argument); +} + +TEST(CudaLossTest, CrossEntropyBackward) { + auto y = TensorFunctions::makeSharedTensor( + {2, 3}, {1.0, 0.0, 0.0, + 0.0, 1.0, 0.0}, Device::CUDA, false); + auto ypred = TensorFunctions::makeSharedTensor( + {2, 3}, {0.7, 0.2, 0.1, + 0.1, 0.8, 0.1}, Device::CUDA, true); + + CrossEntropyLoss loss; + auto result = loss(y, ypred); + result->backward(); + + auto grads = ypred->getGrads(); + ASSERT_NEAR((*grads)[0], -0.7143f, delta); + ASSERT_NEAR((*grads)[1], 0.0f, delta); + ASSERT_NEAR((*grads)[2], 0.0f, delta); + ASSERT_NEAR((*grads)[3], 0.0f, delta); + ASSERT_NEAR((*grads)[4], -0.625f, delta); + ASSERT_NEAR((*grads)[5], 0.0f, delta); +} + +TEST(CudaLossTest, BceForward) { + auto y = TensorFunctions::makeSharedTensor( + {4, 1}, {0.0, 1.0, 1.0, 0.0}, Device::CUDA, false); + + auto ypred = TensorFunctions::makeSharedTensor( + {4, 1}, {0.1, 0.9, 0.8, 0.2}, Device::CUDA, true); + + BceLoss loss; + auto result = loss(y, ypred); + + const ftype expected = -(std::log(0.9f) + std::log(0.9f) + + std::log(0.8f) + std::log(0.8f)) / 4.0f; + ASSERT_NEAR((*result)[0], expected, delta); +} + +TEST(CudaLossTest, BcePerfectPrediction) { + auto y = TensorFunctions::makeSharedTensor( + {2, 1}, {1.0, 0.0}, Device::CUDA, false); + + auto ypred = TensorFunctions::makeSharedTensor( + {2, 1}, {0.999, 0.001}, Device::CUDA, true); + + BceLoss loss; + auto result = loss(y, ypred); + + ASSERT_LT((*result)[0], 0.01f); +} + +TEST(CudaLossTest, BceRandomPrediction) { + auto y = TensorFunctions::makeSharedTensor( + {2, 1}, {1.0, 0.0}, Device::CUDA, false); + + auto ypred = TensorFunctions::makeSharedTensor( + {2, 1}, {0.5, 0.5}, Device::CUDA, true); + + BceLoss loss; + auto result = loss(y, ypred); + + ASSERT_NEAR((*result)[0], std::log(2.0f), delta); +} + +TEST(CudaLossTest, BceThrowsOnDimMismatch) { + auto y = TensorFunctions::makeSharedTensor( + {2, 1}, {1.0, 0.0}, Device::CUDA, false); + auto ypred = TensorFunctions::makeSharedTensor( + {3, 1}, {0.5, 0.5, 0.5}, Device::CUDA, true); + + BceLoss loss; + ASSERT_THROW(loss(y, ypred), std::invalid_argument); +} + +TEST(CudaLossTest, BceNoInfOrNanOnNearZeroPred) { + auto y = TensorFunctions::makeSharedTensor( + {1, 1}, {1.0}, Device::CUDA, false); + auto ypred = TensorFunctions::makeSharedTensor( + {1, 1}, {0.0}, Device::CUDA, true); + + BceLoss loss; + auto result = loss(y, ypred); + + ASSERT_FALSE(std::isinf((*result)[0])); +} + +TEST(CudaLossTest, BceBackward) { + auto y = TensorFunctions::makeSharedTensor( + {2, 1}, {1.0, 0.0}, Device::CUDA, false); + auto ypred = TensorFunctions::makeSharedTensor( + {2, 1}, {0.8, 0.3}, Device::CUDA, true); + + BceLoss loss; + auto result = loss(y, ypred); + result->backward(); + + auto grads = ypred->getGrads(); + ASSERT_NEAR((*grads)[0], -0.625f, delta); + ASSERT_NEAR((*grads)[1], 0.7143f, delta); +} + +TEST(CudaLossTest, RmseForward) { + auto y = TensorFunctions::makeSharedTensor( + {3}, {1.0, 2.0, 3.0}, Device::CUDA, false); + auto ypred = TensorFunctions::makeSharedTensor( + {3}, {1.5, 2.5, 2.5}, Device::CUDA, true); + + RmseLoss loss; + auto result = loss(y, ypred); + + ASSERT_NEAR((*result)[0], 0.5f, delta); +} + +TEST(CudaLossTest, RmsePerfectPrediction) { + auto y = TensorFunctions::makeSharedTensor( + {3}, {1.0, 2.0, 3.0}, Device::CUDA, false); + auto ypred = TensorFunctions::makeSharedTensor( + {3}, {1.0, 2.0, 3.0}, Device::CUDA, true); + + RmseLoss loss; + auto result = loss(y, ypred); + + ASSERT_NEAR((*result)[0], 0.0f, delta); +} + +TEST(CudaLossTest, RmseBackward) { + auto y = TensorFunctions::makeSharedTensor( + {2}, {1.0, 0.0}, Device::CUDA, false); + auto ypred = TensorFunctions::makeSharedTensor( + {2}, {0.5, 0.5}, Device::CUDA, true); + + RmseLoss loss; + auto result = loss(y, ypred); + result->backward(); + + auto grads = ypred->getGrads(); + ASSERT_NEAR((*grads)[0], -0.5f, delta); + ASSERT_NEAR((*grads)[1], 0.5f, delta); +} From 4290ad92fe170e4add6eea5f5855491a5df11575 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Mon, 25 May 2026 15:15:33 +0200 Subject: [PATCH 61/73] Forward crossentropy and RMSE losses --- readme.md | 4 +- .../loss_functions/cuda/loss_functions.cu | 220 +++++++++++++++++- .../training/loss_functions/rmse_loss.cpp | 7 +- src/backend/utility/cuda/cuda_common.cuh | 7 + 4 files changed, 224 insertions(+), 14 deletions(-) diff --git a/readme.md b/readme.md index 3c8d55a..b7b137e 100644 --- a/readme.md +++ b/readme.md @@ -25,12 +25,13 @@ For some examples on Python interface, see tests/python. ## Tech Stack -- C++17/20 +- C++17/20/23 - CMake build system - Boost.Python for Python bindings - Python 3 for library interface and examples - Google Test (GTest) and PyTest for unit testing - GitHub Actions for CI/CD +- CUDA ## Current Status @@ -82,6 +83,7 @@ ctest . - numpy 1.26.4 - pytest and GTest for unit tests (we use pytest=9.0.2) - Google Benchmark for benchmarking +- CUDA (we use CUDA 13.1 on an RTX-5050) ## Troubleshooting diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cu b/src/backend/training/loss_functions/cuda/loss_functions.cu index dd1203a..a1547c9 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cu +++ b/src/backend/training/loss_functions/cuda/loss_functions.cu @@ -15,6 +15,7 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include "loss_functions.cuh" #include "utility/cuda/cuda_common.cuh" +#include "utility/macros.h" #include #include @@ -60,6 +61,7 @@ namespace { __syncthreads(); } + // TODO: warp shuffles volatile ftype* sdata = smem; if(tid < 32) { if(gid + 32 < size) { @@ -128,6 +130,7 @@ namespace { __syncthreads(); } + // TODO: warp shuffles volatile ftype* sdata = smem; if(tid < 32) { if(gid + 32 < size) { @@ -155,11 +158,140 @@ namespace { } } - // TODO: crossEntropyLoss kernel + template + __forceinline__ __device__ ftype crossEntropy(const ftype y, const ftype ypred) { + if constexpr (std::is_same_v) { + return y * __logf(cudaMax(ypred, EPS_CROSSENTROPY)); + } + else if constexpr (std::is_same_v) { + return y * log(cudaMax(ypred, EPS_CROSSENTROPY)); + } + else { + static_assert(always_false, "Encountered unexpected ftype"); + } + } + + /** + * @brief Cross-entropy reduction over a full block. Covers two times blockDim.x + */ + __global__ void crossEntropyLossKernelOneBlock(ftype* const res, const ftype* const y, const ftype* const yPred, const tensorSize_t size) { + const int tid = threadIdx.x; + const int gid = blockIdx.x * blockDim.x + tid; + + extern __shared__ ftype smem[]; + smem[tid] = crossEntropy(y[gid], yPred[gid]); + if(gid + blockDim.x < size) { + smem[tid] += crossEntropy(y[gid + blockDim.x], yPred[gid + blockDim.x]); + } + __syncthreads(); + + for(int offset = blockDim.x / 2; offset > 64; offset >>= 2) { + if(tid < offset) { + smem[tid] += smem[tid + offset]; + } + __syncthreads(); + } + + // TODO: warp shuffle again + volatile ftype* sdata = smem; + if(tid < 32) { + if(gid + 32 < size) { + sdata[tid] += sdata[tid + 32]; + } + if(gid + 16 < size) { + sdata[tid] += sdata[tid + 16]; + } + if(gid + 8 < size) { + sdata[tid] += sdata[tid + 8]; + } + if(gid + 4 < size) { + sdata[tid] += sdata[tid + 4]; + } + if(gid + 2 < size) { + sdata[tid] += sdata[tid + 2]; + } + if(gid + 1 < size) { + sdata[0] = (sdata[0] + sdata[1]) / size; + } + } + + if(threadIdx.x == 0) { + res[blockDim.x] = sdata[0]; + } + } // TODO: crossEntropySoftmaxLoss kernel - // TODO: rmseLoss kernel + /** + * @brief Helper for RMSE loss. + */ + __forceinline__ __device__ ftype diffPow(const ftype y, const ftype ypred) { + auto diff = y - ypred; + return diff * diff; + } + + /** + * @brief RMSE forward loss. + */ + __global__ void rmseKernelOneBlock(ftype* const res, const ftype* const y, const ftype* const yPred, const tensorSize_t size) { + const int tid = threadIdx.x; + const int gid = blockIdx.x * blockDim.x + tid; + + extern __shared__ ftype smem[]; + smem[tid] = diffPow(y[gid], yPred[gid]); + if(gid + blockDim.x < size) { + smem[tid] += diffPow(y[gid + blockDim.x], yPred[gid + blockDim.x]); + } + __syncthreads(); + + for(int offset = blockDim.x / 2; offset > 64; offset >>= 2) { + if(tid < offset) { + smem[tid] += smem[tid + offset]; + } + __syncthreads(); + } + + // TODO: warp shuffle again + volatile ftype* sdata = smem; + if(tid < 32) { + if(gid + 32 < size) { + sdata[tid] += sdata[tid + 32]; + } + if(gid + 16 < size) { + sdata[tid] += sdata[tid + 16]; + } + if(gid + 8 < size) { + sdata[tid] += sdata[tid + 8]; + } + if(gid + 4 < size) { + sdata[tid] += sdata[tid + 4]; + } + if(gid + 2 < size) { + sdata[tid] += sdata[tid + 2]; + } + if(gid + 1 < size) { + sdata[0] = (sdata[0] + sdata[1]) / size; + } + } + + if(threadIdx.x == 0) { + res[blockDim.x] = sdata[0]; + } + } + + template + __global__ void normalizeRmse(ftype* val, ftype divisor) { + const v = val[0]; + if constexpr (std::is_same_v) { + val[0] = __sqrtf(v / divisor); + } + else if constexpr (std::is_same_v) { + val[0] = sqrt(v / divisor); + } + else { + static_assert(always_false, "Encountered unexpected ftype"); + } + } } namespace cuda_impl { @@ -167,9 +299,11 @@ namespace cuda_impl { constexpr int threadsPerBlock = 256; const int blocks = (y.getSize() + threadsPerBlock - 1) / (threadsPerBlock * 2); + // TODO: res = make_shared(std::vector{1}, std::vector{loss / nBatches}, y->getDevice(), true); + if(blocks > 1) { - // we do two passes at max - ftype* tmp; + // two pass solution + ftype* tmp; // TODO: Keep this guy in memory for an instance cudaErrchk(cudaMalloc(&tmp, blocks * sizeof(ftype))); bceLossKernel<<>>( @@ -188,6 +322,10 @@ namespace cuda_impl { res.getData(), y.getData(), yPred.getData(), y.getDims()[0]); cudaErrchk(cudaDeviceSynchronize()); } + + // loss = -loss / nBatches + divideScalarKernel<<<1, 1>>>(res.getData(), -y.getDims()[0]); + cudaErrchk(cudaDeviceSynchronize()); } void bceSigmoidLoss(Tensor& res, const Tensor& y, const Tensor& logits) { @@ -215,13 +353,47 @@ namespace cuda_impl { res.getData(), y.getData(), logits.getData(), y.getDims()[0]); cudaErrchk(cudaDeviceSynchronize()); } + + // loss = -loss / nBatches + divideScalarKernel<<<1, 1>>>(res.getData(), -y.getDims()[0]); + cudaErrchk(cudaDeviceSynchronize()); } void crossEntropyLoss(Tensor& res, const Tensor& y, const Tensor& yPred) { - constexpr int threadsPerBlock = 256; - const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; + constexpr int maxThreadsPerBlock = 256; + + const tensorSize_t stride = y.getDims()[-1]; + const tensorSize_t nSamples = y.getSize() / stride; - // TODO: launch kernel + if(y.getSize() * 2 <= maxThreadsPerBlock) { + int threadsPerBlock = 1; + while(threadsPerBlock < y.getSize()) threadsPerBlock <<= 1; + threadsPerBlock = max(1, threadsPerBlock << 1); + + const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; + + crossEntropyLossKernelOneBlock<<>>(res.getData(), y.getData(), yPred.getData(), y.getSize()); + cudaErrchk(cudaDeviceSynchronize()); + } + else { + const int blocks = (y.getSize() + maxThreadsPerBlock - 1) / (maxThreadsPerBlock * 2); + + ftype* tmp; + cudaErrchk(cudaMalloc(&tmp, blocks * sizeof(ftype))); + + crossEntropyLossKernelOneBlock<<>>(tmp, y.getData(), yPred.getData(), y.getSize()); + cudaErrchk(cudaDeviceSynchronize()); + + // do a sum over the residual array + thrust::device_ptr tmpPtr(tmp); + thrust::device_ptr resPtr(res.getData()); + resPtr[0] = thrust::reduce(tmpPtr, tmpPtr + blocks, static_cast(0.0f), thrust::plus()); + + cudaErrchk(cudaFree(tmp)); + } + + // loss = -loss / nBatches + divideScalarKernel<<<1, 1>>>(res.getData(), -nSamples); cudaErrchk(cudaDeviceSynchronize()); } @@ -234,10 +406,38 @@ namespace cuda_impl { } void rmseLoss(Tensor& res, const Tensor& y, const Tensor& yPred) { - constexpr int threadsPerBlock = 256; - const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; + constexpr int maxThreadsPerBlock = 256; + + const auto nSamples = y.getSize(); - // TODO: launch kernel + if(nSamples * 2 <= maxThreadsPerBlock) { + int threadsPerBlock = 1; + while(threadsPerBlock < nSamples) threadsPerBlock <<= 1; + threadsPerBlock = max(1, threadsPerBlock << 1); + + const int blocks = (nSamples + threadsPerBlock - 1) / threadsPerBlock; + + crossEntropyLossKernelOneBlock<<>>(res.getData(), y.getData(), yPred.getData(), y.getSize()); + cudaErrchk(cudaDeviceSynchronize()); + } + else { + const int blocks = (nSamples + maxThreadsPerBlock - 1) / (maxThreadsPerBlock * 2); + + ftype* tmp; + cudaErrchk(cudaMalloc(&tmp, blocks * sizeof(ftype))); + + crossEntropyLossKernelOneBlock<<>>(tmp, y.getData(), yPred.getData(), y.getSize()); + cudaErrchk(cudaDeviceSynchronize()); + + // do a sum over the residual array + thrust::device_ptr tmpPtr(tmp); + thrust::device_ptr resPtr(res.getData()); + resPtr[0] = thrust::reduce(tmpPtr, tmpPtr + blocks, static_cast(0.0f), thrust::plus()); + + cudaErrchk(cudaFree(tmp)); + } + + normalizeRmse<<<1, 1>>>(res.getData(), nSamples); cudaErrchk(cudaDeviceSynchronize()); } } diff --git a/src/backend/training/loss_functions/rmse_loss.cpp b/src/backend/training/loss_functions/rmse_loss.cpp index 08513cf..2684443 100644 --- a/src/backend/training/loss_functions/rmse_loss.cpp +++ b/src/backend/training/loss_functions/rmse_loss.cpp @@ -48,12 +48,13 @@ shared_ptr RmseLoss::operator()(const shared_ptr y, const shared return diff * diff; }; - const auto nBatches = y->getDims()[0]; + const auto nSamples = y->getSize(); + ftype loss = 0; - for(tensorSize_t i=0; i(std::vector{1}, std::vector{loss}, y->getDevice(), true); break; diff --git a/src/backend/utility/cuda/cuda_common.cuh b/src/backend/utility/cuda/cuda_common.cuh index c78add8..381b00b 100644 --- a/src/backend/utility/cuda/cuda_common.cuh +++ b/src/backend/utility/cuda/cuda_common.cuh @@ -59,6 +59,13 @@ __device__ __forceinline__ ftype cudaSigmoid(ftype x) { return (x >= 0.f) ? s : z * s; // x < 0 => e^x/(e^x+1) } +/** + * @brief For single normalization, e.g. when normalizing with batch-size. + */ +static __global__ void divideScalarKernel(ftype* val, ftype divisor) { + val[0] /= divisor; +} + #define cudaErrchk(ans) { utility::gpuAssert((ans), __FILE__, __LINE__); } namespace cuda_impl { From 01cd521425c459cbc3b95bb7da798b7ca7934152 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Mon, 25 May 2026 20:26:10 +0200 Subject: [PATCH 62/73] Preparing infrastructure for forward crossentropy softmax kernel --- .../activation_functions/cuda/activations.cu | 197 +--------------- src/backend/shared/cuda/common_kernels.cuh | 44 ++++ src/backend/shared/cuda/common_softmax.cuh | 210 ++++++++++++++++++ .../loss_functions/cuda/loss_functions.cu | 8 +- src/backend/utility/cuda/cuda_common.cuh | 28 --- 5 files changed, 267 insertions(+), 220 deletions(-) create mode 100644 src/backend/shared/cuda/common_kernels.cuh create mode 100644 src/backend/shared/cuda/common_softmax.cuh diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index a221b09..57cf5af 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -15,14 +15,19 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include "activations.cuh" -#include "utility/cuda/cuda_common.cuh" +#include "shared/cuda/common_kernels.cuh" +#include "shared/cuda/common_softmax.cuh" + #include "utility/macros.h" +#include "utility/cuda/cuda_common.cuh" #include using namespace std; namespace { + using namespace cuda_impl; + /** * @brief Kernel for forward ReLU function. */ @@ -57,32 +62,6 @@ namespace { res[gid] = cudaSigmoid(input[gid]); } - /** - * @brief Reduction kernel that computes the maximum within the size of 2 * warpsize at maximum. - */ - template - __forceinline__ __device__ void warpMaxReduce(volatile ftype* const input, const tensorSize_t stride, const int offset) { - // TODO: warp shuffle for newer architectures - if(maxoffset == 32) { - if(offset + 32 < stride) input[offset] = max(input[offset], input[offset + 32]); - } - if(maxoffset >= 16) { - if(offset + 16 < stride) input[offset] = max(input[offset], input[offset + 16]); - } - if(maxoffset >= 8) { - if(offset + 8 < stride) input[offset] = max(input[offset], input[offset + 8]); - } - if(maxoffset >= 4) { - if(offset + 4 < stride) input[offset] = max(input[offset], input[offset + 4]); - } - if(maxoffset >= 2) { - if(offset + 2 < stride) input[offset] = max(input[offset], input[offset + 2]); - } - if(maxoffset >= 1) { - if(offset + 1 < stride) input[offset] = max(input[offset], input[offset + 1]); - } - } - /** * @brief Reduction kernel that computes the sum over an array within the size of 2 * warpsize at maximum. */ @@ -124,33 +103,6 @@ namespace { static_assert(always_false, "ftype encountered unexpected type"); } } - - /** - * @brief Here we find the maximum within 'stride'. Assumption: One warp does exactly one element of stride! - * Reduction via warp reduce. res has the maximum values stored. - */ - template - __global__ void findMaxKernelOneWarp(ftype* const res, const ftype* const input, const tensorSize_t stride, const tensorSize_t size) { - assert(blockDim.x % 32 == 0); - - int gid = blockIdx.x * blockDim.x + threadIdx.x; - if(gid >= size) - return; - - int tid = threadIdx.x; - extern __shared__ ftype smem[]; - smem[tid] = input[gid]; - __syncthreads(); - - volatile ftype* const start = smem + (tid / stride) * stride; - const int offset = gid % stride; - warpMaxReduce(start, stride, offset); - - // one warp reduces one 'stride' - if(offset == 0) { - res[tid / 32] = smem[tid]; - } - } /** * @brief Numerically stable version of softmax kernel. Just as in findMaxKernelOneWarp we assume that stride <= 2 * warpsize. @@ -181,53 +133,6 @@ namespace { res[gid] = expVal / start[0]; } - /** - * @brief Like findMaxKernelOneWarp. The difference now is that the input size can be much larger. stride is - * 64 < stride <= threadsPerBlock. res has the maximum values stored. - * - * In this initial version we assume one kernel per stride, to make matters simple to understand. - */ - __global__ void findMaxKernelOneBlock(ftype* const res, const ftype* const input, const tensorSize_t stride) { - assert_debug(blockDim.x / stride == 0, "Kernel built for one stride per block, blockDim.x is < stride"); - - const int tid = threadIdx.x; - const int gid = blockIdx.x * stride + tid; - - extern __shared__ ftype smem[]; // can lead to bank conflicts iff std::is_same_v - - const tensorSize_t maxIdx = tid + blockDim.x; - const bool doPadding = maxIdx >= stride; - if(doPadding) { - smem[tid] = input[gid]; - } - else { - smem[tid] = max(input[gid], input[gid + blockDim.x]); - } - __syncthreads(); - - for(tensorSize_t offset = blockDim.x >> 1; offset > 32; offset >>= 1) { - if(tid < offset) { - smem[tid] = max(smem[tid], smem[tid + offset]); - } - __syncthreads(); - } - - // TODO: warp shuffle for newer architectures - volatile ftype* const start = smem; - if(tid < 32) { - start[tid] = max(start[tid], start[tid + 32]); - start[tid] = max(start[tid], start[tid + 16]); - start[tid] = max(start[tid], start[tid + 8]); - start[tid] = max(start[tid], start[tid + 4]); - start[tid] = max(start[tid], start[tid + 2]); - start[tid] = max(start[tid], start[tid + 1]); - } - - if(tid == 0) { // one block per stride - res[blockIdx.x] = start[0]; - } - } - /** * @brief Just like stableSoftmaxKernelOneWarp, but this one works across a whole block, not just a warp. * @@ -281,96 +186,6 @@ namespace { } } - /** - * @brief Like findMaxKernelOneBlock, but finding partial maximum. This is the case when the stride is too - * large to fit in one block. - */ - __global__ void findMaxKernelLargePass1(ftype* const partialMaxValues, const ftype* const input, - const tensorSize_t stride, const int blocksPerStride) { - const int tid = threadIdx.x; - const int strideIdx = blockIdx.x / blocksPerStride; - const int blockWithinStride = blockIdx.x % blocksPerStride; - - // block 0 within stride handles elements [0, 2*blockDim.x), block 1 within stride handles elements [2*blockDim.x, 4*blockDim.x), ... - const int inputBase = strideIdx * stride + blockWithinStride * 2 * blockDim.x; - - extern __shared__ ftype smem[]; - const tensorSize_t localIdx0 = inputBase + tid; - const tensorSize_t localIdx1 = inputBase + tid + blockDim.x; - - // localIdx0 < (strideIdx + 1) * stride <- checks whether thread idx exceeds bounds of this stride; one stride per block at max - smem[tid] = (localIdx0 < (strideIdx + 1) * stride) ? input[localIdx0] : -INFINITY; - smem[tid + blockDim.x] = (localIdx1 < (strideIdx + 1) * stride) ? input[localIdx1] : -INFINITY; - __syncthreads(); - - // same reduction as findMaxKernelOneBlock from here - for(tensorSize_t offset = blockDim.x; offset > 32; offset >>= 1) { - if(tid < offset){ - smem[tid] = max(smem[tid], smem[tid + offset]); - } - __syncthreads(); - } - - volatile ftype* start = smem; - if(tid < 32) { - start[tid] = max(start[tid], start[tid + 32]); - start[tid] = max(start[tid], start[tid + 16]); - start[tid] = max(start[tid], start[tid + 8]); - start[tid] = max(start[tid], start[tid + 4]); - start[tid] = max(start[tid], start[tid + 2]); - start[tid] = max(start[tid], start[tid + 1]); - } - - if(tid == 0) { - partialMaxValues[blockIdx.x] = start[0]; - } - } - - /** - * @brief Self explanatory following findMaxKernelLargePass1. Assumption: All remaining max values do fit into - * one single block now -> we launch one block per stride this time. - */ - __global__ void findMaxKernelLargePass2(ftype* const maxValues, const ftype* const partialMaxValues, const tensorSize_t blocksPerStride) { - assert_debug(blockDim.x / blocksPerStride == 0, "Kernel built for one stride per block, blockDim.x is < stride"); - - const int tid = threadIdx.x; - const int gid = blockIdx.x * blocksPerStride + tid; - - extern __shared__ ftype smem[]; // can lead to bank conflicts iff std::is_same_v - - const tensorSize_t maxIdx = tid + blockDim.x; - const bool doPadding = maxIdx >= blocksPerStride; - if(doPadding) { - smem[tid] = partialMaxValues[gid]; - } - else { - smem[tid] = max(partialMaxValues[gid], partialMaxValues[gid + blockDim.x]); - } - __syncthreads(); - - for(tensorSize_t offset = blockDim.x >> 1; offset > 32; offset >>= 1) { - if(tid < offset) { - smem[tid] = max(smem[tid], smem[tid + offset]); - } - __syncthreads(); - } - - // TODO: warp shuffle for newer architectures - volatile ftype* const start = smem; - if(tid < 32) { - if(tid + 32 < blockDim.x) start[tid] = max(start[tid], start[tid + 32]); - if(tid + 16 < blockDim.x) start[tid] = max(start[tid], start[tid + 16]); - if(tid + 8 < blockDim.x) start[tid] = max(start[tid], start[tid + 8]); - if(tid + 4 < blockDim.x) start[tid] = max(start[tid], start[tid + 4]); - if(tid + 2 < blockDim.x) start[tid] = max(start[tid], start[tid + 2]); - if(tid + 1 < blockDim.x) start[tid] = max(start[tid], start[tid + 1]); - } - - if(tid == 0) { // one block per stride - maxValues[blockIdx.x] = start[0]; - } - } - /** * @brief Does the first part of stableSoftmaxKernelOneBlock, namely the sums. Because here again we have a partial sum * and assume the stride did not fit into the block entirely, we do a partial sum only. Additionally, write the max-adjusted diff --git a/src/backend/shared/cuda/common_kernels.cuh b/src/backend/shared/cuda/common_kernels.cuh new file mode 100644 index 0000000..2e4cc73 --- /dev/null +++ b/src/backend/shared/cuda/common_kernels.cuh @@ -0,0 +1,44 @@ +/** + * @file common_kernels.cuh + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-05-25 + * + * @copyright Copyright (c) 2026 + * + */ + +#pragma once + +#include "utility/global_params.h" + +namespace cuda_impl { + template + __device__ __forceinline__ ftype cudaMax(const ftype a, const ftype b) { + if constexpr (std::is_same_v) { + return fmaxf(a, b); + } else if(std::is_same_v) { + return fmax(a, b); + } + else { + static_assert(always_false, "Unexpected value for ftype encountered"); + } + } + + /** + * @brief Single sigmoid computation. + */ + __device__ __forceinline__ ftype cudaSigmoid(ftype x) { + ftype z = expf(-fabsf(x)); + ftype s = 1.0f / (1.0f + z); + return (x >= 0.f) ? s : z * s; // x < 0 => e^x/(e^x+1) + } + + /** + * @brief For single normalization, e.g. when normalizing with batch-size. + */ + static __global__ void divideScalarKernel(ftype* val, ftype divisor) { + val[0] /= divisor; + } +} \ No newline at end of file diff --git a/src/backend/shared/cuda/common_softmax.cuh b/src/backend/shared/cuda/common_softmax.cuh new file mode 100644 index 0000000..83eb2a1 --- /dev/null +++ b/src/backend/shared/cuda/common_softmax.cuh @@ -0,0 +1,210 @@ +/** + * @file common_softmax.cuh + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief Common kernels for softmax. + * @version 0.1 + * @date 2026-05-25 + * + * @copyright Copyright (c) 2026 + * + */ + +#pragma once + +#include "common_kernels.cuh" + +#include "utility/global_params.h" +#include "utility/macros.h" + +namespace cuda_impl { + + /** + * @brief Reduction kernel that computes the maximum within the size of 2 * warpsize at maximum. + */ + template + __forceinline__ __device__ void warpMaxReduce(volatile ftype* const input, const tensorSize_t stride, const int offset) { + // TODO: warp shuffle for newer architectures + if(maxoffset == 32) { + if(offset + 32 < stride) input[offset] = cudaMax(input[offset], input[offset + 32]); + } + if(maxoffset >= 16) { + if(offset + 16 < stride) input[offset] = cudaMax(input[offset], input[offset + 16]); + } + if(maxoffset >= 8) { + if(offset + 8 < stride) input[offset] = cudaMax(input[offset], input[offset + 8]); + } + if(maxoffset >= 4) { + if(offset + 4 < stride) input[offset] = cudaMax(input[offset], input[offset + 4]); + } + if(maxoffset >= 2) { + if(offset + 2 < stride) input[offset] = cudaMax(input[offset], input[offset + 2]); + } + if(maxoffset >= 1) { + if(offset + 1 < stride) input[offset] = cudaMax(input[offset], input[offset + 1]); + } + } + + /** + * @brief Here we find the maximum within 'stride'. Assumption: One warp does exactly one element of stride! + * Reduction via warp reduce. res has the maximum values stored. + */ + template + static __global__ void findMaxKernelOneWarp(ftype* const res, const ftype* const input, const tensorSize_t stride, const tensorSize_t size) { + assert(blockDim.x % 32 == 0); + + int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) + return; + + int tid = threadIdx.x; + extern __shared__ ftype smem[]; + smem[tid] = input[gid]; + __syncthreads(); + + volatile ftype* const start = smem + (tid / stride) * stride; + const int offset = gid % stride; + warpMaxReduce(start, stride, offset); + + // one warp reduces one 'stride' + if(offset == 0) { + res[tid / 32] = smem[tid]; + } + } + + /** + * @brief Like findMaxKernelOneWarp. The difference now is that the input size can be much larger. stride is + * 64 < stride <= threadsPerBlock. res has the maximum values stored. + * + * In this initial version we assume one kernel per stride, to make matters simple to understand. + */ + static __global__ void findMaxKernelOneBlock(ftype* const res, const ftype* const input, const tensorSize_t stride) { + assert_debug(blockDim.x / stride == 0, "Kernel built for one stride per block, blockDim.x is < stride"); + + const int tid = threadIdx.x; + const int gid = blockIdx.x * stride + tid; + + extern __shared__ ftype smem[]; // can lead to bank conflicts iff std::is_same_v + + const tensorSize_t maxIdx = tid + blockDim.x; + const bool doPadding = maxIdx >= stride; + if(doPadding) { + smem[tid] = input[gid]; + } + else { + smem[tid] = cudaMax(input[gid], input[gid + blockDim.x]); + } + __syncthreads(); + + for(tensorSize_t offset = blockDim.x >> 1; offset > 32; offset >>= 1) { + if(tid < offset) { + smem[tid] = cudaMax(smem[tid], smem[tid + offset]); + } + __syncthreads(); + } + + // TODO: warp shuffle for newer architectures + volatile ftype* const start = smem; + if(tid < 32) { + start[tid] = cudaMax(start[tid], start[tid + 32]); + start[tid] = cudaMax(start[tid], start[tid + 16]); + start[tid] = cudaMax(start[tid], start[tid + 8]); + start[tid] = cudaMax(start[tid], start[tid + 4]); + start[tid] = cudaMax(start[tid], start[tid + 2]); + start[tid] = cudaMax(start[tid], start[tid + 1]); + } + + if(tid == 0) { // one block per stride + res[blockIdx.x] = start[0]; + } + } + + /** + * @brief Like findMaxKernelOneBlock, but finding partial maximum. This is the case when the stride is too + * large to fit in one block. + */ + static __global__ void findMaxKernelLargePass1(ftype* const partialMaxValues, const ftype* const input, + const tensorSize_t stride, const int blocksPerStride) { + const int tid = threadIdx.x; + const int strideIdx = blockIdx.x / blocksPerStride; + const int blockWithinStride = blockIdx.x % blocksPerStride; + + // block 0 within stride handles elements [0, 2*blockDim.x), block 1 within stride handles elements [2*blockDim.x, 4*blockDim.x), ... + const int inputBase = strideIdx * stride + blockWithinStride * 2 * blockDim.x; + + extern __shared__ ftype smem[]; + const tensorSize_t localIdx0 = inputBase + tid; + const tensorSize_t localIdx1 = inputBase + tid + blockDim.x; + + // localIdx0 < (strideIdx + 1) * stride <- checks whether thread idx exceeds bounds of this stride; one stride per block at cudaMax + smem[tid] = (localIdx0 < (strideIdx + 1) * stride) ? input[localIdx0] : -INFINITY; + smem[tid + blockDim.x] = (localIdx1 < (strideIdx + 1) * stride) ? input[localIdx1] : -INFINITY; + __syncthreads(); + + // same reduction as findMaxKernelOneBlock from here + for(tensorSize_t offset = blockDim.x; offset > 32; offset >>= 1) { + if(tid < offset){ + smem[tid] = cudaMax(smem[tid], smem[tid + offset]); + } + __syncthreads(); + } + + volatile ftype* start = smem; + if(tid < 32) { + start[tid] = cudaMax(start[tid], start[tid + 32]); + start[tid] = cudaMax(start[tid], start[tid + 16]); + start[tid] = cudaMax(start[tid], start[tid + 8]); + start[tid] = cudaMax(start[tid], start[tid + 4]); + start[tid] = cudaMax(start[tid], start[tid + 2]); + start[tid] = cudaMax(start[tid], start[tid + 1]); + } + + if(tid == 0) { + partialMaxValues[blockIdx.x] = start[0]; + } + } + + /** + * @brief Self explanatory following findMaxKernelLargePass1. Assumption: All remaining cudaMax values do fit into + * one single block now -> we launch one block per stride this time. + */ + static __global__ void findMaxKernelLargePass2(ftype* const maxValues, const ftype* const partialMaxValues, const tensorSize_t blocksPerStride) { + assert_debug(blockDim.x / blocksPerStride == 0, "Kernel built for one stride per block, blockDim.x is < stride"); + + const int tid = threadIdx.x; + const int gid = blockIdx.x * blocksPerStride + tid; + + extern __shared__ ftype smem[]; // can lead to bank conflicts iff std::is_same_v + + const tensorSize_t maxIdx = tid + blockDim.x; + const bool doPadding = maxIdx >= blocksPerStride; + if(doPadding) { + smem[tid] = partialMaxValues[gid]; + } + else { + smem[tid] = cudaMax(partialMaxValues[gid], partialMaxValues[gid + blockDim.x]); + } + __syncthreads(); + + for(tensorSize_t offset = blockDim.x >> 1; offset > 32; offset >>= 1) { + if(tid < offset) { + smem[tid] = cudaMax(smem[tid], smem[tid + offset]); + } + __syncthreads(); + } + + // TODO: warp shuffle for newer architectures + volatile ftype* const start = smem; + if(tid < 32) { + if(tid + 32 < blockDim.x) start[tid] = cudaMax(start[tid], start[tid + 32]); + if(tid + 16 < blockDim.x) start[tid] = cudaMax(start[tid], start[tid + 16]); + if(tid + 8 < blockDim.x) start[tid] = cudaMax(start[tid], start[tid + 8]); + if(tid + 4 < blockDim.x) start[tid] = cudaMax(start[tid], start[tid + 4]); + if(tid + 2 < blockDim.x) start[tid] = cudaMax(start[tid], start[tid + 2]); + if(tid + 1 < blockDim.x) start[tid] = cudaMax(start[tid], start[tid + 1]); + } + + if(tid == 0) { // one block per stride + maxValues[blockIdx.x] = start[0]; + } + } +} \ No newline at end of file diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cu b/src/backend/training/loss_functions/cuda/loss_functions.cu index a1547c9..587cc86 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cu +++ b/src/backend/training/loss_functions/cuda/loss_functions.cu @@ -14,8 +14,12 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #endif // __CUDA #include "loss_functions.cuh" -#include "utility/cuda/cuda_common.cuh" + #include "utility/macros.h" +#include "utility/cuda/cuda_common.cuh" + +#include "shared/cuda/common_kernels.cuh" +#include "shared/cuda/common_softmax.cuh" #include #include @@ -23,6 +27,8 @@ static_assert(false, "File should not be compiled without CUDA enabled"); using namespace std; namespace { + using namespace cuda_impl; + template __forceinline__ __device__ T bce(T y, T ypred) { if constexpr (std::is_same_v) { diff --git a/src/backend/utility/cuda/cuda_common.cuh b/src/backend/utility/cuda/cuda_common.cuh index 381b00b..4308e2e 100644 --- a/src/backend/utility/cuda/cuda_common.cuh +++ b/src/backend/utility/cuda/cuda_common.cuh @@ -38,34 +38,6 @@ namespace utility { template constexpr bool always_false = false; -template -__device__ __forceinline__ ftype cudaMax(const ftype a, const ftype b) { - if constexpr (std::is_same_v) { - return fmaxf(a, b); - } else if(std::is_same_v) { - return fmax(a, b); - } - else { - static_assert(always_false, "Unexpected value for ftype encountered"); - } -} - -/** - * @brief Single sigmoid computation. - */ -__device__ __forceinline__ ftype cudaSigmoid(ftype x) { - ftype z = expf(-fabsf(x)); - ftype s = 1.0f / (1.0f + z); - return (x >= 0.f) ? s : z * s; // x < 0 => e^x/(e^x+1) -} - -/** - * @brief For single normalization, e.g. when normalizing with batch-size. - */ -static __global__ void divideScalarKernel(ftype* val, ftype divisor) { - val[0] /= divisor; -} - #define cudaErrchk(ans) { utility::gpuAssert((ans), __FILE__, __LINE__); } namespace cuda_impl { From 8045d393e38848fca9efebfdce7ea4f57be37001 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Mon, 25 May 2026 21:01:36 +0200 Subject: [PATCH 63/73] Crossentropy softmax kernel forward, some restructuring of includes --- .../loss_functions/cuda/loss_nodes.cu | 3 + src/backend/data_modeling/cuda/tensor_ops.cu | 2 +- src/backend/data_modeling/tensor.cpp | 2 +- .../activation_functions/cuda/activations.cu | 20 +-- src/backend/shared/cuda/common_kernels.cuh | 3 +- src/backend/shared/cuda/common_softmax.cuh | 18 ++- .../loss_functions/cuda/loss_functions.cu | 126 ++++++++++++++++-- src/backend/utility/cuda/cuda_common.cuh | 14 -- src/backend/utility/{macros.h => utils.h} | 18 ++- src/python/py_core/py_core.cpp | 6 +- 10 files changed, 161 insertions(+), 51 deletions(-) rename src/backend/utility/{macros.h => utils.h} (60%) diff --git a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu index 3ef7b58..63bbfe5 100644 --- a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu +++ b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu @@ -16,6 +16,7 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include "loss_nodes.cuh" #include "module/activation_functions/softmax.h" +#include "shared/cuda/common_kernels.cuh" #include "utility/cuda/cuda_common.cuh" #include "utility/global_params.h" @@ -24,6 +25,8 @@ static_assert(false, "File should not be compiled without CUDA enabled"); using namespace std; namespace { + using namespace cuda_impl; + /** * @brief Does what you think it does. */ diff --git a/src/backend/data_modeling/cuda/tensor_ops.cu b/src/backend/data_modeling/cuda/tensor_ops.cu index 49701b3..d42b4d5 100644 --- a/src/backend/data_modeling/cuda/tensor_ops.cu +++ b/src/backend/data_modeling/cuda/tensor_ops.cu @@ -17,7 +17,7 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include "tensor_ops.cuh" #include "utility/cuda/cuda_common.cuh" -#include "utility/macros.h" +#include "utility/utils.h" #include #include diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index 3961b98..b6191f2 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -14,7 +14,7 @@ #include "computational_graph/graph_node.h" #include "computational_graph/topological_sort.h" -#include "utility/macros.h" +#include "utility/utils.h" #include #include diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index 57cf5af..f3d8f56 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -18,7 +18,7 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include "shared/cuda/common_kernels.cuh" #include "shared/cuda/common_softmax.cuh" -#include "utility/macros.h" +#include "utility/utils.h" #include "utility/cuda/cuda_common.cuh" #include @@ -27,7 +27,7 @@ using namespace std; namespace { using namespace cuda_impl; - + /** * @brief Kernel for forward ReLU function. */ @@ -88,22 +88,6 @@ namespace { } } - /** - * @brief For the softmax implementations. - */ - template - __forceinline__ __device__ ftype stableExp(const ftype x, const ftype maxValue) { - if constexpr (std::is_same_v) { - return expf(x - maxValue); - } - else if constexpr (std::is_same_v) { - return exp(x - maxValue); - } - else { - static_assert(always_false, "ftype encountered unexpected type"); - } - } - /** * @brief Numerically stable version of softmax kernel. Just as in findMaxKernelOneWarp we assume that stride <= 2 * warpsize. * Numerical stability comes from computing the maximum values per row, see findMaxKernelOneWarp and argument maxValues. diff --git a/src/backend/shared/cuda/common_kernels.cuh b/src/backend/shared/cuda/common_kernels.cuh index 2e4cc73..d393663 100644 --- a/src/backend/shared/cuda/common_kernels.cuh +++ b/src/backend/shared/cuda/common_kernels.cuh @@ -12,6 +12,7 @@ #pragma once #include "utility/global_params.h" +#include "utility/utils.h" namespace cuda_impl { template @@ -38,7 +39,7 @@ namespace cuda_impl { /** * @brief For single normalization, e.g. when normalizing with batch-size. */ - static __global__ void divideScalarKernel(ftype* val, ftype divisor) { + static __global__ void divideScalarKernel(ftype* const val, ftype divisor) { val[0] /= divisor; } } \ No newline at end of file diff --git a/src/backend/shared/cuda/common_softmax.cuh b/src/backend/shared/cuda/common_softmax.cuh index 83eb2a1..9e3098d 100644 --- a/src/backend/shared/cuda/common_softmax.cuh +++ b/src/backend/shared/cuda/common_softmax.cuh @@ -14,10 +14,26 @@ #include "common_kernels.cuh" #include "utility/global_params.h" -#include "utility/macros.h" +#include "utility/utils.h" namespace cuda_impl { + /** + * @brief Basic helper. + */ + template + __forceinline__ __device__ ftype stableExp(const ftype x, const ftype maxVal) { + if constexpr (std::is_same_v) { + return __expf(x - maxVal); + } + else if constexpr (std::is_same_v) { + return exp(x - maxVal); + } + else { + static_assert(always_false, "ftype encountered unexpected type"); + } + } + /** * @brief Reduction kernel that computes the maximum within the size of 2 * warpsize at maximum. */ diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cu b/src/backend/training/loss_functions/cuda/loss_functions.cu index 587cc86..b07c48c 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cu +++ b/src/backend/training/loss_functions/cuda/loss_functions.cu @@ -15,7 +15,7 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include "loss_functions.cuh" -#include "utility/macros.h" +#include "utility/utils.h" #include "utility/cuda/cuda_common.cuh" #include "shared/cuda/common_kernels.cuh" @@ -226,7 +226,64 @@ namespace { } } - // TODO: crossEntropySoftmaxLoss kernel + /** + * @brief Softmax cross-entropy loss kernel. One block per sample. + */ + template + __global__ void crossEntropySoftmaxLossKernel(ftype* const perSampleLoss, const ftype* const y, const ftype* const logits, + const ftype* const maxValues, const tensorSize_t stride) { + const int tid = threadIdx.x; + const int tid2 = tid + blockDim.x; + const int sampleBase = blockIdx.x * stride; + const ftype maxV = maxValues[blockIdx.x]; + + extern __shared__ ftype smem[]; + + const bool inBounds0 = (tid < stride); + const bool inBounds1 = (tid2 < stride); + + constexpr ftype zero = 0; + ftype e0 = inBounds0 ? stableExp(logits[sampleBase + tid], maxV) : zero; + ftype yz0 = inBounds0 ? y[sampleBase + tid] * logits[sampleBase + tid] : zero; + ftype e1 = inBounds1 ? stableExp(logits[sampleBase + tid2], maxV) : zero; + ftype yz1 = inBounds1 ? y[sampleBase + tid2] * logits[sampleBase + tid2] : zero; + + // Phase 1: reduce exp sum across stride (smem has 2 * blockDim.x slots) + smem[tid] = e0; + smem[tid2] = e1; + __syncthreads(); + + for(int offset = blockDim.x; offset >= 1; offset >>= 1) { + if(tid < offset) { + smem[tid] += smem[tid + offset]; + } + __syncthreads(); + } + + const ftype expSum = smem[0]; + + // reduce y*z dot product (y is one-hot, so at most one non-zero per sample) + smem[tid] = yz0 + yz1; + __syncthreads(); + + for(int offset = blockDim.x / 2; offset >= 1; offset >>= 1) { + if(tid < offset) { + smem[tid] += smem[tid + offset]; + } + __syncthreads(); + } + + if(tid == 0) { + ftype logExpSum; + if constexpr (std::is_same_v) { + logExpSum = __logf(expSum); + } + else { + logExpSum = log(expSum); + } + perSampleLoss[blockIdx.x] = maxV + logExpSum - smem[0]; + } + } /** * @brief Helper for RMSE loss. @@ -286,10 +343,10 @@ namespace { } template - __global__ void normalizeRmse(ftype* val, ftype divisor) { - const v = val[0]; + __global__ void normalizeRmse(ftype* const val, ftype divisor) { + const ftype v = val[0]; if constexpr (std::is_same_v) { - val[0] = __sqrtf(v / divisor); + val[0] = sqrtf(v / divisor); } else if constexpr (std::is_same_v) { val[0] = sqrt(v / divisor); @@ -404,11 +461,64 @@ namespace cuda_impl { } void crossEntropySoftmaxLoss(Tensor& res, const Tensor& y, const Tensor& yPred) { - constexpr int threadsPerBlock = 256; - const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; + const tensorSize_t stride = static_cast(yPred.getDims().get(-1)); + const tensorSize_t nSamples = yPred.getSize() / stride; + + ftype* maxValues; + cudaErrchk(cudaMalloc(&maxValues, nSamples * sizeof(ftype))); + + // Find per-sample max values for numerical stability (mirrors softmax dispatch) + static const auto warpSizeT2 = 2 * DeviceProperties::getWarpSize(); + if(stride <= warpSizeT2) { + constexpr int threadsPerBlock = 256; + const int blocks = (yPred.getSize() + threadsPerBlock - 1) / threadsPerBlock; + + if(stride == 2) + findMaxKernelOneWarp<1><<>>(maxValues, yPred.getData(), stride, yPred.getSize()); + else if(stride <= 4) + findMaxKernelOneWarp<2><<>>(maxValues, yPred.getData(), stride, yPred.getSize()); + else if(stride <= 8) + findMaxKernelOneWarp<4><<>>(maxValues, yPred.getData(), stride, yPred.getSize()); + else if(stride <= 16) + findMaxKernelOneWarp<8><<>>(maxValues, yPred.getData(), stride, yPred.getSize()); + else if(stride <= 32) + findMaxKernelOneWarp<16><<>>(maxValues, yPred.getData(), stride, yPred.getSize()); + else + findMaxKernelOneWarp<32><<>>(maxValues, yPred.getData(), stride, yPred.getSize()); + cudaErrchk(cudaDeviceSynchronize()); + } + else if(stride <= 512) { + int threadsPerBlock = 1; + while(threadsPerBlock < stride) threadsPerBlock <<= 1; + threadsPerBlock /= 2; + + findMaxKernelOneBlock<<>>(maxValues, yPred.getData(), stride); + cudaErrchk(cudaDeviceSynchronize()); + } + else { + cudaErrchk(cudaFree(maxValues)); + __throw_invalid_argument("crossEntropySoftmaxLoss: stride > 512 not yet supported on CUDA"); + } + + // one block per sample, each thread covers up to 2 elements + int threadsPerBlock = 1; + while(threadsPerBlock * 2 < stride) threadsPerBlock <<= 1; + threadsPerBlock = max(1, threadsPerBlock); + + ftype* perSampleLoss; + cudaErrchk(cudaMalloc(&perSampleLoss, nSamples * sizeof(ftype))); - // TODO: launch kernel + crossEntropySoftmaxLossKernel<<>>( + perSampleLoss, y.getData(), yPred.getData(), maxValues, stride); cudaErrchk(cudaDeviceSynchronize()); + + thrust::device_ptr lossPtr(perSampleLoss); + thrust::device_ptr resPtr(res.getData()); + resPtr[0] = thrust::reduce(lossPtr, lossPtr + nSamples, static_cast(0), thrust::plus()) + / static_cast(nSamples); + + cudaErrchk(cudaFree(perSampleLoss)); + cudaErrchk(cudaFree(maxValues)); } void rmseLoss(Tensor& res, const Tensor& y, const Tensor& yPred) { diff --git a/src/backend/utility/cuda/cuda_common.cuh b/src/backend/utility/cuda/cuda_common.cuh index 4308e2e..7d505d5 100644 --- a/src/backend/utility/cuda/cuda_common.cuh +++ b/src/backend/utility/cuda/cuda_common.cuh @@ -20,24 +20,10 @@ static_assert(false, "File should not be included without CUDA enabled"); #include -template -struct FtypeWarning { - static constexpr void check() {} -}; - -template<> -struct FtypeWarning { - [[deprecated("ftype=double has serious CUDA performance implications")]] - static constexpr void check() {} -}; - namespace utility { void gpuAssert(cudaError_t code, const char *file, int line, bool abort=true); } -template -constexpr bool always_false = false; - #define cudaErrchk(ans) { utility::gpuAssert((ans), __FILE__, __LINE__); } namespace cuda_impl { diff --git a/src/backend/utility/macros.h b/src/backend/utility/utils.h similarity index 60% rename from src/backend/utility/macros.h rename to src/backend/utility/utils.h index e541a8a..660a4bf 100644 --- a/src/backend/utility/macros.h +++ b/src/backend/utility/utils.h @@ -1,5 +1,5 @@ /** - * @file macros.h + * @file utils.h * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) * @brief * @version 0.1 @@ -22,4 +22,18 @@ abort(); \ } \ } while(0) -#endif \ No newline at end of file +#endif + +template +constexpr bool always_false = false; + +template +struct FtypeWarning { + static constexpr void check() {} +}; + +template<> +struct FtypeWarning { + [[deprecated("ftype=double has serious CUDA performance implications")]] + static constexpr void check() {} +}; \ No newline at end of file diff --git a/src/python/py_core/py_core.cpp b/src/python/py_core/py_core.cpp index d5f44ca..9fbf212 100644 --- a/src/python/py_core/py_core.cpp +++ b/src/python/py_core/py_core.cpp @@ -19,9 +19,7 @@ #include "data_modeling/tensor_functions.h" #include "computational_graph/tensor_ops/graph_creation.h" -#ifdef __CUDA -#include "utility/cuda/cuda_common.cuh" -#endif +#include "utility/utils.h" #include #include @@ -32,9 +30,7 @@ BOOST_PYTHON_MODULE(_core) { -#ifdef __CUDA FtypeWarning::check(); -#endif using namespace boost::python; From e17cef4d0bd918f977316f48de599f83f72c1d55 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sat, 30 May 2026 11:11:32 +0200 Subject: [PATCH 64/73] Fix crossentropy forward kernel, simplify CPU version --- src/backend/shared/cuda/common_kernels.cuh | 6 +-- .../loss_functions/crossentropy_loss.cpp | 14 +---- .../loss_functions/cuda/loss_functions.cu | 52 +++++++++---------- 3 files changed, 31 insertions(+), 41 deletions(-) diff --git a/src/backend/shared/cuda/common_kernels.cuh b/src/backend/shared/cuda/common_kernels.cuh index d393663..b7db3d8 100644 --- a/src/backend/shared/cuda/common_kernels.cuh +++ b/src/backend/shared/cuda/common_kernels.cuh @@ -30,7 +30,7 @@ namespace cuda_impl { /** * @brief Single sigmoid computation. */ - __device__ __forceinline__ ftype cudaSigmoid(ftype x) { + __device__ __forceinline__ ftype cudaSigmoid(const ftype x) { ftype z = expf(-fabsf(x)); ftype s = 1.0f / (1.0f + z); return (x >= 0.f) ? s : z * s; // x < 0 => e^x/(e^x+1) @@ -39,7 +39,7 @@ namespace cuda_impl { /** * @brief For single normalization, e.g. when normalizing with batch-size. */ - static __global__ void divideScalarKernel(ftype* const val, ftype divisor) { - val[0] /= divisor; + static __global__ void divideScalarKernel(ftype* const val, const ftype divisor) { + val[0] /= divisor; } } \ No newline at end of file diff --git a/src/backend/training/loss_functions/crossentropy_loss.cpp b/src/backend/training/loss_functions/crossentropy_loss.cpp index 67e25c5..641ab3e 100644 --- a/src/backend/training/loss_functions/crossentropy_loss.cpp +++ b/src/backend/training/loss_functions/crossentropy_loss.cpp @@ -45,19 +45,9 @@ shared_ptr CrossEntropyLoss::operator()(const shared_ptr y, cons const tensorSize_t stride = y->getDims()[-1]; const tensorSize_t nSamples = y->getSize() / stride; - auto ce = [&y, &ypred, stride](const tensorSize_t offset){ - ftype r = 0; - for(tensorSize_t i = offset; i < offset + stride; i++){ - r += (*y)[i] * log(std::max((*ypred)[i], EPS_CROSSENTROPY)); - } - return r; - }; - ftype loss = 0; - tensorSize_t offset = 0; - while(offset < y->getSize()){ - loss += ce(offset); - offset += stride; + for (tensorSize_t i = 0; i < y->getSize(); i++) { + loss += (*y)[i] * log(std::max((*ypred)[i], EPS_CROSSENTROPY)); } res = make_shared(std::vector{1}, std::vector{-loss / nSamples}, y->getDevice(), true); diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cu b/src/backend/training/loss_functions/cuda/loss_functions.cu index b07c48c..560fc05 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cu +++ b/src/backend/training/loss_functions/cuda/loss_functions.cu @@ -178,20 +178,20 @@ namespace { } /** - * @brief Cross-entropy reduction over a full block. Covers two times blockDim.x + * @brief Cross-entropy reduction over a full block. Covers two times blockDim.x. */ __global__ void crossEntropyLossKernelOneBlock(ftype* const res, const ftype* const y, const ftype* const yPred, const tensorSize_t size) { const int tid = threadIdx.x; const int gid = blockIdx.x * blockDim.x + tid; extern __shared__ ftype smem[]; - smem[tid] = crossEntropy(y[gid], yPred[gid]); - if(gid + blockDim.x < size) { - smem[tid] += crossEntropy(y[gid + blockDim.x], yPred[gid + blockDim.x]); + + if(gid < size) { + smem[tid] = crossEntropy(y[gid], yPred[gid]); } __syncthreads(); - for(int offset = blockDim.x / 2; offset > 64; offset >>= 2) { + for(int offset = blockDim.x / 2; offset > 64; offset >>= 1) { if(tid < offset) { smem[tid] += smem[tid + offset]; } @@ -217,18 +217,19 @@ namespace { sdata[tid] += sdata[tid + 2]; } if(gid + 1 < size) { - sdata[0] = (sdata[0] + sdata[1]) / size; + sdata[tid] += sdata[tid + 1]; } } if(threadIdx.x == 0) { - res[blockDim.x] = sdata[0]; + res[blockIdx.x] = sdata[0]; } } /** * @brief Softmax cross-entropy loss kernel. One block per sample. */ + template __global__ void crossEntropySoftmaxLossKernel(ftype* const perSampleLoss, const ftype* const y, const ftype* const logits, const ftype* const maxValues, const tensorSize_t stride) { @@ -307,7 +308,7 @@ namespace { } __syncthreads(); - for(int offset = blockDim.x / 2; offset > 64; offset >>= 2) { + for(int offset = blockDim.x / 2; offset > 64; offset >>= 1) { if(tid < offset) { smem[tid] += smem[tid + offset]; } @@ -333,12 +334,12 @@ namespace { sdata[tid] += sdata[tid + 2]; } if(gid + 1 < size) { - sdata[0] = (sdata[0] + sdata[1]) / size; + sdata[tid] += sdata[tid + 1]; } } if(threadIdx.x == 0) { - res[blockDim.x] = sdata[0]; + res[blockIdx.x] = sdata[0]; } } @@ -362,8 +363,6 @@ namespace cuda_impl { constexpr int threadsPerBlock = 256; const int blocks = (y.getSize() + threadsPerBlock - 1) / (threadsPerBlock * 2); - // TODO: res = make_shared(std::vector{1}, std::vector{loss / nBatches}, y->getDevice(), true); - if(blocks > 1) { // two pass solution ftype* tmp; // TODO: Keep this guy in memory for an instance @@ -387,7 +386,7 @@ namespace cuda_impl { } // loss = -loss / nBatches - divideScalarKernel<<<1, 1>>>(res.getData(), -y.getDims()[0]); + divideScalarKernel<<<1, 1>>>(res.getData(), -1 * static_cast(y.getDims()[0])); cudaErrchk(cudaDeviceSynchronize()); } @@ -418,7 +417,7 @@ namespace cuda_impl { } // loss = -loss / nBatches - divideScalarKernel<<<1, 1>>>(res.getData(), -y.getDims()[0]); + divideScalarKernel<<<1, 1>>>(res.getData(), -1 * static_cast(y.getDims()[0])); cudaErrchk(cudaDeviceSynchronize()); } @@ -428,14 +427,12 @@ namespace cuda_impl { const tensorSize_t stride = y.getDims()[-1]; const tensorSize_t nSamples = y.getSize() / stride; - if(y.getSize() * 2 <= maxThreadsPerBlock) { + if(y.getSize() <= maxThreadsPerBlock) { + int threadsPerBlock = 1; while(threadsPerBlock < y.getSize()) threadsPerBlock <<= 1; - threadsPerBlock = max(1, threadsPerBlock << 1); - - const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; - crossEntropyLossKernelOneBlock<<>>(res.getData(), y.getData(), yPred.getData(), y.getSize()); + crossEntropyLossKernelOneBlock<<<1, threadsPerBlock, y.getSize() * sizeof(ftype)>>>(res.getData(), y.getData(), yPred.getData(), y.getSize()); cudaErrchk(cudaDeviceSynchronize()); } else { @@ -456,7 +453,7 @@ namespace cuda_impl { } // loss = -loss / nBatches - divideScalarKernel<<<1, 1>>>(res.getData(), -nSamples); + divideScalarKernel<<<1, 1>>>(res.getData(), -1 * static_cast(nSamples)); cudaErrchk(cudaDeviceSynchronize()); } @@ -527,13 +524,16 @@ namespace cuda_impl { const auto nSamples = y.getSize(); if(nSamples * 2 <= maxThreadsPerBlock) { + // ceil(y.getSize() / 2) + const int sizeOverTwo = (y.getSize() + 1) / 2; + assert(sizeOverTwo > 0); + int threadsPerBlock = 1; - while(threadsPerBlock < nSamples) threadsPerBlock <<= 1; - threadsPerBlock = max(1, threadsPerBlock << 1); + while(threadsPerBlock < sizeOverTwo) threadsPerBlock <<= 1; + threadsPerBlock = max(1, threadsPerBlock >> 1); + assert(threadsPerBlock >= y.getSize() / 2); - const int blocks = (nSamples + threadsPerBlock - 1) / threadsPerBlock; - - crossEntropyLossKernelOneBlock<<>>(res.getData(), y.getData(), yPred.getData(), y.getSize()); + rmseKernelOneBlock<<<1, threadsPerBlock, threadsPerBlock * sizeof(ftype)>>>(res.getData(), y.getData(), yPred.getData(), y.getSize()); cudaErrchk(cudaDeviceSynchronize()); } else { @@ -542,7 +542,7 @@ namespace cuda_impl { ftype* tmp; cudaErrchk(cudaMalloc(&tmp, blocks * sizeof(ftype))); - crossEntropyLossKernelOneBlock<<>>(tmp, y.getData(), yPred.getData(), y.getSize()); + rmseKernelOneBlock<<>>(tmp, y.getData(), yPred.getData(), y.getSize()); cudaErrchk(cudaDeviceSynchronize()); // do a sum over the residual array From cffa43168675c440be2bf734b4b3afa5958abfef Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sat, 30 May 2026 11:52:45 +0200 Subject: [PATCH 65/73] Unit tests for larger input in loss functions, bugfixes in crossentropy and rmse for larger inputs --- src/backend/data_modeling/tensor.cpp | 10 ++-- .../loss_functions/cuda/loss_functions.cu | 8 +-- tests/backend/cuda/test_losses.cu | 58 +++++++++++++++++++ 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/src/backend/data_modeling/tensor.cpp b/src/backend/data_modeling/tensor.cpp index b6191f2..3f4c75b 100644 --- a/src/backend/data_modeling/tensor.cpp +++ b/src/backend/data_modeling/tensor.cpp @@ -625,7 +625,7 @@ Tensor Tensor::operator*(const Tensor& other) const { makeContiguous(); other.makeContiguous(); - Tensor res(dims, values->getDevice(), false); + Tensor res(dims, values->getDevice(), requiresGrad); switch(values->getDevice()){ case Device::CPU: @@ -672,7 +672,7 @@ Tensor& Tensor::operator+=(const Tensor& other) { } Tensor Tensor::operator*(const ftype scalar) const { - Tensor res(dims, values->getDevice(), false); + Tensor res(dims, values->getDevice(), requiresGrad); switch(values->getDevice()){ case Device::CPU: for (tensorSize_t i = 0; i < values->getSize(); ++i) { @@ -696,7 +696,7 @@ Tensor Tensor::operator/(const ftype scalar) const { __throw_runtime_error("Cannot divide by zero."); } - Tensor res(dims, values->getDevice(), false); + Tensor res(dims, values->getDevice(), requiresGrad); switch(values->getDevice()){ case Device::CPU: for (tensorSize_t i = 0; i < values->getSize(); ++i) { @@ -716,7 +716,7 @@ Tensor Tensor::operator/(const ftype scalar) const { } Tensor Tensor::operator+(const ftype scalar) const { - Tensor res(dims, values->getDevice(), false); + Tensor res(dims, values->getDevice(), requiresGrad); switch(values->getDevice()){ case Device::CPU: for (tensorSize_t i = 0; i < values->getSize(); ++i) { @@ -736,7 +736,7 @@ Tensor Tensor::operator+(const ftype scalar) const { } Tensor Tensor::operator-(const ftype scalar) const { - Tensor res(dims, values->getDevice(), false); + Tensor res(dims, values->getDevice(), requiresGrad); switch(values->getDevice()){ case Device::CPU: for (tensorSize_t i = 0; i < values->getSize(); ++i) { diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cu b/src/backend/training/loss_functions/cuda/loss_functions.cu index 560fc05..cc6858b 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cu +++ b/src/backend/training/loss_functions/cuda/loss_functions.cu @@ -191,7 +191,7 @@ namespace { } __syncthreads(); - for(int offset = blockDim.x / 2; offset > 64; offset >>= 1) { + for(int offset = blockDim.x / 2; offset > 32; offset >>= 1) { if(tid < offset) { smem[tid] += smem[tid + offset]; } @@ -308,7 +308,7 @@ namespace { } __syncthreads(); - for(int offset = blockDim.x / 2; offset > 64; offset >>= 1) { + for(int offset = blockDim.x / 2; offset > 32; offset >>= 1) { if(tid < offset) { smem[tid] += smem[tid + offset]; } @@ -436,7 +436,7 @@ namespace cuda_impl { cudaErrchk(cudaDeviceSynchronize()); } else { - const int blocks = (y.getSize() + maxThreadsPerBlock - 1) / (maxThreadsPerBlock * 2); + const int blocks = (y.getSize() + maxThreadsPerBlock - 1) / maxThreadsPerBlock; ftype* tmp; cudaErrchk(cudaMalloc(&tmp, blocks * sizeof(ftype))); @@ -537,7 +537,7 @@ namespace cuda_impl { cudaErrchk(cudaDeviceSynchronize()); } else { - const int blocks = (nSamples + maxThreadsPerBlock - 1) / (maxThreadsPerBlock * 2); + const int blocks = (nSamples + maxThreadsPerBlock - 1) / maxThreadsPerBlock; ftype* tmp; cudaErrchk(cudaMalloc(&tmp, blocks * sizeof(ftype))); diff --git a/tests/backend/cuda/test_losses.cu b/tests/backend/cuda/test_losses.cu index c65148a..07badd3 100644 --- a/tests/backend/cuda/test_losses.cu +++ b/tests/backend/cuda/test_losses.cu @@ -23,6 +23,7 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include +using namespace std; using namespace train; static constexpr ftype delta = 1e-4f; @@ -71,6 +72,25 @@ TEST(CudaLossTest, CrossEntropyUniformPrediction) { ASSERT_NEAR((*result)[0], std::log(3.0f), delta); } +TEST(CudaLossTest, CrossEntropyForwardLarge) { + constexpr tensorDim_t nSamples = 500; + constexpr tensorDim_t nClasses = 200; + + auto yCpu = make_shared(TensorFunctions::Ones({nSamples, nClasses}, Device::CPU, true) * 0.5f); + auto yGpu = make_shared(yCpu->createDeepCopy()); + yGpu->setDevice(Device::CUDA); + + auto ypredCpu = make_shared(TensorFunctions::Ones({nSamples, nClasses}, Device::CPU, true) * 0.7f); + auto ypredGpu = make_shared(ypredCpu->createDeepCopy()); + ypredGpu->setDevice(Device::CUDA); + + CrossEntropyLoss loss; + auto resCpu = loss(yCpu, ypredCpu); + auto resGpu = loss(yGpu, ypredGpu); + + EXPECT_NEAR((*resCpu)[0], (*resGpu)[0], delta); +} + TEST(CudaLossTest, CrossEntropyThrowsOnDimMismatch) { auto y = TensorFunctions::makeSharedTensor( {2, 3}, {1.0, 0.0, 0.0, 0.0, 1.0, 0.0}, Device::CUDA, false); @@ -143,6 +163,26 @@ TEST(CudaLossTest, BceRandomPrediction) { ASSERT_NEAR((*result)[0], std::log(2.0f), delta); } +TEST(CudaLossTest, BceForwardLarge) { + constexpr tensorDim_t nSamples = 10000; + + auto yCpu = make_shared(TensorFunctions::Ones({nSamples, 1}) * 0.5f); + auto yGpu = make_shared(yCpu->createDeepCopy()); + yGpu->setDevice(Device::CUDA); + + auto ypredCpu = make_shared(TensorFunctions::Ones({nSamples, 1}) * 0.7f); + ypredCpu->setRequiresGrad(true); + auto ypredGpu = make_shared(ypredCpu->createDeepCopy()); + ypredGpu->setDevice(Device::CUDA); + ypredGpu->setRequiresGrad(true); + + BceLoss loss; + auto resCpu = loss(yCpu, ypredCpu); + auto resGpu = loss(yGpu, ypredGpu); + + EXPECT_NEAR((*resCpu)[0], (*resGpu)[0], delta); +} + TEST(CudaLossTest, BceThrowsOnDimMismatch) { auto y = TensorFunctions::makeSharedTensor( {2, 1}, {1.0, 0.0}, Device::CUDA, false); @@ -192,6 +232,24 @@ TEST(CudaLossTest, RmseForward) { ASSERT_NEAR((*result)[0], 0.5f, delta); } +TEST(CudaLossTest, RmseForwardLarge) { + auto yCpu = make_shared(TensorFunctions::Gaussian({500, 500}, 1.0f)); + auto yGpu = make_shared(yCpu->createDeepCopy()); + yGpu->setDevice(Device::CUDA); + + auto ypredCpu = make_shared(TensorFunctions::Gaussian({500, 500}, 1.0f)); + ypredCpu->setRequiresGrad(true); + auto ypredGpu = make_shared(ypredCpu->createDeepCopy()); + ypredGpu->setDevice(Device::CUDA); + ypredGpu->setRequiresGrad(true); + + RmseLoss loss; + auto resCpu = loss(yCpu, ypredCpu); + auto resGpu = loss(yGpu, ypredGpu); + + EXPECT_NEAR((*resCpu)[0], (*resGpu)[0], delta); +} + TEST(CudaLossTest, RmsePerfectPrediction) { auto y = TensorFunctions::makeSharedTensor( {3}, {1.0, 2.0, 3.0}, Device::CUDA, false); From 6e005d18c3b39c9460aab57ff14f98fa8529cf22 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sat, 30 May 2026 12:49:55 +0200 Subject: [PATCH 66/73] Fixed all forward loss functions and eliminated error of uninitialized shared memory --- .../loss_functions/bce_node.cpp | 6 +- .../loss_functions/cuda/loss_nodes.cu | 20 ++-- .../training/loss_functions/bce_loss.cpp | 2 +- .../loss_functions/cuda/loss_functions.cu | 113 +++++++----------- tests/backend/cuda/test_losses.cu | 40 +++---- 5 files changed, 78 insertions(+), 103 deletions(-) diff --git a/src/backend/computational_graph/loss_functions/bce_node.cpp b/src/backend/computational_graph/loss_functions/bce_node.cpp index 9419e9d..cc09a44 100644 --- a/src/backend/computational_graph/loss_functions/bce_node.cpp +++ b/src/backend/computational_graph/loss_functions/bce_node.cpp @@ -32,10 +32,10 @@ vector< shared_ptr > BceNode::backward(const Tensor& upstreamGrad) { const ftype bSize = yPred->getDims()[0]; for(tensorSize_t i = 0; i < yPred->getDims()[0]; i++){ - auto yi = (*yTrue)[i]; - auto yiHat = (*yPred)[i]; + const auto yi = (*yTrue)[i]; + const auto yiHat = (*yPred)[i]; - auto g = -yi / std::max(yiHat, EPS_BCE) + (1 - yi) / std::max(1 - yiHat, EPS_BCE); + const auto g = -yi / std::max(yiHat, EPS_BCE) + (1 - yi) / std::max(1 - yiHat, EPS_BCE); res->set(g / bSize, i); } break; diff --git a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu index 63bbfe5..329109c 100644 --- a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu +++ b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu @@ -30,17 +30,17 @@ namespace { /** * @brief Does what you think it does. */ - __global__ void bceBackwardKernel(ftype* const res, const ftype* const yPred, const ftype* const yTrue, const tensorSize_t size) { + __global__ void bceBackwardKernel(ftype* const res, const ftype* const yPred, const ftype* const yTrue, const ftype bSize, const tensorSize_t size) { const int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) { return; } - auto yi = yTrue[gid]; - auto yiHat = yPred[gid]; + const auto yi = yTrue[gid]; + const auto yiHat = yPred[gid]; - auto g = -yi / cudaMax(yiHat, EPS_BCE) + (1 - yi) / cudaMax(1 - yiHat, EPS_BCE); - res[gid] = g; + const auto g = -yi / cudaMax(yiHat, EPS_BCE) + (1 - yi) / cudaMax(1 - yiHat, EPS_BCE); + res[gid] = g / bSize; } /** @@ -55,10 +55,10 @@ namespace { return; } - auto y = yTrue[gid]; - auto s = cudaSigmoid(logits[gid]); + const auto y = yTrue[gid]; + const auto s = cudaSigmoid(logits[gid]); - auto g = s - y; + const auto g = s - y; res[gid] = g / bSize; } @@ -73,7 +73,7 @@ namespace { return; } - auto g = -yTrue[gid] / cudaMax(yPred[gid], EPS_CROSSENTROPY); + const auto g = -yTrue[gid] / cudaMax(yPred[gid], EPS_CROSSENTROPY); res[gid] = g / nSamples; } @@ -115,7 +115,7 @@ namespace cuda_impl { constexpr int threadsPerBlock = 256; const int blocks = (yPred.getSize() + threadsPerBlock - 1) / threadsPerBlock; - bceBackwardKernel<<>>(res.getData(), yPred.getData(), yTrue.getData(), yTrue.getSize()); + bceBackwardKernel<<>>(res.getData(), yPred.getData(), yTrue.getData(), yPred.getDims()[0], yTrue.getSize()); cudaErrchk(cudaDeviceSynchronize()); } diff --git a/src/backend/training/loss_functions/bce_loss.cpp b/src/backend/training/loss_functions/bce_loss.cpp index ebd481d..200bd80 100644 --- a/src/backend/training/loss_functions/bce_loss.cpp +++ b/src/backend/training/loss_functions/bce_loss.cpp @@ -43,7 +43,7 @@ shared_ptr BceLoss::operator()(const shared_ptr y, const shared_ switch(y->getDevice()) { case Device::CPU: { - auto bce = [](ftype y, ftype ypred){ + auto bce = [](const ftype y, const ftype ypred){ return y * log(std::max(ypred, EPS_BCE)) + (1 - y) * log(std::max(1-ypred, EPS_BCE)); }; diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cu b/src/backend/training/loss_functions/cuda/loss_functions.cu index cc6858b..84064f1 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cu +++ b/src/backend/training/loss_functions/cuda/loss_functions.cu @@ -46,23 +46,17 @@ namespace { * @brief Forward BCE loss. */ __global__ void bceLossKernel(ftype* const res, const ftype* const y, const ftype* const ypred, tensorSize_t size) { - int gid = blockDim.x * blockIdx.x + threadIdx.x; - if(gid >= size) - return; + const int gid = blockDim.x * blockIdx.x + threadIdx.x; + const int tid = threadIdx.x; - int tid = threadIdx.x; extern __shared__ ftype smem[]; + const ftype tmp = gid < size ? bce(y[gid], ypred[gid]) : 0; + smem[tid] = tmp; + __syncthreads(); - // pre-load first round - { - int i = blockIdx.x * (blockDim.x * 2) + threadIdx.x; - smem[tid] = bce(y[i], ypred[i]) + bce(y[i + blockDim.x], ypred[i + blockDim.x]); - __syncthreads(); - } - - for(tensorSize_t i = blockDim.x / 2; i >= 64; i >>= 1){ - if(tid < i) { - smem[tid] += smem[tid + i]; + for(tensorSize_t offset = blockDim.x / 2; offset > 32; offset >>= 1){ + if(tid < offset) { + smem[tid] += smem[tid + offset]; } __syncthreads(); } @@ -86,7 +80,7 @@ namespace { sdata[tid] += sdata[tid + 2]; } if(gid + 1 < size) { - sdata[0] = (sdata[0] + sdata[1]) / size; + sdata[tid] += sdata[tid + 1]; } } @@ -115,23 +109,19 @@ namespace { * @param logits Forward logits. */ __global__ void bceSigmoidLossKernel(ftype* const res, const ftype* const y, const ftype* const logits, tensorSize_t size) { - int gid = blockDim.x * blockIdx.x + threadIdx.x; - if(gid >= size) - return; - - int tid = threadIdx.x; + const int gid = blockDim.x * blockIdx.x + threadIdx.x; + const int tid = threadIdx.x; + extern __shared__ ftype smem[]; // pre-load first round - { - int i = blockIdx.x * (blockDim.x * 2) + threadIdx.x; - smem[tid] = bceSimplified(y[i], logits[i]) + bceSimplified(y[i + blockDim.x], logits[i + blockDim.x]); - __syncthreads(); - } + const ftype tmp = gid < size ? bceSimplified(y[gid], logits[gid]) : 0; + smem[tid] = tmp; + __syncthreads(); - for(tensorSize_t i = blockDim.x / 2; i >= 64; i >>= 1){ - if(tid < i) { - smem[tid] += smem[tid + i]; + for(tensorSize_t offset = blockDim.x / 2; offset > 32; offset >>= 1){ + if(tid < offset) { + smem[tid] += smem[tid + offset]; } __syncthreads(); } @@ -155,7 +145,7 @@ namespace { sdata[tid] += sdata[tid + 2]; } if(gid + 1 < size) { - sdata[0] = (sdata[0] + sdata[1]) / size; + sdata[tid] += sdata[tid + 1]; } } @@ -186,9 +176,8 @@ namespace { extern __shared__ ftype smem[]; - if(gid < size) { - smem[tid] = crossEntropy(y[gid], yPred[gid]); - } + const ftype tmp = gid < size ? crossEntropy(y[gid], yPred[gid]) : 0; + smem[tid] = tmp; __syncthreads(); for(int offset = blockDim.x / 2; offset > 32; offset >>= 1) { @@ -302,10 +291,8 @@ namespace { const int gid = blockIdx.x * blockDim.x + tid; extern __shared__ ftype smem[]; - smem[tid] = diffPow(y[gid], yPred[gid]); - if(gid + blockDim.x < size) { - smem[tid] += diffPow(y[gid + blockDim.x], yPred[gid + blockDim.x]); - } + const ftype tmp = gid < size ? diffPow(y[gid], yPred[gid]) : 0; + smem[tid] = tmp; __syncthreads(); for(int offset = blockDim.x / 2; offset > 32; offset >>= 1) { @@ -361,7 +348,7 @@ namespace { namespace cuda_impl { void bceLoss(Tensor& res, const Tensor& y, const Tensor& yPred) { constexpr int threadsPerBlock = 256; - const int blocks = (y.getSize() + threadsPerBlock - 1) / (threadsPerBlock * 2); + const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; if(blocks > 1) { // two pass solution @@ -369,7 +356,7 @@ namespace cuda_impl { cudaErrchk(cudaMalloc(&tmp, blocks * sizeof(ftype))); bceLossKernel<<>>( - tmp, y.getData(), yPred.getData(), y.getDims()[0]); + tmp, y.getData(), yPred.getData(), y.getSize()); cudaErrchk(cudaDeviceSynchronize()); // do a sum over the residual array @@ -381,7 +368,7 @@ namespace cuda_impl { } else { bceLossKernel<<>>( - res.getData(), y.getData(), yPred.getData(), y.getDims()[0]); + res.getData(), y.getData(), yPred.getData(), y.getSize()); cudaErrchk(cudaDeviceSynchronize()); } @@ -392,7 +379,7 @@ namespace cuda_impl { void bceSigmoidLoss(Tensor& res, const Tensor& y, const Tensor& logits) { constexpr int threadsPerBlock = 256; - const int blocks = (y.getSize() + threadsPerBlock - 1) / (threadsPerBlock * 2); + const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; if(blocks > 1) { // we do two passes at max @@ -400,7 +387,7 @@ namespace cuda_impl { cudaErrchk(cudaMalloc(&tmp, blocks * sizeof(ftype))); bceSigmoidLossKernel<<>>( - tmp, y.getData(), logits.getData(), y.getDims()[0]); + tmp, y.getData(), logits.getData(), y.getSize()); cudaErrchk(cudaDeviceSynchronize()); // do a sum over the residual array @@ -412,7 +399,7 @@ namespace cuda_impl { } else { bceSigmoidLossKernel<<>>( - res.getData(), y.getData(), logits.getData(), y.getDims()[0]); + res.getData(), y.getData(), logits.getData(), y.getSize()); cudaErrchk(cudaDeviceSynchronize()); } @@ -421,27 +408,22 @@ namespace cuda_impl { cudaErrchk(cudaDeviceSynchronize()); } - void crossEntropyLoss(Tensor& res, const Tensor& y, const Tensor& yPred) { - constexpr int maxThreadsPerBlock = 256; - - const tensorSize_t stride = y.getDims()[-1]; - const tensorSize_t nSamples = y.getSize() / stride; - - if(y.getSize() <= maxThreadsPerBlock) { - + void crossEntropyLoss(Tensor& res, const Tensor& y, const Tensor& yPred) { + if(y.getSize() <= 256) { int threadsPerBlock = 1; - while(threadsPerBlock < y.getSize()) threadsPerBlock <<= 1; - + while(threadsPerBlock < y.getSize()) threadsPerBlock <<= 1; // < 512 threads + crossEntropyLossKernelOneBlock<<<1, threadsPerBlock, y.getSize() * sizeof(ftype)>>>(res.getData(), y.getData(), yPred.getData(), y.getSize()); cudaErrchk(cudaDeviceSynchronize()); } else { - const int blocks = (y.getSize() + maxThreadsPerBlock - 1) / maxThreadsPerBlock; + constexpr int threadsPerBlock = 256; + const int blocks = (y.getSize() + threadsPerBlock - 1) / threadsPerBlock; ftype* tmp; cudaErrchk(cudaMalloc(&tmp, blocks * sizeof(ftype))); - crossEntropyLossKernelOneBlock<<>>(tmp, y.getData(), yPred.getData(), y.getSize()); + crossEntropyLossKernelOneBlock<<>>(tmp, y.getData(), yPred.getData(), y.getSize()); cudaErrchk(cudaDeviceSynchronize()); // do a sum over the residual array @@ -453,6 +435,8 @@ namespace cuda_impl { } // loss = -loss / nBatches + const tensorSize_t stride = y.getDims()[-1]; + const tensorSize_t nSamples = y.getSize() / stride; divideScalarKernel<<<1, 1>>>(res.getData(), -1 * static_cast(nSamples)); cudaErrchk(cudaDeviceSynchronize()); } @@ -518,31 +502,24 @@ namespace cuda_impl { cudaErrchk(cudaFree(maxValues)); } - void rmseLoss(Tensor& res, const Tensor& y, const Tensor& yPred) { - constexpr int maxThreadsPerBlock = 256; - + void rmseLoss(Tensor& res, const Tensor& y, const Tensor& yPred) { const auto nSamples = y.getSize(); - if(nSamples * 2 <= maxThreadsPerBlock) { - // ceil(y.getSize() / 2) - const int sizeOverTwo = (y.getSize() + 1) / 2; - assert(sizeOverTwo > 0); - + if(nSamples <= 256) { int threadsPerBlock = 1; - while(threadsPerBlock < sizeOverTwo) threadsPerBlock <<= 1; - threadsPerBlock = max(1, threadsPerBlock >> 1); - assert(threadsPerBlock >= y.getSize() / 2); + while(threadsPerBlock < nSamples) threadsPerBlock <<= 1; // < 512 threads rmseKernelOneBlock<<<1, threadsPerBlock, threadsPerBlock * sizeof(ftype)>>>(res.getData(), y.getData(), yPred.getData(), y.getSize()); cudaErrchk(cudaDeviceSynchronize()); } else { - const int blocks = (nSamples + maxThreadsPerBlock - 1) / maxThreadsPerBlock; + constexpr int threadsPerBlock = 256; + const int blocks = (nSamples + threadsPerBlock - 1) / threadsPerBlock; ftype* tmp; cudaErrchk(cudaMalloc(&tmp, blocks * sizeof(ftype))); - rmseKernelOneBlock<<>>(tmp, y.getData(), yPred.getData(), y.getSize()); + rmseKernelOneBlock<<>>(tmp, y.getData(), yPred.getData(), y.getSize()); cudaErrchk(cudaDeviceSynchronize()); // do a sum over the residual array @@ -553,7 +530,7 @@ namespace cuda_impl { cudaErrchk(cudaFree(tmp)); } - normalizeRmse<<<1, 1>>>(res.getData(), nSamples); + normalizeRmse<<<1, 1>>>(res.getData(), static_cast(nSamples)); cudaErrchk(cudaDeviceSynchronize()); } } diff --git a/tests/backend/cuda/test_losses.cu b/tests/backend/cuda/test_losses.cu index 07badd3..a428a91 100644 --- a/tests/backend/cuda/test_losses.cu +++ b/tests/backend/cuda/test_losses.cu @@ -26,8 +26,6 @@ static_assert(false, "File should not be compiled without CUDA enabled"); using namespace std; using namespace train; -static constexpr ftype delta = 1e-4f; - TEST(CudaLossTest, CrossEntropyForward) { auto y = TensorFunctions::makeSharedTensor( {2, 3}, {1.0, 0.0, 0.0, @@ -41,7 +39,7 @@ TEST(CudaLossTest, CrossEntropyForward) { auto result = loss(y, ypred); const ftype expected = -(std::log(0.7f) + std::log(0.8f)) / 2.0f; - ASSERT_NEAR((*result)[0], expected, delta); + ASSERT_NEAR((*result)[0], expected, 1e-4); } TEST(CudaLossTest, CrossEntropyPerfectPrediction) { @@ -69,7 +67,7 @@ TEST(CudaLossTest, CrossEntropyUniformPrediction) { CrossEntropyLoss loss; auto result = loss(y, ypred); - ASSERT_NEAR((*result)[0], std::log(3.0f), delta); + ASSERT_NEAR((*result)[0], std::log(3.0f), 1e-4); } TEST(CudaLossTest, CrossEntropyForwardLarge) { @@ -88,7 +86,7 @@ TEST(CudaLossTest, CrossEntropyForwardLarge) { auto resCpu = loss(yCpu, ypredCpu); auto resGpu = loss(yGpu, ypredGpu); - EXPECT_NEAR((*resCpu)[0], (*resGpu)[0], delta); + EXPECT_NEAR((*resCpu)[0], (*resGpu)[0], 0.1); } TEST(CudaLossTest, CrossEntropyThrowsOnDimMismatch) { @@ -114,12 +112,12 @@ TEST(CudaLossTest, CrossEntropyBackward) { result->backward(); auto grads = ypred->getGrads(); - ASSERT_NEAR((*grads)[0], -0.7143f, delta); - ASSERT_NEAR((*grads)[1], 0.0f, delta); - ASSERT_NEAR((*grads)[2], 0.0f, delta); - ASSERT_NEAR((*grads)[3], 0.0f, delta); - ASSERT_NEAR((*grads)[4], -0.625f, delta); - ASSERT_NEAR((*grads)[5], 0.0f, delta); + ASSERT_NEAR((*grads)[0], -0.7143f, 1e-4); + ASSERT_NEAR((*grads)[1], 0.0f, 1e-4); + ASSERT_NEAR((*grads)[2], 0.0f, 1e-4); + ASSERT_NEAR((*grads)[3], 0.0f, 1e-4); + ASSERT_NEAR((*grads)[4], -0.625f, 1e-4); + ASSERT_NEAR((*grads)[5], 0.0f, 1e-4); } TEST(CudaLossTest, BceForward) { @@ -134,7 +132,7 @@ TEST(CudaLossTest, BceForward) { const ftype expected = -(std::log(0.9f) + std::log(0.9f) + std::log(0.8f) + std::log(0.8f)) / 4.0f; - ASSERT_NEAR((*result)[0], expected, delta); + ASSERT_NEAR((*result)[0], expected, 1e-4); } TEST(CudaLossTest, BcePerfectPrediction) { @@ -160,7 +158,7 @@ TEST(CudaLossTest, BceRandomPrediction) { BceLoss loss; auto result = loss(y, ypred); - ASSERT_NEAR((*result)[0], std::log(2.0f), delta); + ASSERT_NEAR((*result)[0], std::log(2.0f), 1e-4); } TEST(CudaLossTest, BceForwardLarge) { @@ -180,7 +178,7 @@ TEST(CudaLossTest, BceForwardLarge) { auto resCpu = loss(yCpu, ypredCpu); auto resGpu = loss(yGpu, ypredGpu); - EXPECT_NEAR((*resCpu)[0], (*resGpu)[0], delta); + EXPECT_NEAR((*resCpu)[0], (*resGpu)[0], 1e-4); } TEST(CudaLossTest, BceThrowsOnDimMismatch) { @@ -216,8 +214,8 @@ TEST(CudaLossTest, BceBackward) { result->backward(); auto grads = ypred->getGrads(); - ASSERT_NEAR((*grads)[0], -0.625f, delta); - ASSERT_NEAR((*grads)[1], 0.7143f, delta); + ASSERT_NEAR((*grads)[0], -0.625f, 1e-4); + ASSERT_NEAR((*grads)[1], 0.7143f, 1e-4); } TEST(CudaLossTest, RmseForward) { @@ -229,7 +227,7 @@ TEST(CudaLossTest, RmseForward) { RmseLoss loss; auto result = loss(y, ypred); - ASSERT_NEAR((*result)[0], 0.5f, delta); + ASSERT_NEAR((*result)[0], 0.5f, 1e-4); } TEST(CudaLossTest, RmseForwardLarge) { @@ -247,7 +245,7 @@ TEST(CudaLossTest, RmseForwardLarge) { auto resCpu = loss(yCpu, ypredCpu); auto resGpu = loss(yGpu, ypredGpu); - EXPECT_NEAR((*resCpu)[0], (*resGpu)[0], delta); + EXPECT_NEAR((*resCpu)[0], (*resGpu)[0], 0.1); } TEST(CudaLossTest, RmsePerfectPrediction) { @@ -259,7 +257,7 @@ TEST(CudaLossTest, RmsePerfectPrediction) { RmseLoss loss; auto result = loss(y, ypred); - ASSERT_NEAR((*result)[0], 0.0f, delta); + ASSERT_NEAR((*result)[0], 0.0f, 1e-4); } TEST(CudaLossTest, RmseBackward) { @@ -273,6 +271,6 @@ TEST(CudaLossTest, RmseBackward) { result->backward(); auto grads = ypred->getGrads(); - ASSERT_NEAR((*grads)[0], -0.5f, delta); - ASSERT_NEAR((*grads)[1], 0.5f, delta); + ASSERT_NEAR((*grads)[0], -0.5f, 1e-4); + ASSERT_NEAR((*grads)[1], 0.5f, 1e-4); } From 438dcd82d7c4678ab81d1a0bcb1ca6481e839905 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sat, 30 May 2026 21:58:06 +0200 Subject: [PATCH 67/73] CUDA versions of optimizers --- .../training/optimizers/cuda/optimizers.cu | 35 ++- .../training/optimizers/cuda/optimizers.cuh | 2 +- src/backend/training/optimizers/rmsprop.cpp | 18 +- src/backend/training/optimizers/sgd.cpp | 4 +- src/backend/utility/global_params.h | 1 + tests/CMakeLists.txt | 1 + tests/backend/cuda/netfacories.cuh | 2 + .../backend/cuda/test_computational_graph.cu | 246 ++++++++++++++++++ tests/backend/net_factories.h | 35 +-- 9 files changed, 307 insertions(+), 37 deletions(-) create mode 100644 tests/backend/cuda/netfacories.cuh create mode 100644 tests/backend/cuda/test_computational_graph.cu diff --git a/src/backend/training/optimizers/cuda/optimizers.cu b/src/backend/training/optimizers/cuda/optimizers.cu index ee050d7..b6d82e5 100644 --- a/src/backend/training/optimizers/cuda/optimizers.cu +++ b/src/backend/training/optimizers/cuda/optimizers.cu @@ -16,30 +16,49 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include "optimizers.cuh" #include "utility/cuda/cuda_common.cuh" +#include "utility/global_params.h" + using namespace std; namespace { - // TODO: sgdStep kernel + __global__ void stepSgdKernel(ftype* const params, const ftype* const grads, const ftype lr, const tensorSize_t size) { + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) { + return; + } + + params[gid] = params[gid] - lr * grads[gid]; + } + + __global__ void stepRmsPropKernel(ftype* const tensor, ftype* const movingAvgs, const ftype* const grads, const ftype lr, const ftype decay, const tensorSize_t size) { + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) { + return; + } - // TODO: rmspropStep kernel + const ftype g = grads[gid]; + const ftype mavg = decay * movingAvgs[gid] + (1 - decay) * g * g; + movingAvgs[gid] = mavg; + + const ftype update = tensor[gid] - lr * g / (mavg + EPS_RMSPROP); + tensor[gid] = update; + } } namespace cuda_impl { - void sgdStep(Tensor& param, const Tensor& grad, ftype lr) { + void sgdStep(Tensor& param, const Tensor& grads, ftype lr) { constexpr int threadsPerBlock = 256; const int blocks = (param.getSize() + threadsPerBlock - 1) / threadsPerBlock; - // TODO: launch kernel - + stepSgdKernel<<>>(param.getData(), grads.getData(), lr, param.getSize()); cudaErrchk(cudaDeviceSynchronize()); } - void rmspropStep(Tensor& param, Tensor& movingAvg, const Tensor& grad, ftype lr, ftype decay, ftype eps) { + void rmspropStep(Tensor& param, Tensor& movingAvg, const Tensor& grad, ftype lr, ftype decay) { constexpr int threadsPerBlock = 256; const int blocks = (param.getSize() + threadsPerBlock - 1) / threadsPerBlock; - // TODO: launch kernel - + stepRmsPropKernel<<>>(param.getData(), movingAvg.getData(), grad.getData(), lr, decay, param.getSize()); cudaErrchk(cudaDeviceSynchronize()); } } diff --git a/src/backend/training/optimizers/cuda/optimizers.cuh b/src/backend/training/optimizers/cuda/optimizers.cuh index 89e0f11..a32562a 100644 --- a/src/backend/training/optimizers/cuda/optimizers.cuh +++ b/src/backend/training/optimizers/cuda/optimizers.cuh @@ -20,5 +20,5 @@ static_assert(false, "File should not be included without CUDA enabled"); namespace cuda_impl { void sgdStep(Tensor& param, const Tensor& grad, ftype lr); - void rmspropStep(Tensor& param, Tensor& movingAvg, const Tensor& grad, ftype lr, ftype decay, ftype eps); + void rmspropStep(Tensor& param, Tensor& movingAvg, const Tensor& grad, ftype lr, ftype decay); } diff --git a/src/backend/training/optimizers/rmsprop.cpp b/src/backend/training/optimizers/rmsprop.cpp index 8cf84a8..ab66caa 100644 --- a/src/backend/training/optimizers/rmsprop.cpp +++ b/src/backend/training/optimizers/rmsprop.cpp @@ -21,7 +21,6 @@ using namespace std; using namespace train; void RmsPropOptimizer::step() { - constexpr ftype eps = 1e-8; for(const auto& param: params){ auto tPtr = param.get(); const auto gPtr = tPtr->getGrads().get(); @@ -30,24 +29,24 @@ void RmsPropOptimizer::step() { case Device::CPU: { auto vPtr = movingAvg[tPtr].get(); - if(vPtr != nullptr) { - for(tensorSize_t i=0; igetSize(); i++){ + if(vPtr != nullptr) [[likely]] { + for(tensorSize_t i = 0; i < gPtr->getSize(); i++){ auto g = (*gPtr)[i]; - auto update = decay * (*vPtr)[i] + (1-decay)*g*g; + auto update = decay * (*vPtr)[i] + (1 - decay) * g * g; vPtr->set(update, i); } } - else { + else [[unlikely]] { movingAvg[tPtr] = make_unique(tPtr->getDims(), tPtr->getDevice(), false); vPtr = movingAvg[tPtr].get(); - for(tensorSize_t i=0; igetSize(); i++) { + for(tensorSize_t i = 0; i < tPtr->getSize(); i++) { auto g = (*gPtr)[i]; - vPtr->set((1-decay)*g*g, i); + vPtr->set((1 - decay) * g * g, i); } } for(tensorSize_t i=0; igetSize(); i++) { - auto update = (*tPtr)[i] - lr * (*gPtr)[i] / ((*vPtr)[i] + eps); + auto update = (*tPtr)[i] - lr * (*gPtr)[i] / ((*vPtr)[i] + EPS_RMSPROP); tPtr->set(update, i); } break; @@ -56,8 +55,9 @@ void RmsPropOptimizer::step() { #ifdef __CUDA if(movingAvg[tPtr] == nullptr) { movingAvg[tPtr] = make_unique(tPtr->getDims(), tPtr->getDevice(), false); + movingAvg[tPtr]->reset(0); } - cuda_impl::rmspropStep(*tPtr, *movingAvg[tPtr], *gPtr, lr, decay, eps); + cuda_impl::rmspropStep(*tPtr, *movingAvg[tPtr], *gPtr, lr, decay); #else __throw_invalid_argument("Attempted to use CUDA tensor"); #endif diff --git a/src/backend/training/optimizers/sgd.cpp b/src/backend/training/optimizers/sgd.cpp index f132dbd..c3dc0f6 100644 --- a/src/backend/training/optimizers/sgd.cpp +++ b/src/backend/training/optimizers/sgd.cpp @@ -25,8 +25,8 @@ void SgdOptimizer::step() { auto grads = t->getGrads(); switch(t->getDevice()) { case Device::CPU: - for(auto idx=0; idxgetSize(); idx++){ - auto updatedWeight = (*t)[idx] - lr*(*grads)[idx]; + for(auto idx = 0; idx < t->getSize(); idx++){ + auto updatedWeight = (*t)[idx] - lr * (*grads)[idx]; t->set(updatedWeight, idx); } break; diff --git a/src/backend/utility/global_params.h b/src/backend/utility/global_params.h index ebd4025..b2861e5 100644 --- a/src/backend/utility/global_params.h +++ b/src/backend/utility/global_params.h @@ -56,6 +56,7 @@ static_assert(sizeof(tensorDim_t)<=sizeof(tensorSize_t)); constexpr ftype EPS_CROSSENTROPY = 1e-5; constexpr ftype EPS_BCE = 1e-5; constexpr ftype EPS_RMSE = 1e-9; +constexpr ftype EPS_RMSPROP = 1e-8; // ----------------- Default values ------------------------ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c661c6a..2774e33 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -43,6 +43,7 @@ if(CMAKE_CUDA_COMPILER) find_package(CUDAToolkit REQUIRED) target_include_directories(unit_tests_cuda PRIVATE ${CUDAToolkit_INCLUDE_DIRS} + ${CMAKE_CURRENT_SOURCE_DIR}/backend ) gtest_discover_tests(unit_tests_cuda) diff --git a/tests/backend/cuda/netfacories.cuh b/tests/backend/cuda/netfacories.cuh new file mode 100644 index 0000000..faba0cd --- /dev/null +++ b/tests/backend/cuda/netfacories.cuh @@ -0,0 +1,2 @@ +#pragma once +#include "net_factories.h" diff --git a/tests/backend/cuda/test_computational_graph.cu b/tests/backend/cuda/test_computational_graph.cu new file mode 100644 index 0000000..20edd45 --- /dev/null +++ b/tests/backend/cuda/test_computational_graph.cu @@ -0,0 +1,246 @@ +/** + * @file test_computational_graph.cu + * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) + * @brief + * @version 0.1 + * @date 2026-05-20 + * + * @copyright Copyright (c) 2026 + * + */ + +#ifndef __CUDA +static_assert(false, "File should not be compiled without CUDA enabled"); +#endif // __CUDA + +#include + +#include "data_modeling/tensor_functions.h" +#include "computational_graph/tensor_ops/graph_creation.h" + +#include "net_factories.h" + +#include "training/optimizers/sgd.h" +#include "training/optimizers/rmsprop.h" + +#include "training/loss_functions/bce_loss.h" +#include "training/loss_functions/crossentropy_loss.h" +#include "training/loss_functions/bce_sigmoid_loss.h" +#include "training/loss_functions/crossentropy_softmax_loss.h" + +#include "training/trainers/base_train_loop.h" + +using namespace std; + +TEST(CudaAutogradTest, ThrowsIfNoGradientSet) { + auto t1 = TensorFunctions::makeSharedTensor({1}, {3.0}, Device::CUDA, false); + auto t2 = TensorFunctions::makeSharedTensor({1}, {2.0}, Device::CUDA, false); + + auto loss = cgraph::add(t1, t2); + + ASSERT_THROW(loss->backward(), std::runtime_error); +} + +TEST(CudaAutogradTest, ChainRule) { + auto x = TensorFunctions::makeSharedTensor({1}, {2.0}, Device::CUDA, true); + + auto y = cgraph::mul(x, x); // y = x^2 + auto z = cgraph::add(x, y); // z = x^2 + x + auto loss = cgraph::mul(z, z); // loss = (x^2 + x)^2 + + loss->backward(); + + // dloss/dx = 2(x^2 + x) * (2x + 1) + // At x=2: 2(4 + 2) * (4 + 1) = 60 + ASSERT_NEAR(x->getGrads()->get(0), 60.0, 1e-4); +} + +TEST(CudaAutogradTest, MultiVariateChainRule) { + auto x = TensorFunctions::makeSharedTensor({2}, {1.0, 2.0}, Device::CUDA, true); + + auto y = cgraph::mul(x, 3.0); // y = [3, 6] + auto loss = TensorFunctions::makeSharedTensor({1}, {0.0}, Device::CUDA, true); + for(int i = 0; i < y->getSize(); i++){ + loss = cgraph::add(loss, cgraph::get(y, i)); + } // loss = 9 + + loss->backward(); + + // dloss/dx = 3 + ASSERT_NEAR(x->getGrads()->get(0), 3.0, 1e-4); + ASSERT_NEAR(x->getGrads()->get(1), 3.0, 1e-4); + + ASSERT_NEAR(y->getGrads()->get(0), 1.0, 1e-4); + ASSERT_NEAR(y->getGrads()->get(1), 1.0, 1e-4); +} + +TEST(CudaOverfitTest, BceSgdOverfitsSmallDataset) { + auto x = TensorFunctions::makeSharedTensor( + {4, 2}, {0.0, 0.0, + 0.0, 1.0, + 1.0, 0.0, + 1.0, 1.0}, Device::CUDA, false); + + auto y = TensorFunctions::makeSharedTensor( + {4, 1}, {0.0, + 1.0, + 1.0, + 0.0}, Device::CUDA, false); + + auto net = makeBinaryNet(Device::CUDA); + auto loss = make_shared(); + auto optim = make_shared( + net->parameters(), /*lr=*/0.05); + + auto trainLoop = train::BaseTrainLoop( + net, loss, optim, /*epochs=*/2000, /*bsize=*/static_cast(4)); + + trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); + + auto pred = (*net)(x); + auto finalLoss = (*loss)(y, pred); + + EXPECT_LT((*finalLoss)[0], 0.05f) + << "Network failed to overfit binary dataset\n" + << "Final loss: " << *finalLoss; +} + +TEST(CudaOverfitTest, BceSgdOverfitsSmallDataset_OptimizedLoss) { + auto x = TensorFunctions::makeSharedTensor( + {4, 2}, {0.0, 0.0, + 0.0, 1.0, + 1.0, 0.0, + 1.0, 1.0}, Device::CUDA, false); + + auto y = TensorFunctions::makeSharedTensor( + {4, 1}, {0.0, + 1.0, + 1.0, + 0.0}, Device::CUDA, false); + + auto net = makeBinaryNet2(Device::CUDA); + auto loss = make_shared(); + auto optim = make_shared( + net->parameters(), /*lr=*/0.05); + + auto trainLoop = train::BaseTrainLoop( + net, loss, optim, /*epochs=*/2000, /*bsize=*/static_cast(4)); + + trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); + + auto pred = (*net)(x); + auto finalLoss = (*loss)(y, pred); + + EXPECT_LT((*finalLoss)[0], 0.05f) + << "Network failed to overfit binary dataset\n" + << "Final loss: " << *finalLoss; +} + +TEST(CudaOverfitTest, CrossEntropyRMSPropOverfitsSmallDataset) { + auto x = TensorFunctions::makeSharedTensor( + {6, 2}, {1.0, 0.0, + 1.0, 0.1, + 0.0, 1.0, + 0.1, 1.0, + 0.5, 0.5, + 0.4, 0.6}, Device::CUDA, false); + + auto y = TensorFunctions::makeSharedTensor( + {6, 3}, {1.0, 0.0, 0.0, + 1.0, 0.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 0.0, 1.0, + 0.0, 0.0, 1.0}, Device::CUDA, false); + + auto net = makeMulticlassNet(Device::CUDA); + auto loss = make_shared(); + auto optim = make_shared( + net->parameters(), /*lr=*/0.0001, /*decay=*/0.95); + + auto trainLoop = train::BaseTrainLoop( + net, loss, optim, /*epochs=*/2000, /*bsize=*/6); + + trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); + + auto pred = (*net)(x); + auto finalLoss = (*loss)(y, pred); + + EXPECT_LT((*finalLoss)[0], 0.05f) + << "Network failed to overfit multiclass dataset\n" + << "Final loss: " << *finalLoss; +} + +TEST(CudaOverfitTest, CrossEntropyRMSPropOverfitsSmallDataset_OptimizedLoss) { + auto x = TensorFunctions::makeSharedTensor( + {6, 2}, {1.0, 0.0, + 1.0, 0.1, + 0.0, 1.0, + 0.1, 1.0, + 0.5, 0.5, + 0.4, 0.6}, Device::CUDA, false); + + auto y = TensorFunctions::makeSharedTensor( + {6, 3}, {1.0, 0.0, 0.0, + 1.0, 0.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 0.0, 1.0, + 0.0, 0.0, 1.0}, Device::CUDA, false); + + auto net = makeMulticlassNet2(Device::CUDA); + auto loss = make_shared(); + auto optim = make_shared( + net->parameters(), /*lr=*/0.0001, /*decay=*/0.95); + + auto trainLoop = train::BaseTrainLoop( + net, loss, optim, /*epochs=*/2000, /*bsize=*/6); + + trainLoop.run(x, y, /*shuffle=*/false, /*verbose=*/false); + + auto pred = (*net)(x); + auto finalLoss = (*loss)(y, pred); + + EXPECT_LT((*finalLoss)[0], 0.05f) + << "Network failed to overfit multiclass dataset\n" + << "Final loss: " << *finalLoss; +} + +TEST(CudaOptimizerTest, ZeroGrad_ClearsAllGradients) { + auto x = TensorFunctions::makeSharedTensor( + {4, 2}, {0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0}, Device::CUDA, false); + auto y = TensorFunctions::makeSharedTensor( + {4, 1}, {0.0, 1.0, 1.0, 0.0}, Device::CUDA, false); + + auto net = makeBinaryNet(Device::CUDA); + auto loss = make_shared(); + auto optim = make_shared(net->parameters(), 0.01f); + + auto pred = (*net)(x); + auto l = (*loss)(y, pred); + l->backward(); + + bool anyNonZero = false; + for(auto& p : net->parameters()) { + if(p->getGrads()) { + for(tensorSize_t i = 0; i < p->getGrads()->getSize(); i++) { + if((*p->getGrads())[i] != 0.0f) { + anyNonZero = true; + break; + } + } + } + } + EXPECT_TRUE(anyNonZero) << "Expected some non-zero gradients before zeroGrad"; + + optim->zeroGrad(); + + for(auto& p : net->parameters()) { + if(p->getGrads()) { + for(tensorSize_t i = 0; i < p->getGrads()->getSize(); i++) { + ASSERT_NEAR((*p->getGrads())[i], 0.0f, 1e-5) + << "Gradient not zeroed at index " << i; + } + } + } +} diff --git a/tests/backend/net_factories.h b/tests/backend/net_factories.h index 45ecbc6..cc6500c 100644 --- a/tests/backend/net_factories.h +++ b/tests/backend/net_factories.h @@ -1,16 +1,17 @@ /** * @file net_factories.h * @author Robert Baumgartner (r.baumgartner-1@tudelft.nl) - * @brief + * @brief * @version 0.1 * @date 2026-05-20 - * + * * @copyright Copyright (c) 2026 - * + * */ #pragma once +#include "data_modeling/device.h" #include "module/networks/sequential.h" #include "module/layers/ff_layer.h" @@ -21,44 +22,44 @@ #include -static std::shared_ptr makeBinaryNet() { +static std::shared_ptr makeBinaryNet(Device d = Device::CPU) { auto net = std::make_shared(); - net->append(std::make_shared(2, 4, true, true)); + net->append(std::make_shared(2, 4, d, true, true)); net->append(std::make_shared(0.01)); - net->append(std::make_shared(4, 1, true, true)); + net->append(std::make_shared(4, 1, d, true, true)); net->append(std::make_shared()); - + return net; } -static std::shared_ptr makeBinaryNet2() { +static std::shared_ptr makeBinaryNet2(Device d = Device::CPU) { auto net = std::make_shared(); - net->append(std::make_shared(2, 4, true, true)); + net->append(std::make_shared(2, 4, d, true, true)); net->append(std::make_shared(0.01)); - net->append(std::make_shared(4, 1, true, true)); + net->append(std::make_shared(4, 1, d, true, true)); return net; } -static std::shared_ptr makeMulticlassNet() { +static std::shared_ptr makeMulticlassNet(Device d = Device::CPU) { auto net = std::make_shared(); - net->append(std::make_shared(2, 8, true, true)); + net->append(std::make_shared(2, 8, d, true, true)); net->append(std::make_shared(0.01)); - net->append(std::make_shared(8, 3, true, true)); + net->append(std::make_shared(8, 3, d, true, true)); net->append(std::make_shared()); return net; } -static std::shared_ptr makeMulticlassNet2() { +static std::shared_ptr makeMulticlassNet2(Device d = Device::CPU) { auto net = std::make_shared(); - net->append(std::make_shared(2, 8, true, true)); + net->append(std::make_shared(2, 8, d, true, true)); net->append(std::make_shared(0.01)); - net->append(std::make_shared(8, 3, true, true)); + net->append(std::make_shared(8, 3, d, true, true)); return net; -} \ No newline at end of file +} From 9e48d7bbaa548010b556551a1d3c388cffbf0957 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 31 May 2026 10:38:55 +0200 Subject: [PATCH 68/73] Fix bug in loss --- .../computational_graph/loss_functions/bce_sigmoid_node.cpp | 2 +- .../computational_graph/loss_functions/cuda/loss_nodes.cu | 2 +- src/backend/training/loss_functions/cuda/loss_functions.cu | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/computational_graph/loss_functions/bce_sigmoid_node.cpp b/src/backend/computational_graph/loss_functions/bce_sigmoid_node.cpp index ed4aa25..2c1420b 100644 --- a/src/backend/computational_graph/loss_functions/bce_sigmoid_node.cpp +++ b/src/backend/computational_graph/loss_functions/bce_sigmoid_node.cpp @@ -33,7 +33,7 @@ vector< shared_ptr > BceSigmoidNode::backward(const Tensor& upstreamGrad case Device::CPU: { auto sigmoid = [](ftype x){ constexpr ftype one = 1.0; - if(x>=0){ + if(x >= 0) { return one / (one + exp(-x)); } auto e = exp(x); diff --git a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu index 329109c..1dfa8d2 100644 --- a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu +++ b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu @@ -123,7 +123,7 @@ namespace cuda_impl { constexpr int threadsPerBlock = 256; const int blocks = (logits.getSize() + threadsPerBlock - 1) / threadsPerBlock; - bceSigmoidBackwardKernel<<>>(res.getData(), logits.getData(), yTrue.getData(), logits.getDims()[0], res.getSize()); + bceSigmoidBackwardKernel<<>>(res.getData(), logits.getData(), yTrue.getData(), logits.getDims()[0], logits.getSize()); cudaErrchk(cudaDeviceSynchronize()); } diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cu b/src/backend/training/loss_functions/cuda/loss_functions.cu index 84064f1..a3adc0c 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cu +++ b/src/backend/training/loss_functions/cuda/loss_functions.cu @@ -93,10 +93,10 @@ namespace { __forceinline__ __device__ T bceSimplified(T y, T logit) { constexpr T zero = 0; if constexpr (std::is_same_v) { - return cudaMax(logit, zero) - logit * y + __logf(1 + __expf(abs(logit))); + return cudaMax(logit, zero) - logit * y + __logf(1 + __expf(-abs(logit))); } else if constexpr (std::is_same_v) { - return cudaMax(logit, zero) - logit * y + log(1 + exp(abs(logit))); + return cudaMax(logit, zero) - logit * y + log(1 + exp(-abs(logit))); } else { static_assert(always_false, "Unexpected value for ftype"); From df332d049f7a77e95fbaedd56e1fe39e047c9fdb Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 31 May 2026 10:57:43 +0200 Subject: [PATCH 69/73] CUDA Python unit tests --- tests/CMakeLists.txt | 12 ++ tests/python/cuda/test_autograd_cuda.py | 111 +++++++++++++++++ tests/python/cuda/test_tensorops_cuda.py | 41 +++++++ tests/python/cuda/test_training_cuda.py | 150 +++++++++++++++++++++++ 4 files changed, 314 insertions(+) create mode 100644 tests/python/cuda/test_autograd_cuda.py create mode 100644 tests/python/cuda/test_tensorops_cuda.py create mode 100644 tests/python/cuda/test_training_cuda.py diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2774e33..08fb29c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -47,6 +47,18 @@ if(CMAKE_CUDA_COMPILER) ) gtest_discover_tests(unit_tests_cuda) + + find_package(Python3 COMPONENTS Interpreter) + if(Python3_FOUND) + add_test( + NAME python_tests_cuda + COMMAND ${Python3_EXECUTABLE} -m pytest + ${CMAKE_CURRENT_SOURCE_DIR}/python/cuda + ) + set_tests_properties(python_tests_cuda PROPERTIES + ENVIRONMENT "PYTHONPATH=${PYTHON_MODULE_DIR}:$ENV{PYTHONPATH}" + ) + endif() endif() find_package(Python3 COMPONENTS Interpreter) diff --git a/tests/python/cuda/test_autograd_cuda.py b/tests/python/cuda/test_autograd_cuda.py new file mode 100644 index 0000000..1fa6c51 --- /dev/null +++ b/tests/python/cuda/test_autograd_cuda.py @@ -0,0 +1,111 @@ +""" +Robert Baumgartner, r.baumgartner-1@tudelft.nl +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "python_lib")) + +from dl_lib import Tensor, Device +import pytest + + +class TestAutogradCuda: + def test_backward(self): + t = Tensor([2, 2], Device.CUDA, True) + + loss = (t * 2).sum() + loss.backward() + + assert t.grads is not None + + def test_nograd_throws(self): + t1 = Tensor([1], [3.0], Device.CUDA, False) + t2 = Tensor([1], [3.0], Device.CUDA, False) + + t3 = t1 * t2 + + assert not t3.requiresGrad + with pytest.raises(RuntimeError): + t3.backward() + + def test_add(self): + t1 = Tensor([1], [3.0], Device.CUDA, True) + t2 = Tensor([1], [2.0], Device.CUDA, True) + + t3 = t1 + t2 + loss = t3 * t3 + + loss.backward() + + assert t1.grads.getitem(0) == pytest.approx(10.0, abs=1e-4) + assert t2.grads.getitem(0) == pytest.approx(10.0, abs=1e-4) + + def test_scalar_mul(self): + t1 = Tensor([1], [2.0], Device.CUDA, True) + t2 = Tensor([1], [3.0], Device.CUDA, True) + + t3 = t1 * t2 + loss = t3 * t3 + + loss.backward() + + assert t1.grads.getitem(0) == pytest.approx(36.0, abs=1e-4) + assert t2.grads.getitem(0) == pytest.approx(24.0, abs=1e-4) + + def test_matmul(self): + t1 = Tensor([2, 3], [1, 2, 3, 4, 5, 6], Device.CUDA, True) + t2 = Tensor([3, 2], [1, 2, 3, 4, 5, 6], Device.CUDA, True) + + t3 = t1 @ t2 + loss = t3.sum() + + loss.backward() + + # dL/dt1 = dloss/dt3 @ t2^T = Ones({2, 2}) @ t2^T + assert t1.grads.getitem([0, 0]) == pytest.approx(3.0, abs=1e-4) + assert t1.grads.getitem([0, 1]) == pytest.approx(7.0, abs=1e-4) + assert t1.grads.getitem([0, 2]) == pytest.approx(11.0, abs=1e-4) + assert t1.grads.getitem([1, 0]) == pytest.approx(3.0, abs=1e-4) + assert t1.grads.getitem([1, 1]) == pytest.approx(7.0, abs=1e-4) + assert t1.grads.getitem([1, 2]) == pytest.approx(11.0, abs=1e-4) + + # dL/dt2 = t1^T @ dloss/dt3 = t1^T @ Ones({2, 2}) + assert t2.grads.getitem([0, 0]) == pytest.approx(5.0, abs=1e-4) + assert t2.grads.getitem([0, 1]) == pytest.approx(5.0, abs=1e-4) + assert t2.grads.getitem([1, 0]) == pytest.approx(7.0, abs=1e-4) + assert t2.grads.getitem([1, 1]) == pytest.approx(7.0, abs=1e-4) + assert t2.grads.getitem([2, 0]) == pytest.approx(9.0, abs=1e-4) + assert t2.grads.getitem([2, 1]) == pytest.approx(9.0, abs=1e-4) + + def test_chainrule(self): + x = Tensor([1], [2.0], Device.CUDA, True) + + y = x * x # y = x^2 + z = x + y # z = x^2 + x + loss = z * z # loss = (x^2 + x)^2 + + loss.backward() + + # dloss/dx = 2(x^2 + x)(2x + 1) = 60 at x=2 + assert x.grads.getitem(0) == pytest.approx(60.0, abs=1e-4) + + def test_multivariate_chainrule(self): + x = Tensor([2], [1.0, 2.0], Device.CUDA, True) + y = x * 3 + + loss = Tensor([1], [0.0], Device.CUDA, True) + for i in range(len(y)): + loss = loss + y[i] + loss.backward() + + assert x.grads.getitem(0) == pytest.approx(3.0, abs=1e-4) + assert x.grads.getitem(1) == pytest.approx(3.0, abs=1e-4) + + assert y.grads.getitem(0) == pytest.approx(1.0, abs=1e-4) + assert y.grads.getitem(1) == pytest.approx(1.0, abs=1e-4) + + +if __name__ == '__main__': + raise RuntimeError("Not a standalone script") diff --git a/tests/python/cuda/test_tensorops_cuda.py b/tests/python/cuda/test_tensorops_cuda.py new file mode 100644 index 0000000..44503da --- /dev/null +++ b/tests/python/cuda/test_tensorops_cuda.py @@ -0,0 +1,41 @@ +""" +Robert Baumgartner, r.baumgartner-1@tudelft.nl +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "python_lib")) + +from dl_lib import Tensor, Device + + +class TestTensorOpsCuda: + def test_ones(self): + t = Tensor.ones([2, 2], Device.CUDA) + assert t.dims == [2, 2] + assert t.device == Device.CUDA + + def test_ctor(self): + t = Tensor([2], [1.0, 2.0], Device.CUDA, False) + + assert t.getitem(0) == 1.0 + assert t.getitem(1) == 2.0 + + assert t.device == Device.CUDA + assert t.requiresGrad == False + + def test_multiplication(self): + a = Tensor.ones([2, 2], Device.CUDA) * 3 + b = Tensor.ones([2, 2], Device.CUDA) * 0.5 + c = a * b + + assert c.dims == [2, 2] + assert c.getitem([0, 0]) == 1.5 + assert c.getitem([0, 1]) == 1.5 + assert c.getitem([1, 0]) == 1.5 + assert c.getitem([1, 1]) == 1.5 + + +if __name__ == '__main__': + raise RuntimeError("Not a standalone script") diff --git a/tests/python/cuda/test_training_cuda.py b/tests/python/cuda/test_training_cuda.py new file mode 100644 index 0000000..044a479 --- /dev/null +++ b/tests/python/cuda/test_training_cuda.py @@ -0,0 +1,150 @@ +""" +Robert Baumgartner, r.baumgartner-1@tudelft.nl +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "python_lib")) + +from dl_lib import Tensor, Device +from dl_lib.nn import FfLayer, Sequential +from dl_lib.nn.activation import LeakyReLU +from dl_lib.train.loss import BceWithSigmoid, CrossEntropyWithSoftmax +from dl_lib.train.optim import SGD, RmsProp + +from dl_lib.sys import setSeed +import pytest + +setSeed(42) + + +def train(net, loss_fn, optim, x, y, epochs): + for _ in range(epochs): + ypred = net.forward(x) + loss = loss_fn(y, ypred) + + loss.backward() + optim.step() + optim.zeroGrad() + + return loss + + +def make_binary_net(): + net = Sequential() + net.append(FfLayer(2, 4, Device.CUDA)) + net.append(LeakyReLU(0.01)) + net.append(FfLayer(4, 1, Device.CUDA)) + return net + + +def make_multiclass_net(): + net = Sequential() + net.append(FfLayer(2, 8, Device.CUDA)) + net.append(LeakyReLU(0.01)) + net.append(FfLayer(8, 3, Device.CUDA)) + return net + + +def make_xor_data(): + x = Tensor([4, 2], [0.0, 0.0, + 0.0, 1.0, + 1.0, 0.0, + 1.0, 1.0], Device.CUDA, False) + y = Tensor([4, 1], [0.0, + 1.0, + 1.0, + 0.0], Device.CUDA, False) + return x, y + + +def make_multiclass_data(): + x = Tensor([6, 2], [1.0, 0.0, + 1.0, 0.1, + 0.0, 1.0, + 0.1, 1.0, + 0.5, 0.5, + 0.4, 0.6], Device.CUDA, False) + y = Tensor([6, 3], [1.0, 0.0, 0.0, + 1.0, 0.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 0.0, 1.0, + 0.0, 0.0, 1.0], Device.CUDA, False) + return x, y + + +class TestOverfitBinaryCuda: + def test_binary_sgd_overfits(self): + x, y = make_xor_data() + net = make_binary_net() + loss_fn = BceWithSigmoid() + optim = SGD(net.parameters(), 0.05) + + final_loss = train(net, loss_fn, optim, x, y, epochs=2000) + + assert final_loss.getitem(0) < 0.05, \ + f"SGD failed to overfit XOR on CUDA, loss={final_loss.getitem(0)}" + + def test_binary_rmsprop_overfits(self): + x, y = make_xor_data() + net = make_binary_net() + loss_fn = BceWithSigmoid() + optim = RmsProp(net.parameters(), 0.0001, 0.95) + + final_loss = train(net, loss_fn, optim, x, y, epochs=5000) + + assert final_loss.getitem(0) < 0.05, \ + f"RmsProp failed to overfit XOR on CUDA, loss={final_loss.getitem(0)}" + + def test_multiclass_rmsprop_overfits(self): + x, y = make_multiclass_data() + net = make_multiclass_net() + loss_fn = CrossEntropyWithSoftmax() + optim = RmsProp(net.parameters(), 0.0001, 0.95) + + final_loss = train(net, loss_fn, optim, x, y, epochs=2000) + + assert final_loss.getitem(0) < 0.05, \ + f"RmsProp failed to overfit multiclass on CUDA, loss={final_loss.getitem(0)}" + + def test_loss_decreases(self): + x, y = make_xor_data() + net = make_binary_net() + loss_fn = BceWithSigmoid() + optim = SGD(net.parameters(), 0.001) + + initial_pred = net.forward(x) + initial_loss = loss_fn(y, initial_pred).getitem(0) + train(net, loss_fn, optim, x, y, epochs=2000) + + final_pred = net.forward(x) + final_loss = loss_fn(y, final_pred).getitem(0) + + assert final_loss < initial_loss, \ + f"Loss did not decrease on CUDA: {initial_loss} -> {final_loss}" + + def test_gradients_are_zeroed_between_steps(self): + x, y = make_xor_data() + net = make_binary_net() + loss_fn = BceWithSigmoid() + optim = SGD(net.parameters(), 0.01) + + for _ in range(2): + pred = net.forward(x) + loss = loss_fn(y, pred) + + loss.backward() + optim.step() + optim.zeroGrad() + + for p in net.parameters(): + if p.grads is not None: + for i in range(p.grads.size): + assert p.grads.getitem(i) == pytest.approx(0.0, abs=1e-5), \ + f"Gradient not zeroed at index {i}" + + +if __name__ == '__main__': + raise RuntimeError("Not a standalone script") From e6c097841528149f39cc816a63852223a81553f9 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 31 May 2026 12:15:27 +0200 Subject: [PATCH 70/73] Fixing some subtle bugs, updated some kernels with type traits, cleaned code, added FfLayer large backward unit tests --- .../cuda/activation_nodes.cu | 35 +++--- .../loss_functions/cuda/loss_nodes.cu | 2 +- .../activation_functions/cuda/activations.cu | 13 +-- .../module/activation_functions/sigmoid.cpp | 2 +- src/backend/shared/cuda/common_kernels.cuh | 24 +++- .../training/loss_functions/bce_loss.cpp | 2 +- .../loss_functions/cuda/loss_functions.cu | 5 +- tests/backend/cuda/main.cu | 3 + tests/backend/cuda/test_module_cuda.cu | 107 +++++++++++++++++- 9 files changed, 157 insertions(+), 36 deletions(-) diff --git a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu index 966f173..05a6d8d 100644 --- a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu +++ b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu @@ -23,8 +23,10 @@ namespace { * @brief Relu backward kernel. */ __global__ void reluBackwardKernel(ftype* const res, const ftype* const upstreamGrad, const ftype* const parent, const tensorSize_t size) { - int gid = blockIdx.x * blockDim.x + threadIdx.x; - if(gid >= size) return; + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) { + return; + } res[gid] = parent[gid] > 0 ? upstreamGrad[gid] : 0; } @@ -33,8 +35,10 @@ namespace { * @brief Leaky relu backward kernel. */ __global__ void leakyReluBackwardKernel(ftype* const res, const ftype* const upstreamGrad, const ftype* const parent, const ftype eps, const tensorSize_t size) { - int gid = blockIdx.x * blockDim.x + threadIdx.x; - if(gid >= size) return; + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + if(gid >= size) { + return; + } res[gid] = parent[gid] > 0 ? upstreamGrad[gid] : eps * upstreamGrad[gid]; } @@ -43,20 +47,13 @@ namespace { * @brief Sigmoid backward kernel, optimized by using the forward sigmoid. */ __global__ void sigmoidBackwardKernel(ftype* const res, const ftype* const upstreamGrad, const ftype* const sigmoid, const tensorSize_t size) { - int gid = blockIdx.x * blockDim.x + threadIdx.x; - if(gid >= size) return; - - ftype si = sigmoid[gid]; - res[gid] = si * (1 - si) * upstreamGrad[gid]; - } - - __global__ void softmaxBackwardKernelOneWarp(ftype* const res, const ftype* const softmax, const ftype* const upstreamGrad, - const tensorSize_t stride, tensorSize_t size) { const int gid = blockIdx.x * blockDim.x + threadIdx.x; - if(gid >= size) + if(gid >= size) { return; + } - const int tid = threadIdx.x; + ftype si = sigmoid[gid]; + res[gid] = si * (1 - si) * upstreamGrad[gid]; } /** @@ -70,10 +67,10 @@ namespace { const int tid = threadIdx.x; const int withinStrideOffset = tid % threadsPerStride; - const bool isPadded = withinStrideOffset >= stride; // padded threads only exists to align warps with strides - const int strideOffset = (tid / stride) * stride; + const int gid = blockIdx.x * stridesWidthPerBlock + strideOffset + withinStrideOffset; + const bool isPadded = (withinStrideOffset >= stride) || (gid > size); // padded threads only exists to align warps with strides ftype yi = 0; const int smemOffset = strideOffset + withinStrideOffset; @@ -86,7 +83,7 @@ namespace { } __syncthreads(); - if(isPadded || gid > size) { + if(isPadded) { return; } @@ -122,7 +119,7 @@ namespace { ftype grad = 0; for(int offset = 0; offset < stride; offset += blockDim.x) { - // load int smem + // load into smem { const int j = offset + tid; if(j < stride) { diff --git a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu index 1dfa8d2..35d49d0 100644 --- a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu +++ b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu @@ -56,7 +56,7 @@ namespace { } const auto y = yTrue[gid]; - const auto s = cudaSigmoid(logits[gid]); + const auto s = cudaSigmoid(logits[gid]); const auto g = s - y; res[gid] = g / bSize; diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index f3d8f56..0548453 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -32,34 +32,33 @@ namespace { * @brief Kernel for forward ReLU function. */ __global__ void reluKernel(ftype* const res, const ftype* const input, const tensorSize_t size) { - int gid = blockIdx.x * blockDim.x + threadIdx.x; + const int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; - constexpr ftype zero = 0; - res[gid] = fmaxf(input[gid], zero); + res[gid] = cudaMax(input[gid], 0); } /** * @brief Kernel for forward Leaky-ReLU function. */ __global__ void leakyReluKernel(ftype* const res, const ftype* const input, const ftype eps, const tensorSize_t size) { - int gid = blockIdx.x * blockDim.x + threadIdx.x; + const int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; - res[gid] = fmaxf(input[gid], eps * input[gid]); // eps < 1 + res[gid] = cudaMax(input[gid], eps * input[gid]); // eps < 1 } /** * @brief Kernel for forward Sigmoid function. */ __global__ void sigmoidKernel(ftype* const res, const ftype* const input, const tensorSize_t size) { - int gid = blockIdx.x * blockDim.x + threadIdx.x; + const int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) return; - res[gid] = cudaSigmoid(input[gid]); + res[gid] = cudaSigmoid(input[gid]); } /** diff --git a/src/backend/module/activation_functions/sigmoid.cpp b/src/backend/module/activation_functions/sigmoid.cpp index 603ffb9..11be184 100644 --- a/src/backend/module/activation_functions/sigmoid.cpp +++ b/src/backend/module/activation_functions/sigmoid.cpp @@ -33,7 +33,7 @@ Tensor Sigmoid::operator()(const Tensor& t) const { case Device::CPU: { constexpr ftype one = 1.0; auto compute = [](ftype x){ - if(x>=0){ + if(x >= 0){ return one / (one + exp(-x)); } auto e = exp(x); diff --git a/src/backend/shared/cuda/common_kernels.cuh b/src/backend/shared/cuda/common_kernels.cuh index b7db3d8..46c4925 100644 --- a/src/backend/shared/cuda/common_kernels.cuh +++ b/src/backend/shared/cuda/common_kernels.cuh @@ -19,7 +19,7 @@ namespace cuda_impl { __device__ __forceinline__ ftype cudaMax(const ftype a, const ftype b) { if constexpr (std::is_same_v) { return fmaxf(a, b); - } else if(std::is_same_v) { + } else if constexpr (std::is_same_v) { return fmax(a, b); } else { @@ -30,10 +30,26 @@ namespace cuda_impl { /** * @brief Single sigmoid computation. */ + template __device__ __forceinline__ ftype cudaSigmoid(const ftype x) { - ftype z = expf(-fabsf(x)); - ftype s = 1.0f / (1.0f + z); - return (x >= 0.f) ? s : z * s; // x < 0 => e^x/(e^x+1) + if constexpr (std::is_same_v) { + ftype z = expf(-fabsf(x)); + ftype s = 1.0f / (1.0f + z); + return (x >= 0.f) ? s : z * s; // x < 0 => e^x/(e^x+1) + } + else if constexpr (std::is_same_v) { + ftype z = exp(-abs(x)); + ftype s = 1.0 / (1.0 + z); + return (x >= 0.0) ? s : z * s; // x < 0 => e^x/(e^x+1) + } + else { + static_assert(always_false, "Unexpected value for ftype encountered"); + } + + #ifndef NDEBUG + printf("This line should never be reached!"); + #endif + return 0; } /** diff --git a/src/backend/training/loss_functions/bce_loss.cpp b/src/backend/training/loss_functions/bce_loss.cpp index 200bd80..8176784 100644 --- a/src/backend/training/loss_functions/bce_loss.cpp +++ b/src/backend/training/loss_functions/bce_loss.cpp @@ -44,7 +44,7 @@ shared_ptr BceLoss::operator()(const shared_ptr y, const shared_ case Device::CPU: { auto bce = [](const ftype y, const ftype ypred){ - return y * log(std::max(ypred, EPS_BCE)) + (1 - y) * log(std::max(1-ypred, EPS_BCE)); + return y * log(std::max(ypred, EPS_BCE)) + (1 - y) * log(std::max(1 - ypred, EPS_BCE)); }; const auto nBatches = y->getDims()[0]; diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cu b/src/backend/training/loss_functions/cuda/loss_functions.cu index a3adc0c..810e416 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cu +++ b/src/backend/training/loss_functions/cuda/loss_functions.cu @@ -268,9 +268,12 @@ namespace { if constexpr (std::is_same_v) { logExpSum = __logf(expSum); } - else { + else if constexpr (std::is_same_v) { logExpSum = log(expSum); } + else { + static_assert(always_false, "Unexpected value for ftype encountered"); + } perSampleLoss[blockIdx.x] = maxV + logExpSum - smem[0]; } } diff --git a/tests/backend/cuda/main.cu b/tests/backend/cuda/main.cu index b8f92e9..8cca788 100644 --- a/tests/backend/cuda/main.cu +++ b/tests/backend/cuda/main.cu @@ -15,6 +15,8 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include +#include "system/sys_functions.h" + class CudaEnvironment : public ::testing::Environment { public: void SetUp() override { @@ -29,5 +31,6 @@ public: int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); ::testing::AddGlobalTestEnvironment(new CudaEnvironment()); + sys::setRandomSeed(42); return RUN_ALL_TESTS(); } \ No newline at end of file diff --git a/tests/backend/cuda/test_module_cuda.cu b/tests/backend/cuda/test_module_cuda.cu index fb78d18..22f035d 100644 --- a/tests/backend/cuda/test_module_cuda.cu +++ b/tests/backend/cuda/test_module_cuda.cu @@ -185,7 +185,7 @@ TEST(CudaActivationTest, SoftmaxForwardNumericalStability) { ASSERT_NEAR(rowsum, 1.0, 1e-5); } -TEST(CudaActivationTest, SoftmaxMediumLargeInput) { +TEST(CudaActivationTest, SoftmaxMediumLarge) { constexpr tensorDim_t testDim = 190; assert(testDim <= 256 && testDim > 64); // see the kernel call @@ -210,7 +210,7 @@ TEST(CudaActivationTest, SoftmaxMediumLargeInput) { } } -TEST(CudaActivationTest, SoftmaxLargeInput) { +TEST(CudaActivationTest, SoftmaxLarge) { constexpr tensorDim_t testDim = 1500; assert(testDim > 256); // see the kernel call @@ -364,4 +364,107 @@ TEST(CudaAutogradTest, FfLayerBackwardWithBias) { ASSERT_NE(bGrads, nullptr); ASSERT_NEAR((*bGrads)[0], 2.0, 1e-5); ASSERT_NEAR((*bGrads)[1], 2.0, 1e-5); +} + +TEST(CudaAutogradTest, FfLayerBackwardLarge) { + constexpr tensorDim_t inDim = 200; + constexpr tensorDim_t outDim = 10; + + auto xCpu = make_shared(TensorFunctions::Gaussian({30, inDim}, 1.0f, true)); + auto xGpu = make_shared(xCpu->createDeepCopy()); + xGpu->setDevice(Device::CUDA); + + auto layerCpu = module::FfLayer(inDim, outDim, Device::CPU, false, true); + auto layerGpu = module::FfLayer(inDim, outDim, Device::CUDA, false, true); + + // give both layers identical weights + auto wCpu = layerCpu.getWeights(); + auto wGpu = layerGpu.getWeights(); + for(tensorSize_t i = 0; i < wCpu->getSize(); i++) { + const ftype val = 1.0f / inDim; + wCpu->set(val, i); + wGpu->set(val, i); + } + + auto resCpu = layerCpu(xCpu); + auto resGpu = layerGpu(xGpu); + + auto upstreamCpu = make_shared(TensorFunctions::Ones(resCpu->getDims().toVector())); + auto upstreamGpu = make_shared(TensorFunctions::Ones(resCpu->getDims().toVector(), Device::CUDA)); + + resCpu->setGrads(upstreamCpu); + resGpu->setGrads(upstreamGpu); + + resCpu->backward(); + resGpu->backward(); + + auto xGradsCpu = xCpu->getGrads(); + auto xGradsGpu = xGpu->getGrads(); + for(int i = 0; i < xCpu->getSize(); i++) { + EXPECT_NEAR((*xGradsCpu)[i], (*xGradsGpu)[i], 1e-4); + } + + auto wGradsCpu = layerCpu.getWeights()->getGrads(); + auto wGradsGpu = layerGpu.getWeights()->getGrads(); + for(int i = 0; i < wCpu->getSize(); i++) { + EXPECT_NEAR((*wGradsCpu)[i], (*wGradsGpu)[i], 1e-4); + } +} + +TEST(CudaAutogradTest, FfLayerBackwardWithBiasLarge) { + constexpr tensorDim_t inDim = 200; + constexpr tensorDim_t outDim = 10; + + auto xCpu = make_shared(TensorFunctions::Gaussian({30, inDim}, 1.0f, true)); + auto xGpu = make_shared(xCpu->createDeepCopy()); + xGpu->setDevice(Device::CUDA); + + auto layerCpu = module::FfLayer(inDim, outDim, Device::CPU, true, true); + auto layerGpu = module::FfLayer(inDim, outDim, Device::CUDA, true, true); + + // give both layers identical weights and biases + auto wCpu = layerCpu.getWeights(); + auto wGpu = layerGpu.getWeights(); + for(tensorSize_t i = 0; i < wCpu->getSize(); i++) { + const ftype val = 1.0f / inDim; + wCpu->set(val, i); + wGpu->set(val, i); + } + + auto bCpu = layerCpu.getBias(); + auto bGpu = layerGpu.getBias(); + for(tensorSize_t i = 0; i < bCpu->getSize(); i++) { + bCpu->set(0.1f, i); + bGpu->set(0.1f, i); + } + + auto resCpu = layerCpu(xCpu); + auto resGpu = layerGpu(xGpu); + + auto upstreamCpu = make_shared(TensorFunctions::Ones(resCpu->getDims().toVector())); + auto upstreamGpu = make_shared(TensorFunctions::Ones(resCpu->getDims().toVector(), Device::CUDA)); + + resCpu->setGrads(upstreamCpu); + resGpu->setGrads(upstreamGpu); + + resCpu->backward(); + resGpu->backward(); + + auto xGradsCpu = xCpu->getGrads(); + auto xGradsGpu = xGpu->getGrads(); + for(int i = 0; i < xCpu->getSize(); i++) { + EXPECT_NEAR((*xGradsCpu)[i], (*xGradsGpu)[i], 1e-4); + } + + auto wGradsCpu = layerCpu.getWeights()->getGrads(); + auto wGradsGpu = layerGpu.getWeights()->getGrads(); + for(int i = 0; i < wCpu->getSize(); i++) { + EXPECT_NEAR((*wGradsCpu)[i], (*wGradsGpu)[i], 1e-4); + } + + auto bGradsCpu = layerCpu.getBias()->getGrads(); + auto bGradsGpu = layerGpu.getBias()->getGrads(); + for(int i = 0; i < bCpu->getSize(); i++) { + EXPECT_NEAR((*bGradsCpu)[i], (*bGradsGpu)[i], 1e-4); + } } \ No newline at end of file From 4995c969a00b24a1b7f628848626d9b040c27b09 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 31 May 2026 13:07:09 +0200 Subject: [PATCH 71/73] Fixed RMSProp and bug in RMSE backward, added unit tests for large loss inputs --- .../loss_functions/cuda/loss_nodes.cu | 2 +- .../loss_functions/rmse_node.cpp | 2 +- src/backend/shared/cuda/common_kernels.cuh | 21 +++-- .../training/optimizers/cuda/optimizers.cu | 15 ++-- src/backend/training/optimizers/rmsprop.cpp | 21 +++-- tests/backend/cuda/test_losses.cu | 89 +++++++++++++++++++ tests/backend/test_computational_graph.cpp | 4 +- 7 files changed, 129 insertions(+), 25 deletions(-) diff --git a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu index 35d49d0..e2b0508 100644 --- a/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu +++ b/src/backend/computational_graph/loss_functions/cuda/loss_nodes.cu @@ -104,7 +104,7 @@ namespace { const ftype yiHat = yPred[gid]; const ftype denom = rmse * bSize + EPS_RMSE; - const ftype g = (yiHat-yi) / denom; + const ftype g = (yiHat - yi) / denom; res[gid] = g; } diff --git a/src/backend/computational_graph/loss_functions/rmse_node.cpp b/src/backend/computational_graph/loss_functions/rmse_node.cpp index 9deeefc..b171ef7 100644 --- a/src/backend/computational_graph/loss_functions/rmse_node.cpp +++ b/src/backend/computational_graph/loss_functions/rmse_node.cpp @@ -33,7 +33,7 @@ vector< shared_ptr > RmseNode::backward(const Tensor& upstreamGrad) { case Device::CPU: { const ftype bSize = yPred->getDims()[0]; - for(tensorSize_t i = 0; i < yPred->getDims()[0]; i++){ + for(tensorSize_t i = 0; i < yPred->getSize(); i++){ auto yi = (*yTrue)[i]; auto yiHat = (*yPred)[i]; diff --git a/src/backend/shared/cuda/common_kernels.cuh b/src/backend/shared/cuda/common_kernels.cuh index 46c4925..1abf5e7 100644 --- a/src/backend/shared/cuda/common_kernels.cuh +++ b/src/backend/shared/cuda/common_kernels.cuh @@ -19,7 +19,8 @@ namespace cuda_impl { __device__ __forceinline__ ftype cudaMax(const ftype a, const ftype b) { if constexpr (std::is_same_v) { return fmaxf(a, b); - } else if constexpr (std::is_same_v) { + } + else if constexpr (std::is_same_v) { return fmax(a, b); } else { @@ -45,11 +46,19 @@ namespace cuda_impl { else { static_assert(always_false, "Unexpected value for ftype encountered"); } - - #ifndef NDEBUG - printf("This line should never be reached!"); - #endif - return 0; + } + + template + __device__ __forceinline__ ftype cudaSqrt(const ftype x) { + if constexpr (std::is_same_v) { + return sqrtf(x); + } + else if constexpr (std::is_same_v) { + return sqrt(x); + } + else { + static_assert(always_false, "Unexpected value for ftype encountered"); + } } /** diff --git a/src/backend/training/optimizers/cuda/optimizers.cu b/src/backend/training/optimizers/cuda/optimizers.cu index b6d82e5..5038e93 100644 --- a/src/backend/training/optimizers/cuda/optimizers.cu +++ b/src/backend/training/optimizers/cuda/optimizers.cu @@ -15,12 +15,15 @@ static_assert(false, "File should not be compiled without CUDA enabled"); #include "optimizers.cuh" #include "utility/cuda/cuda_common.cuh" +#include "shared/cuda/common_kernels.cuh" #include "utility/global_params.h" using namespace std; namespace { + using namespace cuda_impl; + __global__ void stepSgdKernel(ftype* const params, const ftype* const grads, const ftype lr, const tensorSize_t size) { const int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) { @@ -30,17 +33,17 @@ namespace { params[gid] = params[gid] - lr * grads[gid]; } - __global__ void stepRmsPropKernel(ftype* const tensor, ftype* const movingAvgs, const ftype* const grads, const ftype lr, const ftype decay, const tensorSize_t size) { + __global__ void stepRmsPropKernel(ftype* const tensor, ftype* const v, const ftype* const grads, const ftype lr, const ftype decay, const tensorSize_t size) { const int gid = blockIdx.x * blockDim.x + threadIdx.x; if(gid >= size) { return; } const ftype g = grads[gid]; - const ftype mavg = decay * movingAvgs[gid] + (1 - decay) * g * g; - movingAvgs[gid] = mavg; + const ftype mavg = decay * v[gid] + (1 - decay) * g * g; + v[gid] = mavg; - const ftype update = tensor[gid] - lr * g / (mavg + EPS_RMSPROP); + const ftype update = tensor[gid] - (lr * g / (cudaSqrt(mavg) + EPS_RMSPROP)); tensor[gid] = update; } } @@ -54,11 +57,11 @@ namespace cuda_impl { cudaErrchk(cudaDeviceSynchronize()); } - void rmspropStep(Tensor& param, Tensor& movingAvg, const Tensor& grad, ftype lr, ftype decay) { + void rmspropStep(Tensor& param, Tensor& movingAvg, const Tensor& grads, ftype lr, ftype decay) { constexpr int threadsPerBlock = 256; const int blocks = (param.getSize() + threadsPerBlock - 1) / threadsPerBlock; - stepRmsPropKernel<<>>(param.getData(), movingAvg.getData(), grad.getData(), lr, decay, param.getSize()); + stepRmsPropKernel<<>>(param.getData(), movingAvg.getData(), grads.getData(), lr, decay, param.getSize()); cudaErrchk(cudaDeviceSynchronize()); } } diff --git a/src/backend/training/optimizers/rmsprop.cpp b/src/backend/training/optimizers/rmsprop.cpp index ab66caa..2d3f792 100644 --- a/src/backend/training/optimizers/rmsprop.cpp +++ b/src/backend/training/optimizers/rmsprop.cpp @@ -22,11 +22,11 @@ using namespace train; void RmsPropOptimizer::step() { for(const auto& param: params){ - auto tPtr = param.get(); - const auto gPtr = tPtr->getGrads().get(); - - switch(tPtr->getDevice()) { + switch(param->getDevice()) { case Device::CPU: { + auto tPtr = param.get(); + const auto gPtr = tPtr->getGrads().get(); + auto vPtr = movingAvg[tPtr].get(); if(vPtr != nullptr) [[likely]] { @@ -37,7 +37,7 @@ void RmsPropOptimizer::step() { } } else [[unlikely]] { - movingAvg[tPtr] = make_unique(tPtr->getDims(), tPtr->getDevice(), false); + movingAvg[tPtr] = make_unique(tPtr->getDims(), Device::CPU, false); vPtr = movingAvg[tPtr].get(); for(tensorSize_t i = 0; i < tPtr->getSize(); i++) { auto g = (*gPtr)[i]; @@ -45,19 +45,22 @@ void RmsPropOptimizer::step() { } } - for(tensorSize_t i=0; igetSize(); i++) { - auto update = (*tPtr)[i] - lr * (*gPtr)[i] / ((*vPtr)[i] + EPS_RMSPROP); + for(tensorSize_t i = 0; i < tPtr->getSize(); i++) { + auto update = (*tPtr)[i] - (lr * (*gPtr)[i] / (sqrt((*vPtr)[i]) + EPS_RMSPROP)); tPtr->set(update, i); } break; } case Device::CUDA: #ifdef __CUDA + { + auto tPtr = param.get(); if(movingAvg[tPtr] == nullptr) { - movingAvg[tPtr] = make_unique(tPtr->getDims(), tPtr->getDevice(), false); + movingAvg[tPtr] = make_unique(tPtr->getDims(), Device::CUDA, false); movingAvg[tPtr]->reset(0); } - cuda_impl::rmspropStep(*tPtr, *movingAvg[tPtr], *gPtr, lr, decay); + cuda_impl::rmspropStep(*param, *movingAvg[tPtr], *(param->getGrads()), lr, decay); + } #else __throw_invalid_argument("Attempted to use CUDA tensor"); #endif diff --git a/tests/backend/cuda/test_losses.cu b/tests/backend/cuda/test_losses.cu index a428a91..58d9a8a 100644 --- a/tests/backend/cuda/test_losses.cu +++ b/tests/backend/cuda/test_losses.cu @@ -120,6 +120,37 @@ TEST(CudaLossTest, CrossEntropyBackward) { ASSERT_NEAR((*grads)[5], 0.0f, 1e-4); } +TEST(CudaLossTest, CrossEntropyBackwardLarge) { + constexpr tensorDim_t nSamples = 500; + constexpr tensorDim_t nClasses = 200; + + auto yCpu = make_shared(TensorFunctions::Ones({nSamples, nClasses}) * 0.5f); + auto yGpu = make_shared(yCpu->createDeepCopy()); + yGpu->setDevice(Device::CUDA); + + auto ypredCpu = make_shared(TensorFunctions::Ones({nSamples, nClasses}, Device::CPU, true) * 0.7f); + auto ypredGpu = make_shared(ypredCpu->createDeepCopy()); + ypredGpu->setDevice(Device::CUDA); + + CrossEntropyLoss loss; + auto resCpu = loss(yCpu, ypredCpu); + auto resGpu = loss(yGpu, ypredGpu); + + resCpu->backward(); + resGpu->backward(); + + auto gradsCpu = ypredCpu->getGrads(); + auto gradsGpu = ypredGpu->getGrads(); + gradsGpu->setDevice(Device::CPU); + + for(int i = 0; i < ypredCpu->getSize(); i++) { + EXPECT_NEAR((*gradsCpu)[i], (*gradsGpu)[i], 1e-4) + << "Failed at index " << i + << "- GradsCpu[i]: " << (*gradsCpu)[i] + << "- GradsGpu[i]: " << (*gradsGpu)[i]; + } +} + TEST(CudaLossTest, BceForward) { auto y = TensorFunctions::makeSharedTensor( {4, 1}, {0.0, 1.0, 1.0, 0.0}, Device::CUDA, false); @@ -218,6 +249,36 @@ TEST(CudaLossTest, BceBackward) { ASSERT_NEAR((*grads)[1], 0.7143f, 1e-4); } +TEST(CudaLossTest, BceBackwardLarge) { + constexpr tensorDim_t nSamples = 10000; + + auto yCpu = make_shared(TensorFunctions::Ones({nSamples, 1}) * 0.5f); + auto yGpu = make_shared(yCpu->createDeepCopy()); + yGpu->setDevice(Device::CUDA); + + auto ypredCpu = make_shared(TensorFunctions::Ones({nSamples, 1}, Device::CPU, true) * 0.7f); + auto ypredGpu = make_shared(ypredCpu->createDeepCopy()); + ypredGpu->setDevice(Device::CUDA); + + BceLoss loss; + auto resCpu = loss(yCpu, ypredCpu); + auto resGpu = loss(yGpu, ypredGpu); + + resCpu->backward(); + resGpu->backward(); + + auto gradsCpu = ypredCpu->getGrads(); + auto gradsGpu = ypredGpu->getGrads(); + gradsGpu->setDevice(Device::CPU); + + for(int i = 0; i < ypredCpu->getSize(); i++) { + EXPECT_NEAR((*gradsCpu)[i], (*gradsGpu)[i], 1e-4) + << "Failed at index " << i + << "- GradsCpu[i]: " << (*gradsCpu)[i] + << "- GradsGpu[i]: " << (*gradsGpu)[i]; + } +} + TEST(CudaLossTest, RmseForward) { auto y = TensorFunctions::makeSharedTensor( {3}, {1.0, 2.0, 3.0}, Device::CUDA, false); @@ -274,3 +335,31 @@ TEST(CudaLossTest, RmseBackward) { ASSERT_NEAR((*grads)[0], -0.5f, 1e-4); ASSERT_NEAR((*grads)[1], 0.5f, 1e-4); } + +TEST(CudaLossTest, RmseBackwardLarge) { + auto yCpu = make_shared(TensorFunctions::Gaussian({500, 500}, 1.0f)); + auto yGpu = make_shared(yCpu->createDeepCopy()); + yGpu->setDevice(Device::CUDA); + + auto ypredCpu = make_shared(TensorFunctions::Gaussian({500, 500}, 1.0f, true)); + auto ypredGpu = make_shared(ypredCpu->createDeepCopy()); + ypredGpu->setDevice(Device::CUDA); + + RmseLoss loss; + auto resCpu = loss(yCpu, ypredCpu); + auto resGpu = loss(yGpu, ypredGpu); + + resCpu->backward(); + resGpu->backward(); + + auto gradsCpu = ypredCpu->getGrads(); + auto gradsGpu = ypredGpu->getGrads(); + gradsGpu->setDevice(Device::CPU); + + for(int i = 0; i < ypredCpu->getSize(); i++) { + EXPECT_NEAR((*gradsCpu)[i], (*gradsGpu)[i], 1e-4) + << "Failed at index " << i + << "- GradsCpu[i]: " << (*gradsCpu)[i] + << "- GradsGpu[i]: " << (*gradsGpu)[i]; + } +} diff --git a/tests/backend/test_computational_graph.cpp b/tests/backend/test_computational_graph.cpp index d3993d9..489b944 100644 --- a/tests/backend/test_computational_graph.cpp +++ b/tests/backend/test_computational_graph.cpp @@ -159,7 +159,7 @@ TEST(OverfitTest, CrossEntropyRMSPropOverfitsSmallDataset) { auto net = makeMulticlassNet(); auto loss = make_shared(); auto optim = make_shared( - net->parameters(), /*lr=*/0.0001, /*decay=*/0.95); + net->parameters(), /*lr=*/0.001, /*decay=*/0.95); auto trainLoop = train::BaseTrainLoop( net, loss, optim, /*epochs=*/2000, /*bsize=*/6); @@ -196,7 +196,7 @@ TEST(OverfitTest, CrossEntropyRMSPropOverfitsSmallDataset_OptimizedLoss) { auto net = makeMulticlassNet2(); auto loss = make_shared(); auto optim = make_shared( - net->parameters(), /*lr=*/0.0001, /*decay=*/0.95); + net->parameters(), /*lr=*/0.001, /*decay=*/0.95); auto trainLoop = train::BaseTrainLoop( net, loss, optim, /*epochs=*/2000, /*bsize=*/6); From 10877a3f517f8b9206619a2a360e50856dde6047 Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 31 May 2026 13:38:59 +0200 Subject: [PATCH 72/73] Fix bug in CUDA softmax backward, add unit test to capture gap, fix overfitting unit tests for CUDA --- .../cuda/activation_nodes.cu | 8 ++-- .../activation_functions/cuda/activations.cu | 4 +- .../backend/cuda/test_computational_graph.cu | 5 +- tests/backend/cuda/test_losses.cu | 12 ++--- tests/backend/cuda/test_module_cuda.cu | 47 +++++++++++-------- 5 files changed, 43 insertions(+), 33 deletions(-) diff --git a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu index 05a6d8d..e19165c 100644 --- a/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu +++ b/src/backend/computational_graph/activation_functions/cuda/activation_nodes.cu @@ -67,10 +67,10 @@ namespace { const int tid = threadIdx.x; const int withinStrideOffset = tid % threadsPerStride; - const int strideOffset = (tid / stride) * stride; + const int strideOffset = (tid / threadsPerStride) * stride; const int gid = blockIdx.x * stridesWidthPerBlock + strideOffset + withinStrideOffset; - const bool isPadded = (withinStrideOffset >= stride) || (gid > size); // padded threads only exists to align warps with strides + const bool isPadded = (withinStrideOffset >= stride) || (gid >= size); // padded threads only exists to align warps with strides ftype yi = 0; const int smemOffset = strideOffset + withinStrideOffset; @@ -79,7 +79,7 @@ namespace { if(!isPadded) { yi = softmax[gid]; smem[smemOffset] = yi; - smem[smemOffset + stride] = upstreamGrad[gid]; + smem[smemOffset + stridesWidthPerBlock] = upstreamGrad[gid]; } __syncthreads(); @@ -91,7 +91,7 @@ namespace { for(int j = 0; j < stride; j++) { // warp alignment -> smem-reads are broadcasted per warp -> no bank conflicts ftype yj = smem[strideOffset + j]; - ftype gj = smem[strideOffset + j + stride]; + ftype gj = smem[strideOffset + j + stridesWidthPerBlock]; auto jacobian = (withinStrideOffset == j) ? yi * (1 - yj) : -yi * yj; grad += gj * jacobian; diff --git a/src/backend/module/activation_functions/cuda/activations.cu b/src/backend/module/activation_functions/cuda/activations.cu index 0548453..889350c 100644 --- a/src/backend/module/activation_functions/cuda/activations.cu +++ b/src/backend/module/activation_functions/cuda/activations.cu @@ -65,7 +65,7 @@ namespace { * @brief Reduction kernel that computes the sum over an array within the size of 2 * warpsize at maximum. */ template - __forceinline__ __device__ void warpSumReduce(volatile ftype* const input, const tensorSize_t stride, const int offset) { + __forceinline__ __device__ void softmaxWarpSumReduce(volatile ftype* const input, const tensorSize_t stride, const int offset) { // TODO: warp shuffle for newer architectures if(maxoffset == 32) { if(offset + 32 < stride) input[offset] += input[offset + 32]; @@ -111,7 +111,7 @@ namespace { volatile ftype* const start = smem + (tid / stride) * stride; const int offset = gid % stride; - warpSumReduce(start, stride, offset); + softmaxWarpSumReduce(start, stride, offset); res[gid] = expVal / start[0]; } diff --git a/tests/backend/cuda/test_computational_graph.cu b/tests/backend/cuda/test_computational_graph.cu index 20edd45..31950da 100644 --- a/tests/backend/cuda/test_computational_graph.cu +++ b/tests/backend/cuda/test_computational_graph.cu @@ -156,7 +156,7 @@ TEST(CudaOverfitTest, CrossEntropyRMSPropOverfitsSmallDataset) { auto net = makeMulticlassNet(Device::CUDA); auto loss = make_shared(); auto optim = make_shared( - net->parameters(), /*lr=*/0.0001, /*decay=*/0.95); + net->parameters(), /*lr=*/0.001, /*decay=*/0.95); auto trainLoop = train::BaseTrainLoop( net, loss, optim, /*epochs=*/2000, /*bsize=*/6); @@ -166,6 +166,7 @@ TEST(CudaOverfitTest, CrossEntropyRMSPropOverfitsSmallDataset) { auto pred = (*net)(x); auto finalLoss = (*loss)(y, pred); + EXPECT_LT((*finalLoss)[0], 0.05f) << "Network failed to overfit multiclass dataset\n" << "Final loss: " << *finalLoss; @@ -191,7 +192,7 @@ TEST(CudaOverfitTest, CrossEntropyRMSPropOverfitsSmallDataset_OptimizedLoss) { auto net = makeMulticlassNet2(Device::CUDA); auto loss = make_shared(); auto optim = make_shared( - net->parameters(), /*lr=*/0.0001, /*decay=*/0.95); + net->parameters(), /*lr=*/0.001, /*decay=*/0.95); auto trainLoop = train::BaseTrainLoop( net, loss, optim, /*epochs=*/2000, /*bsize=*/6); diff --git a/tests/backend/cuda/test_losses.cu b/tests/backend/cuda/test_losses.cu index 58d9a8a..5acc3f2 100644 --- a/tests/backend/cuda/test_losses.cu +++ b/tests/backend/cuda/test_losses.cu @@ -146,8 +146,8 @@ TEST(CudaLossTest, CrossEntropyBackwardLarge) { for(int i = 0; i < ypredCpu->getSize(); i++) { EXPECT_NEAR((*gradsCpu)[i], (*gradsGpu)[i], 1e-4) << "Failed at index " << i - << "- GradsCpu[i]: " << (*gradsCpu)[i] - << "- GradsGpu[i]: " << (*gradsGpu)[i]; + << " - GradsCpu[i]: " << (*gradsCpu)[i] + << " - GradsGpu[i]: " << (*gradsGpu)[i]; } } @@ -274,8 +274,8 @@ TEST(CudaLossTest, BceBackwardLarge) { for(int i = 0; i < ypredCpu->getSize(); i++) { EXPECT_NEAR((*gradsCpu)[i], (*gradsGpu)[i], 1e-4) << "Failed at index " << i - << "- GradsCpu[i]: " << (*gradsCpu)[i] - << "- GradsGpu[i]: " << (*gradsGpu)[i]; + << " - GradsCpu[i]: " << (*gradsCpu)[i] + << " - GradsGpu[i]: " << (*gradsGpu)[i]; } } @@ -359,7 +359,7 @@ TEST(CudaLossTest, RmseBackwardLarge) { for(int i = 0; i < ypredCpu->getSize(); i++) { EXPECT_NEAR((*gradsCpu)[i], (*gradsGpu)[i], 1e-4) << "Failed at index " << i - << "- GradsCpu[i]: " << (*gradsCpu)[i] - << "- GradsGpu[i]: " << (*gradsGpu)[i]; + << " - GradsCpu[i]: " << (*gradsCpu)[i] + << " - GradsGpu[i]: " << (*gradsGpu)[i]; } } diff --git a/tests/backend/cuda/test_module_cuda.cu b/tests/backend/cuda/test_module_cuda.cu index 22f035d..7626e21 100644 --- a/tests/backend/cuda/test_module_cuda.cu +++ b/tests/backend/cuda/test_module_cuda.cu @@ -251,6 +251,30 @@ TEST(CudaAutogradTest, SoftmaxBackward) { ASSERT_NEAR((*grads)[2], -0.0599, 1e-4); } +TEST(CudaAutogradTest, SoftmaxBackwardBatched) { + // 4 samples x 3 classes — exercises the one-block path with multiple strides + auto tCpu = make_shared(TensorFunctions::Gaussian({4, 3}, 1.0f, true)); + auto tGpu = make_shared(tCpu->createDeepCopy()); + tGpu->setDevice(Device::CUDA); + + module::Softmax sm; + auto resCpu = sm(tCpu); + auto resGpu = sm(tGpu); + + resCpu->backward(); + resGpu->backward(); + + auto gradsCpu = tCpu->getGrads(); + auto gradsGpu = tGpu->getGrads(); + + for(int i = 0; i < tCpu->getSize(); i++) { + EXPECT_NEAR((*gradsCpu)[i], (*gradsGpu)[i], 1e-4) + << "Failed at index " << i + << " - GradsCpu[i]: " << (*gradsCpu)[i] + << " - GradsGpu[i]: " << (*gradsGpu)[i]; + } +} + TEST(CudaAutogradTest, SoftmaxBackwardLarge) { constexpr tensorDim_t testDim = 1300; auto tCpu = make_shared(TensorFunctions::Gaussian({2, 2, testDim}, 2.0f, true)); @@ -261,12 +285,6 @@ TEST(CudaAutogradTest, SoftmaxBackwardLarge) { auto resPtrCpu = sm(tCpu); auto resPtrGpu = sm(tGpu); - auto upstreamGradCpu = make_shared(TensorFunctions::Ones(tCpu->getDims().toVector())); - auto upstreamGradGpu = make_shared(TensorFunctions::Ones(tCpu->getDims().toVector(), Device::CUDA)); - - resPtrCpu->setGrads(upstreamGradCpu); - resPtrGpu->setGrads(upstreamGradGpu); - resPtrCpu->backward(); resPtrGpu->backward(); @@ -274,7 +292,10 @@ TEST(CudaAutogradTest, SoftmaxBackwardLarge) { auto gradsGpu = tGpu->getGrads(); for(int i = 0; i < tCpu->getSize(); i++) { - EXPECT_NEAR((*gradsCpu)[i], (*gradsGpu)[i], 1e-4); + EXPECT_NEAR((*gradsCpu)[i], (*gradsGpu)[i], 1e-4) + << "Failed at index " << i + << " - GradsCpu[i]: " << (*gradsCpu)[i] + << " - GradsGpu[i]: " << (*gradsGpu)[i]; } } @@ -389,12 +410,6 @@ TEST(CudaAutogradTest, FfLayerBackwardLarge) { auto resCpu = layerCpu(xCpu); auto resGpu = layerGpu(xGpu); - auto upstreamCpu = make_shared(TensorFunctions::Ones(resCpu->getDims().toVector())); - auto upstreamGpu = make_shared(TensorFunctions::Ones(resCpu->getDims().toVector(), Device::CUDA)); - - resCpu->setGrads(upstreamCpu); - resGpu->setGrads(upstreamGpu); - resCpu->backward(); resGpu->backward(); @@ -441,12 +456,6 @@ TEST(CudaAutogradTest, FfLayerBackwardWithBiasLarge) { auto resCpu = layerCpu(xCpu); auto resGpu = layerGpu(xGpu); - auto upstreamCpu = make_shared(TensorFunctions::Ones(resCpu->getDims().toVector())); - auto upstreamGpu = make_shared(TensorFunctions::Ones(resCpu->getDims().toVector(), Device::CUDA)); - - resCpu->setGrads(upstreamCpu); - resGpu->setGrads(upstreamGpu); - resCpu->backward(); resGpu->backward(); From 895500aed75b4756eb094992d03bed4ddf0dbd9d Mon Sep 17 00:00:00 2001 From: Robert Baumgartner Date: Sun, 31 May 2026 14:01:13 +0200 Subject: [PATCH 73/73] Fix bug and python unit tests --- .../training/loss_functions/cuda/loss_functions.cu | 4 ++-- src/python/py_core/py_core.cpp | 6 +++--- tests/python/cuda/test_training_cuda.py | 11 +++++------ tests/python/test_training.py | 10 +++++----- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/backend/training/loss_functions/cuda/loss_functions.cu b/src/backend/training/loss_functions/cuda/loss_functions.cu index 810e416..717e5b6 100644 --- a/src/backend/training/loss_functions/cuda/loss_functions.cu +++ b/src/backend/training/loss_functions/cuda/loss_functions.cu @@ -406,8 +406,8 @@ namespace cuda_impl { cudaErrchk(cudaDeviceSynchronize()); } - // loss = -loss / nBatches - divideScalarKernel<<<1, 1>>>(res.getData(), -1 * static_cast(y.getDims()[0])); + // loss = loss / nBatches + divideScalarKernel<<<1, 1>>>(res.getData(), static_cast(y.getDims()[0])); cudaErrchk(cudaDeviceSynchronize()); } diff --git a/src/python/py_core/py_core.cpp b/src/python/py_core/py_core.cpp index 9fbf212..8c1bec8 100644 --- a/src/python/py_core/py_core.cpp +++ b/src/python/py_core/py_core.cpp @@ -164,20 +164,20 @@ BOOST_PYTHON_MODULE(_core) // static creation methods .def("ones", WRAP_FREE_FUNC_1(Py_DataModeling::Ones0, std::vector)) - .def("ones", WRAP_FREE_FUNC_2(Py_DataModeling::Ones1, std::vector, Device)) .def("ones", WRAP_FREE_FUNC_2(Py_DataModeling::Ones2, std::vector, const bool)) + .def("ones", WRAP_FREE_FUNC_2(Py_DataModeling::Ones1, std::vector, Device)) .def("ones", WRAP_FREE_FUNC_3(Py_DataModeling::Ones3, std::vector, Device, const bool)) .staticmethod("ones") .def("zeros", WRAP_FREE_FUNC_1(Py_DataModeling::Zeros0, std::vector)) - .def("zeros", WRAP_FREE_FUNC_2(Py_DataModeling::Zeros1, std::vector, Device)) .def("zeros", WRAP_FREE_FUNC_2(Py_DataModeling::Zeros2, std::vector, const bool)) + .def("zeros", WRAP_FREE_FUNC_2(Py_DataModeling::Zeros1, std::vector, Device)) .def("zeros", WRAP_FREE_FUNC_3(Py_DataModeling::Zeros3, std::vector, Device, const bool)) .staticmethod("zeros") .def("gauss", WRAP_FREE_FUNC_2(Py_DataModeling::Gaussian0, std::vector, ftype)) - .def("gauss", WRAP_FREE_FUNC_3(Py_DataModeling::Gaussian1, std::vector, ftype, Device)) .def("gauss", WRAP_FREE_FUNC_3(Py_DataModeling::Gaussian2, std::vector, ftype, const bool)) + .def("gauss", WRAP_FREE_FUNC_3(Py_DataModeling::Gaussian1, std::vector, ftype, Device)) .def("gauss", WRAP_FREE_FUNC_8(Py_DataModeling::Gaussian3, std::vector, ftype, Device, const bool)) .staticmethod("gauss") diff --git a/tests/python/cuda/test_training_cuda.py b/tests/python/cuda/test_training_cuda.py index 044a479..bb48d92 100644 --- a/tests/python/cuda/test_training_cuda.py +++ b/tests/python/cuda/test_training_cuda.py @@ -18,7 +18,6 @@ setSeed(42) - def train(net, loss_fn, optim, x, y, epochs): for _ in range(epochs): ypred = net.forward(x) @@ -82,7 +81,7 @@ def test_binary_sgd_overfits(self): loss_fn = BceWithSigmoid() optim = SGD(net.parameters(), 0.05) - final_loss = train(net, loss_fn, optim, x, y, epochs=2000) + final_loss = train(net, loss_fn, optim, x, y, epochs=3000) assert final_loss.getitem(0) < 0.05, \ f"SGD failed to overfit XOR on CUDA, loss={final_loss.getitem(0)}" @@ -91,9 +90,9 @@ def test_binary_rmsprop_overfits(self): x, y = make_xor_data() net = make_binary_net() loss_fn = BceWithSigmoid() - optim = RmsProp(net.parameters(), 0.0001, 0.95) + optim = RmsProp(net.parameters(), 0.001, 0.95) - final_loss = train(net, loss_fn, optim, x, y, epochs=5000) + final_loss = train(net, loss_fn, optim, x, y, epochs=3000) assert final_loss.getitem(0) < 0.05, \ f"RmsProp failed to overfit XOR on CUDA, loss={final_loss.getitem(0)}" @@ -102,9 +101,9 @@ def test_multiclass_rmsprop_overfits(self): x, y = make_multiclass_data() net = make_multiclass_net() loss_fn = CrossEntropyWithSoftmax() - optim = RmsProp(net.parameters(), 0.0001, 0.95) + optim = RmsProp(net.parameters(), 0.001, 0.95) - final_loss = train(net, loss_fn, optim, x, y, epochs=2000) + final_loss = train(net, loss_fn, optim, x, y, epochs=3000) assert final_loss.getitem(0) < 0.05, \ f"RmsProp failed to overfit multiclass on CUDA, loss={final_loss.getitem(0)}" diff --git a/tests/python/test_training.py b/tests/python/test_training.py index 1cef568..bf4bb97 100644 --- a/tests/python/test_training.py +++ b/tests/python/test_training.py @@ -77,7 +77,7 @@ def test_binary_sgd_overfits(self): loss_fn = BceWithSigmoid() optim = SGD(net.parameters(), 0.05) - final_loss = train(net, loss_fn, optim, x, y, epochs=2000) + final_loss = train(net, loss_fn, optim, x, y, epochs=3000) assert final_loss.getitem(0) < 0.05, \ f"SGD failed to overfit XOR, loss={final_loss.getitem(0)}" @@ -86,9 +86,9 @@ def test_binary_rmsprop_overfits(self): x, y = make_xor_data() net = make_binary_net() loss_fn = BceWithSigmoid() - optim = RmsProp(net.parameters(), 0.0001, 0.95) + optim = RmsProp(net.parameters(), 0.001, 0.95) - final_loss = train(net, loss_fn, optim, x, y, epochs=5000) + final_loss = train(net, loss_fn, optim, x, y, epochs=3000) assert final_loss.getitem(0) < 0.05, \ f"RmsProp failed to overfit XOR, loss={final_loss.getitem(0)}" @@ -97,9 +97,9 @@ def test_multiclass_rmsprop_overfits(self): x, y = make_multiclass_data() net = make_multiclass_net() loss_fn = CrossEntropyWithSoftmax() - optim = RmsProp(net.parameters(), 0.0001, 0.95) + optim = RmsProp(net.parameters(), 0.001, 0.95) - final_loss = train(net, loss_fn, optim, x, y, epochs=2000) + final_loss = train(net, loss_fn, optim, x, y, epochs=3000) assert final_loss.getitem(0) < 0.05, \ f"RmsProp failed to overfit multiclass, loss={final_loss.getitem(0)}"