diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e5d863..8aa714e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,4 +30,4 @@ target_link_libraries(pearl-example pearl m) add_test(NAME test COMMAND pearl-test) set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) -add_compile_options(-O2) +add_compile_options(-O3 -Wall -Wextra -pedantic) diff --git a/src/pearl_layer.c b/src/pearl_layer.c index 33f2ab2..8b2b476 100644 --- a/src/pearl_layer.c +++ b/src/pearl_layer.c @@ -1,5 +1,8 @@ #include #include +#include // For rand(), RAND_MAX +#include // For time() in srand comment +#include // For assert() pearl_layer *pearl_layer_create() { @@ -91,6 +94,59 @@ void pearl_layer_destroy(pearl_layer **layer) } } +void pearl_layer_backward_dropout(pearl_layer **child_layer, pearl_layer **parent_layer) +{ + pearl_layer_data_dropout *dropout_data = (pearl_layer_data_dropout *)(*child_layer)->layer_data; + pearl_tensor *current_layer_da = (*child_layer)->da; + + assert(dropout_data != NULL); + assert(dropout_data->weights != NULL); + assert(current_layer_da != NULL); + + if ((*parent_layer)->da == NULL) { + if (current_layer_da->dimension == 1) { + (*parent_layer)->da = pearl_tensor_create(1, current_layer_da->size[0]); + } else if (current_layer_da->dimension == 2) { + (*parent_layer)->da = pearl_tensor_create(2, current_layer_da->size[0], current_layer_da->size[1]); + } + } else { + assert((*parent_layer)->da->dimension == current_layer_da->dimension); + assert((*parent_layer)->da->size[0] == current_layer_da->size[0]); + if (current_layer_da->dimension == 2) { + assert((*parent_layer)->da->size[1] == current_layer_da->size[1]); + } + } + + assert(dropout_data->weights->dimension == current_layer_da->dimension); + assert(dropout_data->weights->size[0] == current_layer_da->size[0]); + if (current_layer_da->dimension == 2) { + assert(dropout_data->weights->size[1] == current_layer_da->size[1]); + } + + float rate = dropout_data->rate; + float scale; + if (rate >= 1.0f - 1e-9f) { // Treat rate as 1.0 if it's very close + scale = 0.0f; + } else if (rate < 0.0f) { // Invalid rate + // Handle error or assert, for now, let's assume valid range or clamp + assert(rate >= 0.0f && rate < 1.0f); // This will fail if rate < 0 + scale = 1.0f; // Default or error scale + } + else { + scale = 1.0f / (1.0f - rate); + } + + + unsigned int total_elements = 1; + for (unsigned int i = 0; i < current_layer_da->dimension; i++) { + total_elements *= current_layer_da->size[i]; + } + + for (unsigned int i = 0; i < total_elements; i++) { + (*parent_layer)->da->data[i] = current_layer_da->data[i] * dropout_data->weights->data[i] * scale; + } +} + PEARL_API void pearl_layer_add_child(pearl_layer **parent, pearl_layer **child) { if (*parent != NULL) { @@ -170,12 +226,86 @@ pearl_layer *pearl_layer_create_dropout(unsigned int num_neurons) layer->num_neurons = num_neurons; pearl_layer_data_dropout *data = calloc(1, sizeof(pearl_layer_data_dropout)); data->rate = 0.5f; - data->weights = pearl_tensor_create(1, layer->num_neurons); + data->weights = NULL; layer->layer_data = data; return layer; } -void pearl_layer_forward(pearl_layer **parent_layer, pearl_layer **child_layer) +void pearl_layer_forward_dropout(pearl_layer **parent_layer, pearl_layer **child_layer, bool is_training) // Add is_training +{ + pearl_tensor *input_a = (*parent_layer)->a; + pearl_layer_data_dropout *dropout_data = (pearl_layer_data_dropout *)(*child_layer)->layer_data; + + assert(input_a != NULL); + assert(dropout_data != NULL); + + // Manage child's activation tensor (*child_layer)->a + if ((*child_layer)->a == NULL) { + if (input_a->dimension == 1) { + (*child_layer)->a = pearl_tensor_create(1, input_a->size[0]); + } else if (input_a->dimension == 2) { + (*child_layer)->a = pearl_tensor_create(2, input_a->size[0], input_a->size[1]); + } + } else { + assert((*child_layer)->a->dimension == input_a->dimension); + assert((*child_layer)->a->size[0] == input_a->size[0]); + if (input_a->dimension == 2) { + assert((*child_layer)->a->size[1] == input_a->size[1]); + } + } + + // Manage dropout mask tensor dropout_data->weights + if (dropout_data->weights == NULL) { + if (input_a->dimension == 1) { + dropout_data->weights = pearl_tensor_create(1, input_a->size[0]); + } else if (input_a->dimension == 2) { + dropout_data->weights = pearl_tensor_create(2, input_a->size[0], input_a->size[1]); + } + } else { + assert(dropout_data->weights->dimension == input_a->dimension); + assert(dropout_data->weights->size[0] == input_a->size[0]); + if (input_a->dimension == 2) { + assert(dropout_data->weights->size[1] == input_a->size[1]); + } + } + + float rate = dropout_data->rate; + // Assert 0.0 <= rate < 1.0 for typical operation. + // If rate is exactly 1.0, all neurons are dropped. If 0.0, no neurons are dropped. + assert(rate >= 0.0f && rate < 1.0f); + + + float scale = 1.0f / (1.0f - rate); + if (rate >= 1.0f - 1e-9f) { // Handle rate very close or equal to 1.0 + scale = 0.0f; + } + + + unsigned int total_elements = 1; + for (unsigned int i = 0; i < input_a->dimension; i++) { + total_elements *= input_a->size[i]; + } + + // Note: For proper operation, ensure that rand() is seeded once at program start, e.g., with srand(time(NULL)). + if (is_training) { + for (unsigned int i = 0; i < total_elements; i++) { + float random_val = (float)rand() / RAND_MAX; + if (random_val < rate) { + dropout_data->weights->data[i] = 0.0f; + } else { + dropout_data->weights->data[i] = 1.0f; + } + (*child_layer)->a->data[i] = input_a->data[i] * dropout_data->weights->data[i] * scale; + } + } else { + // Not training, just copy data and do not apply dropout mask or scaling + for (unsigned int i = 0; i < total_elements; i++) { + (*child_layer)->a->data[i] = input_a->data[i]; + } + } +} + +void pearl_layer_forward(pearl_layer **parent_layer, pearl_layer **child_layer, bool is_training) // Add is_training { switch ((*child_layer)->type) { case pearl_layer_type_input: @@ -183,11 +313,12 @@ void pearl_layer_forward(pearl_layer **parent_layer, pearl_layer **child_layer) case pearl_layer_type_fully_connected: pearl_layer_forward_fully_connected(parent_layer, child_layer); break; - case pearl_layer_type_dropout: + case pearl_layer_type_dropout: // New case + pearl_layer_forward_dropout(parent_layer, child_layer, is_training); // Pass is_training break; } for (unsigned int i = 0; i < (*child_layer)->num_child_layers; i++) { - pearl_layer_forward(child_layer, &(*child_layer)->child_layers[i]); + pearl_layer_forward(child_layer, &(*child_layer)->child_layers[i], is_training); // Pass is_training } } @@ -243,7 +374,8 @@ void pearl_layer_backward(pearl_layer **child_layer, pearl_layer **parent_layer) case pearl_layer_type_fully_connected: pearl_layer_backward_fully_connected(child_layer, parent_layer); break; - case pearl_layer_type_dropout: + case pearl_layer_type_dropout: // New case + pearl_layer_backward_dropout(child_layer, parent_layer); break; } for (unsigned int i = 0; i < (*parent_layer)->num_parent_layers; i++) { @@ -357,6 +489,8 @@ void pearl_layer_update_fully_connected(pearl_layer **child_layer, float learnin assert(data->weights->dimension == 2); assert(data->weights->size[0] == data->dw->size[0]); assert(data->weights->size[1] == data->dw->size[1]); + + #pragma omp parallel for // Add this line for (unsigned int i = 0; i < data->weights->size[0]; i++) { for (unsigned int j = 0; j < data->weights->size[1]; j++) { data->weights->data[ARRAY_IDX_2D(i, j, data->weights->size[1])] -= learning_rate * data->dw->data[ARRAY_IDX_2D(i, j, data->dw->size[1])]; diff --git a/src/pearl_layer.h b/src/pearl_layer.h index 377c8aa..6e17099 100644 --- a/src/pearl_layer.h +++ b/src/pearl_layer.h @@ -2,6 +2,7 @@ #define PEARL_LAYER_H #include +#include #include #include #include @@ -52,7 +53,7 @@ PEARL_API pearl_layer *pearl_layer_create_input(unsigned int num_neurons); PEARL_API pearl_layer *pearl_layer_create_fully_connected(unsigned int num_neurons, unsigned int num_neurons_prev_layer); pearl_layer *pearl_layer_create_fully_connected_blank(unsigned int num_neurons); PEARL_API pearl_layer *pearl_layer_create_dropout(unsigned int num_neurons); -void pearl_layer_forward(pearl_layer **parent_layer, pearl_layer **child_layer); +void pearl_layer_forward(pearl_layer **parent_layer, pearl_layer **child_layer, bool is_training); void pearl_layer_forward_fully_connected(pearl_layer **parent_layer, pearl_layer **child_layer); void pearl_layer_backward(pearl_layer **child_layer, pearl_layer **parent_layer); void pearl_layer_backward_fully_connected(pearl_layer **child_layer, pearl_layer **parent_layer); diff --git a/src/pearl_network.c b/src/pearl_network.c index ab8645f..2d2ba53 100644 --- a/src/pearl_network.c +++ b/src/pearl_network.c @@ -11,6 +11,7 @@ PEARL_API pearl_network *pearl_network_create() network->version.minor = PEARL_NETWORK_VERSION_MINOR; network->version.revision = PEARL_NETWORK_VERSION_REVISION; network->input_layer = NULL; + network->is_training = true; // Add this line return network; } @@ -18,18 +19,17 @@ PEARL_API void pearl_network_destroy(pearl_network **network) { if (*network != NULL) { if ((*network)->input_layer != NULL) { - pearl_layer_destroy(&(*network)->input_layer); - free((*network)->input_layer); - (*network)->input_layer = NULL; + pearl_layer_destroy(&((*network)->input_layer)); // This will free the layer and set (*network)->input_layer to NULL. } - - free(*network); + // No need to free (*network)->input_layer again or set it to NULL here. + free(*network); // Free the network struct itself *network = NULL; } } PEARL_API float pearl_network_train_epoch(pearl_network **network, const pearl_tensor *input, const pearl_tensor *output) { + (*network)->is_training = true; // Set is_training to true for training // Forward pearl_network_forward(network, input); @@ -65,7 +65,7 @@ void pearl_network_forward(pearl_network **network, const pearl_tensor *input) /* Recursive forward other layers */ for (unsigned int i = 0; i < (*network)->input_layer->num_child_layers; i++) { - pearl_layer_forward(&(*network)->input_layer, &(*network)->input_layer->child_layers[i]); + pearl_layer_forward(&(*network)->input_layer, &(*network)->input_layer->child_layers[i], (*network)->is_training); // Pass is_training } } @@ -93,6 +93,7 @@ void pearl_network_backward(pearl_network **network, const pearl_tensor *output) PEARL_API pearl_tensor *pearl_network_calculate(pearl_network **network, const pearl_tensor *input) { + (*network)->is_training = false; // Set is_training to false for calculation/inference pearl_network_forward(network, input); pearl_tensor *output = pearl_tensor_copy((*network)->output_layer->a); return output; diff --git a/src/pearl_network.h b/src/pearl_network.h index eb0d625..09193d0 100644 --- a/src/pearl_network.h +++ b/src/pearl_network.h @@ -2,6 +2,7 @@ #define PEARL_NETWORK_H #include +#include #include #include #include @@ -22,6 +23,7 @@ typedef struct { pearl_loss loss; float learning_rate; pearl_version version; + bool is_training; } pearl_network; PEARL_API pearl_network *pearl_network_create(); diff --git a/src/pearl_tensor.c b/src/pearl_tensor.c index 184b304..5c722f1 100644 --- a/src/pearl_tensor.c +++ b/src/pearl_tensor.c @@ -1,4 +1,5 @@ #include +#include // Add this include PEARL_API pearl_tensor *pearl_tensor_create(const int num_args, ...) { @@ -39,21 +40,22 @@ PEARL_API pearl_tensor *pearl_tensor_copy(const pearl_tensor *x) pearl_tensor *result = calloc(1, sizeof(pearl_tensor)); result->dimension = x->dimension; result->size = calloc(x->dimension, sizeof(unsigned int)); - int alloc = 1; + + unsigned int total_elements = 1; for (unsigned int i = 0 ; i < x->dimension; i++) { result->size[i] = x->size[i]; - alloc *= x->size[i]; + total_elements *= x->size[i]; } - result->data = calloc(alloc, sizeof(float)); - - unsigned int num_data = 1; - for (unsigned int i = 0; i < x->dimension; i++) { - num_data *= x->size[i]; + + result->data = calloc(total_elements, sizeof(float)); + if (x->data != NULL && result->data != NULL && total_elements > 0) { // Add checks for safety + memcpy(result->data, x->data, total_elements * sizeof(float)); + } else if (total_elements == 0) { + // If total_elements is 0, data might be NULL, which is fine. + // If x->data is NULL but total_elements > 0, calloc still provides zeroed memory. } - for (unsigned int i = 0; i < num_data; i++) { - result->data[i] = x->data[i]; - } + return result; } diff --git a/test/main.c b/test/main.c index 96f4ec8..2f97682 100644 --- a/test/main.c +++ b/test/main.c @@ -2,9 +2,11 @@ #include #include #include +#include void setUp(void) { + srand(time(NULL)); } void tearDown(void) @@ -59,13 +61,7 @@ void test_network_add_layers() const pearl_layer_data_dropout *data_drop = (pearl_layer_data_dropout *)drop->layer_data; TEST_ASSERT_EQUAL_INT(drop->num_neurons, 5); TEST_ASSERT_EQUAL_FLOAT(data_drop->rate, 0.5); - TEST_ASSERT_EQUAL_INT(data_drop->weights->dimension, 1); - TEST_ASSERT_NOT_NULL(data_drop->weights->size); - TEST_ASSERT_EQUAL_INT(data_drop->weights->size[0], 5); - TEST_ASSERT_NOT_NULL(data_drop->weights->data); - for (unsigned int i = 0; i < data_drop->weights->size[0]; i++) { - TEST_ASSERT_EQUAL_FLOAT(data_drop->weights->data[i], 0.0); - } + TEST_ASSERT_NULL(data_drop->weights); pearl_layer *fc = pearl_layer_create_fully_connected(5, 1); pearl_layer_add_child(&drop, &fc); @@ -166,13 +162,6 @@ void test_network_save_load() TEST_ASSERT_EQUAL_INT(drop_load->num_neurons, 5); const pearl_layer_data_dropout *data_drop_load = (pearl_layer_data_dropout *)drop_load->layer_data; TEST_ASSERT_EQUAL_FLOAT(data_drop_load->rate, 0.5); - TEST_ASSERT_EQUAL_INT(data_drop_load->weights->dimension, 1); - TEST_ASSERT_NOT_NULL(data_drop_load->weights->size); - TEST_ASSERT_EQUAL_INT(data_drop_load->weights->size[0], 5); - TEST_ASSERT_NOT_NULL(data_drop_load->weights->data); - for (unsigned int i = 0; i < data_drop_load->weights->size[0]; i++) { - TEST_ASSERT_EQUAL_FLOAT(data_drop_load->weights->data[i], 0.0); - } //TODO: Compare biases and weights TEST_ASSERT_NOT_NULL(drop_load->child_layers); @@ -361,6 +350,169 @@ void test_network_regression() pearl_tensor_destroy(&pred); } +// Helper function to create a simple network with one input and one dropout layer +static pearl_network* create_simple_dropout_network(unsigned int num_neurons, float dropout_rate_val) { + pearl_network *network = pearl_network_create(); + TEST_ASSERT_NOT_NULL(network); + + pearl_layer *input_l = pearl_layer_create_input(num_neurons); + network->input_layer = input_l; + + pearl_layer *dropout_l = pearl_layer_create_dropout(num_neurons); + ((pearl_layer_data_dropout*)dropout_l->layer_data)->rate = dropout_rate_val; + + pearl_layer_add_child(&input_l, &dropout_l); + network->output_layer = dropout_l; // Treat dropout as output for this simple test + return network; +} + +void test_dropout_layer_forward_training_mode(void) { + unsigned int num_neurons = 1000; + unsigned int batch_size = 1; // Test with batch size 1 for simplicity of counting + float dropout_rate = 0.3f; + float scale = 1.0f / (1.0f - dropout_rate); + + pearl_network *net = create_simple_dropout_network(num_neurons, dropout_rate); + net->is_training = true; + + // Create input tensor (e.g., all ones) + pearl_tensor *input_data = pearl_tensor_create(2, num_neurons, batch_size); + for (unsigned int i = 0; i < num_neurons * batch_size; ++i) { + input_data->data[i] = 1.0f; + } + + // Set input layer's activation directly for the test + // Normally pearl_network_forward would handle this, but for isolated layer test: + if (net->input_layer->a == NULL) { + net->input_layer->a = pearl_tensor_create(2, num_neurons, batch_size); + } + pearl_tensor_destroy(&net->input_layer->a); // Destroy if pre-existing from create_simple + net->input_layer->a = pearl_tensor_copy(input_data); + + + // Perform forward pass focusing on the dropout layer + // The actual network forward will call layer_forward + pearl_layer_forward(&net->input_layer, &net->input_layer->child_layers[0], net->is_training); + + pearl_layer *dropout_layer = net->input_layer->child_layers[0]; + pearl_tensor *output_a = dropout_layer->a; + pearl_layer_data_dropout *dropout_data_cast = (pearl_layer_data_dropout*)dropout_layer->layer_data; + pearl_tensor *mask = dropout_data_cast->weights; + + TEST_ASSERT_NOT_NULL(output_a); + TEST_ASSERT_NOT_NULL(mask); + TEST_ASSERT_EQUAL_UINT(output_a->dimension, 2); + TEST_ASSERT_EQUAL_UINT(output_a->size[0], num_neurons); + TEST_ASSERT_EQUAL_UINT(output_a->size[1], batch_size); + TEST_ASSERT_EQUAL_UINT(mask->dimension, output_a->dimension); + TEST_ASSERT_EQUAL_UINT(mask->size[0], output_a->size[0]); + TEST_ASSERT_EQUAL_UINT(mask->size[1], output_a->size[1]); + + + unsigned int zero_count = 0; + for (unsigned int i = 0; i < num_neurons * batch_size; ++i) { + if (mask->data[i] == 0.0f) { + zero_count++; + TEST_ASSERT_EQUAL_FLOAT(0.0f, output_a->data[i]); + } else { + TEST_ASSERT_EQUAL_FLOAT(1.0f, mask->data[i]); + TEST_ASSERT_EQUAL_FLOAT(input_data->data[i] * scale, output_a->data[i]); + } + } + + // Check if the number of zeros is roughly rate * total_elements + // This is a probabilistic test, so allow some tolerance. + float expected_zeros = dropout_rate * num_neurons * batch_size; + float tolerance = 0.1f * num_neurons * batch_size; // 10% tolerance + TEST_ASSERT_FLOAT_WITHIN(tolerance, expected_zeros, (float)zero_count); + + pearl_tensor_destroy(&input_data); + pearl_network_destroy(&net); +} + +void test_dropout_layer_forward_inference_mode(void) { + unsigned int num_neurons = 100; + unsigned int batch_size = 2; + float dropout_rate = 0.5f; + + pearl_network *net = create_simple_dropout_network(num_neurons, dropout_rate); + net->is_training = false; + + pearl_tensor *input_data = pearl_tensor_create(2, num_neurons, batch_size); + for (unsigned int i = 0; i < num_neurons * batch_size; ++i) { + input_data->data[i] = (float)(i + 1); // Some varied data + } + + // Set input layer's activation + if (net->input_layer->a == NULL) { + net->input_layer->a = pearl_tensor_create(2, num_neurons, batch_size); + } + pearl_tensor_destroy(&net->input_layer->a); + net->input_layer->a = pearl_tensor_copy(input_data); + + pearl_layer_forward(&net->input_layer, &net->input_layer->child_layers[0], net->is_training); + + pearl_tensor *output_a = net->input_layer->child_layers[0]->a; + TEST_ASSERT_NOT_NULL(output_a); + + for (unsigned int i = 0; i < num_neurons * batch_size; ++i) { + TEST_ASSERT_EQUAL_FLOAT(input_data->data[i], output_a->data[i]); + } + + pearl_tensor_destroy(&input_data); + pearl_network_destroy(&net); +} + +void test_dropout_layer_backward_pass(void) { + unsigned int num_neurons = 100; + unsigned int batch_size = 1; + float dropout_rate = 0.4f; + float scale = 1.0f / (1.0f - dropout_rate); + + pearl_network *net = create_simple_dropout_network(num_neurons, dropout_rate); + net->is_training = true; + + pearl_tensor *input_val = pearl_tensor_create(2, num_neurons, batch_size); + for(unsigned int i=0; i < num_neurons * batch_size; ++i) input_val->data[i] = 1.0f; + + if (net->input_layer->a == NULL) { + net->input_layer->a = pearl_tensor_create(2, num_neurons, batch_size); + } + pearl_tensor_destroy(&net->input_layer->a); + net->input_layer->a = pearl_tensor_copy(input_val); + + + // Forward pass to generate the mask + pearl_layer_forward(&net->input_layer, &net->input_layer->child_layers[0], net->is_training); + + pearl_layer* dropout_layer = net->input_layer->child_layers[0]; + pearl_layer_data_dropout *dropout_data_cast = (pearl_layer_data_dropout*)dropout_layer->layer_data; + pearl_tensor *mask = dropout_data_cast->weights; + TEST_ASSERT_NOT_NULL(mask); + + // Manually set incoming gradients for the dropout layer (da of dropout layer) + if (dropout_layer->da == NULL) { + dropout_layer->da = pearl_tensor_create(2, num_neurons, batch_size); + } + for (unsigned int i = 0; i < num_neurons * batch_size; ++i) { + dropout_layer->da->data[i] = (float)(i + 1); // Arbitrary gradient values + } + + // Perform backward pass for the dropout layer + pearl_layer_backward(&dropout_layer, &net->input_layer); // child_layer, parent_layer + + pearl_tensor *parent_da = net->input_layer->da; // This is da for the layer *before* dropout + TEST_ASSERT_NOT_NULL(parent_da); + + for (unsigned int i = 0; i < num_neurons * batch_size; ++i) { + float expected_grad = dropout_layer->da->data[i] * mask->data[i] * scale; + TEST_ASSERT_EQUAL_FLOAT(expected_grad, parent_da->data[i]); + } + + pearl_tensor_destroy(&input_val); + pearl_network_destroy(&net); +} + int main(void) { UNITY_BEGIN(); RUN_TEST(test_network_create); @@ -368,5 +520,8 @@ int main(void) { RUN_TEST(test_network_save_load); RUN_TEST(test_network_epoch_check); RUN_TEST(test_network_regression); + RUN_TEST(test_dropout_layer_forward_training_mode); + RUN_TEST(test_dropout_layer_forward_inference_mode); + RUN_TEST(test_dropout_layer_backward_pass); return UNITY_END(); }