From 57ff2ae875163802d5bfff1705a86a1bdf3976ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Felipe=20Santos?= Date: Tue, 13 Jan 2026 16:11:59 -0800 Subject: [PATCH 1/9] Added gating activation classes --- NAM/gating_activations.h | 104 +++++++++++ tools/run_tests.cpp | 12 ++ tools/test/test_gating_activations.cpp | 234 +++++++++++++++++++++++++ 3 files changed, 350 insertions(+) create mode 100644 NAM/gating_activations.h create mode 100644 tools/test/test_gating_activations.cpp diff --git a/NAM/gating_activations.h b/NAM/gating_activations.h new file mode 100644 index 0000000..39b9cd4 --- /dev/null +++ b/NAM/gating_activations.h @@ -0,0 +1,104 @@ +#pragma once + +#include +#include // expf +#include +#include +#include +#include +#include "activations.h" + +namespace nam +{ +namespace gating_activations +{ + +class GatingActivation +{ +public: + GatingActivation(activations::Activation* input_act = nullptr, activations::Activation* gating_act = nullptr) + : input_activation(input_act ? input_act : &default_activation), + gating_activation(gating_act ? gating_act : &default_activation) + { + } + + ~GatingActivation() = default; + + void apply(Eigen::MatrixXf& input, Eigen::MatrixXf& output) + { + // Validate input dimensions + if (input.rows() < 2) { + throw std::invalid_argument("GatingActivation: input matrix must have at least 2 rows"); + } + + // Ensure output has correct dimensions + if (output.rows() != 1 || output.cols() != input.cols()) { + output.resize(1, input.cols()); + } + + // Apply activations to the two rows + Eigen::MatrixXf input_row = input.row(0); + Eigen::MatrixXf gating_row = input.row(1); + + input_activation->apply(input_row); + gating_activation->apply(gating_row); + + // Element-wise multiplication + output = input_row.array() * gating_row.array(); + } + +private: + activations::Activation* input_activation; + activations::Activation* gating_activation; + activations::Activation default_activation; // Default activation for safety +}; + +class BlendingActivation +{ +public: + BlendingActivation(activations::Activation* input_act = nullptr, activations::Activation* blend_act = nullptr, float alpha_val = 0.5f) + : input_activation(input_act ? input_act : &default_activation), + blending_activation(blend_act ? blend_act : &default_activation), + alpha(alpha_val) + { + // Validate alpha is in valid range + if (alpha < 0.0f || alpha > 1.0f) { + throw std::invalid_argument("BlendingActivation: alpha must be between 0.0 and 1.0"); + } + } + + ~BlendingActivation() = default; + + void apply(Eigen::MatrixXf& input, Eigen::MatrixXf& output) + { + // Validate input dimensions + if (input.rows() < 2) { + throw std::invalid_argument("BlendingActivation: input matrix must have at least 2 rows"); + } + + // Ensure output has correct dimensions + if (output.rows() != 1 || output.cols() != input.cols()) { + output.resize(1, input.cols()); + } + + // Apply activations to the two rows + Eigen::MatrixXf input_row = input.row(0); + Eigen::MatrixXf blend_row = input.row(1); + + input_activation->apply(input_row); + blending_activation->apply(blend_row); + + // Weighted blending + output = alpha * input_row + (1.0f - alpha) * blend_row; + } + +private: + activations::Activation* input_activation; + activations::Activation* blending_activation; + float alpha; + activations::Activation default_activation; // Default activation for safety +}; + + +}; // namespace gating_activations +}; // namespace nam diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp index 8d622d8..978de81 100644 --- a/tools/run_tests.cpp +++ b/tools/run_tests.cpp @@ -7,6 +7,7 @@ #include "test/test_get_dsp.cpp" #include "test/test_wavenet.cpp" #include "test/test_fast_lut.cpp" +#include "test/test_gating_activations.cpp" int main() { @@ -39,6 +40,17 @@ int main() test_lut::TestFastLUT::test_sigmoid(); test_lut::TestFastLUT::test_tanh(); + // Gating activations tests + test_gating_activations::TestGatingActivation::test_basic_functionality(); + test_gating_activations::TestGatingActivation::test_with_custom_activations(); + test_gating_activations::TestGatingActivation::test_error_handling(); + + test_gating_activations::TestBlendingActivation::test_basic_functionality(); + test_gating_activations::TestBlendingActivation::test_different_alpha_values(); + test_gating_activations::TestBlendingActivation::test_with_custom_activations(); + test_gating_activations::TestBlendingActivation::test_error_handling(); + test_gating_activations::TestBlendingActivation::test_edge_cases(); + std::cout << "Success!" << std::endl; return 0; } diff --git a/tools/test/test_gating_activations.cpp b/tools/test/test_gating_activations.cpp new file mode 100644 index 0000000..a068788 --- /dev/null +++ b/tools/test/test_gating_activations.cpp @@ -0,0 +1,234 @@ +// Tests for gating activation functions + +#include +#include +#include +#include +#include + +#include "NAM/gating_activations.h" +#include "NAM/activations.h" + +namespace test_gating_activations +{ + +class TestGatingActivation +{ +public: + static void test_basic_functionality() + { + // Create test input data (2 rows, 3 columns) + Eigen::MatrixXf input(2, 3); + input << 1.0f, -1.0f, 0.0f, + 0.5f, 0.8f, 1.0f; + + Eigen::MatrixXf output; + + // Create gating activation with default activations + nam::gating_activations::GatingActivation gating_act; + + // Apply the activation + gating_act.apply(input, output); + + // Basic checks + assert(output.rows() == 1); + assert(output.cols() == 3); + + // The output should be element-wise multiplication of the two rows + // after applying activations + std::cout << "GatingActivation basic test passed" << std::endl; + } + + static void test_with_custom_activations() + { + // Create custom activations + nam::activations::ActivationLeakyReLU leaky_relu(0.01f); + nam::activations::ActivationLeakyReLU leaky_relu2(0.05f); + + // Create test input data + Eigen::MatrixXf input(2, 2); + input << -1.0f, 1.0f, + -2.0f, 0.5f; + + Eigen::MatrixXf output; + + // Create gating activation with custom activations + nam::gating_activations::GatingActivation gating_act(&leaky_relu, &leaky_relu2); + + // Apply the activation + gating_act.apply(input, output); + + // Verify dimensions + assert(output.rows() == 1); + assert(output.cols() == 2); + + std::cout << "GatingActivation custom activations test passed" << std::endl; + } + + static void test_error_handling() + { + // Test with insufficient rows + Eigen::MatrixXf input(1, 3); // Only 1 row + Eigen::MatrixXf output; + + nam::gating_activations::GatingActivation gating_act; + + try { + gating_act.apply(input, output); + assert(false); // Should not reach here + } catch (const std::invalid_argument& e) { + std::cout << "GatingActivation error handling test passed: " << e.what() << std::endl; + } + } +}; + +class TestBlendingActivation +{ +public: + static void test_basic_functionality() + { + // Create test input data (2 rows, 3 columns) + Eigen::MatrixXf input(2, 3); + input << 1.0f, -1.0f, 0.0f, + 0.5f, 0.8f, 1.0f; + + Eigen::MatrixXf output; + + // Create blending activation with default alpha (0.5) + nam::gating_activations::BlendingActivation blending_act; + + // Apply the activation + blending_act.apply(input, output); + + // Basic checks + assert(output.rows() == 1); + assert(output.cols() == 3); + + std::cout << "BlendingActivation basic test passed" << std::endl; + } + + static void test_different_alpha_values() + { + // Test with alpha = 0.0 (should use only second row) + Eigen::MatrixXf input(2, 2); + input << 1.0f, -1.0f, + 2.0f, 3.0f; + + Eigen::MatrixXf output; + + nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 0.0f); + blending_act.apply(input, output); + + // With alpha=0.0, output should be close to second row + assert(fabs(output(0, 0) - 2.0f) < 1e-6); + assert(fabs(output(0, 1) - 3.0f) < 1e-6); + + // Test with alpha = 1.0 (should use only first row) + nam::gating_activations::BlendingActivation blending_act2(nullptr, nullptr, 1.0f); + blending_act2.apply(input, output); + + // With alpha=1.0, output should be close to first row + assert(fabs(output(0, 0) - 1.0f) < 1e-6); + assert(fabs(output(0, 1) - (-1.0f)) < 1e-6); + + // Test with alpha = 0.3 + nam::gating_activations::BlendingActivation blending_act3(nullptr, nullptr, 0.3f); + blending_act3.apply(input, output); + + // With alpha=0.3, output should be 0.3*row1 + 0.7*row2 + float expected0 = 0.3f * 1.0f + 0.7f * 2.0f; + float expected1 = 0.3f * (-1.0f) + 0.7f * 3.0f; + + assert(fabs(output(0, 0) - expected0) < 1e-6); + assert(fabs(output(0, 1) - expected1) < 1e-6); + + std::cout << "BlendingActivation alpha values test passed" << std::endl; + } + + static void test_with_custom_activations() + { + // Create custom activations + nam::activations::ActivationLeakyReLU leaky_relu(0.01f); + nam::activations::ActivationLeakyReLU leaky_relu2(0.05f); + + // Create test input data + Eigen::MatrixXf input(2, 2); + input << -1.0f, 1.0f, + -2.0f, 0.5f; + + Eigen::MatrixXf output; + + // Create blending activation with custom activations and alpha = 0.7 + nam::gating_activations::BlendingActivation blending_act(&leaky_relu, &leaky_relu2, 0.7f); + + // Apply the activation + blending_act.apply(input, output); + + // Verify dimensions + assert(output.rows() == 1); + assert(output.cols() == 2); + + std::cout << "BlendingActivation custom activations test passed" << std::endl; + } + + static void test_error_handling() + { + // Test with insufficient rows + Eigen::MatrixXf input(1, 2); // Only 1 row + Eigen::MatrixXf output; + + nam::gating_activations::BlendingActivation blending_act; + + try { + blending_act.apply(input, output); + assert(false); // Should not reach here + } catch (const std::invalid_argument& e) { + std::cout << "BlendingActivation error handling test passed: " << e.what() << std::endl; + } + + // Test with invalid alpha value + try { + nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 1.5f); + assert(false); // Should not reach here + } catch (const std::invalid_argument& e) { + std::cout << "BlendingActivation alpha validation test passed: " << e.what() << std::endl; + } + + try { + nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, -0.1f); + assert(false); // Should not reach here + } catch (const std::invalid_argument& e) { + std::cout << "BlendingActivation alpha validation test passed: " << e.what() << std::endl; + } + } + + static void test_edge_cases() + { + // Test with zero input + Eigen::MatrixXf input(2, 1); + input << 0.0f, + 0.0f; + + Eigen::MatrixXf output; + + nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 0.5f); + blending_act.apply(input, output); + + assert(fabs(output(0, 0) - 0.0f) < 1e-6); + + // Test with large values + Eigen::MatrixXf input2(2, 1); + input2 << 1000.0f, + -1000.0f; + + blending_act.apply(input2, output); + + // Should handle large values without issues + assert(output.rows() == 1); + assert(output.cols() == 1); + + std::cout << "BlendingActivation edge cases test passed" << std::endl; + } +}; + +}; // namespace test_gating_activations \ No newline at end of file From 7e48751892bf80fd9f860e7cf3be5a03b117ae6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Felipe=20Santos?= Date: Wed, 14 Jan 2026 09:46:15 -0800 Subject: [PATCH 2/9] Throwing exceptions instead of resizing, removed default activation --- NAM/gating_activations.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/NAM/gating_activations.h b/NAM/gating_activations.h index 39b9cd4..90a251a 100644 --- a/NAM/gating_activations.h +++ b/NAM/gating_activations.h @@ -33,7 +33,7 @@ class GatingActivation // Ensure output has correct dimensions if (output.rows() != 1 || output.cols() != input.cols()) { - output.resize(1, input.cols()); + throw std::invalid_argument("GatingActivation: output matrix must have at least 1 row"); } // Apply activations to the two rows @@ -50,7 +50,6 @@ class GatingActivation private: activations::Activation* input_activation; activations::Activation* gating_activation; - activations::Activation default_activation; // Default activation for safety }; class BlendingActivation @@ -78,7 +77,7 @@ class BlendingActivation // Ensure output has correct dimensions if (output.rows() != 1 || output.cols() != input.cols()) { - output.resize(1, input.cols()); + throw std::invalid_argument("BlendingActivation: output matrix must have at least 1 row"); } // Apply activations to the two rows @@ -96,7 +95,6 @@ class BlendingActivation activations::Activation* input_activation; activations::Activation* blending_activation; float alpha; - activations::Activation default_activation; // Default activation for safety }; From 844ab27a2a647cefcbf847b586d6851c82346473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Felipe=20Santos?= Date: Wed, 14 Jan 2026 10:16:50 -0800 Subject: [PATCH 3/9] Updated to generalize to 2*channels in, channels out. Added tests to compare to wavenet implementation --- NAM/gating_activations.h | 176 ++++++++++++++--- tools/run_tests.cpp | 7 + tools/test/test_gating_activations.cpp | 30 +-- .../test_wavenet_gating_compatibility.cpp | 183 ++++++++++++++++++ 4 files changed, 349 insertions(+), 47 deletions(-) create mode 100644 tools/test/test_wavenet_gating_compatibility.cpp diff --git a/NAM/gating_activations.h b/NAM/gating_activations.h index 90a251a..b8ab16f 100644 --- a/NAM/gating_activations.h +++ b/NAM/gating_activations.h @@ -13,88 +13,200 @@ namespace nam namespace gating_activations { +// Default linear activation (identity function) +class LinearActivation : public nam::activations::Activation +{ +public: + LinearActivation() = default; + ~LinearActivation() = default; + // Inherit the default apply methods which do nothing (linear/identity) +}; + +// Static instance for default activation +static LinearActivation default_activation; + class GatingActivation { public: - GatingActivation(activations::Activation* input_act = nullptr, activations::Activation* gating_act = nullptr) + /** + * Constructor for GatingActivation + * @param input_act Activation function for input channels (default: linear) + * @param gating_act Activation function for gating channels (default: sigmoid) + * @param input_channels Number of input channels (default: 1) + * @param gating_channels Number of gating channels (default: 1) + */ + GatingActivation(activations::Activation* input_act = nullptr, + activations::Activation* gating_act = nullptr, + int input_channels = 1, + int gating_channels = 1) : input_activation(input_act ? input_act : &default_activation), - gating_activation(gating_act ? gating_act : &default_activation) + gating_activation(gating_act ? gating_act : activations::Activation::get_activation("Sigmoid")), + num_input_channels(input_channels), + num_gating_channels(gating_channels) { + if (num_input_channels <= 0 || num_gating_channels <= 0) { + throw std::invalid_argument("GatingActivation: number of channels must be positive"); + } } ~GatingActivation() = default; + /** + * Apply gating activation to input matrix + * @param input Input matrix with shape (input_channels + gating_channels) x num_samples + * @param output Output matrix with shape input_channels x num_samples + */ void apply(Eigen::MatrixXf& input, Eigen::MatrixXf& output) { // Validate input dimensions - if (input.rows() < 2) { - throw std::invalid_argument("GatingActivation: input matrix must have at least 2 rows"); + const int total_channels = num_input_channels + num_gating_channels; + if (input.rows() != total_channels) { + throw std::invalid_argument("GatingActivation: input matrix must have " + + std::to_string(total_channels) + " rows"); } - // Ensure output has correct dimensions - if (output.rows() != 1 || output.cols() != input.cols()) { - throw std::invalid_argument("GatingActivation: output matrix must have at least 1 row"); + // Validate output dimensions + if (output.rows() != num_input_channels || output.cols() != input.cols()) { + throw std::invalid_argument("GatingActivation: output matrix must have " + + std::to_string(num_input_channels) + " rows and " + + std::to_string(input.cols()) + " columns"); } - // Apply activations to the two rows - Eigen::MatrixXf input_row = input.row(0); - Eigen::MatrixXf gating_row = input.row(1); - - input_activation->apply(input_row); - gating_activation->apply(gating_row); - - // Element-wise multiplication - output = input_row.array() * gating_row.array(); + // Process column-by-column to ensure memory contiguity (important for column-major matrices) + const int num_samples = input.cols(); + for (int i = 0; i < num_samples; i++) + { + // Apply activation to input channels + Eigen::MatrixXf input_block = input.block(0, i, num_input_channels, 1); + input_activation->apply(input_block); + + // Apply activation to gating channels + Eigen::MatrixXf gating_block = input.block(num_input_channels, i, num_gating_channels, 1); + gating_activation->apply(gating_block); + + // Element-wise multiplication and store result + // For wavenet compatibility, we assume one-to-one mapping + assert(num_input_channels == num_gating_channels); + output.block(0, i, num_input_channels, 1) = + input_block.array() * gating_block.array(); + } + } + + /** + * Get the total number of input channels required + */ + int get_total_input_channels() const { + return num_input_channels + num_gating_channels; + } + + /** + * Get the number of output channels + */ + int get_output_channels() const { + return num_input_channels; } private: activations::Activation* input_activation; activations::Activation* gating_activation; + int num_input_channels; + int num_gating_channels; }; class BlendingActivation { public: - BlendingActivation(activations::Activation* input_act = nullptr, activations::Activation* blend_act = nullptr, float alpha_val = 0.5f) + /** + * Constructor for BlendingActivation + * @param input_act Activation function for input channels + * @param blend_act Activation function for blending channels + * @param alpha_val Blending factor (0.0 to 1.0) + * @param input_channels Number of input channels + * @param blend_channels Number of blending channels + */ + BlendingActivation(activations::Activation* input_act = nullptr, + activations::Activation* blend_act = nullptr, + float alpha_val = 0.5f, + int input_channels = 1, + int blend_channels = 1) : input_activation(input_act ? input_act : &default_activation), blending_activation(blend_act ? blend_act : &default_activation), - alpha(alpha_val) + alpha(alpha_val), + num_input_channels(input_channels), + num_blend_channels(blend_channels) { // Validate alpha is in valid range if (alpha < 0.0f || alpha > 1.0f) { throw std::invalid_argument("BlendingActivation: alpha must be between 0.0 and 1.0"); } + if (num_input_channels <= 0 || num_blend_channels <= 0) { + throw std::invalid_argument("BlendingActivation: number of channels must be positive"); + } } ~BlendingActivation() = default; + /** + * Apply blending activation to input matrix + * @param input Input matrix with shape (input_channels + blend_channels) x num_samples + * @param output Output matrix with shape input_channels x num_samples + */ void apply(Eigen::MatrixXf& input, Eigen::MatrixXf& output) { // Validate input dimensions - if (input.rows() < 2) { - throw std::invalid_argument("BlendingActivation: input matrix must have at least 2 rows"); + const int total_channels = num_input_channels + num_blend_channels; + if (input.rows() != total_channels) { + throw std::invalid_argument("BlendingActivation: input matrix must have " + + std::to_string(total_channels) + " rows"); } - // Ensure output has correct dimensions - if (output.rows() != 1 || output.cols() != input.cols()) { - throw std::invalid_argument("BlendingActivation: output matrix must have at least 1 row"); + // Validate output dimensions + if (output.rows() != num_input_channels || output.cols() != input.cols()) { + throw std::invalid_argument("BlendingActivation: output matrix must have " + + std::to_string(num_input_channels) + " rows and " + + std::to_string(input.cols()) + " columns"); } - // Apply activations to the two rows - Eigen::MatrixXf input_row = input.row(0); - Eigen::MatrixXf blend_row = input.row(1); - - input_activation->apply(input_row); - blending_activation->apply(blend_row); - - // Weighted blending - output = alpha * input_row + (1.0f - alpha) * blend_row; + // Process column-by-column to ensure memory contiguity + const int num_samples = input.cols(); + for (int i = 0; i < num_samples; i++) + { + // Apply activation to input channels + Eigen::MatrixXf input_block = input.block(0, i, num_input_channels, 1); + input_activation->apply(input_block); + + // Apply activation to blend channels + Eigen::MatrixXf blend_block = input.block(num_input_channels, i, num_blend_channels, 1); + blending_activation->apply(blend_block); + + // Weighted blending + // For wavenet compatibility, we assume one-to-one mapping + assert(num_input_channels == num_blend_channels); + output.block(0, i, num_input_channels, 1) = + alpha * input_block + (1.0f - alpha) * blend_block; + } + } + + /** + * Get the total number of input channels required + */ + int get_total_input_channels() const { + return num_input_channels + num_blend_channels; + } + + /** + * Get the number of output channels + */ + int get_output_channels() const { + return num_input_channels; } private: activations::Activation* input_activation; activations::Activation* blending_activation; float alpha; + int num_input_channels; + int num_blend_channels; }; diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp index 978de81..883772b 100644 --- a/tools/run_tests.cpp +++ b/tools/run_tests.cpp @@ -8,6 +8,7 @@ #include "test/test_wavenet.cpp" #include "test/test_fast_lut.cpp" #include "test/test_gating_activations.cpp" +#include "test/test_wavenet_gating_compatibility.cpp" int main() { @@ -51,6 +52,12 @@ int main() test_gating_activations::TestBlendingActivation::test_error_handling(); test_gating_activations::TestBlendingActivation::test_edge_cases(); + // Wavenet gating compatibility tests + test_wavenet_gating_compatibility::TestWavenetGatingCompatibility::test_wavenet_style_gating(); + test_wavenet_gating_compatibility::TestWavenetGatingCompatibility::test_column_by_column_processing(); + test_wavenet_gating_compatibility::TestWavenetGatingCompatibility::test_memory_contiguity(); + test_wavenet_gating_compatibility::TestWavenetGatingCompatibility::test_multiple_channels(); + std::cout << "Success!" << std::endl; return 0; } diff --git a/tools/test/test_gating_activations.cpp b/tools/test/test_gating_activations.cpp index a068788..2f6227b 100644 --- a/tools/test/test_gating_activations.cpp +++ b/tools/test/test_gating_activations.cpp @@ -22,10 +22,10 @@ class TestGatingActivation input << 1.0f, -1.0f, 0.0f, 0.5f, 0.8f, 1.0f; - Eigen::MatrixXf output; + Eigen::MatrixXf output(1, 3); - // Create gating activation with default activations - nam::gating_activations::GatingActivation gating_act; + // Create gating activation with default activations (1 input channel, 1 gating channel) + nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, 1, 1); // Apply the activation gating_act.apply(input, output); @@ -50,10 +50,10 @@ class TestGatingActivation input << -1.0f, 1.0f, -2.0f, 0.5f; - Eigen::MatrixXf output; + Eigen::MatrixXf output(1, 2); // Create gating activation with custom activations - nam::gating_activations::GatingActivation gating_act(&leaky_relu, &leaky_relu2); + nam::gating_activations::GatingActivation gating_act(&leaky_relu, &leaky_relu2, 1, 1); // Apply the activation gating_act.apply(input, output); @@ -92,10 +92,10 @@ class TestBlendingActivation input << 1.0f, -1.0f, 0.0f, 0.5f, 0.8f, 1.0f; - Eigen::MatrixXf output; + Eigen::MatrixXf output(1, 3); // Create blending activation with default alpha (0.5) - nam::gating_activations::BlendingActivation blending_act; + nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 0.5f, 1, 1); // Apply the activation blending_act.apply(input, output); @@ -114,9 +114,9 @@ class TestBlendingActivation input << 1.0f, -1.0f, 2.0f, 3.0f; - Eigen::MatrixXf output; + Eigen::MatrixXf output(1, 2); - nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 0.0f); + nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 0.0f, 1, 1); blending_act.apply(input, output); // With alpha=0.0, output should be close to second row @@ -124,7 +124,7 @@ class TestBlendingActivation assert(fabs(output(0, 1) - 3.0f) < 1e-6); // Test with alpha = 1.0 (should use only first row) - nam::gating_activations::BlendingActivation blending_act2(nullptr, nullptr, 1.0f); + nam::gating_activations::BlendingActivation blending_act2(nullptr, nullptr, 1.0f, 1, 1); blending_act2.apply(input, output); // With alpha=1.0, output should be close to first row @@ -132,7 +132,7 @@ class TestBlendingActivation assert(fabs(output(0, 1) - (-1.0f)) < 1e-6); // Test with alpha = 0.3 - nam::gating_activations::BlendingActivation blending_act3(nullptr, nullptr, 0.3f); + nam::gating_activations::BlendingActivation blending_act3(nullptr, nullptr, 0.3f, 1, 1); blending_act3.apply(input, output); // With alpha=0.3, output should be 0.3*row1 + 0.7*row2 @@ -156,10 +156,10 @@ class TestBlendingActivation input << -1.0f, 1.0f, -2.0f, 0.5f; - Eigen::MatrixXf output; + Eigen::MatrixXf output(1, 2); // Create blending activation with custom activations and alpha = 0.7 - nam::gating_activations::BlendingActivation blending_act(&leaky_relu, &leaky_relu2, 0.7f); + nam::gating_activations::BlendingActivation blending_act(&leaky_relu, &leaky_relu2, 0.7f, 1, 1); // Apply the activation blending_act.apply(input, output); @@ -209,9 +209,9 @@ class TestBlendingActivation input << 0.0f, 0.0f; - Eigen::MatrixXf output; + Eigen::MatrixXf output(1, 1); - nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 0.5f); + nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 0.5f, 1, 1); blending_act.apply(input, output); assert(fabs(output(0, 0) - 0.0f) < 1e-6); diff --git a/tools/test/test_wavenet_gating_compatibility.cpp b/tools/test/test_wavenet_gating_compatibility.cpp new file mode 100644 index 0000000..618b5ed --- /dev/null +++ b/tools/test/test_wavenet_gating_compatibility.cpp @@ -0,0 +1,183 @@ +// Test to verify that our gating implementation matches the wavenet behavior + +#include +#include +#include +#include +#include + +#include "NAM/gating_activations.h" +#include "NAM/activations.h" + +namespace test_wavenet_gating_compatibility +{ + +class TestWavenetGatingCompatibility +{ +public: + static void test_wavenet_style_gating() + { + // Simulate wavenet scenario: 2 channels (input + gating), multiple samples + const int channels = 2; + const int num_samples = 3; + + // Create input matrix similar to wavenet's _z matrix + // First 'channels' rows are input, next 'channels' rows are gating + Eigen::MatrixXf input(2 * channels, num_samples); + input << 1.0f, -0.5f, 0.2f, // Input channel 1 + 0.3f, 0.1f, -0.4f, // Input channel 2 + 0.8f, 0.6f, 0.9f, // Gating channel 1 + 0.4f, 0.2f, 0.7f; // Gating channel 2 + + Eigen::MatrixXf output(channels, num_samples); + + // Create gating activation that matches wavenet behavior + // Wavenet uses: input activation (default/linear) and sigmoid for gating + nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels, channels); + + // Apply the activation + gating_act.apply(input, output); + + // Verify dimensions + assert(output.rows() == channels); + assert(output.cols() == num_samples); + + // Verify that the output is the element-wise product of input and gating channels + // after applying activations + for (int c = 0; c < channels; c++) + { + for (int s = 0; s < num_samples; s++) + { + // Input channel value (no activation applied - linear) + float input_val = input(c, s); + + // Gating channel value (sigmoid activation applied) + float gating_val = input(channels + c, s); + float sigmoid_gating = 1.0f / (1.0f + expf(-gating_val)); + + // Expected output + float expected = input_val * sigmoid_gating; + + // Check if they match + if (fabs(output(c, s) - expected) > 1e-6) + { + std::cerr << "Mismatch at channel " << c << ", sample " << s + << ": expected " << expected << ", got " << output(c, s) << std::endl; + assert(false); + } + } + } + + std::cout << "Wavenet gating compatibility test passed" << std::endl; + } + + static void test_column_by_column_processing() + { + // Test that our implementation processes column-by-column like wavenet + const int channels = 1; + const int num_samples = 4; + + Eigen::MatrixXf input(2, num_samples); + input << 1.0f, 2.0f, 3.0f, 4.0f, + 0.1f, 0.2f, 0.3f, 0.4f; + + Eigen::MatrixXf output(channels, num_samples); + + nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels, channels); + gating_act.apply(input, output); + + // Verify each column was processed independently + for (int s = 0; s < num_samples; s++) + { + float input_val = input(0, s); + float gating_val = input(1, s); + float sigmoid_gating = 1.0f / (1.0f + expf(-gating_val)); + float expected = input_val * sigmoid_gating; + + assert(fabs(output(0, s) - expected) < 1e-6); + } + + std::cout << "Column-by-column processing test passed" << std::endl; + } + + static void test_memory_contiguity() + { + // Test that our implementation handles memory contiguity correctly + // This is important for column-major matrices + const int channels = 3; + const int num_samples = 2; + + Eigen::MatrixXf input(2 * channels, num_samples); + // Fill with some values + for (int i = 0; i < 2 * channels; i++) + { + for (int j = 0; j < num_samples; j++) + { + input(i, j) = static_cast(i * num_samples + j + 1); + } + } + + Eigen::MatrixXf output(channels, num_samples); + + nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels, channels); + + // This should not crash or produce incorrect results due to memory contiguity issues + gating_act.apply(input, output); + + // Verify the results are correct + for (int c = 0; c < channels; c++) + { + for (int s = 0; s < num_samples; s++) + { + float input_val = input(c, s); + float gating_val = input(channels + c, s); + float sigmoid_gating = 1.0f / (1.0f + expf(-gating_val)); + float expected = input_val * sigmoid_gating; + + assert(fabs(output(c, s) - expected) < 1e-6); + } + } + + std::cout << "Memory contiguity test passed" << std::endl; + } + + static void test_multiple_channels() + { + // Test with multiple equal input and gating channels (wavenet style) + const int channels = 2; + const int num_samples = 2; + + Eigen::MatrixXf input(2 * channels, num_samples); + input << 1.0f, 2.0f, // Input channels + 3.0f, 4.0f, + 5.0f, 6.0f, // Gating channels + 7.0f, 8.0f; + + Eigen::MatrixXf output(channels, num_samples); + + nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels, channels); + gating_act.apply(input, output); + + // Verify dimensions + assert(output.rows() == channels); + assert(output.cols() == num_samples); + + // Verify that each input channel is multiplied by corresponding gating channel + for (int c = 0; c < channels; c++) + { + for (int s = 0; s < num_samples; s++) + { + float input_val = input(c, s); + float gating_val = input(channels + c, s); + float sigmoid_gating = 1.0f / (1.0f + expf(-gating_val)); + float expected = input_val * sigmoid_gating; + + assert(fabs(output(c, s) - expected) < 1e-6); + } + } + + std::cout << "Multiple channels test passed" << std::endl; + } +}; + +}; // namespace test_wavenet_gating_compatibility From 2caf327b85ccee0588a78351536ae31f1231cd25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Felipe=20Santos?= Date: Wed, 14 Jan 2026 10:29:22 -0800 Subject: [PATCH 4/9] Formatting --- NAM/activations.cpp | 30 ++-- NAM/activations.h | 111 +++++++------ NAM/gating_activations.h | 124 +++++++-------- tools/run_tests.cpp | 2 +- tools/test/test_fast_lut.cpp | 46 +++--- tools/test/test_gating_activations.cpp | 149 +++++++++--------- .../test_wavenet_gating_compatibility.cpp | 88 +++++------ 7 files changed, 285 insertions(+), 265 deletions(-) diff --git a/NAM/activations.cpp b/NAM/activations.cpp index daa05c5..2bfe6c8 100644 --- a/NAM/activations.cpp +++ b/NAM/activations.cpp @@ -4,7 +4,8 @@ nam::activations::ActivationTanh _TANH = nam::activations::ActivationTanh(); nam::activations::ActivationFastTanh _FAST_TANH = nam::activations::ActivationFastTanh(); nam::activations::ActivationHardTanh _HARD_TANH = nam::activations::ActivationHardTanh(); nam::activations::ActivationReLU _RELU = nam::activations::ActivationReLU(); -nam::activations::ActivationLeakyReLU _LEAKY_RELU = nam::activations::ActivationLeakyReLU(0.01); //FIXME does not parameterize LeakyReLU +nam::activations::ActivationLeakyReLU _LEAKY_RELU = + nam::activations::ActivationLeakyReLU(0.01); // FIXME does not parameterize LeakyReLU nam::activations::ActivationSigmoid _SIGMOID = nam::activations::ActivationSigmoid(); nam::activations::ActivationSwish _SWISH = nam::activations::ActivationSwish(); nam::activations::ActivationHardSwish _HARD_SWISH = nam::activations::ActivationHardSwish(); @@ -13,8 +14,8 @@ nam::activations::ActivationLeakyHardTanh _LEAKY_HARD_TANH = nam::activations::A bool nam::activations::Activation::using_fast_tanh = false; std::unordered_map nam::activations::Activation::_activations = { - {"Tanh", &_TANH}, {"Hardtanh", &_HARD_TANH}, {"Fasttanh", &_FAST_TANH}, - {"ReLU", &_RELU}, {"LeakyReLU", &_LEAKY_RELU}, {"Sigmoid", &_SIGMOID}, + {"Tanh", &_TANH}, {"Hardtanh", &_HARD_TANH}, {"Fasttanh", &_FAST_TANH}, + {"ReLU", &_RELU}, {"LeakyReLU", &_LEAKY_RELU}, {"Sigmoid", &_SIGMOID}, {"SiLU", &_SWISH}, {"Hardswish", &_HARD_SWISH}, {"LeakyHardtanh", &_LEAKY_HARD_TANH}}; nam::activations::Activation* tanh_bak = nullptr; @@ -52,13 +53,18 @@ void nam::activations::Activation::disable_fast_tanh() void nam::activations::Activation::enable_lut(std::string function_name, float min, float max, std::size_t n_points) { std::function fn; - if (function_name == "Tanh"){ + if (function_name == "Tanh") + { fn = [](float x) { return std::tanh(x); }; tanh_bak = _activations["Tanh"]; - } else if (function_name == "Sigmoid") { + } + else if (function_name == "Sigmoid") + { fn = sigmoid; sigmoid_bak = _activations["Sigmoid"]; - } else { + } + else + { throw std::runtime_error("Tried to enable LUT for a function other than Tanh or Sigmoid"); } FastLUTActivation lut_activation(min, max, n_points, fn); @@ -67,12 +73,16 @@ void nam::activations::Activation::enable_lut(std::string function_name, float m void nam::activations::Activation::disable_lut(std::string function_name) { - if (function_name == "Tanh"){ + if (function_name == "Tanh") + { _activations["Tanh"] = tanh_bak; - } else if (function_name == "Sigmoid") { + } + else if (function_name == "Sigmoid") + { _activations["Sigmoid"] = sigmoid_bak; - } else { + } + else + { throw std::runtime_error("Tried to disable LUT for a function other than Tanh or Sigmoid"); } } - diff --git a/NAM/activations.h b/NAM/activations.h index d8a5591..13fed4a 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -28,11 +28,16 @@ inline float hard_tanh(float x) inline float leaky_hardtanh(float x, float min_val, float max_val, float min_slope, float max_slope) { - if (x < min_val) { + if (x < min_val) + { return (x - min_val) * min_slope + min_val; - } else if (x > max_val) { + } + else if (x > max_val) + { return (x - max_val) * max_slope + max_val; - } else { + } + else + { return x; } } @@ -50,7 +55,7 @@ inline float fast_sigmoid(const float x) { return 0.5f * (fast_tanh(x * 0.5f) + 1.0f); } - + inline float leaky_relu(float x, float negative_slope) { return x > 0.0f ? x : negative_slope * x; @@ -68,12 +73,17 @@ inline float swish(float x) inline float hardswish(float x) { - if (x <= -3.0) { + if (x <= -3.0) + { return 0; - } else if (x >= 3.0) { + } + else if (x >= 3.0) + { return x; - } else { - return x * (x + 3.0)/6.0; + } + else + { + return x * (x + 3.0) / 6.0; } } @@ -129,7 +139,8 @@ class ActivationLeakyHardTanh : public Activation { public: ActivationLeakyHardTanh() = default; - ActivationLeakyHardTanh(float min_val_, float max_val_, float min_slope_, float max_slope_) { + ActivationLeakyHardTanh(float min_val_, float max_val_, float min_slope_, float max_slope_) + { min_val = min_val_; max_val = max_val_; min_slope = min_slope_; @@ -142,6 +153,7 @@ class ActivationLeakyHardTanh : public Activation data[pos] = leaky_hardtanh(data[pos], min_val, max_val, min_slope, max_slope); } } + private: float min_val = -1.0; float max_val = 1.0; @@ -177,9 +189,7 @@ class ActivationLeakyReLU : public Activation { public: ActivationLeakyReLU() = default; - ActivationLeakyReLU(float ns) { - negative_slope = ns; - } + ActivationLeakyReLU(float ns) { negative_slope = ns; } void apply(float* data, long size) override { for (long pos = 0; pos < size; pos++) @@ -187,6 +197,7 @@ class ActivationLeakyReLU : public Activation data[pos] = leaky_relu(data[pos], negative_slope); } } + private: float negative_slope = 0.01; }; @@ -230,46 +241,54 @@ class ActivationHardSwish : public Activation class FastLUTActivation : public Activation { public: - FastLUTActivation(float min_x, float max_x, std::size_t size, std::function f) - : min_x_(min_x), max_x_(max_x), size_(size) { - - step_ = (max_x - min_x) / (size - 1); - inv_step_ = 1.0f / step_; - table_.reserve(size); - - for (std::size_t i = 0; i < size; ++i) { - table_.push_back(f(min_x + i * step_)); - } - } + FastLUTActivation(float min_x, float max_x, std::size_t size, std::function f) + : min_x_(min_x) + , max_x_(max_x) + , size_(size) + { - // Fast lookup with linear interpolation - inline float lookup(float x) const { - // Clamp input to range - x = std::clamp(x, min_x_, max_x_); - - // Calculate float index - float f_idx = (x - min_x_) * inv_step_; - std::size_t i = static_cast(f_idx); - - // Handle edge case at max_x_ - if (i >= size_ - 1) return table_.back(); - - // Linear interpolation: y = y0 + (y1 - y0) * fractional_part - float frac = f_idx - static_cast(i); - return table_[i] + (table_[i + 1] - table_[i]) * frac; + step_ = (max_x - min_x) / (size - 1); + inv_step_ = 1.0f / step_; + table_.reserve(size); + + for (std::size_t i = 0; i < size; ++i) + { + table_.push_back(f(min_x + i * step_)); } + } + + // Fast lookup with linear interpolation + inline float lookup(float x) const + { + // Clamp input to range + x = std::clamp(x, min_x_, max_x_); - // Vector application (Batch processing) - void apply(std::vector& data) const { - for (float& val : data) { - val = lookup(val); - } + // Calculate float index + float f_idx = (x - min_x_) * inv_step_; + std::size_t i = static_cast(f_idx); + + // Handle edge case at max_x_ + if (i >= size_ - 1) + return table_.back(); + + // Linear interpolation: y = y0 + (y1 - y0) * fractional_part + float frac = f_idx - static_cast(i); + return table_[i] + (table_[i + 1] - table_[i]) * frac; + } + + // Vector application (Batch processing) + void apply(std::vector& data) const + { + for (float& val : data) + { + val = lookup(val); } + } private: - float min_x_, max_x_, step_, inv_step_; - size_t size_; - std::vector table_; + float min_x_, max_x_, step_, inv_step_; + size_t size_; + std::vector table_; }; }; // namespace activations diff --git a/NAM/gating_activations.h b/NAM/gating_activations.h index b8ab16f..c01ef4c 100644 --- a/NAM/gating_activations.h +++ b/NAM/gating_activations.h @@ -35,22 +35,21 @@ class GatingActivation * @param input_channels Number of input channels (default: 1) * @param gating_channels Number of gating channels (default: 1) */ - GatingActivation(activations::Activation* input_act = nullptr, - activations::Activation* gating_act = nullptr, - int input_channels = 1, - int gating_channels = 1) - : input_activation(input_act ? input_act : &default_activation), - gating_activation(gating_act ? gating_act : activations::Activation::get_activation("Sigmoid")), - num_input_channels(input_channels), - num_gating_channels(gating_channels) + GatingActivation(activations::Activation* input_act = nullptr, activations::Activation* gating_act = nullptr, + int input_channels = 1, int gating_channels = 1) + : input_activation(input_act ? input_act : &default_activation) + , gating_activation(gating_act ? gating_act : activations::Activation::get_activation("Sigmoid")) + , num_input_channels(input_channels) + , num_gating_channels(gating_channels) { - if (num_input_channels <= 0 || num_gating_channels <= 0) { + if (num_input_channels <= 0 || num_gating_channels <= 0) + { throw std::invalid_argument("GatingActivation: number of channels must be positive"); } } - + ~GatingActivation() = default; - + /** * Apply gating activation to input matrix * @param input Input matrix with shape (input_channels + gating_channels) x num_samples @@ -60,18 +59,19 @@ class GatingActivation { // Validate input dimensions const int total_channels = num_input_channels + num_gating_channels; - if (input.rows() != total_channels) { - throw std::invalid_argument("GatingActivation: input matrix must have " + - std::to_string(total_channels) + " rows"); + if (input.rows() != total_channels) + { + throw std::invalid_argument("GatingActivation: input matrix must have " + std::to_string(total_channels) + + " rows"); } - + // Validate output dimensions - if (output.rows() != num_input_channels || output.cols() != input.cols()) { - throw std::invalid_argument("GatingActivation: output matrix must have " + - std::to_string(num_input_channels) + " rows and " + - std::to_string(input.cols()) + " columns"); + if (output.rows() != num_input_channels || output.cols() != input.cols()) + { + throw std::invalid_argument("GatingActivation: output matrix must have " + std::to_string(num_input_channels) + + " rows and " + std::to_string(input.cols()) + " columns"); } - + // Process column-by-column to ensure memory contiguity (important for column-major matrices) const int num_samples = input.cols(); for (int i = 0; i < num_samples; i++) @@ -79,32 +79,27 @@ class GatingActivation // Apply activation to input channels Eigen::MatrixXf input_block = input.block(0, i, num_input_channels, 1); input_activation->apply(input_block); - + // Apply activation to gating channels Eigen::MatrixXf gating_block = input.block(num_input_channels, i, num_gating_channels, 1); gating_activation->apply(gating_block); - + // Element-wise multiplication and store result // For wavenet compatibility, we assume one-to-one mapping assert(num_input_channels == num_gating_channels); - output.block(0, i, num_input_channels, 1) = - input_block.array() * gating_block.array(); + output.block(0, i, num_input_channels, 1) = input_block.array() * gating_block.array(); } } - + /** * Get the total number of input channels required */ - int get_total_input_channels() const { - return num_input_channels + num_gating_channels; - } - + int get_total_input_channels() const { return num_input_channels + num_gating_channels; } + /** * Get the number of output channels */ - int get_output_channels() const { - return num_input_channels; - } + int get_output_channels() const { return num_input_channels; } private: activations::Activation* input_activation; @@ -124,28 +119,27 @@ class BlendingActivation * @param input_channels Number of input channels * @param blend_channels Number of blending channels */ - BlendingActivation(activations::Activation* input_act = nullptr, - activations::Activation* blend_act = nullptr, - float alpha_val = 0.5f, - int input_channels = 1, - int blend_channels = 1) - : input_activation(input_act ? input_act : &default_activation), - blending_activation(blend_act ? blend_act : &default_activation), - alpha(alpha_val), - num_input_channels(input_channels), - num_blend_channels(blend_channels) + BlendingActivation(activations::Activation* input_act = nullptr, activations::Activation* blend_act = nullptr, + float alpha_val = 0.5f, int input_channels = 1, int blend_channels = 1) + : input_activation(input_act ? input_act : &default_activation) + , blending_activation(blend_act ? blend_act : &default_activation) + , alpha(alpha_val) + , num_input_channels(input_channels) + , num_blend_channels(blend_channels) { // Validate alpha is in valid range - if (alpha < 0.0f || alpha > 1.0f) { + if (alpha < 0.0f || alpha > 1.0f) + { throw std::invalid_argument("BlendingActivation: alpha must be between 0.0 and 1.0"); } - if (num_input_channels <= 0 || num_blend_channels <= 0) { + if (num_input_channels <= 0 || num_blend_channels <= 0) + { throw std::invalid_argument("BlendingActivation: number of channels must be positive"); } } - + ~BlendingActivation() = default; - + /** * Apply blending activation to input matrix * @param input Input matrix with shape (input_channels + blend_channels) x num_samples @@ -155,18 +149,19 @@ class BlendingActivation { // Validate input dimensions const int total_channels = num_input_channels + num_blend_channels; - if (input.rows() != total_channels) { - throw std::invalid_argument("BlendingActivation: input matrix must have " + - std::to_string(total_channels) + " rows"); + if (input.rows() != total_channels) + { + throw std::invalid_argument("BlendingActivation: input matrix must have " + std::to_string(total_channels) + + " rows"); } - + // Validate output dimensions - if (output.rows() != num_input_channels || output.cols() != input.cols()) { - throw std::invalid_argument("BlendingActivation: output matrix must have " + - std::to_string(num_input_channels) + " rows and " + - std::to_string(input.cols()) + " columns"); + if (output.rows() != num_input_channels || output.cols() != input.cols()) + { + throw std::invalid_argument("BlendingActivation: output matrix must have " + std::to_string(num_input_channels) + + " rows and " + std::to_string(input.cols()) + " columns"); } - + // Process column-by-column to ensure memory contiguity const int num_samples = input.cols(); for (int i = 0; i < num_samples; i++) @@ -174,32 +169,27 @@ class BlendingActivation // Apply activation to input channels Eigen::MatrixXf input_block = input.block(0, i, num_input_channels, 1); input_activation->apply(input_block); - + // Apply activation to blend channels Eigen::MatrixXf blend_block = input.block(num_input_channels, i, num_blend_channels, 1); blending_activation->apply(blend_block); - + // Weighted blending // For wavenet compatibility, we assume one-to-one mapping assert(num_input_channels == num_blend_channels); - output.block(0, i, num_input_channels, 1) = - alpha * input_block + (1.0f - alpha) * blend_block; + output.block(0, i, num_input_channels, 1) = alpha * input_block + (1.0f - alpha) * blend_block; } } - + /** * Get the total number of input channels required */ - int get_total_input_channels() const { - return num_input_channels + num_blend_channels; - } - + int get_total_input_channels() const { return num_input_channels + num_blend_channels; } + /** * Get the number of output channels */ - int get_output_channels() const { - return num_input_channels; - } + int get_output_channels() const { return num_input_channels; } private: activations::Activation* input_activation; diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp index 883772b..32bd4df 100644 --- a/tools/run_tests.cpp +++ b/tools/run_tests.cpp @@ -45,7 +45,7 @@ int main() test_gating_activations::TestGatingActivation::test_basic_functionality(); test_gating_activations::TestGatingActivation::test_with_custom_activations(); test_gating_activations::TestGatingActivation::test_error_handling(); - + test_gating_activations::TestBlendingActivation::test_basic_functionality(); test_gating_activations::TestBlendingActivation::test_different_alpha_values(); test_gating_activations::TestBlendingActivation::test_with_custom_activations(); diff --git a/tools/test/test_fast_lut.cpp b/tools/test/test_fast_lut.cpp index 29aee29..f7b1d83 100644 --- a/tools/test/test_fast_lut.cpp +++ b/tools/test/test_fast_lut.cpp @@ -5,35 +5,33 @@ #include "NAM/activations.h" -namespace test_lut { +namespace test_lut +{ -float sigmoid(float x) { - return 1.0f / (1.0f + std::exp(-x)); +float sigmoid(float x) +{ + return 1.0f / (1.0f + std::exp(-x)); } class TestFastLUT { - public: - static void test_sigmoid() - { - // create a lut for sigmoid from -8.0 to 8.0 with 1024 samples - nam::activations::FastLUTActivation lut_sigmoid(-8.0f, 8.0f, 1024, [](float x) { - return 1.0f / (1.0f + expf(-x)); - }); +public: + static void test_sigmoid() + { + // create a lut for sigmoid from -8.0 to 8.0 with 1024 samples + nam::activations::FastLUTActivation lut_sigmoid( + -8.0f, 8.0f, 1024, [](float x) { return 1.0f / (1.0f + expf(-x)); }); - float input = 1.25f; - assert(abs(sigmoid(input) - lut_sigmoid.lookup(input)) < 1e-3); - } - static void test_tanh() - { - // create a lut for sigmoid from -8.0 to 8.0 with 1024 samples - nam::activations::FastLUTActivation lut_tanh(-8.0f, 8.0f, 1024, [](float x) { - return std::tanh(x); - }); + float input = 1.25f; + assert(abs(sigmoid(input) - lut_sigmoid.lookup(input)) < 1e-3); + } + static void test_tanh() + { + // create a lut for sigmoid from -8.0 to 8.0 with 1024 samples + nam::activations::FastLUTActivation lut_tanh(-8.0f, 8.0f, 1024, [](float x) { return std::tanh(x); }); - float input = 1.25f; - assert(abs(std::tanh(input) - lut_tanh.lookup(input)) < 1e-3); - } + float input = 1.25f; + assert(abs(std::tanh(input) - lut_tanh.lookup(input)) < 1e-3); + } }; -} - +} // namespace test_lut diff --git a/tools/test/test_gating_activations.cpp b/tools/test/test_gating_activations.cpp index 2f6227b..ce402ea 100644 --- a/tools/test/test_gating_activations.cpp +++ b/tools/test/test_gating_activations.cpp @@ -19,64 +19,65 @@ class TestGatingActivation { // Create test input data (2 rows, 3 columns) Eigen::MatrixXf input(2, 3); - input << 1.0f, -1.0f, 0.0f, - 0.5f, 0.8f, 1.0f; - + input << 1.0f, -1.0f, 0.0f, 0.5f, 0.8f, 1.0f; + Eigen::MatrixXf output(1, 3); - + // Create gating activation with default activations (1 input channel, 1 gating channel) nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, 1, 1); - + // Apply the activation gating_act.apply(input, output); - + // Basic checks assert(output.rows() == 1); assert(output.cols() == 3); - + // The output should be element-wise multiplication of the two rows // after applying activations std::cout << "GatingActivation basic test passed" << std::endl; } - + static void test_with_custom_activations() { // Create custom activations nam::activations::ActivationLeakyReLU leaky_relu(0.01f); nam::activations::ActivationLeakyReLU leaky_relu2(0.05f); - + // Create test input data Eigen::MatrixXf input(2, 2); - input << -1.0f, 1.0f, - -2.0f, 0.5f; - + input << -1.0f, 1.0f, -2.0f, 0.5f; + Eigen::MatrixXf output(1, 2); - + // Create gating activation with custom activations nam::gating_activations::GatingActivation gating_act(&leaky_relu, &leaky_relu2, 1, 1); - + // Apply the activation gating_act.apply(input, output); - + // Verify dimensions assert(output.rows() == 1); assert(output.cols() == 2); - + std::cout << "GatingActivation custom activations test passed" << std::endl; } - + static void test_error_handling() { // Test with insufficient rows Eigen::MatrixXf input(1, 3); // Only 1 row Eigen::MatrixXf output; - + nam::gating_activations::GatingActivation gating_act; - - try { + + try + { gating_act.apply(input, output); assert(false); // Should not reach here - } catch (const std::invalid_argument& e) { + } + catch (const std::invalid_argument& e) + { std::cout << "GatingActivation error handling test passed: " << e.what() << std::endl; } } @@ -89,144 +90,148 @@ class TestBlendingActivation { // Create test input data (2 rows, 3 columns) Eigen::MatrixXf input(2, 3); - input << 1.0f, -1.0f, 0.0f, - 0.5f, 0.8f, 1.0f; - + input << 1.0f, -1.0f, 0.0f, 0.5f, 0.8f, 1.0f; + Eigen::MatrixXf output(1, 3); - + // Create blending activation with default alpha (0.5) nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 0.5f, 1, 1); - + // Apply the activation blending_act.apply(input, output); - + // Basic checks assert(output.rows() == 1); assert(output.cols() == 3); - + std::cout << "BlendingActivation basic test passed" << std::endl; } - + static void test_different_alpha_values() { // Test with alpha = 0.0 (should use only second row) Eigen::MatrixXf input(2, 2); - input << 1.0f, -1.0f, - 2.0f, 3.0f; - + input << 1.0f, -1.0f, 2.0f, 3.0f; + Eigen::MatrixXf output(1, 2); - + nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 0.0f, 1, 1); blending_act.apply(input, output); - + // With alpha=0.0, output should be close to second row assert(fabs(output(0, 0) - 2.0f) < 1e-6); assert(fabs(output(0, 1) - 3.0f) < 1e-6); - + // Test with alpha = 1.0 (should use only first row) nam::gating_activations::BlendingActivation blending_act2(nullptr, nullptr, 1.0f, 1, 1); blending_act2.apply(input, output); - + // With alpha=1.0, output should be close to first row assert(fabs(output(0, 0) - 1.0f) < 1e-6); assert(fabs(output(0, 1) - (-1.0f)) < 1e-6); - + // Test with alpha = 0.3 nam::gating_activations::BlendingActivation blending_act3(nullptr, nullptr, 0.3f, 1, 1); blending_act3.apply(input, output); - + // With alpha=0.3, output should be 0.3*row1 + 0.7*row2 float expected0 = 0.3f * 1.0f + 0.7f * 2.0f; float expected1 = 0.3f * (-1.0f) + 0.7f * 3.0f; - + assert(fabs(output(0, 0) - expected0) < 1e-6); assert(fabs(output(0, 1) - expected1) < 1e-6); - + std::cout << "BlendingActivation alpha values test passed" << std::endl; } - + static void test_with_custom_activations() { // Create custom activations nam::activations::ActivationLeakyReLU leaky_relu(0.01f); nam::activations::ActivationLeakyReLU leaky_relu2(0.05f); - + // Create test input data Eigen::MatrixXf input(2, 2); - input << -1.0f, 1.0f, - -2.0f, 0.5f; - + input << -1.0f, 1.0f, -2.0f, 0.5f; + Eigen::MatrixXf output(1, 2); - + // Create blending activation with custom activations and alpha = 0.7 nam::gating_activations::BlendingActivation blending_act(&leaky_relu, &leaky_relu2, 0.7f, 1, 1); - + // Apply the activation blending_act.apply(input, output); - + // Verify dimensions assert(output.rows() == 1); assert(output.cols() == 2); - + std::cout << "BlendingActivation custom activations test passed" << std::endl; } - + static void test_error_handling() { // Test with insufficient rows Eigen::MatrixXf input(1, 2); // Only 1 row Eigen::MatrixXf output; - + nam::gating_activations::BlendingActivation blending_act; - - try { + + try + { blending_act.apply(input, output); assert(false); // Should not reach here - } catch (const std::invalid_argument& e) { + } + catch (const std::invalid_argument& e) + { std::cout << "BlendingActivation error handling test passed: " << e.what() << std::endl; } - + // Test with invalid alpha value - try { + try + { nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 1.5f); assert(false); // Should not reach here - } catch (const std::invalid_argument& e) { + } + catch (const std::invalid_argument& e) + { std::cout << "BlendingActivation alpha validation test passed: " << e.what() << std::endl; } - - try { + + try + { nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, -0.1f); assert(false); // Should not reach here - } catch (const std::invalid_argument& e) { + } + catch (const std::invalid_argument& e) + { std::cout << "BlendingActivation alpha validation test passed: " << e.what() << std::endl; } } - + static void test_edge_cases() { // Test with zero input Eigen::MatrixXf input(2, 1); - input << 0.0f, - 0.0f; - + input << 0.0f, 0.0f; + Eigen::MatrixXf output(1, 1); - + nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 0.5f, 1, 1); blending_act.apply(input, output); - + assert(fabs(output(0, 0) - 0.0f) < 1e-6); - + // Test with large values Eigen::MatrixXf input2(2, 1); - input2 << 1000.0f, - -1000.0f; - + input2 << 1000.0f, -1000.0f; + blending_act.apply(input2, output); - + // Should handle large values without issues assert(output.rows() == 1); assert(output.cols() == 1); - + std::cout << "BlendingActivation edge cases test passed" << std::endl; } }; diff --git a/tools/test/test_wavenet_gating_compatibility.cpp b/tools/test/test_wavenet_gating_compatibility.cpp index 618b5ed..93e7046 100644 --- a/tools/test/test_wavenet_gating_compatibility.cpp +++ b/tools/test/test_wavenet_gating_compatibility.cpp @@ -20,28 +20,28 @@ class TestWavenetGatingCompatibility // Simulate wavenet scenario: 2 channels (input + gating), multiple samples const int channels = 2; const int num_samples = 3; - + // Create input matrix similar to wavenet's _z matrix // First 'channels' rows are input, next 'channels' rows are gating Eigen::MatrixXf input(2 * channels, num_samples); - input << 1.0f, -0.5f, 0.2f, // Input channel 1 - 0.3f, 0.1f, -0.4f, // Input channel 2 - 0.8f, 0.6f, 0.9f, // Gating channel 1 - 0.4f, 0.2f, 0.7f; // Gating channel 2 - + input << 1.0f, -0.5f, 0.2f, // Input channel 1 + 0.3f, 0.1f, -0.4f, // Input channel 2 + 0.8f, 0.6f, 0.9f, // Gating channel 1 + 0.4f, 0.2f, 0.7f; // Gating channel 2 + Eigen::MatrixXf output(channels, num_samples); - + // Create gating activation that matches wavenet behavior // Wavenet uses: input activation (default/linear) and sigmoid for gating nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels, channels); - + // Apply the activation gating_act.apply(input, output); - + // Verify dimensions assert(output.rows() == channels); assert(output.cols() == num_samples); - + // Verify that the output is the element-wise product of input and gating channels // after applying activations for (int c = 0; c < channels; c++) @@ -50,42 +50,41 @@ class TestWavenetGatingCompatibility { // Input channel value (no activation applied - linear) float input_val = input(c, s); - + // Gating channel value (sigmoid activation applied) float gating_val = input(channels + c, s); float sigmoid_gating = 1.0f / (1.0f + expf(-gating_val)); - + // Expected output float expected = input_val * sigmoid_gating; - + // Check if they match if (fabs(output(c, s) - expected) > 1e-6) { - std::cerr << "Mismatch at channel " << c << ", sample " << s - << ": expected " << expected << ", got " << output(c, s) << std::endl; + std::cerr << "Mismatch at channel " << c << ", sample " << s << ": expected " << expected << ", got " + << output(c, s) << std::endl; assert(false); } } } - + std::cout << "Wavenet gating compatibility test passed" << std::endl; } - + static void test_column_by_column_processing() { // Test that our implementation processes column-by-column like wavenet const int channels = 1; const int num_samples = 4; - + Eigen::MatrixXf input(2, num_samples); - input << 1.0f, 2.0f, 3.0f, 4.0f, - 0.1f, 0.2f, 0.3f, 0.4f; - + input << 1.0f, 2.0f, 3.0f, 4.0f, 0.1f, 0.2f, 0.3f, 0.4f; + Eigen::MatrixXf output(channels, num_samples); - + nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels, channels); gating_act.apply(input, output); - + // Verify each column was processed independently for (int s = 0; s < num_samples; s++) { @@ -93,20 +92,20 @@ class TestWavenetGatingCompatibility float gating_val = input(1, s); float sigmoid_gating = 1.0f / (1.0f + expf(-gating_val)); float expected = input_val * sigmoid_gating; - + assert(fabs(output(0, s) - expected) < 1e-6); } - + std::cout << "Column-by-column processing test passed" << std::endl; } - + static void test_memory_contiguity() { // Test that our implementation handles memory contiguity correctly // This is important for column-major matrices const int channels = 3; const int num_samples = 2; - + Eigen::MatrixXf input(2 * channels, num_samples); // Fill with some values for (int i = 0; i < 2 * channels; i++) @@ -116,14 +115,14 @@ class TestWavenetGatingCompatibility input(i, j) = static_cast(i * num_samples + j + 1); } } - + Eigen::MatrixXf output(channels, num_samples); - + nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels, channels); - + // This should not crash or produce incorrect results due to memory contiguity issues gating_act.apply(input, output); - + // Verify the results are correct for (int c = 0; c < channels; c++) { @@ -133,35 +132,34 @@ class TestWavenetGatingCompatibility float gating_val = input(channels + c, s); float sigmoid_gating = 1.0f / (1.0f + expf(-gating_val)); float expected = input_val * sigmoid_gating; - + assert(fabs(output(c, s) - expected) < 1e-6); } } - + std::cout << "Memory contiguity test passed" << std::endl; } - + static void test_multiple_channels() { // Test with multiple equal input and gating channels (wavenet style) const int channels = 2; const int num_samples = 2; - + Eigen::MatrixXf input(2 * channels, num_samples); - input << 1.0f, 2.0f, // Input channels - 3.0f, 4.0f, - 5.0f, 6.0f, // Gating channels - 7.0f, 8.0f; - + input << 1.0f, 2.0f, // Input channels + 3.0f, 4.0f, 5.0f, 6.0f, // Gating channels + 7.0f, 8.0f; + Eigen::MatrixXf output(channels, num_samples); - + nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels, channels); gating_act.apply(input, output); - + // Verify dimensions assert(output.rows() == channels); assert(output.cols() == num_samples); - + // Verify that each input channel is multiplied by corresponding gating channel for (int c = 0; c < channels; c++) { @@ -171,11 +169,11 @@ class TestWavenetGatingCompatibility float gating_val = input(channels + c, s); float sigmoid_gating = 1.0f / (1.0f + expf(-gating_val)); float expected = input_val * sigmoid_gating; - + assert(fabs(output(c, s) - expected) < 1e-6); } } - + std::cout << "Multiple channels test passed" << std::endl; } }; From dd65affd8694d85c018e8d2458f86871ca36cfa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Felipe=20Santos?= Date: Wed, 14 Jan 2026 13:49:04 -0800 Subject: [PATCH 5/9] Fixed issue with blending activation, addressed comments for gating activation. Removed all runtime checks and replaced with asserts. --- NAM/gating_activations.h | 95 +++++-------- tools/run_tests.cpp | 22 ++- tools/test/test_blending_detailed.cpp | 112 +++++++++++++++ tools/test/test_gating_activations.cpp | 132 +++++++----------- tools/test/test_input_buffer_verification.cpp | 88 ++++++++++++ 5 files changed, 303 insertions(+), 146 deletions(-) create mode 100644 tools/test/test_blending_detailed.cpp create mode 100644 tools/test/test_input_buffer_verification.cpp diff --git a/NAM/gating_activations.h b/NAM/gating_activations.h index c01ef4c..18afb0b 100644 --- a/NAM/gating_activations.h +++ b/NAM/gating_activations.h @@ -14,16 +14,16 @@ namespace gating_activations { // Default linear activation (identity function) -class LinearActivation : public nam::activations::Activation +class IdentityActivation : public nam::activations::Activation { public: - LinearActivation() = default; - ~LinearActivation() = default; + IdentityActivation() = default; + ~IdentityActivation() = default; // Inherit the default apply methods which do nothing (linear/identity) }; // Static instance for default activation -static LinearActivation default_activation; +static IdentityActivation default_activation; class GatingActivation { @@ -40,12 +40,8 @@ class GatingActivation : input_activation(input_act ? input_act : &default_activation) , gating_activation(gating_act ? gating_act : activations::Activation::get_activation("Sigmoid")) , num_input_channels(input_channels) - , num_gating_channels(gating_channels) { - if (num_input_channels <= 0 || num_gating_channels <= 0) - { - throw std::invalid_argument("GatingActivation: number of channels must be positive"); - } + assert(num_input_channels > 0); } ~GatingActivation() = default; @@ -57,20 +53,11 @@ class GatingActivation */ void apply(Eigen::MatrixXf& input, Eigen::MatrixXf& output) { - // Validate input dimensions - const int total_channels = num_input_channels + num_gating_channels; - if (input.rows() != total_channels) - { - throw std::invalid_argument("GatingActivation: input matrix must have " + std::to_string(total_channels) - + " rows"); - } - - // Validate output dimensions - if (output.rows() != num_input_channels || output.cols() != input.cols()) - { - throw std::invalid_argument("GatingActivation: output matrix must have " + std::to_string(num_input_channels) - + " rows and " + std::to_string(input.cols()) + " columns"); - } + // Validate input dimensions (assert for real-time performance) + const int total_channels = 2 * num_input_channels; + assert(input.rows() == total_channels); + assert(output.rows() == num_input_channels); + assert(output.cols() == input.cols()); // Process column-by-column to ensure memory contiguity (important for column-major matrices) const int num_samples = input.cols(); @@ -81,12 +68,11 @@ class GatingActivation input_activation->apply(input_block); // Apply activation to gating channels - Eigen::MatrixXf gating_block = input.block(num_input_channels, i, num_gating_channels, 1); + Eigen::MatrixXf gating_block = input.block(num_input_channels, i, num_input_channels, 1); gating_activation->apply(gating_block); // Element-wise multiplication and store result // For wavenet compatibility, we assume one-to-one mapping - assert(num_input_channels == num_gating_channels); output.block(0, i, num_input_channels, 1) = input_block.array() * gating_block.array(); } } @@ -94,7 +80,7 @@ class GatingActivation /** * Get the total number of input channels required */ - int get_total_input_channels() const { return num_input_channels + num_gating_channels; } + int get_total_input_channels() const { return 2 * num_input_channels; } /** * Get the number of output channels @@ -105,7 +91,6 @@ class GatingActivation activations::Activation* input_activation; activations::Activation* gating_activation; int num_input_channels; - int num_gating_channels; }; class BlendingActivation @@ -115,27 +100,21 @@ class BlendingActivation * Constructor for BlendingActivation * @param input_act Activation function for input channels * @param blend_act Activation function for blending channels - * @param alpha_val Blending factor (0.0 to 1.0) * @param input_channels Number of input channels - * @param blend_channels Number of blending channels */ BlendingActivation(activations::Activation* input_act = nullptr, activations::Activation* blend_act = nullptr, - float alpha_val = 0.5f, int input_channels = 1, int blend_channels = 1) + int input_channels = 1) : input_activation(input_act ? input_act : &default_activation) , blending_activation(blend_act ? blend_act : &default_activation) - , alpha(alpha_val) , num_input_channels(input_channels) - , num_blend_channels(blend_channels) { - // Validate alpha is in valid range - if (alpha < 0.0f || alpha > 1.0f) - { - throw std::invalid_argument("BlendingActivation: alpha must be between 0.0 and 1.0"); - } - if (num_input_channels <= 0 || num_blend_channels <= 0) + if (num_input_channels <= 0) { - throw std::invalid_argument("BlendingActivation: number of channels must be positive"); + throw std::invalid_argument("BlendingActivation: number of input channels must be positive"); } + // Initialize input buffer with correct size + // Note: current code copies column-by-column so we only need (num_input_channels, 1) + input_buffer.resize(num_input_channels, 1); } ~BlendingActivation() = default; @@ -147,44 +126,37 @@ class BlendingActivation */ void apply(Eigen::MatrixXf& input, Eigen::MatrixXf& output) { - // Validate input dimensions - const int total_channels = num_input_channels + num_blend_channels; - if (input.rows() != total_channels) - { - throw std::invalid_argument("BlendingActivation: input matrix must have " + std::to_string(total_channels) - + " rows"); - } - - // Validate output dimensions - if (output.rows() != num_input_channels || output.cols() != input.cols()) - { - throw std::invalid_argument("BlendingActivation: output matrix must have " + std::to_string(num_input_channels) - + " rows and " + std::to_string(input.cols()) + " columns"); - } + // Validate input dimensions (assert for real-time performance) + const int total_channels = num_input_channels * 2; // 2*channels in, channels out + assert(input.rows() == total_channels); + assert(output.rows() == num_input_channels); + assert(output.cols() == input.cols()); // Process column-by-column to ensure memory contiguity const int num_samples = input.cols(); for (int i = 0; i < num_samples; i++) { + // Store pre-activation input values in buffer + input_buffer = input.block(0, i, num_input_channels, 1); + // Apply activation to input channels Eigen::MatrixXf input_block = input.block(0, i, num_input_channels, 1); input_activation->apply(input_block); - // Apply activation to blend channels - Eigen::MatrixXf blend_block = input.block(num_input_channels, i, num_blend_channels, 1); + // Apply activation to blend channels to compute alpha + Eigen::MatrixXf blend_block = input.block(num_input_channels, i, num_input_channels, 1); blending_activation->apply(blend_block); - // Weighted blending - // For wavenet compatibility, we assume one-to-one mapping - assert(num_input_channels == num_blend_channels); - output.block(0, i, num_input_channels, 1) = alpha * input_block + (1.0f - alpha) * blend_block; + // Weighted blending: alpha * activated_input + (1 - alpha) * pre_activation_input + output.block(0, i, num_input_channels, 1) = + blend_block.array() * input_block.array() + (1.0f - blend_block.array()) * input_buffer.array(); } } /** * Get the total number of input channels required */ - int get_total_input_channels() const { return num_input_channels + num_blend_channels; } + int get_total_input_channels() const { return 2 * num_input_channels; } /** * Get the number of output channels @@ -194,9 +166,8 @@ class BlendingActivation private: activations::Activation* input_activation; activations::Activation* blending_activation; - float alpha; int num_input_channels; - int num_blend_channels; + Eigen::MatrixXf input_buffer; }; diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp index 32bd4df..ae1922a 100644 --- a/tools/run_tests.cpp +++ b/tools/run_tests.cpp @@ -9,6 +9,8 @@ #include "test/test_fast_lut.cpp" #include "test/test_gating_activations.cpp" #include "test/test_wavenet_gating_compatibility.cpp" +#include "test/test_blending_detailed.cpp" +#include "test/test_input_buffer_verification.cpp" int main() { @@ -46,18 +48,26 @@ int main() test_gating_activations::TestGatingActivation::test_with_custom_activations(); test_gating_activations::TestGatingActivation::test_error_handling(); - test_gating_activations::TestBlendingActivation::test_basic_functionality(); - test_gating_activations::TestBlendingActivation::test_different_alpha_values(); - test_gating_activations::TestBlendingActivation::test_with_custom_activations(); - test_gating_activations::TestBlendingActivation::test_error_handling(); - test_gating_activations::TestBlendingActivation::test_edge_cases(); - // Wavenet gating compatibility tests test_wavenet_gating_compatibility::TestWavenetGatingCompatibility::test_wavenet_style_gating(); test_wavenet_gating_compatibility::TestWavenetGatingCompatibility::test_column_by_column_processing(); test_wavenet_gating_compatibility::TestWavenetGatingCompatibility::test_memory_contiguity(); test_wavenet_gating_compatibility::TestWavenetGatingCompatibility::test_multiple_channels(); + test_gating_activations::TestBlendingActivation::test_basic_functionality(); + test_gating_activations::TestBlendingActivation::test_blending_behavior(); + test_gating_activations::TestBlendingActivation::test_with_custom_activations(); + test_gating_activations::TestBlendingActivation::test_error_handling(); + test_gating_activations::TestBlendingActivation::test_edge_cases(); + + // Detailed blending tests + test_blending_detailed::TestBlendingDetailed::test_blending_with_different_activations(); + test_blending_detailed::TestBlendingDetailed::test_input_buffer_usage(); + + // Input buffer verification tests + test_input_buffer_verification::TestInputBufferVerification::test_buffer_stores_pre_activation_values(); + test_input_buffer_verification::TestInputBufferVerification::test_buffer_with_different_activations(); + std::cout << "Success!" << std::endl; return 0; } diff --git a/tools/test/test_blending_detailed.cpp b/tools/test/test_blending_detailed.cpp new file mode 100644 index 0000000..2505982 --- /dev/null +++ b/tools/test/test_blending_detailed.cpp @@ -0,0 +1,112 @@ +// Detailed test for BlendingActivation behavior + +#include +#include +#include +#include +#include + +#include "NAM/gating_activations.h" +#include "NAM/activations.h" + +namespace test_blending_detailed +{ + +class TestBlendingDetailed +{ +public: + static void test_blending_with_different_activations() + { + // Test case: 2 input channels, so we need 4 total input channels (2*channels in) + Eigen::MatrixXf input(4, 2); // 4 rows (2 input + 2 blending), 2 samples + input << 1.0f, 2.0f, // Input channel 1 + 3.0f, 4.0f, // Input channel 2 + 0.5f, 0.8f, // Blending channel 1 + 0.3f, 0.6f; // Blending channel 2 + + Eigen::MatrixXf output(2, 2); // 2 output channels, 2 samples + + // Test with default (linear) activations + nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 2); + blending_act.apply(input, output); + + std::cout << "Blending with linear activations:" << std::endl; + std::cout << "Input:" << std::endl << input << std::endl; + std::cout << "Output:" << std::endl << output << std::endl; + + // With linear activations: + // alpha = blend_input (since linear activation does nothing) + // output = alpha * input + (1 - alpha) * input = input + // So output should equal the input channels after activation (which is the same as input) + assert(fabs(output(0, 0) - 1.0f) < 1e-6); + assert(fabs(output(1, 0) - 3.0f) < 1e-6); + assert(fabs(output(0, 1) - 2.0f) < 1e-6); + assert(fabs(output(1, 1) - 4.0f) < 1e-6); + + // Test with sigmoid blending activation + nam::activations::Activation* sigmoid_act = nam::activations::Activation::get_activation("Sigmoid"); + nam::gating_activations::BlendingActivation blending_act_sigmoid(nullptr, sigmoid_act, 2); + + Eigen::MatrixXf output_sigmoid(2, 2); + blending_act_sigmoid.apply(input, output_sigmoid); + + std::cout << "Blending with sigmoid blending activation:" << std::endl; + std::cout << "Output:" << std::endl << output_sigmoid << std::endl; + + // With sigmoid blending, alpha values should be between 0 and 1 + // For blend input 0.5, sigmoid(0.5) ≈ 0.622 + // For blend input 0.8, sigmoid(0.8) ≈ 0.690 + // For blend input 0.3, sigmoid(0.3) ≈ 0.574 + // For blend input 0.6, sigmoid(0.6) ≈ 0.646 + + float alpha0_0 = 1.0f / (1.0f + expf(-0.5f)); // sigmoid(0.5) + float alpha1_0 = 1.0f / (1.0f + expf(-0.8f)); // sigmoid(0.8) + float alpha0_1 = 1.0f / (1.0f + expf(-0.3f)); // sigmoid(0.3) + float alpha1_1 = 1.0f / (1.0f + expf(-0.6f)); // sigmoid(0.6) + + // Expected output: alpha * activated_input + (1 - alpha) * pre_activation_input + // Since input activation is linear, activated_input = pre_activation_input = input + // So output = alpha * input + (1 - alpha) * input = input + // This should be the same as with linear activations + assert(fabs(output_sigmoid(0, 0) - 1.0f) < 1e-6); + assert(fabs(output_sigmoid(1, 0) - 3.0f) < 1e-6); + assert(fabs(output_sigmoid(0, 1) - 2.0f) < 1e-6); + assert(fabs(output_sigmoid(1, 1) - 4.0f) < 1e-6); + + std::cout << "Blending detailed test passed" << std::endl; + } + + static void test_input_buffer_usage() + { + // Test that the input buffer is correctly storing pre-activation values + Eigen::MatrixXf input(2, 1); + input << 2.0f, 0.5f; + + Eigen::MatrixXf output(1, 1); + + // Test with ReLU activation on input (which will change values < 0 to 0) + nam::activations::ActivationReLU relu_act; + nam::gating_activations::BlendingActivation blending_act(&relu_act, nullptr, 1); + + blending_act.apply(input, output); + + // With input=2.0, ReLU(2.0)=2.0, blend=0.5 + // output = 0.5 * 2.0 + (1 - 0.5) * 2.0 = 0.5 * 2.0 + 0.5 * 2.0 = 2.0 + assert(fabs(output(0, 0) - 2.0f) < 1e-6); + + // Test with negative input value + Eigen::MatrixXf input2(2, 1); + input2 << -1.0f, 0.5f; + + Eigen::MatrixXf output2(1, 1); + blending_act.apply(input2, output2); + + // With input=-1.0, ReLU(-1.0)=0.0, blend=0.5 + // output = 0.5 * 0.0 + (1 - 0.5) * (-1.0) = 0.0 + 0.5 * (-1.0) = -0.5 + assert(fabs(output2(0, 0) - (-0.5f)) < 1e-6); + + std::cout << "Input buffer usage test passed" << std::endl; + } +}; + +}; // namespace test_blending_detailed \ No newline at end of file diff --git a/tools/test/test_gating_activations.cpp b/tools/test/test_gating_activations.cpp index ce402ea..31229ba 100644 --- a/tools/test/test_gating_activations.cpp +++ b/tools/test/test_gating_activations.cpp @@ -17,6 +17,7 @@ class TestGatingActivation public: static void test_basic_functionality() { + std::cout << "test_basic_functionality" << std::endl; // Create test input data (2 rows, 3 columns) Eigen::MatrixXf input(2, 3); input << 1.0f, -1.0f, 0.0f, 0.5f, 0.8f, 1.0f; @@ -40,6 +41,7 @@ class TestGatingActivation static void test_with_custom_activations() { + std::cout << "test_with_custom_activations" << std::endl; // Create custom activations nam::activations::ActivationLeakyReLU leaky_relu(0.01f); nam::activations::ActivationLeakyReLU leaky_relu2(0.05f); @@ -65,21 +67,12 @@ class TestGatingActivation static void test_error_handling() { - // Test with insufficient rows - Eigen::MatrixXf input(1, 3); // Only 1 row - Eigen::MatrixXf output; - - nam::gating_activations::GatingActivation gating_act; - - try - { - gating_act.apply(input, output); - assert(false); // Should not reach here - } - catch (const std::invalid_argument& e) - { - std::cout << "GatingActivation error handling test passed: " << e.what() << std::endl; - } + // Test with insufficient rows - should assert + // In real-time code, we use asserts instead of exceptions for performance + // These tests would normally crash the program due to asserts + // In production, these conditions should never occur if the code is used correctly + + std::cout << "GatingActivation error handling tests skipped (asserts in real-time code)" << std::endl; } }; @@ -94,8 +87,8 @@ class TestBlendingActivation Eigen::MatrixXf output(1, 3); - // Create blending activation with default alpha (0.5) - nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 0.5f, 1, 1); + // Create blending activation (1 input channel) + nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 1); // Apply the activation blending_act.apply(input, output); @@ -107,41 +100,45 @@ class TestBlendingActivation std::cout << "BlendingActivation basic test passed" << std::endl; } - static void test_different_alpha_values() + static void test_blending_behavior() { - // Test with alpha = 0.0 (should use only second row) + // Test blending with different activation functions + // Create test input data (2 rows, 2 columns) Eigen::MatrixXf input(2, 2); - input << 1.0f, -1.0f, 2.0f, 3.0f; + input << 1.0f, -1.0f, 0.5f, 0.8f; Eigen::MatrixXf output(1, 2); - nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 0.0f, 1, 1); + // Test with default (linear) activations + nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 1); blending_act.apply(input, output); - // With alpha=0.0, output should be close to second row - assert(fabs(output(0, 0) - 2.0f) < 1e-6); - assert(fabs(output(0, 1) - 3.0f) < 1e-6); - - // Test with alpha = 1.0 (should use only first row) - nam::gating_activations::BlendingActivation blending_act2(nullptr, nullptr, 1.0f, 1, 1); - blending_act2.apply(input, output); - - // With alpha=1.0, output should be close to first row + // With linear activations, blending should be: + // alpha = blend_input (since linear activation does nothing) + // output = alpha * input + (1 - alpha) * input = input + // So output should equal the first row (input after activation) assert(fabs(output(0, 0) - 1.0f) < 1e-6); assert(fabs(output(0, 1) - (-1.0f)) < 1e-6); - // Test with alpha = 0.3 - nam::gating_activations::BlendingActivation blending_act3(nullptr, nullptr, 0.3f, 1, 1); - blending_act3.apply(input, output); + // Test with sigmoid blending activation + nam::activations::Activation* sigmoid_act = nam::activations::Activation::get_activation("Sigmoid"); + nam::gating_activations::BlendingActivation blending_act2(nullptr, sigmoid_act, 1); + blending_act2.apply(input, output); - // With alpha=0.3, output should be 0.3*row1 + 0.7*row2 - float expected0 = 0.3f * 1.0f + 0.7f * 2.0f; - float expected1 = 0.3f * (-1.0f) + 0.7f * 3.0f; + // With sigmoid blending, alpha values should be between 0 and 1 + // For input 0.5, sigmoid(0.5) ≈ 0.622 + // For input 0.8, sigmoid(0.8) ≈ 0.690 + float alpha0 = 1.0f / (1.0f + expf(-0.5f)); // sigmoid(0.5) + float alpha1 = 1.0f / (1.0f + expf(-0.8f)); // sigmoid(0.8) - assert(fabs(output(0, 0) - expected0) < 1e-6); - assert(fabs(output(0, 1) - expected1) < 1e-6); + // Expected output: alpha * activated_input + (1 - alpha) * pre_activation_input + // Since input activation is linear, activated_input = pre_activation_input = input + // So output = alpha * input + (1 - alpha) * input = input + // This is the same as with linear activations + assert(fabs(output(0, 0) - 1.0f) < 1e-6); + assert(fabs(output(0, 1) - (-1.0f)) < 1e-6); - std::cout << "BlendingActivation alpha values test passed" << std::endl; + std::cout << "BlendingActivation blending behavior test passed" << std::endl; } static void test_with_custom_activations() @@ -156,8 +153,8 @@ class TestBlendingActivation Eigen::MatrixXf output(1, 2); - // Create blending activation with custom activations and alpha = 0.7 - nam::gating_activations::BlendingActivation blending_act(&leaky_relu, &leaky_relu2, 0.7f, 1, 1); + // Create blending activation with custom activations + nam::gating_activations::BlendingActivation blending_act(&leaky_relu, &leaky_relu2, 1); // Apply the activation blending_act.apply(input, output); @@ -171,42 +168,21 @@ class TestBlendingActivation static void test_error_handling() { - // Test with insufficient rows + // Test with insufficient rows - should assert Eigen::MatrixXf input(1, 2); // Only 1 row - Eigen::MatrixXf output; - - nam::gating_activations::BlendingActivation blending_act; - - try - { - blending_act.apply(input, output); - assert(false); // Should not reach here - } - catch (const std::invalid_argument& e) - { - std::cout << "BlendingActivation error handling test passed: " << e.what() << std::endl; - } - - // Test with invalid alpha value - try - { - nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 1.5f); - assert(false); // Should not reach here - } - catch (const std::invalid_argument& e) - { - std::cout << "BlendingActivation alpha validation test passed: " << e.what() << std::endl; - } - - try - { - nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, -0.1f); - assert(false); // Should not reach here - } - catch (const std::invalid_argument& e) - { - std::cout << "BlendingActivation alpha validation test passed: " << e.what() << std::endl; - } + Eigen::MatrixXf output(1, 2); + + nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 1); + + // This should trigger an assert and terminate the program + // We can't easily test asserts in a unit test framework without special handling + // For real-time code, we rely on the asserts to catch issues during development + + // Test with invalid number of channels - should assert in constructor + // These tests would normally crash the program due to asserts + // In production, these conditions should never occur if the code is used correctly + + std::cout << "BlendingActivation error handling tests skipped (asserts in real-time code)" << std::endl; } static void test_edge_cases() @@ -217,7 +193,7 @@ class TestBlendingActivation Eigen::MatrixXf output(1, 1); - nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 0.5f, 1, 1); + nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 1); blending_act.apply(input, output); assert(fabs(output(0, 0) - 0.0f) < 1e-6); @@ -236,4 +212,4 @@ class TestBlendingActivation } }; -}; // namespace test_gating_activations \ No newline at end of file +}; // namespace test_gating_activations diff --git a/tools/test/test_input_buffer_verification.cpp b/tools/test/test_input_buffer_verification.cpp new file mode 100644 index 0000000..3a57264 --- /dev/null +++ b/tools/test/test_input_buffer_verification.cpp @@ -0,0 +1,88 @@ +// Test to verify that input buffer correctly stores pre-activation values + +#include +#include +#include +#include +#include + +#include "NAM/gating_activations.h" +#include "NAM/activations.h" + +namespace test_input_buffer_verification +{ + +class TestInputBufferVerification +{ +public: + static void test_buffer_stores_pre_activation_values() + { + // Create a test case where input activation changes the values + Eigen::MatrixXf input(2, 1); + input << -2.0f, 0.5f; // Negative input value + + Eigen::MatrixXf output(1, 1); + + // Use ReLU activation which will set negative values to 0 + nam::activations::ActivationReLU relu_act; + nam::gating_activations::BlendingActivation blending_act(&relu_act, nullptr, 1); + + // Apply the activation + blending_act.apply(input, output); + + std::cout << "Input buffer verification test:" << std::endl; + std::cout << "Input: " << input(0, 0) << " (will be modified by ReLU)" << std::endl; + std::cout << "Blend value: " << input(1, 0) << std::endl; + std::cout << "Output: " << output(0, 0) << std::endl; + + // Expected behavior: + // 1. Store pre-activation input in buffer: input_buffer = -2.0f + // 2. Apply ReLU to input: activated_input = max(-2.0f, 0) = 0.0f + // 3. Apply linear activation to blend: alpha = 0.5f (no change) + // 4. Compute output: alpha * activated_input + (1 - alpha) * input_buffer + // = 0.5f * 0.0f + 0.5f * (-2.0f) = -1.0f + + float expected = 0.5f * 0.0f + 0.5f * (-2.0f); // = -1.0f + assert(fabs(output(0, 0) - expected) < 1e-6); + + std::cout << "Expected: " << expected << std::endl; + std::cout << "Input buffer verification test passed!" << std::endl; + } + + static void test_buffer_with_different_activations() + { + // Test with LeakyReLU which modifies negative values differently + Eigen::MatrixXf input(2, 1); + input << -1.0f, 0.8f; + + Eigen::MatrixXf output(1, 1); + + // Use LeakyReLU with slope 0.1 + nam::activations::ActivationLeakyReLU leaky_relu(0.1f); + nam::gating_activations::BlendingActivation blending_act(&leaky_relu, nullptr, 1); + + blending_act.apply(input, output); + + std::cout << "LeakyReLU buffer test:" << std::endl; + std::cout << "Input: " << input(0, 0) << std::endl; + std::cout << "Blend value: " << input(1, 0) << std::endl; + std::cout << "Output: " << output(0, 0) << std::endl; + + // Expected behavior: + // 1. Store pre-activation input in buffer: input_buffer = -1.0f + // 2. Apply LeakyReLU: activated_input = (-1.0f > 0) ? -1.0f : 0.1f * -1.0f = -0.1f + // 3. Apply linear activation to blend: alpha = 0.8f + // 4. Compute output: alpha * activated_input + (1 - alpha) * input_buffer + // = 0.8f * (-0.1f) + 0.2f * (-1.0f) = -0.08f - 0.2f = -0.28f + + float activated_input = (-1.0f > 0) ? -1.0f : 0.1f * -1.0f; // = -0.1f + float expected = 0.8f * activated_input + 0.2f * (-1.0f); // = -0.28f + + assert(fabs(output(0, 0) - expected) < 1e-6); + + std::cout << "Expected: " << expected << std::endl; + std::cout << "LeakyReLU buffer test passed!" << std::endl; + } +}; + +}; // namespace test_input_buffer_verification \ No newline at end of file From bd3e03fab9276c2b9765ff38c057504cb8327ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Felipe=20Santos?= Date: Wed, 14 Jan 2026 14:11:24 -0800 Subject: [PATCH 6/9] Formatting --- tools/test/test_blending_detailed.cpp | 14 +++++++------- tools/test/test_gating_activations.cpp | 2 -- tools/test/test_input_buffer_verification.cpp | 12 ++++++------ 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/tools/test/test_blending_detailed.cpp b/tools/test/test_blending_detailed.cpp index 2505982..a0e4962 100644 --- a/tools/test/test_blending_detailed.cpp +++ b/tools/test/test_blending_detailed.cpp @@ -19,10 +19,10 @@ class TestBlendingDetailed { // Test case: 2 input channels, so we need 4 total input channels (2*channels in) Eigen::MatrixXf input(4, 2); // 4 rows (2 input + 2 blending), 2 samples - input << 1.0f, 2.0f, // Input channel 1 - 3.0f, 4.0f, // Input channel 2 - 0.5f, 0.8f, // Blending channel 1 - 0.3f, 0.6f; // Blending channel 2 + input << 1.0f, 2.0f, // Input channel 1 + 3.0f, 4.0f, // Input channel 2 + 0.5f, 0.8f, // Blending channel 1 + 0.3f, 0.6f; // Blending channel 2 Eigen::MatrixXf output(2, 2); // 2 output channels, 2 samples @@ -58,12 +58,12 @@ class TestBlendingDetailed // For blend input 0.8, sigmoid(0.8) ≈ 0.690 // For blend input 0.3, sigmoid(0.3) ≈ 0.574 // For blend input 0.6, sigmoid(0.6) ≈ 0.646 - + float alpha0_0 = 1.0f / (1.0f + expf(-0.5f)); // sigmoid(0.5) float alpha1_0 = 1.0f / (1.0f + expf(-0.8f)); // sigmoid(0.8) float alpha0_1 = 1.0f / (1.0f + expf(-0.3f)); // sigmoid(0.3) float alpha1_1 = 1.0f / (1.0f + expf(-0.6f)); // sigmoid(0.6) - + // Expected output: alpha * activated_input + (1 - alpha) * pre_activation_input // Since input activation is linear, activated_input = pre_activation_input = input // So output = alpha * input + (1 - alpha) * input = input @@ -87,7 +87,7 @@ class TestBlendingDetailed // Test with ReLU activation on input (which will change values < 0 to 0) nam::activations::ActivationReLU relu_act; nam::gating_activations::BlendingActivation blending_act(&relu_act, nullptr, 1); - + blending_act.apply(input, output); // With input=2.0, ReLU(2.0)=2.0, blend=0.5 diff --git a/tools/test/test_gating_activations.cpp b/tools/test/test_gating_activations.cpp index 31229ba..96ef517 100644 --- a/tools/test/test_gating_activations.cpp +++ b/tools/test/test_gating_activations.cpp @@ -17,7 +17,6 @@ class TestGatingActivation public: static void test_basic_functionality() { - std::cout << "test_basic_functionality" << std::endl; // Create test input data (2 rows, 3 columns) Eigen::MatrixXf input(2, 3); input << 1.0f, -1.0f, 0.0f, 0.5f, 0.8f, 1.0f; @@ -41,7 +40,6 @@ class TestGatingActivation static void test_with_custom_activations() { - std::cout << "test_with_custom_activations" << std::endl; // Create custom activations nam::activations::ActivationLeakyReLU leaky_relu(0.01f); nam::activations::ActivationLeakyReLU leaky_relu2(0.05f); diff --git a/tools/test/test_input_buffer_verification.cpp b/tools/test/test_input_buffer_verification.cpp index 3a57264..b9d8ffd 100644 --- a/tools/test/test_input_buffer_verification.cpp +++ b/tools/test/test_input_buffer_verification.cpp @@ -19,14 +19,14 @@ class TestInputBufferVerification { // Create a test case where input activation changes the values Eigen::MatrixXf input(2, 1); - input << -2.0f, 0.5f; // Negative input value + input << -2.0f, 0.5f; // Negative input value Eigen::MatrixXf output(1, 1); // Use ReLU activation which will set negative values to 0 nam::activations::ActivationReLU relu_act; nam::gating_activations::BlendingActivation blending_act(&relu_act, nullptr, 1); - + // Apply the activation blending_act.apply(input, output); @@ -41,7 +41,7 @@ class TestInputBufferVerification // 3. Apply linear activation to blend: alpha = 0.5f (no change) // 4. Compute output: alpha * activated_input + (1 - alpha) * input_buffer // = 0.5f * 0.0f + 0.5f * (-2.0f) = -1.0f - + float expected = 0.5f * 0.0f + 0.5f * (-2.0f); // = -1.0f assert(fabs(output(0, 0) - expected) < 1e-6); @@ -60,7 +60,7 @@ class TestInputBufferVerification // Use LeakyReLU with slope 0.1 nam::activations::ActivationLeakyReLU leaky_relu(0.1f); nam::gating_activations::BlendingActivation blending_act(&leaky_relu, nullptr, 1); - + blending_act.apply(input, output); std::cout << "LeakyReLU buffer test:" << std::endl; @@ -74,10 +74,10 @@ class TestInputBufferVerification // 3. Apply linear activation to blend: alpha = 0.8f // 4. Compute output: alpha * activated_input + (1 - alpha) * input_buffer // = 0.8f * (-0.1f) + 0.2f * (-1.0f) = -0.08f - 0.2f = -0.28f - + float activated_input = (-1.0f > 0) ? -1.0f : 0.1f * -1.0f; // = -0.1f float expected = 0.8f * activated_input + 0.2f * (-1.0f); // = -0.28f - + assert(fabs(output(0, 0) - expected) < 1e-6); std::cout << "Expected: " << expected << std::endl; From d14a527668235ef472658d872bab32f750e0b499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Felipe=20Santos?= Date: Wed, 14 Jan 2026 14:13:46 -0800 Subject: [PATCH 7/9] Removed extra argument that wasn't being used --- NAM/gating_activations.h | 2 +- tools/test/test_gating_activations.cpp | 4 ++-- tools/test/test_wavenet_gating_compatibility.cpp | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NAM/gating_activations.h b/NAM/gating_activations.h index 18afb0b..0d32ac4 100644 --- a/NAM/gating_activations.h +++ b/NAM/gating_activations.h @@ -36,7 +36,7 @@ class GatingActivation * @param gating_channels Number of gating channels (default: 1) */ GatingActivation(activations::Activation* input_act = nullptr, activations::Activation* gating_act = nullptr, - int input_channels = 1, int gating_channels = 1) + int input_channels = 1) : input_activation(input_act ? input_act : &default_activation) , gating_activation(gating_act ? gating_act : activations::Activation::get_activation("Sigmoid")) , num_input_channels(input_channels) diff --git a/tools/test/test_gating_activations.cpp b/tools/test/test_gating_activations.cpp index 96ef517..d42e06f 100644 --- a/tools/test/test_gating_activations.cpp +++ b/tools/test/test_gating_activations.cpp @@ -24,7 +24,7 @@ class TestGatingActivation Eigen::MatrixXf output(1, 3); // Create gating activation with default activations (1 input channel, 1 gating channel) - nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, 1, 1); + nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, 1); // Apply the activation gating_act.apply(input, output); @@ -51,7 +51,7 @@ class TestGatingActivation Eigen::MatrixXf output(1, 2); // Create gating activation with custom activations - nam::gating_activations::GatingActivation gating_act(&leaky_relu, &leaky_relu2, 1, 1); + nam::gating_activations::GatingActivation gating_act(&leaky_relu, &leaky_relu2, 1); // Apply the activation gating_act.apply(input, output); diff --git a/tools/test/test_wavenet_gating_compatibility.cpp b/tools/test/test_wavenet_gating_compatibility.cpp index 93e7046..b3fd8c2 100644 --- a/tools/test/test_wavenet_gating_compatibility.cpp +++ b/tools/test/test_wavenet_gating_compatibility.cpp @@ -33,7 +33,7 @@ class TestWavenetGatingCompatibility // Create gating activation that matches wavenet behavior // Wavenet uses: input activation (default/linear) and sigmoid for gating - nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels, channels); + nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels); // Apply the activation gating_act.apply(input, output); @@ -82,7 +82,7 @@ class TestWavenetGatingCompatibility Eigen::MatrixXf output(channels, num_samples); - nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels, channels); + nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels); gating_act.apply(input, output); // Verify each column was processed independently @@ -118,7 +118,7 @@ class TestWavenetGatingCompatibility Eigen::MatrixXf output(channels, num_samples); - nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels, channels); + nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels); // This should not crash or produce incorrect results due to memory contiguity issues gating_act.apply(input, output); @@ -153,7 +153,7 @@ class TestWavenetGatingCompatibility Eigen::MatrixXf output(channels, num_samples); - nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels, channels); + nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels); gating_act.apply(input, output); // Verify dimensions From c7ed1de17c7f23091d32b83d8fe624b0f9f9127b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Felipe=20Santos?= Date: Wed, 14 Jan 2026 15:01:59 -0800 Subject: [PATCH 8/9] Fixed small nitpicks --- NAM/gating_activations.h | 46 +++++++++++++------------- tools/run_tests.cpp | 4 +-- tools/test/test_gating_activations.cpp | 6 ---- 3 files changed, 25 insertions(+), 31 deletions(-) diff --git a/NAM/gating_activations.h b/NAM/gating_activations.h index 0d32ac4..7637788 100644 --- a/NAM/gating_activations.h +++ b/NAM/gating_activations.h @@ -39,9 +39,9 @@ class GatingActivation int input_channels = 1) : input_activation(input_act ? input_act : &default_activation) , gating_activation(gating_act ? gating_act : activations::Activation::get_activation("Sigmoid")) - , num_input_channels(input_channels) + , num_channels(input_channels) { - assert(num_input_channels > 0); + assert(num_channels > 0); } ~GatingActivation() = default; @@ -54,9 +54,9 @@ class GatingActivation void apply(Eigen::MatrixXf& input, Eigen::MatrixXf& output) { // Validate input dimensions (assert for real-time performance) - const int total_channels = 2 * num_input_channels; + const int total_channels = 2 * num_channels; assert(input.rows() == total_channels); - assert(output.rows() == num_input_channels); + assert(output.rows() == num_channels); assert(output.cols() == input.cols()); // Process column-by-column to ensure memory contiguity (important for column-major matrices) @@ -64,33 +64,33 @@ class GatingActivation for (int i = 0; i < num_samples; i++) { // Apply activation to input channels - Eigen::MatrixXf input_block = input.block(0, i, num_input_channels, 1); + Eigen::MatrixXf input_block = input.block(0, i, num_channels, 1); input_activation->apply(input_block); // Apply activation to gating channels - Eigen::MatrixXf gating_block = input.block(num_input_channels, i, num_input_channels, 1); + Eigen::MatrixXf gating_block = input.block(num_channels, i, num_channels, 1); gating_activation->apply(gating_block); // Element-wise multiplication and store result // For wavenet compatibility, we assume one-to-one mapping - output.block(0, i, num_input_channels, 1) = input_block.array() * gating_block.array(); + output.block(0, i, num_channels, 1) = input_block.array() * gating_block.array(); } } /** * Get the total number of input channels required */ - int get_total_input_channels() const { return 2 * num_input_channels; } + int get_input_channels() const { return 2 * num_channels; } /** * Get the number of output channels */ - int get_output_channels() const { return num_input_channels; } + int get_output_channels() const { return num_channels; } private: activations::Activation* input_activation; activations::Activation* gating_activation; - int num_input_channels; + int num_channels; }; class BlendingActivation @@ -106,15 +106,15 @@ class BlendingActivation int input_channels = 1) : input_activation(input_act ? input_act : &default_activation) , blending_activation(blend_act ? blend_act : &default_activation) - , num_input_channels(input_channels) + , num_channels(input_channels) { - if (num_input_channels <= 0) + if (num_channels <= 0) { throw std::invalid_argument("BlendingActivation: number of input channels must be positive"); } // Initialize input buffer with correct size - // Note: current code copies column-by-column so we only need (num_input_channels, 1) - input_buffer.resize(num_input_channels, 1); + // Note: current code copies column-by-column so we only need (num_channels, 1) + input_buffer.resize(num_channels, 1); } ~BlendingActivation() = default; @@ -127,9 +127,9 @@ class BlendingActivation void apply(Eigen::MatrixXf& input, Eigen::MatrixXf& output) { // Validate input dimensions (assert for real-time performance) - const int total_channels = num_input_channels * 2; // 2*channels in, channels out + const int total_channels = num_channels * 2; // 2*channels in, channels out assert(input.rows() == total_channels); - assert(output.rows() == num_input_channels); + assert(output.rows() == num_channels); assert(output.cols() == input.cols()); // Process column-by-column to ensure memory contiguity @@ -137,18 +137,18 @@ class BlendingActivation for (int i = 0; i < num_samples; i++) { // Store pre-activation input values in buffer - input_buffer = input.block(0, i, num_input_channels, 1); + input_buffer = input.block(0, i, num_channels, 1); // Apply activation to input channels - Eigen::MatrixXf input_block = input.block(0, i, num_input_channels, 1); + Eigen::MatrixXf input_block = input.block(0, i, num_channels, 1); input_activation->apply(input_block); // Apply activation to blend channels to compute alpha - Eigen::MatrixXf blend_block = input.block(num_input_channels, i, num_input_channels, 1); + Eigen::MatrixXf blend_block = input.block(num_channels, i, num_channels, 1); blending_activation->apply(blend_block); // Weighted blending: alpha * activated_input + (1 - alpha) * pre_activation_input - output.block(0, i, num_input_channels, 1) = + output.block(0, i, num_channels, 1) = blend_block.array() * input_block.array() + (1.0f - blend_block.array()) * input_buffer.array(); } } @@ -156,17 +156,17 @@ class BlendingActivation /** * Get the total number of input channels required */ - int get_total_input_channels() const { return 2 * num_input_channels; } + int get_input_channels() const { return 2 * num_channels; } /** * Get the number of output channels */ - int get_output_channels() const { return num_input_channels; } + int get_output_channels() const { return num_channels; } private: activations::Activation* input_activation; activations::Activation* blending_activation; - int num_input_channels; + int num_channels; Eigen::MatrixXf input_buffer; }; diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp index ae1922a..26b791c 100644 --- a/tools/run_tests.cpp +++ b/tools/run_tests.cpp @@ -46,7 +46,7 @@ int main() // Gating activations tests test_gating_activations::TestGatingActivation::test_basic_functionality(); test_gating_activations::TestGatingActivation::test_with_custom_activations(); - test_gating_activations::TestGatingActivation::test_error_handling(); + // test_gating_activations::TestGatingActivation::test_error_handling(); // Wavenet gating compatibility tests test_wavenet_gating_compatibility::TestWavenetGatingCompatibility::test_wavenet_style_gating(); @@ -57,7 +57,7 @@ int main() test_gating_activations::TestBlendingActivation::test_basic_functionality(); test_gating_activations::TestBlendingActivation::test_blending_behavior(); test_gating_activations::TestBlendingActivation::test_with_custom_activations(); - test_gating_activations::TestBlendingActivation::test_error_handling(); + // test_gating_activations::TestBlendingActivation::test_error_handling(); test_gating_activations::TestBlendingActivation::test_edge_cases(); // Detailed blending tests diff --git a/tools/test/test_gating_activations.cpp b/tools/test/test_gating_activations.cpp index d42e06f..47de837 100644 --- a/tools/test/test_gating_activations.cpp +++ b/tools/test/test_gating_activations.cpp @@ -69,8 +69,6 @@ class TestGatingActivation // In real-time code, we use asserts instead of exceptions for performance // These tests would normally crash the program due to asserts // In production, these conditions should never occur if the code is used correctly - - std::cout << "GatingActivation error handling tests skipped (asserts in real-time code)" << std::endl; } }; @@ -179,8 +177,6 @@ class TestBlendingActivation // Test with invalid number of channels - should assert in constructor // These tests would normally crash the program due to asserts // In production, these conditions should never occur if the code is used correctly - - std::cout << "BlendingActivation error handling tests skipped (asserts in real-time code)" << std::endl; } static void test_edge_cases() @@ -205,8 +201,6 @@ class TestBlendingActivation // Should handle large values without issues assert(output.rows() == 1); assert(output.cols() == 1); - - std::cout << "BlendingActivation edge cases test passed" << std::endl; } }; From 5d48e3abb48c6f8a115776caaf59742bd4e60c8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Felipe=20Santos?= Date: Wed, 14 Jan 2026 15:25:13 -0800 Subject: [PATCH 9/9] Removed default activations, moved ActivationIdentity to activations.h --- NAM/activations.h | 9 ++++++++ NAM/gating_activations.h | 21 +++++++----------- tools/run_tests.cpp | 2 +- tools/test/test_blending_detailed.cpp | 9 +++++--- tools/test/test_gating_activations.cpp | 22 ++++++++++++++----- tools/test/test_input_buffer_verification.cpp | 6 +++-- .../test_wavenet_gating_compatibility.cpp | 16 ++++++++++---- 7 files changed, 56 insertions(+), 29 deletions(-) diff --git a/NAM/activations.h b/NAM/activations.h index 3e77614..4429964 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -111,6 +111,15 @@ class Activation static std::unordered_map _activations; }; +// identity function activation +class ActivationIdentity : public nam::activations::Activation +{ +public: + ActivationIdentity() = default; + ~ActivationIdentity() = default; + // Inherit the default apply methods which do nothing +}; + class ActivationTanh : public Activation { public: diff --git a/NAM/gating_activations.h b/NAM/gating_activations.h index 7637788..3436cdd 100644 --- a/NAM/gating_activations.h +++ b/NAM/gating_activations.h @@ -22,23 +22,19 @@ class IdentityActivation : public nam::activations::Activation // Inherit the default apply methods which do nothing (linear/identity) }; -// Static instance for default activation -static IdentityActivation default_activation; - class GatingActivation { public: /** * Constructor for GatingActivation - * @param input_act Activation function for input channels (default: linear) - * @param gating_act Activation function for gating channels (default: sigmoid) + * @param input_act Activation function for input channels + * @param gating_act Activation function for gating channels * @param input_channels Number of input channels (default: 1) * @param gating_channels Number of gating channels (default: 1) */ - GatingActivation(activations::Activation* input_act = nullptr, activations::Activation* gating_act = nullptr, - int input_channels = 1) - : input_activation(input_act ? input_act : &default_activation) - , gating_activation(gating_act ? gating_act : activations::Activation::get_activation("Sigmoid")) + GatingActivation(activations::Activation* input_act, activations::Activation* gating_act, int input_channels = 1) + : input_activation(input_act) + , gating_activation(gating_act) , num_channels(input_channels) { assert(num_channels > 0); @@ -102,10 +98,9 @@ class BlendingActivation * @param blend_act Activation function for blending channels * @param input_channels Number of input channels */ - BlendingActivation(activations::Activation* input_act = nullptr, activations::Activation* blend_act = nullptr, - int input_channels = 1) - : input_activation(input_act ? input_act : &default_activation) - , blending_activation(blend_act ? blend_act : &default_activation) + BlendingActivation(activations::Activation* input_act, activations::Activation* blend_act, int input_channels = 1) + : input_activation(input_act) + , blending_activation(blend_act) , num_channels(input_channels) { if (num_channels <= 0) diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp index 1079a9c..2aa66ec 100644 --- a/tools/run_tests.cpp +++ b/tools/run_tests.cpp @@ -28,7 +28,7 @@ int main() test_activations::TestPReLU::test_core_function(); test_activations::TestPReLU::test_per_channel_behavior(); // This is enforced by an assert so it doesn't need to be tested - //test_activations::TestPReLU::test_wrong_number_of_channels(); + // test_activations::TestPReLU::test_wrong_number_of_channels(); test_dsp::test_construct(); test_dsp::test_get_input_level(); diff --git a/tools/test/test_blending_detailed.cpp b/tools/test/test_blending_detailed.cpp index a0e4962..b526ae9 100644 --- a/tools/test/test_blending_detailed.cpp +++ b/tools/test/test_blending_detailed.cpp @@ -27,7 +27,9 @@ class TestBlendingDetailed Eigen::MatrixXf output(2, 2); // 2 output channels, 2 samples // Test with default (linear) activations - nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 2); + nam::activations::ActivationIdentity identity_act; + nam::activations::ActivationIdentity identity_blend_act; + nam::gating_activations::BlendingActivation blending_act(&identity_act, &identity_blend_act, 2); blending_act.apply(input, output); std::cout << "Blending with linear activations:" << std::endl; @@ -45,7 +47,7 @@ class TestBlendingDetailed // Test with sigmoid blending activation nam::activations::Activation* sigmoid_act = nam::activations::Activation::get_activation("Sigmoid"); - nam::gating_activations::BlendingActivation blending_act_sigmoid(nullptr, sigmoid_act, 2); + nam::gating_activations::BlendingActivation blending_act_sigmoid(&identity_act, sigmoid_act, 2); Eigen::MatrixXf output_sigmoid(2, 2); blending_act_sigmoid.apply(input, output_sigmoid); @@ -86,7 +88,8 @@ class TestBlendingDetailed // Test with ReLU activation on input (which will change values < 0 to 0) nam::activations::ActivationReLU relu_act; - nam::gating_activations::BlendingActivation blending_act(&relu_act, nullptr, 1); + nam::activations::ActivationIdentity identity_act; + nam::gating_activations::BlendingActivation blending_act(&relu_act, &identity_act, 1); blending_act.apply(input, output); diff --git a/tools/test/test_gating_activations.cpp b/tools/test/test_gating_activations.cpp index 47de837..43e414f 100644 --- a/tools/test/test_gating_activations.cpp +++ b/tools/test/test_gating_activations.cpp @@ -24,7 +24,9 @@ class TestGatingActivation Eigen::MatrixXf output(1, 3); // Create gating activation with default activations (1 input channel, 1 gating channel) - nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, 1); + nam::activations::ActivationIdentity identity_act; + nam::activations::ActivationSigmoid sigmoid_act; + nam::gating_activations::GatingActivation gating_act(&identity_act, &sigmoid_act, 1); // Apply the activation gating_act.apply(input, output); @@ -84,7 +86,9 @@ class TestBlendingActivation Eigen::MatrixXf output(1, 3); // Create blending activation (1 input channel) - nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 1); + nam::activations::ActivationIdentity identity_act; + nam::activations::ActivationIdentity identity_blend_act; + nam::gating_activations::BlendingActivation blending_act(&identity_act, &identity_blend_act, 1); // Apply the activation blending_act.apply(input, output); @@ -106,7 +110,9 @@ class TestBlendingActivation Eigen::MatrixXf output(1, 2); // Test with default (linear) activations - nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 1); + nam::activations::ActivationIdentity identity_act; + nam::activations::ActivationIdentity identity_blend_act; + nam::gating_activations::BlendingActivation blending_act(&identity_act, &identity_blend_act, 1); blending_act.apply(input, output); // With linear activations, blending should be: @@ -118,7 +124,7 @@ class TestBlendingActivation // Test with sigmoid blending activation nam::activations::Activation* sigmoid_act = nam::activations::Activation::get_activation("Sigmoid"); - nam::gating_activations::BlendingActivation blending_act2(nullptr, sigmoid_act, 1); + nam::gating_activations::BlendingActivation blending_act2(&identity_act, sigmoid_act, 1); blending_act2.apply(input, output); // With sigmoid blending, alpha values should be between 0 and 1 @@ -168,7 +174,9 @@ class TestBlendingActivation Eigen::MatrixXf input(1, 2); // Only 1 row Eigen::MatrixXf output(1, 2); - nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 1); + nam::activations::ActivationIdentity identity_act; + nam::activations::ActivationIdentity identity_blend_act; + nam::gating_activations::BlendingActivation blending_act(&identity_act, &identity_blend_act, 1); // This should trigger an assert and terminate the program // We can't easily test asserts in a unit test framework without special handling @@ -187,7 +195,9 @@ class TestBlendingActivation Eigen::MatrixXf output(1, 1); - nam::gating_activations::BlendingActivation blending_act(nullptr, nullptr, 1); + nam::activations::ActivationIdentity identity_act; + nam::activations::ActivationIdentity identity_blend_act; + nam::gating_activations::BlendingActivation blending_act(&identity_act, &identity_blend_act, 1); blending_act.apply(input, output); assert(fabs(output(0, 0) - 0.0f) < 1e-6); diff --git a/tools/test/test_input_buffer_verification.cpp b/tools/test/test_input_buffer_verification.cpp index b9d8ffd..d7d280e 100644 --- a/tools/test/test_input_buffer_verification.cpp +++ b/tools/test/test_input_buffer_verification.cpp @@ -25,7 +25,8 @@ class TestInputBufferVerification // Use ReLU activation which will set negative values to 0 nam::activations::ActivationReLU relu_act; - nam::gating_activations::BlendingActivation blending_act(&relu_act, nullptr, 1); + nam::activations::ActivationIdentity identity_act; + nam::gating_activations::BlendingActivation blending_act(&relu_act, &identity_act, 1); // Apply the activation blending_act.apply(input, output); @@ -59,7 +60,8 @@ class TestInputBufferVerification // Use LeakyReLU with slope 0.1 nam::activations::ActivationLeakyReLU leaky_relu(0.1f); - nam::gating_activations::BlendingActivation blending_act(&leaky_relu, nullptr, 1); + nam::activations::ActivationIdentity identity_act; + nam::gating_activations::BlendingActivation blending_act(&leaky_relu, &identity_act, 1); blending_act.apply(input, output); diff --git a/tools/test/test_wavenet_gating_compatibility.cpp b/tools/test/test_wavenet_gating_compatibility.cpp index b3fd8c2..44af69b 100644 --- a/tools/test/test_wavenet_gating_compatibility.cpp +++ b/tools/test/test_wavenet_gating_compatibility.cpp @@ -33,7 +33,9 @@ class TestWavenetGatingCompatibility // Create gating activation that matches wavenet behavior // Wavenet uses: input activation (default/linear) and sigmoid for gating - nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels); + nam::activations::ActivationIdentity identity_act; + nam::activations::ActivationSigmoid sigmoid_act; + nam::gating_activations::GatingActivation gating_act(&identity_act, &sigmoid_act, channels); // Apply the activation gating_act.apply(input, output); @@ -82,7 +84,9 @@ class TestWavenetGatingCompatibility Eigen::MatrixXf output(channels, num_samples); - nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels); + nam::activations::ActivationIdentity identity_act; + nam::activations::ActivationSigmoid sigmoid_act; + nam::gating_activations::GatingActivation gating_act(&identity_act, &sigmoid_act, channels); gating_act.apply(input, output); // Verify each column was processed independently @@ -118,7 +122,9 @@ class TestWavenetGatingCompatibility Eigen::MatrixXf output(channels, num_samples); - nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels); + nam::activations::ActivationIdentity identity_act; + nam::activations::ActivationSigmoid sigmoid_act; + nam::gating_activations::GatingActivation gating_act(&identity_act, &sigmoid_act, channels); // This should not crash or produce incorrect results due to memory contiguity issues gating_act.apply(input, output); @@ -153,7 +159,9 @@ class TestWavenetGatingCompatibility Eigen::MatrixXf output(channels, num_samples); - nam::gating_activations::GatingActivation gating_act(nullptr, nullptr, channels); + nam::activations::ActivationIdentity identity_act; + nam::activations::ActivationSigmoid sigmoid_act; + nam::gating_activations::GatingActivation gating_act(&identity_act, &sigmoid_act, channels); gating_act.apply(input, output); // Verify dimensions