LeakyReLU.hpp

Go to the documentation of this file.

 
#pragma once
 
#include <Eigen/Dense>
 
namespace FlexNN::Activations::detail {
 
// Default kept for backward compat; prefer Activations::kDefaultLeakyAlpha.
inline constexpr double kLeakyAlpha = 0.01;
 
inline Eigen::MatrixXd leaky_relu_forward(const Eigen::MatrixXd& Z,
                                          double alpha = kLeakyAlpha) {
  return Z.unaryExpr([alpha](double x) { return x > 0.0 ? x : alpha * x; });
}
 
inline Eigen::MatrixXd leaky_relu_backward(const Eigen::MatrixXd& dA,
                                           const Eigen::MatrixXd& Z,
                                           const Eigen::MatrixXd& /*A*/,
                                           double alpha = kLeakyAlpha) noexcept {
  return dA.array() * (Z.array() > 0.0).cast<double>() +
         dA.array() * (Z.array() <= 0.0).cast<double>() * alpha;
  // Equivalent to: (Z>0 ? 1 : alpha) elementwise, keeps Eigen vectorization.
}
 
} // namespace FlexNN::Activations::detail