Sam 11 si lu

src/NeuralNet/ActivationFunctions/GELU/GELU.php

@@ -33,13 +33,17 @@ class GELU implements ActivationFunction, IBufferDerivative
      * @var float
      */
     protected const ALPHA = 0.7978845608;
+    /** @var float 0.5 * ALPHA */
+    protected const HALF_ALPHA = 0.3989422804;
 
     /**
      * Gaussian error function approximation term.
      *
      * @var float
      */
     protected const BETA = 0.044715;
+    /** @var float 3 * BETA */
+    protected const TRIPLE_BETA = 0.134145;
 
     /**
      * Apply the GeLU activation function to the input.
@@ -57,21 +61,21 @@ public function activate(NDArray $input) : NDArray
         // Calculate inner term: x + BETA * x^3
         $innerTerm = NumPower::add(
             $input,
-            NumPower::multiply(self::BETA, $cubed)
+            NumPower::multiply($cubed, self::BETA)
         );
 
         // Apply tanh(ALPHA * innerTerm)
         $tanhTerm = NumPower::tanh(
-            NumPower::multiply(self::ALPHA, $innerTerm)
+            NumPower::multiply($innerTerm, self::ALPHA)
         );
 
         // Calculate 1 + tanhTerm
         $onePlusTanh = NumPower::add(1.0, $tanhTerm);
 
         // Calculate 0.5 * x * (1 + tanhTerm)
         return NumPower::multiply(
-            0.5,
-            NumPower::multiply($input, $onePlusTanh)
+            NumPower::multiply($input, $onePlusTanh),
+            0.5
         );
     }
 
@@ -97,11 +101,11 @@ public function differentiate(NDArray $input) : NDArray
 
         // Calculate inner term: ALPHA * (x + BETA * x^3)
         $innerTerm = NumPower::multiply(
-            self::ALPHA,
             NumPower::add(
                 $input,
-                NumPower::multiply(self::BETA, $cubed)
-            )
+                NumPower::multiply($cubed, self::BETA)
+            ),
+            self::ALPHA
         );
 
         // Calculate cosh and sech^2
@@ -113,24 +117,24 @@ public function differentiate(NDArray $input) : NDArray
 
         // Calculate 0.5 * (1 + tanh(innerTerm))
         $firstTerm = NumPower::multiply(
-            0.5,
-            NumPower::add(1.0, NumPower::tanh($innerTerm))
+            NumPower::add(1.0, NumPower::tanh($innerTerm)),
+            0.5
         );
 
         // Calculate 0.5 * x * sech^2 * ALPHA * (1 + 3 * BETA * x^2)
         $secondTerm = NumPower::multiply(
             NumPower::multiply(
                 NumPower::multiply(
-                    0.5 * self::ALPHA,
-                    $input
+                    $input,
+                    self::HALF_ALPHA
                 ),
                 $sech2
             ),
             NumPower::add(
                 1.0,
                 NumPower::multiply(
-                    3.0 * self::BETA,
-                    NumPower::pow($input, 2)
+                    NumPower::pow($input, 2),
+                    self::TRIPLE_BETA
                 )
             )
         );

src/NeuralNet/ActivationFunctions/HardSigmoid/HardSigmoid.php

@@ -65,7 +65,7 @@ public function activate(NDArray $input) : NDArray
     {
         // Calculate 0.2 * x + 0.5
         $linear = NumPower::add(
-            NumPower::multiply(self::SLOPE, $input),
+            NumPower::multiply($input, self::SLOPE),
             self::INTERCEPT
         );
 

src/NeuralNet/ActivationFunctions/SELU/SELU.php

@@ -61,15 +61,15 @@ public function activate(NDArray $input) : NDArray
     {
         // Calculate positive part: λ * x for x > 0
         $positive = NumPower::multiply(
-            self::LAMBDA,
-            NumPower::maximum($input, 0)
+            NumPower::maximum($input, 0),
+            self::LAMBDA
         );
 
         // Calculate negative part: λ * α * (e^x - 1) for x <= 0
         $negativeMask = NumPower::minimum($input, 0);
         $negative = NumPower::multiply(
-            self::BETA,
-            NumPower::expm1($negativeMask)
+            NumPower::expm1($negativeMask),
+            self::BETA
         );
 
         // Combine both parts

src/NeuralNet/ActivationFunctions/SiLU/SiLU.php

@@ -1,8 +1,14 @@
 <?php
 
+declare(strict_types=1);
+
 namespace Rubix\ML\NeuralNet\ActivationFunctions\SiLU;
 
-use Tensor\Matrix;
+use NumPower;
+use NDArray;
+use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\ActivationFunction;
+use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\IBufferDerivative;
+use Rubix\ML\NeuralNet\ActivationFunctions\Sigmoid\Sigmoid;
 
 /**
  * SiLU
@@ -17,34 +23,64 @@
  * @category    Machine Learning
  * @package     Rubix/ML
  * @author      Andrew DalPino
+ * @author      Samuel Akopyan <[email protected]>
  */
-class SiLU implements ActivationFunction
+class SiLU implements ActivationFunction, IBufferDerivative
 {
+    /**
+     * The Sigmoid activation function.
+     *
+     * @var Sigmoid
+     */
+    protected Sigmoid $sigmoid;
+
+    /**
+     * Class constructor.
+     */
+    public function __construct()
+    {
+        $this->sigmoid = new Sigmoid();
+    }
+
     /**
      * Compute the activation.
      *
-     * @internal
+     * f(x) = x * sigmoid(x) = x / (1 + e^(-x))
      *
-     * @param Matrix $input
-     * @return Matrix
+     * @param NDArray $input
+     * @return NDArray
      */
-    public function activate(Matrix $input) : Matrix
+    public function activate(NDArray $input) : NDArray
     {
-        return $input / (1.0 + NumPower::exp(-$input));
+        // Calculate sigmoid(x) using the Sigmoid activation function
+        $sigmoid = $this->sigmoid->activate($input);
+
+        // Calculate x * sigmoid(x)
+        return NumPower::multiply($input, $sigmoid);
     }
 
     /**
      * Calculate the derivative of the activation.
      *
-     * @internal
+     * f'(x) = sigmoid(x) + x * sigmoid(x) * (1 - sigmoid(x))
+     *        = sigmoid(x) + x * sigmoid'(x)
      *
-     * @param Matrix $input
-     * @param Matrix $output
-     * @return Matrix
+     * @param NDArray $input Input matrix
+     * @return NDArray Derivative matrix
      */
-    public function differentiate(Matrix $input, Matrix $output) : Matrix
+    public function differentiate(NDArray $input) : NDArray
     {
-        return $output / $input * NumPower::ones($output->shape()) / $output * 2;
+        // Calculate sigmoid(x) using the Sigmoid activation function
+        $sigmoid = $this->sigmoid->activate($input);
+
+        // Calculate sigmoid'(x) = sigmoid(x) * (1 - sigmoid(x))
+        $sigmoidDerivative = $this->sigmoid->differentiate($sigmoid);
+
+        // Calculate x * sigmoid'(x)
+        $xTimesSigmoidDerivative = NumPower::multiply($input, $sigmoidDerivative);
+
+        // Calculate sigmoid(x) + x * sigmoid'(x)
+        return NumPower::add($sigmoid, $xTimesSigmoidDerivative);
     }
 
     /**

src/NeuralNet/ActivationFunctions/Sigmoid/Sigmoid.php

@@ -34,11 +34,11 @@ class Sigmoid implements ActivationFunction, OBufferDerivative
     public function activate(NDArray $input) : NDArray
     {
         // Calculate e^(-x)
-        $negExp = NumPower::exp(NumPower::multiply(-1.0, $input));
-        
+        $negExp = NumPower::exp(NumPower::multiply($input, -1.0));
+
         // Calculate 1 + e^(-x)
         $denominator = NumPower::add(1.0, $negExp);
-        
+
         // Calculate 1 / (1 + e^(-x))
         return NumPower::divide(1.0, $denominator);
     }
@@ -57,7 +57,7 @@ public function differentiate(NDArray $output) : NDArray
     {
         // Calculate (1 - output)
         $oneMinusOutput = NumPower::subtract(1.0, $output);
-        
+
         // Calculate output * (1 - output)
         return NumPower::multiply($output, $oneMinusOutput);
     }

tests/NeuralNet/ActivationFunctions/SELU/SELUTest.php

@@ -19,11 +19,6 @@
 #[CoversClass(SELU::class)]
 class SELUTest extends TestCase
 {
-    /**
-     * @var SELU
-     */
-    protected SELU $activationFn;
-
     /**
      * The value at which leakage starts to saturate.
      *
@@ -45,6 +40,11 @@ class SELUTest extends TestCase
      */
     protected const BETA = self::LAMBDA * self::ALPHA;
 
+    /**
+     * @var SELU
+     */
+    protected SELU $activationFn;
+
     /**
      * @return Generator<array>
      */
@@ -55,14 +55,7 @@ public static function computeProvider() : Generator
                 [2.0, 1.0, -0.5, 0.0, 20.0, -10.0],
             ]),
             [
-                [
-                    2.10140180,
-                    1.05070090,
-                    -0.6917580,
-                    0.0,
-                    21.0140190,
-                    -1.7580193
-                ],
+                [2.10140180, 1.05070090, -0.6917580, 0.0, 21.0140190, -1.7580193],
             ],
         ];
 
@@ -76,17 +69,17 @@ public static function computeProvider() : Generator
                 [
                     self::BETA * (exp(-0.12) - 1.0),
                     0.31 * self::LAMBDA,
-                    self::BETA * (exp(-0.49) - 1.0)
+                    self::BETA * (exp(-0.49) - 1.0),
                 ],
                 [
                     0.99 * self::LAMBDA,
                     0.08 * self::LAMBDA,
-                    self::BETA * (exp(-0.03) - 1.0)
+                    self::BETA * (exp(-0.03) - 1.0),
                 ],
                 [
                     0.05 * self::LAMBDA,
                     self::BETA * (exp(-0.52) - 1.0),
-                    0.54 * self::LAMBDA
+                    0.54 * self::LAMBDA,
                 ],
             ],
         ];
@@ -102,15 +95,7 @@ public static function differentiateProvider() : Generator
                 [2.0, 1.0, -0.5, 0.0, 20.0, -10.0, -20],
             ]),
             [
-                [
-                    self::LAMBDA,
-                    self::LAMBDA,
-                    1.0663410,
-                    1.7580991,
-                    self::LAMBDA,
-                    0.0000798,
-                    0.0
-                ],
+                [self::LAMBDA, self::LAMBDA, 1.0663410, 1.7580991, self::LAMBDA, 0.0000798, 0.0],
             ],
         ];
 
@@ -121,21 +106,9 @@ public static function differentiateProvider() : Generator
                 [0.05, -0.52, 0.54],
             ]),
             [
-                [
-                    self::BETA * exp(-0.12),
-                    self::LAMBDA,
-                    self::BETA * exp(-0.49)
-                ],
-                [
-                    self::LAMBDA,
-                    self::LAMBDA,
-                    self::BETA * exp(-0.03)
-                ],
-                [
-                    self::LAMBDA,
-                    self::BETA * exp(-0.52),
-                    self::LAMBDA
-                ],
+                [self::BETA * exp(-0.12), self::LAMBDA, self::BETA * exp(-0.49)],
+                [self::LAMBDA, self::LAMBDA, self::BETA * exp(-0.03)],
+                [self::LAMBDA, self::BETA * exp(-0.52), self::LAMBDA],
             ],
         ];
     }
@@ -163,18 +136,10 @@ public static function zeroRegionProvider() : Generator
         yield [
             NumPower::array([[-1e-15, -1e-10, -1e-7]]),
             [
-                [
-                    self::BETA * (exp(-1e-15) - 1.0),
-                    self::BETA * (exp(-1e-10) - 1.0),
-                    self::BETA * (exp(-1e-7) - 1.0),
-                ],
+                [self::BETA * (exp(-1e-15) - 1.0), self::BETA * (exp(-1e-10) - 1.0), self::BETA * (exp(-1e-7) - 1.0)],
             ],
             [
-                [
-                    self::BETA * exp(-1e-15),
-                    self::BETA * exp(-1e-10),
-                    self::BETA * exp(-1e-7),
-                ],
+                [self::BETA * exp(-1e-15), self::BETA * exp(-1e-10), self::BETA * exp(-1e-7)],
             ],
         ];