From 08d394126c9d46fc9d227a0dc1e343ad1fa70679 Mon Sep 17 00:00:00 2001 From: Kausthub Kannan Date: Sun, 8 Oct 2023 21:18:22 +0530 Subject: [PATCH] Changed Mish Activation Function to use Softplus (#10111) --- neural_network/activation_functions/mish.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neural_network/activation_functions/mish.py b/neural_network/activation_functions/mish.py index e4f98307f..e51655df8 100644 --- a/neural_network/activation_functions/mish.py +++ b/neural_network/activation_functions/mish.py @@ -7,6 +7,7 @@ https://en.wikipedia.org/wiki/Rectifier_(neural_networks)#Mish """ import numpy as np +from softplus import softplus def mish(vector: np.ndarray) -> np.ndarray: @@ -30,7 +31,7 @@ def mish(vector: np.ndarray) -> np.ndarray: array([-0.00092952, -0.15113318, 0.33152014, -0.04745745]) """ - return vector * np.tanh(np.log(1 + np.exp(vector))) + return vector * np.tanh(softplus(vector)) if __name__ == "__main__":