mindspore.nn.layer.activation 源代码

# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""activation"""
from __future__ import absolute_import

import numpy as np

from mindspore import _checkparam as validator
from mindspore._extends import cell_attr_register
from mindspore.common import dtype as mstype
from mindspore.common.parameter import Parameter
from mindspore.common.tensor import Tensor
from mindspore.ops import functional as F
from mindspore.ops import operations as P
from mindspore.ops.operations import nn_ops as NN_OPS
from mindspore.nn.cell import Cell
from mindspore import ops
from mindspore.ops.primitive import _primexpr

__all__ = ['Softmin',
           'Softmax',
           'Softmax2d',
           'LogSoftmax',
           'ReLU',
           'ReLU6',
           'RReLU',
           'SeLU',
           'SiLU',
           'Tanh',
           'Tanhshrink',
           'Hardtanh',
           'GELU',
           'FastGelu',
           'Sigmoid',
           'Softsign',
           'PReLU',
           'get_activation',
           'LeakyReLU',
           'HSigmoid',
           'HSwish',
           'ELU',
           'LogSigmoid',
           'LRN',
           'SoftShrink',
           'HShrink',
           'CELU',
           'Threshold',
           'Mish',
           'GLU'
           ]


[文档]class CELU(Cell): r""" CELU Activation Operator. Applies the continuously differentiable exponential linear units function element-wise. .. math:: \text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1)) For more details, refer to `CELU <https://arxiv.org/abs/1704.07483>`_ . CELU Activation Function Graph: .. image:: ../images/CELU.png :align: center Args: alpha (float): The :math:`\alpha` value for the Celu formulation. Default: ``1.0`` . Inputs: - **x** (Tensor) - The input of CELU. The required dtype is float16 or float32. The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions. Outputs: Tensor, with the same type and shape as the `x`. Raises: TypeError: If `alpha` is not a float. ValueError: If `alpha` has the value of 0. TypeError: If `x` is not a Tensor. TypeError: If the dtype of `x` is neither float16 nor float32. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([-2.0, -1.0, 1.0, 2.0]), mindspore.float32) >>> celu = nn.CELU() >>> output = celu(x) >>> print(output) [-0.86466473 -0.63212055 1. 2. ] """ def __init__(self, alpha=1.0): """Initialize CELU.""" super(CELU, self).__init__() self.celu = P.CeLU(alpha=alpha) def construct(self, x): return self.celu(x)
[文档]class Softmin(Cell): r""" Softmin activation function, which is a two-category function :class:`mindspore.nn.Sigmoid` in the promotion of multi-classification, and the purpose is to show the results of multi-classification in the form of probability. Calculate the value of the exponential function for the elements of the input Tensor on the `axis`, and then normalized to lie in range [0, 1] and sum up to 1. Softmin is defined as: .. math:: \text{softmin}(x_{i}) = \frac{\exp(-x_i)}{\sum_{j=0}^{n-1}\exp(-x_j)}, where :math:`x_{i}` is the :math:`i`-th slice in the given dimension of the input Tensor. Args: axis (Union[int, tuple[int]]): The axis to apply Softmin operation, if the dimension of input `x` is x.ndim, the range of axis is `[-x.ndim, x.ndim)`. -1 means the last dimension. Default: ``-1`` . Inputs: - **x** (Tensor) - Tensor for computing Softmin functions with data type of float16 or float32. Outputs: Tensor, which has the same type and shape as `x` with values in the range [0,1]. Raises: TypeError: If `axis` is neither an int nor a tuple. TypeError: If dtype of `x` is neither float16 nor float32. ValueError: If `axis` is a tuple whose length is less than 1. ValueError: If `axis` is a tuple whose elements are not all in the range [-x.ndim, x.ndim). Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> # axis = -1(default), and the sum of return value is 1.0. >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16) >>> softmin = nn.Softmin() >>> output = softmin(x) >>> print(output) [0.2341 0.636 0.0862 0.01165 0.03168 ] """ def __init__(self, axis=-1): """Initialize Softmin.""" super(Softmin, self).__init__() self.axis = axis def construct(self, x): return ops.function.softmin(x, self.axis)
[文档]class Softmax2d(Cell): r""" Softmax function applied to 2D features data. Applies `Softmax` to each location :math:`(c, h, w)` with an input Tensor of shape :math:`(C, H, W)` . Inputs: - **x** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})` or :math:`(C_{in}, H_{in}, W_{in})`. The input of Softmax with data type of float16 or float32. Outputs: Tensor, which has the same type and shape as `x` with values in the range[0,1]. Raises: TypeError: If dtype of `x` is neither float16 nor float32. ValueError: If `data_format` is neither 'NCHW' nor 'CHW'. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([[[[0.1, 0.2]], [[0.3, 0.4]], [[0.6, 0.5]]]]), mindspore.float32) >>> softmax2d = nn.Softmax2d() >>> output = softmax2d(x) >>> print(output) [[[[0.25838965 0.28001308]] [[0.31559783 0.34200877]] [[0.42601252 0.37797815]]]] """ def __init__(self): """Initialize Softmax2d.""" super(Softmax2d, self).__init__() self.softmax = P.Softmax(axis=-3) self.shape = P.Shape() @staticmethod @_primexpr def _check_input_dim(shape, cls_name): dim = len(shape) if dim not in (3, 4): raise ValueError(f"For '{cls_name}', the in_shape must have 3 or 4 dims, but got {dim}.") def construct(self, x): x_shape = self.shape(x) self._check_input_dim(x_shape, self.cls_name) return self.softmax(x)
[文档]class Softmax(Cell): r""" Softmax activation function, which is a two-category function :class:`mindspore.nn.Sigmoid` in the promotion of multi-classification, the purpose is to show the results of multi-classification in the form of probability. Calculate the value of the exponential function for the elements of the input Tensor on the `axis`, and then normalized to lie in range [0, 1] and sum up to 1. Softmax is defined as: .. math:: \text{softmax}(input_{i}) = \frac{\exp(input_i)}{\sum_{j=0}^{n-1}\exp(input_j)}, where :math:`input_{i}` is the :math:`i`-th slice in the given dimension of the input Tensor. Args: axis (int, optional): The axis to apply Softmax operation, if the dimension of `input` is input.ndim, the range of axis is `[-input.ndim, input.ndim)`, -1 means the last dimension. Default: ``-1`` . Inputs: - **input** (Tensor) - The input of Softmax. Outputs: Tensor, which has the same type and shape as `input` with values in the range[0, 1]. Raises: TypeError: If `axis` is neither an int nor a tuple. ValueError: If `axis` is a tuple whose length is less than 1. ValueError: If `axis` is a tuple whose elements are not all in range `[-input.ndim, input.ndim)`. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> # axis = -1(default), and the sum of return value is 1.0. >>> input = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16) >>> softmax = nn.Softmax() >>> output = softmax(input) >>> print(output) [0.03168 0.01166 0.0861 0.636 0.2341 ] """ def __init__(self, axis=-1): """Initialize Softmax.""" super(Softmax, self).__init__() self.softmax = P.Softmax(axis) def construct(self, input): return self.softmax(input)
[文档]class LogSoftmax(Cell): r""" Applies the LogSoftmax function to n-dimensional input tensor element-wise. The input is transformed by the Softmax function and then by the log function to lie in range[-inf,0). Logsoftmax is defined as: .. math:: \text{logsoftmax}(x_i) = \log \left(\frac{\exp(x_i)}{\sum_{j=0}^{n-1} \exp(x_j)}\right) Args: axis (int): The axis to apply LogSoftmax operation, -1 means the last dimension. Default: ``-1`` . Inputs: - **x** (Tensor) - The input of LogSoftmax, with float16 or float32 data type. Outputs: Tensor, which has the same type and shape as `x` with output values in the range[-inf,0). Raises: TypeError: If `axis` is not an int. TypeError: If dtype of `x` is neither float16 nor float32. ValueError: If `axis` is not in range [-len(x), len(x)). Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32) >>> log_softmax = nn.LogSoftmax() >>> output = log_softmax(x) >>> print(output) [[-5.00672150e+00 -6.72150636e-03 -1.20067215e+01] [-7.00091219e+00 -1.40009127e+01 -9.12250078e-04]] """ def __init__(self, axis=-1): """Initialize LogSoftmax.""" super(LogSoftmax, self).__init__() self.log_softmax = P.LogSoftmax(axis) def construct(self, x): return self.log_softmax(x)
[文档]class ELU(Cell): r""" Applies the exponential linear unit function element-wise. The activation function is defined as: .. math:: E_{i} = \begin{cases} x_i, &\text{if } x_i \geq 0; \cr \alpha * (\exp(x_i) - 1), &\text{otherwise.} \end{cases} where :math:`x_i` represents the element of the input and :math:`\alpha` represents the `alpha` parameter. ELU Activation Function Graph: .. image:: ../images/ELU.png :align: center Args: alpha (float): The alpha value of ELU, the data type is float. Default: ``1.0`` . Only alpha equal to ``1.0`` is supported currently. Inputs: - **input_x** (Tensor) - The input of ELU is a Tensor of any dimension with data type of float16 or float32. Outputs: Tensor, with the same type and shape as the `input_x`. Raises: TypeError: If `alpha` is not a float. TypeError: If dtype of `input_x` is neither float16 nor float32. ValueError: If `alpha` is not equal to 1.0. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float32) >>> elu = nn.ELU() >>> result = elu(x) >>> print(result) [-0.63212055 -0.86466473 0. 2. 1.] """ def __init__(self, alpha=1.0): """Initialize ELU.""" super(ELU, self).__init__() self.elu = P.Elu(alpha) def construct(self, x): return self.elu(x)
[文档]class ReLU(Cell): r""" Applies ReLU (Rectified Linear Unit activation function) element-wise. .. math:: \text{ReLU}(input) = (input)^+ = \max(0, input), It returns element-wise :math:`\max(0, input)`. .. note:: The neurons with the negative output will be suppressed and the active neurons will stay the same. ReLU Activation Function Graph: .. image:: ../images/ReLU.png :align: center Inputs: - **input** (Tensor) - The input of ReLU is a Tensor of any dimension. Outputs: Tensor, with the same type and shape as the `input`. Raises: TypeError: If dtype of `input` is not supported. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import numpy as np >>> import mindspore >>> from mindspore import Tensor, nn >>> input = Tensor(np.array([-1, 2, -3, 2, -1]), mindspore.float16) >>> relu = nn.ReLU() >>> output = relu(input) >>> print(output) [0. 2. 0. 2. 0.] """ def __init__(self): """Initialize ReLU.""" super(ReLU, self).__init__() self.relu = P.ReLU() def construct(self, x): return self.relu(x)
[文档]class ReLU6(Cell): r""" Compute ReLU6 activation function element-wise. ReLU6 is similar to ReLU with a upper limit of 6, which if the inputs are greater than 6, the outputs will be suppressed to 6. It computes element-wise as .. math:: Y = \min(\max(0, x), 6) ReLU6 Activation Function Graph: .. image:: ../images/ReLU6.png :align: center Inputs: - **x** (Tensor) - The input of ReLU6 with data type of float16 or float32 and that is a Tensor of any valid shape. Outputs: Tensor, which has the same type as `x`. Raises: TypeError: If dtype of `x` is neither float16 nor float32. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16) >>> relu6 = nn.ReLU6() >>> output = relu6(x) >>> print(output) [0. 0. 0. 2. 1.] """ def __init__(self): """Initialize ReLU6.""" super(ReLU6, self).__init__() self.relu6 = P.ReLU6() def construct(self, x): return self.relu6(x)
[文档]class LeakyReLU(Cell): r""" Leaky ReLU activation function. The activation function is defined as: .. math:: \text{leaky_relu}(x) = \begin{cases}x, &\text{if } x \geq 0; \cr {\alpha} * x, &\text{otherwise.}\end{cases} where :math:`\alpha` represents the `alpha` parameter. For more details, see `Rectifier Nonlinearities Improve Neural Network Acoustic Models <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`_. LeakyReLU Activation Function Graph: .. image:: ../images/LeakyReLU.png :align: center Args: alpha (Union[int, float]): Slope of the activation function at x < 0. Default: ``0.2`` . Inputs: - **x** (Tensor) - The input of LeakyReLU is a Tensor of any dimension. Outputs: Tensor, has the same type and shape as the `x`. Raises: TypeError: If `alpha` is not a float or an int. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32) >>> leaky_relu = nn.LeakyReLU() >>> output = leaky_relu(x) >>> print(output) [[-0.2 4. -1.6] [ 2. -1. 9. ]] """ def __init__(self, alpha=0.2): """Initialize LeakyReLU.""" super(LeakyReLU, self).__init__() self.alpha = alpha def construct(self, x): out = ops.leaky_relu(x, self.alpha) return out
[文档]class RReLU(Cell): r""" Applies RReLU (Randomized Leaky ReLU activation function) element-wise. The activation function is defined as: .. math:: \text{RReLU}(x_{ji}) = \begin{cases}x_{ji}, &\text{if } x_{ji} \geq 0; \cr {\alpha_{ji}} * x_{ji}, &\text{otherwise.}\end{cases} where :math:`\alpha_{ji}` ~ :math:`U(l, u)`, :math:`l \le u`. Applies the RReLU function elementally, as described in the paper: `Empirical Evaluation of Rectified Activations in Convolution Network <https://arxiv.org/pdf/1505.00853.pdf>`_ . Args: lower (Union[int, float]): Slope of the activation function at x < 0. Default: ``1 / 8`` . upper (Union[int, float]): Slope of the activation function at x < 0. Default: ``1 / 3`` . Inputs: - **x** (Tensor) - The input of RReLU is a Tensor of any dimension. Outputs: Tensor, after RReLU, has the same type and shape as the `x`. Raises: TypeError: If `lower` is not a float or an int. TypeError: If `upper` is not a float or an int. TypeError: If `x` is not a Tensor. TypeError: If `x` is not a Tensor of mindspore.float16 or mindspore.float32. ValueError: If `lower` is greater than upper. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([[-1.0, 4.0], [2.0, 0]]), mindspore.float32) >>> r_relu = nn.RReLU() >>> output = r_relu(x) >>> print(output) [[-0.31465699 4. ] [ 2. 0. ]] """ def __init__(self, lower=1 / 8, upper=1 / 3): super(RReLU, self).__init__() validator.check_value_type('upper', upper, [float, int], self.cls_name) validator.check_value_type('lower', lower, [float, int], self.cls_name) if lower > upper: raise ValueError(f"For {self.cls_name}, the value of 'upper' must be greater than or equal to 'lower', " f"but got upper: {upper}, lower: {lower}. ") self.lower = Tensor(lower, dtype=mstype.float32) self.upper = Tensor(upper, dtype=mstype.float32) self.sign = P.Sign() def construct(self, x): if not isinstance(x, Tensor): raise TypeError(f"For 'rrelu', the input must be a Tensor, but got {type(x)}.") _size = x.shape _dtype = x.dtype sign_matrix = self.sign(x) negative_filter = sign_matrix.clip(None, 0) positive_filter = sign_matrix.clip(0, None) mask = ops.uniform(_size, self.lower, self.upper).astype(_dtype) negative_mask = negative_filter * mask * -1 total_mask = negative_mask + positive_filter out = total_mask * x return out
[文档]class SeLU(Cell): r""" Applies activation function SeLU (Scaled exponential Linear Unit) element-wise. SeLU Activation Function Graph: .. image:: ../images/SeLU.png :align: center Refer to :func:`mindspore.ops.selu` for more details. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> input_x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32) >>> selu = nn.SeLU() >>> output = selu(input_x) >>> print(output) [[-1.1113307 4.202804 -1.7575096] [ 2.101402 -1.7462534 9.456309 ]] """ def __init__(self): """Initialize SeLU""" super(SeLU, self).__init__() self.selu = P.SeLU() def construct(self, input_x): return self.selu(input_x)
[文档]class SiLU(Cell): r""" Applies the silu linear unit function element-wise. .. math:: \text{SiLU}(x) = x * \sigma(x), where :math:`x_i` is an element of the input, :math:`\sigma(x)` is Sigmoid function. .. math:: \text{sigmoid}(x_i) = \frac{1}{1 + \exp(-x_i)}, SiLU Activation Function Graph: .. image:: ../images/SiLU.png :align: center Inputs: - **input** (Tensor) - `input` is :math:`x` in the preceding formula. Input with the data type float16 or float32. Tensor of any dimension. Outputs: Tensor, with the same type and shape as the `input`. Raises: TypeError: If dtype of `input` is neither float16 nor float32. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> input = Tensor(np.array([-1, 2, -3, 2, -1]), mindspore.float16) >>> silu = nn.SiLU() >>> output = silu(input) >>> print(output) [-0.269 1.762 -0.1423 1.762 -0.269] """ def __init__(self): """Initialize SiLU.""" super(SiLU, self).__init__() def construct(self, x): return ops.function.silu(x)
[文档]class Tanh(Cell): r""" Applies the Tanh function element-wise, returns a new tensor with the hyperbolic tangent of the elements of input, The input is a Tensor with any valid shape. Tanh function is defined as: .. math:: tanh(x_i) = \frac{\exp(x_i) - \exp(-x_i)}{\exp(x_i) + \exp(-x_i)} = \frac{\exp(2x_i) - 1}{\exp(2x_i) + 1}, where :math:`x_i` is an element of the input Tensor. Tanh Activation Function Graph: .. image:: ../images/Tanh.png :align: center Inputs: - **x** (Tensor) - Tensor of any dimension, input with data type of float16 or float32. Outputs: Tensor, with the same type and shape as the `x`. Raises: TypeError: If dtype of `x` is neither float16 nor float32. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([1, 2, 3, 2, 1]), mindspore.float16) >>> tanh = nn.Tanh() >>> output = tanh(x) >>> print(output) [0.7617 0.964 0.995 0.964 0.7617] """ def __init__(self): """Initialize Tanh.""" super(Tanh, self).__init__() self.tanh = P.Tanh() def construct(self, x): return self.tanh(x)
[文档]class Tanhshrink(Cell): r""" Applies Tanhshrink activation function element-wise and returns a new tensor. Tanh function is defined as: .. math:: tanhshrink(x_i) =x_i- \frac{\exp(x_i) - \exp(-x_i)}{\exp(x_i) + \exp(-x_i)} = x_i-\frac{\exp(2x_i) - 1}{\exp(2x_i) + 1}, where :math:`x_i` is an element of the input Tensor. Inputs: - **x** (Tensor) - Tensor of any dimension. Outputs: Tensor, with the same shape as the `x`. Raises: TypeError: If `x` is not a Tensor. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore as ms >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([1, 2, 3, 2, 1]), ms.float16) >>> tanhshrink = nn.Tanhshrink() >>> output = tanhshrink(x) >>> print(output) [0.2383 1.036 2.004 1.036 0.2383] """ def __init__(self): """Initialize Tanhshrink.""" super(Tanhshrink, self).__init__() def construct(self, x): return F.tanhshrink(x)
[文档]class Hardtanh(Cell): r""" Applies the Hardtanh function element-wise. The activation function is defined as: .. math:: \text{Hardtanh}(x) = \begin{cases} 1, & \text{ if } x > 1; \\ -1, & \text{ if } x < -1; \\ x, & \text{ otherwise. } \end{cases} Linear region range :math:`[-1, 1]` can be adjusted using `min_val` and `max_val`. Hardtanh Activation Function Graph: .. image:: ../images/Hardtanh.png :align: center Note: On Ascend, data type of float16 might lead to accidental accuracy problem. Args: min_val (Union[int, float]): Minimum value of the linear region range. Default: ``-1.0`` . max_val (Union[int, float]): Maximum value of the linear region range. Default: ``1.0`` . Inputs: - **x** (Tensor) - Input Tensor with data type of float16 or float32. On CPU and Ascend support dimension 0-7D. On GPU support dimension 0-4D. Outputs: Tensor, with the same dtype and shape as `x`. Raises: TypeError: If `x` is not a Tensor. TypeError: If dtype of `x` is neither float16 nor float32. TypeError: If dtype of `min_val` is neither float nor int. TypeError: If dtype of `max_val` is neither float nor int. ValueError: If `min_val` is not less than `max_val`. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16) >>> hardtanh = nn.Hardtanh(min_val=-1.0, max_val=1.0) >>> output = hardtanh(x) >>> print(output) [-1. -1. 0. 1. 1.] """ def __init__(self, min_val=-1.0, max_val=1.0): """Initialize Hardtanh.""" super(Hardtanh, self).__init__() self.min_val = min_val self.max_val = max_val if self.min_val >= self.max_val: raise ValueError(f"For Hardtanh, min_val should be less than max_val," f"but got {self.min_val} and {self.max_val}") def construct(self, x): return F.hardtanh(x, self.min_val, self.max_val)
[文档]class GELU(Cell): r""" Applies GELU function to each element of the input. The input is a Tensor with any valid shape. GELU is defined as: .. math:: GELU(x_i) = x_i*P(X < x_i), where :math:`P` is the cumulative distribution function of standard Gaussian distribution and :math:`x_i` is the element of the input. GELU Activation Function Graph: .. image:: ../images/GELU.png :align: center Args: approximate (bool): Whether to enable approximation. Default: ``True`` . If `approximate` is ``True``, The gaussian error linear activation is: :math:`0.5 * x * (1 + tanh(\sqrt(2 / \pi) * (x + 0.044715 * x^3)))` else, it is: :math:`x * P(X <= x) = 0.5 * x * (1 + erf(x / \sqrt(2)))`, where P(X) ~ N(0, 1). Inputs: - **x** (Tensor) - The input of GELU with data type of float16, float32, or float64. The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions. Outputs: Tensor, with the same type and shape as the `x`. Raises: TypeError: If dtype of `x` is not one of float16, float32, or float64. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32) >>> gelu = nn.GELU() >>> output = gelu(x) >>> print(output) [[-1.5880802e-01 3.9999299e+00 -3.1077917e-21] [ 1.9545976e+00 -2.2918017e-07 9.0000000e+00]] >>> gelu = nn.GELU(approximate=False) >>> # CPU not support "approximate=False", using "approximate=True" instead >>> output = gelu(x) >>> print(output) [[-1.5865526e-01 3.9998732e+00 -0.0000000e+00] [ 1.9544997e+00 -1.4901161e-06 9.0000000e+00]] """ def __init__(self, approximate=True): """Initialize GELU.""" super(GELU, self).__init__() validator.check_bool(approximate, 'approximate', self.cls_name) self.approximate = 'tanh' if not approximate: self.approximate = 'none' def construct(self, x): return ops.gelu(x, approximate=self.approximate)
[文档]class FastGelu(Cell): r""" Applies FastGelu function to each element of the input. The input is a Tensor with any valid shape. FastGelu is defined as: .. math:: FastGelu(x_i) = \frac {x_i} {1 + \exp(-1.702 * \left| x_i \right|)} * \exp(0.851 * (x_i - \left| x_i \right|)) where :math:`x_i` is the element of the input. FastGelu Activation Function Graph: .. image:: ../images/FastGelu.png :align: center Inputs: - **x** (Tensor) - The input of FastGelu with data type of float16 or float32. The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions. Outputs: Tensor, with the same type and shape as the `x`. Raises: TypeError: If dtype of `x` is neither float16 nor float32. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32) >>> fast_gelu = nn.FastGelu() >>> output = fast_gelu(x) >>> print(output) [[-1.5418735e-01 3.9921875e+00 -9.7473649e-06] [ 1.9375000e+00 -1.0052517e-03 8.9824219e+00]] """ def __init__(self): """Initialize FastGelu.""" super(FastGelu, self).__init__() self.fast_gelu = P.FastGeLU() def construct(self, x): return self.fast_gelu(x)
[文档]class Sigmoid(Cell): r""" Applies sigmoid activation function element-wise. Sigmoid function is defined as: .. math:: \text{sigmoid}(x_i) = \frac{1}{1 + \exp(-x_i)}, where :math:`x_i` is the element of `x`. Sigmoid Activation Function Graph: .. image:: ../images/Sigmoid.png :align: center Inputs: - **input** (Tensor) - `input` is :math:`x` in the preceding formula. Tensor of any dimension, the data type is float16, float32, float64, complex64 or complex128. Outputs: Tensor, with the same type and shape as the `input`. Raises: TypeError: If dtype of `input` is not float16, float32, float64, complex64 or complex128. TypeError: If `input` is not a Tensor. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> input = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16) >>> sigmoid = nn.Sigmoid() >>> output = sigmoid(input) >>> print(output) [0.2688 0.11914 0.5 0.881 0.7305 ] """ def __init__(self): """Initialize Sigmoid.""" super(Sigmoid, self).__init__() self.sigmoid = P.Sigmoid() def construct(self, x): return self.sigmoid(x)
[文档]class Softsign(Cell): r""" Applies softsign activation function element-wise. Softsign Activation Function Graph: .. image:: ../images/Softsign.png :align: center Refer to :func:`mindspore.ops.softsign` for more details. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([0, -1, 2, 30, -30]), mindspore.float32) >>> softsign = nn.Softsign() >>> output = softsign(x) >>> print(output) [ 0. -0.5 0.6666667 0.9677419 -0.9677419] """ def __init__(self): """Initialize Softsign.""" super(Softsign, self).__init__() self.softsign = P.Softsign() def construct(self, x): return self.softsign(x)
[文档]class PReLU(Cell): r""" Applies PReLU activation function element-wise. PReLU is defined as: .. math:: PReLU(x_i)= \max(0, x_i) + w * \min(0, x_i), where :math:`x_i` is an element of an channel of the input. Here :math:`w` is a learnable parameter with a default initial value 0.25. Parameter :math:`w` has dimensionality of the argument channel. If called without argument channel, a single parameter :math:`w` will be shared across all channels. PReLU Activation Function Graph: .. image:: ../images/PReLU.png :align: center Args: channel (int): The elements number of parameter :math:`w`. It could be an int, and the value is 1 or the channels number of input tensor `x`. Default: ``1`` . w (Union[float, list, Tensor]): The initial value of parameter. It could be a float, a float list or a tensor has the same dtype as the input tensor `x`. Default: ``0.25`` . Inputs: - **x** (Tensor) - The input of PReLU with data type of float16 or float32. The shape is :math:`(N, *)` where :math:`*` means, any number of additional dimensions. Outputs: Tensor, with the same dtype and shape as the `x`. Raises: TypeError: If `channel` is not an int. TypeError: If `w` is not one of a float, a float list, a float Tensor. TypeError: If dtype of `x` is neither float16 nor float32. ValueError: If the `x` is a 0-D or 1-D Tensor on Ascend. ValueError: If `channel` is less than 1. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([[[[0.1, 0.6], [0.9, 0.9]]]]), mindspore.float32) >>> prelu = nn.PReLU() >>> output = prelu(x) >>> print(output) [[[[0.1 0.6] [0.9 0.9]]]] """ @cell_attr_register(attrs="") def __init__(self, channel=1, w=0.25): """Initialize PReLU.""" super(PReLU, self).__init__() validator.check_positive_int(channel, 'channel', self.cls_name) if isinstance(w, (float, np.float32)): tmp = np.empty((channel,), dtype=np.float32) tmp.fill(w) w = Tensor(tmp, dtype=mstype.float32) elif isinstance(w, list): if len(w) != channel: raise ValueError(f"For '{self.cls_name}', the length of 'w' must be equal to the 'channel' when " f"the 'w' is a list, but got the length of 'w': {len(w)}, the 'channel': {channel}.") for i in w: if not isinstance(i, (float, np.float32)): raise ValueError(f"For '{self.cls_name}', all elements in 'w' must be " f"float when the 'w' is a list, but got {i}.") w = Tensor(w, dtype=mstype.float32) elif isinstance(w, Tensor): if w.dtype not in (mstype.float16, mstype.float32): raise ValueError(f"For '{self.cls_name}', the dtype of 'w' must be float16 or " f"float32 when the 'w' is a tensor, but got {w.dtype}.") if len(w.shape) != 1 or w.shape[0] != channel: raise ValueError(f"For '{self.cls_name}', the dimension of 'w' must be 1, and the elements number " f"should be equal to the 'channel' when the 'w' is a tensor, " f"but got 'w' shape {w.shape}, the 'channel' {channel}.") else: raise TypeError(f"For '{self.cls_name}', the 'w' only supported float, list and tensor, " f"but got {type(w).__name__}.") self.w = Parameter(w, name='a') self.prelu = P.PReLU() def construct(self, x): return self.prelu(x, F.cast(self.w, x.dtype))
[文档]class HSwish(Cell): r""" Applies hswish-type activation element-wise. Hard swish is defined as: .. math:: \text{hswish}(x_{i}) = x_{i} * \frac{ReLU6(x_{i} + 3)}{6}, HSwish Activation Function Graph: .. image:: ../images/HSwish.png :align: center Inputs: - **x** (Tensor) - The input of HSwish, data type must be float16 or float32. The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions. Outputs: Tensor, with the same type and shape as the `x`. Raises: TypeError: If dtype of `x` is neither float16 nor float32. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16) >>> hswish = nn.HSwish() >>> result = hswish(x) >>> print(result) [-0.3333 -0.3333 0. 1.667 0.6665] """ def __init__(self): """Initialize HSwish.""" super(HSwish, self).__init__() self.hswish = P.HSwish() def construct(self, x): return self.hswish(x)
[文档]class HSigmoid(Cell): r""" Applies Hard sigmoid activation function element-wise. Hard sigmoid is defined as: .. math:: \text{hsigmoid}(x_{i}) = \max(0, \min(1, \frac{x_{i} + 3}{6})), HSigmoid Activation Function Graph: .. image:: ../images/HSigmoid.png :align: center Inputs: - **input_x** (Tensor) - The input of HSigmoid. Tensor of any dimension. Outputs: Tensor, with the same type and shape as the `input_x`. Raises: TypeError: If `input_x` is not a Tensor. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16) >>> hsigmoid = nn.HSigmoid() >>> result = hsigmoid(x) >>> print(result) [0.3333 0.1666 0.5 0.8335 0.6665] """ def __init__(self): """Initialize HSigmoid.""" super(HSigmoid, self).__init__() self.hsigmoid = P.HSigmoid() def construct(self, input_x): return self.hsigmoid(input_x)
[文档]class LogSigmoid(Cell): r""" Applies logsigmoid activation element-wise. The input is a Tensor with any valid shape. Logsigmoid is defined as: .. math:: \text{logsigmoid}(x_{i}) = \log(\frac{1}{1 + \exp(-x_i)}), where :math:`x_{i}` is the element of the input. LogSigmoid Activation Function Graph: .. image:: ../images/LogSigmoid.png :align: center Inputs: - **x** (Tensor) - The input of LogSigmoid with data type of float16 or float32. The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions. Outputs: Tensor, with the same type and shape as the `x`. Raises: TypeError: If dtype of `x` is neither float16 nor float32. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> net = nn.LogSigmoid() >>> x = Tensor(np.array([1.0, 2.0, 3.0]), mindspore.float32) >>> output = net(x) >>> print(output) [-0.31326166 -0.12692806 -0.04858734] """ def __init__(self): """Initialize LogSigmoid.""" super(LogSigmoid, self).__init__() self.mul = P.Mul() self.exp = P.Exp() self.add = P.Add() self.rec = P.Reciprocal() self.log = P.Log() def construct(self, input_x): neg_input = self.mul(input_x, -1) exp_neg_input = self.exp(neg_input) exp_neg_input_1 = self.add(exp_neg_input, 1) rec_exp_neg_input_1 = self.rec(exp_neg_input_1) ret = self.log(rec_exp_neg_input_1) return ret
[文档]class LRN(Cell): r""" Local Response Normalization. .. warning:: LRN is deprecated on Ascend due to potential accuracy problem. It's recommended to use other normalization methods, e.g. :class:`mindspore.nn.BatchNorm1d` , :class:`mindspore.nn.BatchNorm2d` , :class:`mindspore.nn.BatchNorm3d`. Refer to :func:`mindspore.ops.lrn` for more details. Supported Platforms: ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> input_x = Tensor(np.array([[[[0.1], [0.2]], ... [[0.3], [0.4]]]]), mindspore.float32) >>> output = nn.LRN()(input_x) >>> print(output) [[[[0.09534626] [0.1825742 ]] [[0.2860388 ] [0.3651484 ]]]] """ def __init__(self, depth_radius=5, bias=1.0, alpha=1.0, beta=0.5, norm_region="ACROSS_CHANNELS"): """Initialize LRN.""" super(LRN, self).__init__() self.lrn_op = NN_OPS.LRN(depth_radius, bias, alpha, beta, norm_region) def construct(self, input_x): return self.lrn_op(input_x)
[文档]class SoftShrink(Cell): r""" Applies the SoftShrink function element-wise. .. math:: \text{SoftShrink}(x) = \begin{cases} x - \lambda, & \text{ if } x > \lambda \\ x + \lambda, & \text{ if } x < -\lambda \\ 0, & \text{ otherwise } \end{cases} SoftShrink Activation Function Graph: .. image:: ../images/Softshrink.png :align: center Args: lambd (float): the :math:`\lambda` must be no less than zero for the SoftShrink formulation. Default: ``0.5`` . Inputs: - **input_x** (Tensor) - The input of SoftShrink with data type of float16 or float32. Any number of additional dimensions. Outputs: Tensor, has the same shape and data type as `input_x`. Raises: TypeError: If lambd is not a float. TypeError: If input_x is not a Tensor. TypeError: If dtype of input_x is neither float16 nor float32. ValueError: If lambd is less than 0. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> input_x = Tensor(np.array([[ 0.5297, 0.7871, 1.1754], [ 0.7836, 0.6218, -1.1542]]), mindspore.float16) >>> softshrink = nn.SoftShrink() >>> output = softshrink(input_x) >>> print(output) [[ 0.02979 0.287 0.676 ] [ 0.2837 0.1216 -0.6543 ]] """ def __init__(self, lambd=0.5): super(SoftShrink, self).__init__() self.softshrink = P.SoftShrink(lambd) def construct(self, input_x): output = self.softshrink(input_x) return output
[文档]class HShrink(Cell): r""" Applies Hard Shrink activation function element-wise. The formula is defined as follows: .. math:: \text{HardShrink}(x) = \begin{cases} x, & \text{ if } x > \lambda \\ x, & \text{ if } x < -\lambda \\ 0, & \text{ otherwise } \end{cases} HShrink Activation Function Graph: .. image:: ../images/HShrink.png :align: center Args: lambd (float): The threshold :math:`\lambda` defined by the Hard Shrink formula. Default: ``0.5`` . Inputs: - **input_x** (Tensor) - The input of Hard Shrink with data type of float16 or float32. Outputs: Tensor, the same shape and data type as the input. Raises: TypeError: If `lambd` is not a float. TypeError: If dtype of `input_x` is neither float16 nor float32. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> input_x = Tensor(np.array([[ 0.5, 1, 2.0], [0.0533,0.0776,-2.1233]]), mindspore.float32) >>> hshrink = nn.HShrink() >>> output = hshrink(input_x) >>> print(output) [[ 0. 1. 2. ] [ 0. 0. -2.1233]] """ def __init__(self, lambd=0.5): super(HShrink, self).__init__() self.hshrink = P.HShrink(lambd) def construct(self, input_x): return self.hshrink(input_x)
[文档]class Threshold(Cell): r""" Thresholds each element of the input Tensor. The formula is defined as follows: .. math:: y = \begin{cases} x, &\text{ if } x > \text{threshold} \\ \text{value}, &\text{ otherwise } \end{cases} Args: threshold (Union[int, float]): The value to threshold at. value (Union[int, float]): The value to replace with when element is less than threshold. Inputs: - **input_x** (Tensor) - The input of Threshold with data type of float16 or float32. Outputs: Tensor, the same shape and data type as the `input_x`. Raises: TypeError: If `threshold` is not a float or an int. TypeError: If `value` is not a float or an int. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> m = nn.Threshold(0.1, 20) >>> inputs = Tensor([0.1, 0.2, 0.3], mindspore.float32) >>> outputs = m(inputs) >>> print(outputs) [ 20.0 0.2 0.3] """ def __init__(self, threshold, value): """Initialize Threshold.""" super(Threshold, self).__init__() self.threshold = threshold self.value = value def construct(self, input_x): return F.threshold(input_x, self.threshold, self.value)
[文档]class Mish(Cell): r""" Computes MISH (A Self Regularized Non-Monotonic Neural Activation Function) of input tensors element-wise. Refer to :func:`mindspore.ops.mish` for more details. Mish Activation Function Graph: .. image:: ../images/Mish.png :align: center Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore >>> from mindspore import Tensor, nn >>> import numpy as np >>> x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32) >>> mish = nn.Mish() >>> output = mish(x) >>> print(output) [[-3.03401530e-01 3.99741292e+00 -2.68321624e-03] [ 1.94395900e+00 -3.35762873e-02 9.00000000e+00]] """ def __init__(self): """Initialize Mish.""" super().__init__("Mish") self.mish = NN_OPS.Mish() def construct(self, input_x): return self.mish(input_x)
[文档]class GLU(Cell): r""" The gated linear unit function. .. math:: {GLU}(a, b)= a \otimes \sigma(b) where :math:`a` is the first half of the input matrices and :math:`b` is the second half. Here :math:`\sigma` is the sigmoid function, and :math:`\otimes` is the Hadamard product. Args: axis (int): the axis to split the input. Default: ``-1`` , the last axis in `x`. Inputs: - **x** (Tensor) - :math:`(\ast_1, N, \ast_2)` where `*` means, any number of additional dimensions. Outputs: Tensor, the same dtype as the `x`, with the shape :math:`(\ast_1, M, \ast_2)` where :math:`M=N/2`. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore as ms >>> m = ms.nn.GLU() >>> input = ms.Tensor([[0.1,0.2,0.3,0.4],[0.5,0.6,0.7,0.8]]) >>> output = m(input) >>> print(output) [[0.05744425 0.11973753] [0.33409387 0.41398472]] """ def __init__(self, axis=-1): """Initialize GLU.""" super().__init__("GLU") self.dim = axis self.spilt = P.Split(axis=axis, output_num=2) self.sigmoid = P.Sigmoid() def construct(self, x): x1, x2 = self.spilt(x) x2 = self.sigmoid(x2) return x1 * x2
_activation = { 'softmin': Softmin, 'softmax': Softmax, 'softmax2d': Softmax2d, 'logsoftmax': LogSoftmax, 'relu': ReLU, 'relu6': ReLU6, 'rrelu': RReLU, 'silu': SiLU, 'tanh': Tanh, 'tanhshrink': Tanhshrink, 'hardtanh': Hardtanh, 'gelu': GELU, 'fast_gelu': FastGelu, 'elu': ELU, 'sigmoid': Sigmoid, 'softsign': Softsign, 'prelu': PReLU, 'leakyrelu': LeakyReLU, 'hswish': HSwish, 'hsigmoid': HSigmoid, 'logsigmoid': LogSigmoid, 'softshrink': SoftShrink, 'hshrink': HShrink, 'threshold': Threshold, 'mish': Mish, }
[文档]def get_activation(name, prim_name=None): """ Gets the activation function. Args: name (str): The name of the activation function. prim_name (Union[str, None]): The name of primitive. Default: ``None`` . Returns: Function, the activation function. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` Examples: >>> import mindspore.nn as nn >>> sigmoid = nn.get_activation('sigmoid') >>> print(sigmoid) Sigmoid<> """ msg_prefix = f"For '{prim_name}', the" if prim_name else "The" if name is None: return None if name not in _activation: raise KeyError(f"{msg_prefix} 'name' must be in {list(_activation.keys())}, but got {name}.") return _activation[name]()